crawlee-one 2.0.0 → 2.0.2

package/dist/cjs/lib/io/pushRequests.js.map

@@ -1 +1 @@
-{"version":3,"file":"pushRequests.js","sourceRoot":"","sources":["../../../../src/lib/io/pushRequests.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,qCAA4D;AAK5D,iDAAgD;AAChD,iDAAyD;AAoCzD,MAAM,aAAa,GAAG,CACpB,OAAY,EACZ,QAAgB,EAChB,OAAmE,EACnE,EAAE;IACF,MAAM,EAAE,cAAc,EAAE,GAAG,EAAE,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC;IAE9C,MAAM,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IAErE,MAAM,WAAW,GAAG,IAAA,sCAAuB,EAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAE/D,uDAAuD;IACvD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,CAAC;IAC1C,IAAI,MAAM,EAAE;QACV,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,CAAC,iBAAiB,SAAS,sBAAsB,QAAQ,cAAc,OAAO,CAAC,MAAM,6BAA6B,CAAC,CAAC;QAChI,OAAO,EAAE,CAAC;KACX,CAAC,kBAAkB;IAEpB,8EAA8E;IAC9E,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAC/D,IAAI,aAAa,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE;QAC3C,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,CAAC,iBAAiB,SAAS,sBAAsB,QAAQ,cAAc,OAAO,CAAC,MAAM,6BAA6B,CAAC,CAAC;QAChI,OAAO,EAAE,CAAC;KACX,CAAC,kBAAkB;IAEpB,OAAO,aAAa,CAAC;AACvB,CAAC,CAAA,CAAC;AAEF;;;;;;GAMG;AACI,MAAM,YAAY,GAAG,CAC1B,cAAuB,EACvB,OAAgC,EAChC,EAAE;IACF,MAAM,EACJ,EAAE,GAAG,eAAuB,EAC5B,GAAG,GAAG,IAAI,aAAG,EAAE,EACf,QAAQ,EACR,SAAS,EACT,MAAM,EACN,cAAc,EACd,YAAY,GACb,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC;IAElB,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IACpF,MAAM,KAAK,GACT,QAAQ,IAAI,IAAI;QACd,CAAC,CAAC,MAAM,aAAa,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,CAAC;QACvE,CAAC,CAAC,SAAS,CAAC;IAEhB,GAAG,CAAC,KAAK,CAAC,qBAAqB,KAAK,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC,kBAAkB;IAEpF,MAAM,aAAa,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAO,UAAU,EAAE,IAAI,EAAE,EAAE;QAClE,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC;QAE7B,MAAM,eAAe,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACjE,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEnE,IAAI,YAAY;YAAE,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAE5C,OAAO,GAAG,CAAC;IACb,CAAC,CAAA,EAAE,OAAO,CAAC,OAAO,CAAC,EAAe,CAAC,CAAC,CAAC;IAErC,wCAAwC;IACxC,GAAG,CAAC,IAAI,CAAC,WAAW,aAAa,CAAC,MAAM,oBAAoB,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;IAC3D,MAAM,QAAQ,CAAC,WAAW,CAAC,aAAsB,EAAE,YAAY,CAAC,CAAC;IACjE,GAAG,CAAC,IAAI,CAAC,gBAAgB,aAAa,CAAC,MAAM,oBAAoB,CAAC,CAAC;IAEnE,OAAO,aAAa,CAAC;AACvB,CAAC,CAAA,CAAC;AAxCW,QAAA,YAAY,gBAwCvB","sourcesContent":["import { Log, RequestQueueOperationOptions } from 'crawlee';\n\nimport type { MaybePromise } from '../../utils/types';\nimport type { CrawlerUrl } from '../../types';\nimport type { CrawleeOneIO } from '../integrations/types';\nimport { apifyIO } from '../integrations/apify';\nimport { requestQueueSizeMonitor } from './requestQueue';\n\nexport interface PushRequestsOptions<\n  T extends Exclude<CrawlerUrl, string> = Exclude<CrawlerUrl, string>\n> {\n  io?: CrawleeOneIO<any, any>;\n  log?: Log;\n  /**\n   * If set, only at most this many requests will be added to the RequestQueue.\n   *\n   * The count is determined from the RequestQueue that's used for the crawler run.\n   *\n   * This means that if `maxCount` is set to 50, but the\n   * associated RequestQueue already handled 40 requests, then only 10 new requests\n   * will be processed.\n   */\n  maxCount?: number;\n  /**\n   * Option to freely transform a request before pushing it to the RequestQueue.\n   *\n   * This serves mainly to allow users to transform the requests from actor input UI.\n   */\n  transform?: (req: T) => MaybePromise<T>;\n  /**\n   * Option to filter a request before pushing it to the RequestQueue.\n   *\n   * This serves mainly to allow users to filter the requests from actor input UI.\n   */\n  filter?: (req: T) => MaybePromise<unknown>;\n  /** ID of the RequestQueue to which the data should be pushed */\n  requestQueueId?: string;\n\n  // Pass-through options\n  queueOptions?: RequestQueueOperationOptions;\n}\n\nconst shortenToSize = async <T>(\n  entries: T[],\n  maxCount: number,\n  options?: { io?: CrawleeOneIO; requestQueueId?: string; log?: Log }\n) => {\n  const { requestQueueId, log } = options ?? {};\n\n  const queueName = requestQueueId ? `\"${requestQueueId}\"` : 'DEFAULT';\n\n  const sizeMonitor = requestQueueSizeMonitor(maxCount, options);\n\n  // Ignore incoming entries if the queue is already full\n  const isFull = await sizeMonitor.isFull();\n  if (isFull) {\n    log?.warning(`RequestQueue (${queueName}) is already full (${maxCount} entries), ${entries.length} entries will be discarded.`);\n    return [];\n  } // prettier-ignore\n\n  // Show warning when only part of the incoming requests made it into the queue\n  const slicedEntries = await sizeMonitor.shortenToSize(entries);\n  if (slicedEntries.length !== entries.length) {\n    log?.warning(`RequestQueue (${queueName}) has become full (${maxCount} entries), ${entries.length} entries will be discarded.`);\n    return [];\n  } // prettier-ignore\n\n  return slicedEntries;\n};\n\n/**\n * Similar to `Actor.openRequestQueue().addRequests`, but with extra features:\n *\n * - Data can be sent elsewhere, not just to Apify. This is set by the `io` options. By default data is sent using Apify (cloud/local).\n * - Limit the max size of the RequestQueue. No requests are added when RequestQueue is at or above the limit.\n * - Transform and filter requests. Requests that did not pass the filter are not added to the RequestQueue.\n */\nexport const pushRequests = async <T extends Exclude<CrawlerUrl, string>>(\n  oneOrManyItems: T | T[],\n  options?: PushRequestsOptions<T>\n) => {\n  const {\n    io = apifyIO as CrawleeOneIO,\n    log = new Log(),\n    maxCount,\n    transform,\n    filter,\n    requestQueueId,\n    queueOptions,\n  } = options ?? {};\n\n  const manyItems = Array.isArray(oneOrManyItems) ? oneOrManyItems : [oneOrManyItems];\n  const items =\n    maxCount != null\n      ? await shortenToSize(manyItems, maxCount, { io, requestQueueId, log })\n      : manyItems;\n\n  log.debug(`Preparing to push ${items.length} requests to queue`); // prettier-ignore\n\n  const adjustedItems = await items.reduce(async (aggPromise, item) => {\n    const agg = await aggPromise;\n\n    const transformedItem = transform ? await transform(item) : item;\n    const passedFilter = filter ? await filter(transformedItem) : true;\n\n    if (passedFilter) agg.push(transformedItem);\n\n    return agg;\n  }, Promise.resolve([] as unknown[]));\n\n  // Push requests to primary RequestQueue\n  log.info(`Pushing ${adjustedItems.length} requests to queue`);\n  const reqQueue = await io.openRequestQueue(requestQueueId);\n  await reqQueue.addRequests(adjustedItems as any[], queueOptions);\n  log.info(`Done pushing ${adjustedItems.length} requests to queue`);\n\n  return adjustedItems;\n};\n"]}
+{"version":3,"file":"pushRequests.js","sourceRoot":"","sources":["../../../../src/lib/io/pushRequests.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,qCAA4D;AAK5D,iDAAgD;AAChD,iDAAyD;AAoCzD,MAAM,aAAa,GAAG,CACpB,OAAY,EACZ,QAAgB,EAChB,OAAmE,EACnE,EAAE;IACF,MAAM,EAAE,cAAc,EAAE,GAAG,EAAE,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC;IAE9C,MAAM,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IAErE,MAAM,WAAW,GAAG,IAAA,sCAAuB,EAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAE/D,uDAAuD;IACvD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,CAAC;IAC1C,IAAI,MAAM,EAAE,CAAC;QACX,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,CAAC,iBAAiB,SAAS,sBAAsB,QAAQ,cAAc,OAAO,CAAC,MAAM,6BAA6B,CAAC,CAAC;QAChI,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,kBAAkB;IAEpB,8EAA8E;IAC9E,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAC/D,IAAI,aAAa,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;QAC5C,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,CAAC,iBAAiB,SAAS,sBAAsB,QAAQ,cAAc,OAAO,CAAC,MAAM,6BAA6B,CAAC,CAAC;QAChI,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,kBAAkB;IAEpB,OAAO,aAAa,CAAC;AACvB,CAAC,CAAA,CAAC;AAEF;;;;;;GAMG;AACI,MAAM,YAAY,GAAG,CAC1B,cAAuB,EACvB,OAAgC,EAChC,EAAE;IACF,MAAM,EACJ,EAAE,GAAG,eAAuB,EAC5B,GAAG,GAAG,IAAI,aAAG,EAAE,EACf,QAAQ,EACR,SAAS,EACT,MAAM,EACN,cAAc,EACd,YAAY,GACb,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC;IAElB,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IACpF,MAAM,KAAK,GACT,QAAQ,IAAI,IAAI;QACd,CAAC,CAAC,MAAM,aAAa,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,CAAC;QACvE,CAAC,CAAC,SAAS,CAAC;IAEhB,GAAG,CAAC,KAAK,CAAC,qBAAqB,KAAK,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC,kBAAkB;IAEpF,MAAM,aAAa,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAO,UAAU,EAAE,IAAI,EAAE,EAAE;QAClE,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC;QAE7B,MAAM,eAAe,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACjE,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEnE,IAAI,YAAY;YAAE,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAE5C,OAAO,GAAG,CAAC;IACb,CAAC,CAAA,EAAE,OAAO,CAAC,OAAO,CAAC,EAAe,CAAC,CAAC,CAAC;IAErC,wCAAwC;IACxC,GAAG,CAAC,IAAI,CAAC,WAAW,aAAa,CAAC,MAAM,oBAAoB,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;IAC3D,MAAM,QAAQ,CAAC,WAAW,CAAC,aAAsB,EAAE,YAAY,CAAC,CAAC;IACjE,GAAG,CAAC,IAAI,CAAC,gBAAgB,aAAa,CAAC,MAAM,oBAAoB,CAAC,CAAC;IAEnE,OAAO,aAAa,CAAC;AACvB,CAAC,CAAA,CAAC;AAxCW,QAAA,YAAY,gBAwCvB","sourcesContent":["import { Log, RequestQueueOperationOptions } from 'crawlee';\n\nimport type { MaybePromise } from '../../utils/types';\nimport type { CrawlerUrl } from '../../types';\nimport type { CrawleeOneIO } from '../integrations/types';\nimport { apifyIO } from '../integrations/apify';\nimport { requestQueueSizeMonitor } from './requestQueue';\n\nexport interface PushRequestsOptions<\n  T extends Exclude<CrawlerUrl, string> = Exclude<CrawlerUrl, string>\n> {\n  io?: CrawleeOneIO<any, any>;\n  log?: Log;\n  /**\n   * If set, only at most this many requests will be added to the RequestQueue.\n   *\n   * The count is determined from the RequestQueue that's used for the crawler run.\n   *\n   * This means that if `maxCount` is set to 50, but the\n   * associated RequestQueue already handled 40 requests, then only 10 new requests\n   * will be processed.\n   */\n  maxCount?: number;\n  /**\n   * Option to freely transform a request before pushing it to the RequestQueue.\n   *\n   * This serves mainly to allow users to transform the requests from actor input UI.\n   */\n  transform?: (req: T) => MaybePromise<T>;\n  /**\n   * Option to filter a request before pushing it to the RequestQueue.\n   *\n   * This serves mainly to allow users to filter the requests from actor input UI.\n   */\n  filter?: (req: T) => MaybePromise<unknown>;\n  /** ID of the RequestQueue to which the data should be pushed */\n  requestQueueId?: string;\n\n  // Pass-through options\n  queueOptions?: RequestQueueOperationOptions;\n}\n\nconst shortenToSize = async <T>(\n  entries: T[],\n  maxCount: number,\n  options?: { io?: CrawleeOneIO; requestQueueId?: string; log?: Log }\n) => {\n  const { requestQueueId, log } = options ?? {};\n\n  const queueName = requestQueueId ? `\"${requestQueueId}\"` : 'DEFAULT';\n\n  const sizeMonitor = requestQueueSizeMonitor(maxCount, options);\n\n  // Ignore incoming entries if the queue is already full\n  const isFull = await sizeMonitor.isFull();\n  if (isFull) {\n    log?.warning(`RequestQueue (${queueName}) is already full (${maxCount} entries), ${entries.length} entries will be discarded.`);\n    return [];\n  } // prettier-ignore\n\n  // Show warning when only part of the incoming requests made it into the queue\n  const slicedEntries = await sizeMonitor.shortenToSize(entries);\n  if (slicedEntries.length !== entries.length) {\n    log?.warning(`RequestQueue (${queueName}) has become full (${maxCount} entries), ${entries.length} entries will be discarded.`);\n    return [];\n  } // prettier-ignore\n\n  return slicedEntries;\n};\n\n/**\n * Similar to `Actor.openRequestQueue().addRequests`, but with extra features:\n *\n * - Data can be sent elsewhere, not just to Apify. This is set by the `io` options. By default data is sent using Apify (cloud/local).\n * - Limit the max size of the RequestQueue. No requests are added when RequestQueue is at or above the limit.\n * - Transform and filter requests. Requests that did not pass the filter are not added to the RequestQueue.\n */\nexport const pushRequests = async <T extends Exclude<CrawlerUrl, string>>(\n  oneOrManyItems: T | T[],\n  options?: PushRequestsOptions<T>\n) => {\n  const {\n    io = apifyIO as CrawleeOneIO,\n    log = new Log(),\n    maxCount,\n    transform,\n    filter,\n    requestQueueId,\n    queueOptions,\n  } = options ?? {};\n\n  const manyItems = Array.isArray(oneOrManyItems) ? oneOrManyItems : [oneOrManyItems];\n  const items =\n    maxCount != null\n      ? await shortenToSize(manyItems, maxCount, { io, requestQueueId, log })\n      : manyItems;\n\n  log.debug(`Preparing to push ${items.length} requests to queue`); // prettier-ignore\n\n  const adjustedItems = await items.reduce(async (aggPromise, item) => {\n    const agg = await aggPromise;\n\n    const transformedItem = transform ? await transform(item) : item;\n    const passedFilter = filter ? await filter(transformedItem) : true;\n\n    if (passedFilter) agg.push(transformedItem);\n\n    return agg;\n  }, Promise.resolve([] as unknown[]));\n\n  // Push requests to primary RequestQueue\n  log.info(`Pushing ${adjustedItems.length} requests to queue`);\n  const reqQueue = await io.openRequestQueue(requestQueueId);\n  await reqQueue.addRequests(adjustedItems as any[], queueOptions);\n  log.info(`Done pushing ${adjustedItems.length} requests to queue`);\n\n  return adjustedItems;\n};\n"]}

package/dist/cjs/lib/log.d.ts

@@ -36,4 +36,4 @@ export declare const logLevelToCrawlee: Record<LogLevel, CrawleeLogLevel>;
  * });
  * ```
  */
-export declare const logLevelHandlerWrapper: <T extends CrawleeOneCtx<import("crawlee").CrawlingContext<import("crawlee").BasicCrawler<import("crawlee").BasicCrawlingContext<import("crawlee").Dictionary>> | import("crawlee").PuppeteerCrawler | import("crawlee").PlaywrightCrawler | import("crawlee").JSDOMCrawler | import("crawlee").CheerioCrawler | import("crawlee").HttpCrawler<import("crawlee").InternalHttpCrawlingContext<any, any, import("crawlee").HttpCrawler<any>>>, import("crawlee").Dictionary>, string, Record<string, any>, import("..").CrawleeOneIO<object, object, object>, import("..").CrawleeOneTelemetry<any, any>>, RouterCtx extends Record<string, any> = CrawleeOneRouteCtx<T>>(logLevel: LogLevel) => CrawleeOneRouteWrapper<T, RouterCtx>;
+export declare const logLevelHandlerWrapper: <T extends CrawleeOneCtx<import("crawlee").CrawlingContext<import("crawlee").JSDOMCrawler | import("crawlee").CheerioCrawler | import("crawlee").PlaywrightCrawler | import("crawlee").PuppeteerCrawler | import("crawlee").BasicCrawler<import("crawlee").BasicCrawlingContext<import("crawlee").Dictionary>> | import("crawlee").HttpCrawler<import("crawlee").InternalHttpCrawlingContext<any, any, import("crawlee").HttpCrawler<any>>>, import("crawlee").Dictionary>, string, Record<string, any>, import("..").CrawleeOneIO<object, object, object>, import("..").CrawleeOneTelemetry<any, any>>, RouterCtx extends Record<string, any> = CrawleeOneRouteCtx<T>>(logLevel: LogLevel) => CrawleeOneRouteWrapper<T, RouterCtx>;

package/dist/cjs/lib/migrate/localMigrator.js.map

@@ -1 +1 @@
-{"version":3,"file":"localMigrator.js","sourceRoot":"","sources":["../../../../src/lib/migrate/localMigrator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,gDAAwB;AACxB,oDAA4B;AAC5B,+BAA4B;AAC5B,+CAA2C;AAIpC,MAAM,mBAAmB,GAAG,CAAC,EAClC,aAAa,EACb,SAAS,GAAG,KAAK,EACjB,SAAS,GAAG,GAAG,GAOhB,EAAE,EAAE;IACH,MAAM,+BAA+B,GAAG,CAAO,OAAe,EAAmB,EAAE;QACjF,mDAAmD;QACnD,MAAM,KAAK,GAAG,MAAM,IAAA,WAAI,EAAC,cAAI,CAAC,IAAI,CAAC,aAAa,EAAE,GAAG,OAAO,GAAG,SAAS,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC;QAC1F,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YACjB,MAAM,KAAK,CAAC,sCAAsC,OAAO,GAAG,CAAC,CAAC;SAC/D;QACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;YACpB,MAAM,KAAK,CACT,yCAAyC,OAAO,oCAAoC,CACrF,CAAC;SACH;QACD,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAA,CAAC;IAEF,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,gBAAM,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,IAAI,0BAAW,CAAC;YAC7B,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW;SAC/B,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,CAAC;IACpB,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,CAAO,OAAe,EAAE,EAAE;QACxC,MAAM,OAAO,GAAG,MAAM,+BAA+B,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;QAC3B,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,OAAoB,CAAC;QAC1D,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC,CAAA,CAAC;IAEF,MAAM,SAAS,GAAG,CAAO,OAAe,EAAE,EAAE;QAC1C,MAAM,OAAO,GAAG,MAAM,+BAA+B,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,OAAoB,CAAC;QAC5D,MAAM,SAAS,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC,CAAA,CAAC;IAEF,OAAO;QACL,OAAO;QACP,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AAnDW,QAAA,mBAAmB,uBAmD9B","sourcesContent":["import path from 'path';\nimport dotenv from 'dotenv';\nimport { glob } from 'glob';\nimport { ApifyClient } from 'apify-client';\n\nimport type { Migration } from './types';\n\nexport const createLocalMigrator = ({\n  migrationsDir,\n  extension = '.js',\n  delimeter = '_',\n}: {\n  migrationsDir: string;\n  /** Extension glob */\n  extension: string;\n  /** Delimeter between version and rest of file name */\n  delimeter: string;\n}) => {\n  const findLocalMigrationFileByVersion = async (version: string): Promise<string> => {\n    // Find files like \"v1_bla_bla_bla.js\" (by default)\n    const files = await glob(path.join(migrationsDir, `${version}${delimeter}*${extension}`));\n    if (!files.length) {\n      throw Error(`No migration file matched version \"${version}\"`);\n    }\n    if (files.length > 1) {\n      throw Error(\n        `Ambiguous migration version. Version \"${version}\" matched multiple migration files`\n      );\n    }\n    return files[0];\n  };\n\n  const setup = () => {\n    dotenv.config();\n    const client = new ApifyClient({\n      token: process.env.APIFY_TOKEN,\n    });\n    return { client };\n  };\n\n  const migrate = async (version: string) => {\n    const migFile = await findLocalMigrationFileByVersion(version);\n    const { client } = setup();\n    const { migrate } = require(migFile).default as Migration;\n    await migrate(client);\n  };\n\n  const unmigrate = async (version: string) => {\n    const migFile = await findLocalMigrationFileByVersion(version);\n    const { client } = setup();\n    const { unmigrate } = require(migFile).default as Migration;\n    await unmigrate(client);\n  };\n\n  return {\n    migrate,\n    unmigrate,\n  };\n};\n"]}
+{"version":3,"file":"localMigrator.js","sourceRoot":"","sources":["../../../../src/lib/migrate/localMigrator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,gDAAwB;AACxB,oDAA4B;AAC5B,+BAA4B;AAC5B,+CAA2C;AAIpC,MAAM,mBAAmB,GAAG,CAAC,EAClC,aAAa,EACb,SAAS,GAAG,KAAK,EACjB,SAAS,GAAG,GAAG,GAOhB,EAAE,EAAE;IACH,MAAM,+BAA+B,GAAG,CAAO,OAAe,EAAmB,EAAE;QACjF,mDAAmD;QACnD,MAAM,KAAK,GAAG,MAAM,IAAA,WAAI,EAAC,cAAI,CAAC,IAAI,CAAC,aAAa,EAAE,GAAG,OAAO,GAAG,SAAS,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC;QAC1F,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAClB,MAAM,KAAK,CAAC,sCAAsC,OAAO,GAAG,CAAC,CAAC;QAChE,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,KAAK,CACT,yCAAyC,OAAO,oCAAoC,CACrF,CAAC;QACJ,CAAC;QACD,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAA,CAAC;IAEF,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,gBAAM,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,IAAI,0BAAW,CAAC;YAC7B,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW;SAC/B,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,CAAC;IACpB,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,CAAO,OAAe,EAAE,EAAE;QACxC,MAAM,OAAO,GAAG,MAAM,+BAA+B,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;QAC3B,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,OAAoB,CAAC;QAC1D,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC,CAAA,CAAC;IAEF,MAAM,SAAS,GAAG,CAAO,OAAe,EAAE,EAAE;QAC1C,MAAM,OAAO,GAAG,MAAM,+BAA+B,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;QAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,OAAoB,CAAC;QAC5D,MAAM,SAAS,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC,CAAA,CAAC;IAEF,OAAO;QACL,OAAO;QACP,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AAnDW,QAAA,mBAAmB,uBAmD9B","sourcesContent":["import path from 'path';\nimport dotenv from 'dotenv';\nimport { glob } from 'glob';\nimport { ApifyClient } from 'apify-client';\n\nimport type { Migration } from './types';\n\nexport const createLocalMigrator = ({\n  migrationsDir,\n  extension = '.js',\n  delimeter = '_',\n}: {\n  migrationsDir: string;\n  /** Extension glob */\n  extension: string;\n  /** Delimeter between version and rest of file name */\n  delimeter: string;\n}) => {\n  const findLocalMigrationFileByVersion = async (version: string): Promise<string> => {\n    // Find files like \"v1_bla_bla_bla.js\" (by default)\n    const files = await glob(path.join(migrationsDir, `${version}${delimeter}*${extension}`));\n    if (!files.length) {\n      throw Error(`No migration file matched version \"${version}\"`);\n    }\n    if (files.length > 1) {\n      throw Error(\n        `Ambiguous migration version. Version \"${version}\" matched multiple migration files`\n      );\n    }\n    return files[0];\n  };\n\n  const setup = () => {\n    dotenv.config();\n    const client = new ApifyClient({\n      token: process.env.APIFY_TOKEN,\n    });\n    return { client };\n  };\n\n  const migrate = async (version: string) => {\n    const migFile = await findLocalMigrationFileByVersion(version);\n    const { client } = setup();\n    const { migrate } = require(migFile).default as Migration;\n    await migrate(client);\n  };\n\n  const unmigrate = async (version: string) => {\n    const migFile = await findLocalMigrationFileByVersion(version);\n    const { client } = setup();\n    const { unmigrate } = require(migFile).default as Migration;\n    await unmigrate(client);\n  };\n\n  return {\n    migrate,\n    unmigrate,\n  };\n};\n"]}

package/dist/cjs/lib/router/router.d.ts

@@ -1,4 +1,4 @@
-import type { RouterHandler as CrawlerRouter, Request as CrawlerRequest } from 'crawlee';
+import type { RouterHandler as CrawlerRouter } from 'crawlee';
 import type { CrawleeOneRouteWrapper, CrawleeOneRouteHandler, CrawleeOneRouteCtx, CrawleeOneRoute } from './types';
 import type { CrawleeOneCtx } from '../actor/types';
 /**
@@ -26,13 +26,11 @@ import type { CrawleeOneCtx } from '../actor/types';
  *
  * The entries on the `routerContext` object will be made available to all handlers.
  */
-export declare const registerHandlers: <T extends CrawleeOneCtx<import("crawlee").CrawlingContext<import("crawlee").BasicCrawler<import("crawlee").BasicCrawlingContext<import("crawlee").Dictionary>> | import("crawlee").PuppeteerCrawler | import("crawlee").PlaywrightCrawler | import("crawlee").JSDOMCrawler | import("crawlee").CheerioCrawler | import("crawlee").HttpCrawler<import("crawlee").InternalHttpCrawlingContext<any, any, import("crawlee").HttpCrawler<any>>>, import("crawlee").Dictionary>, string, Record<string, any>, import("../..").CrawleeOneIO<object, object, object>, import("../..").CrawleeOneTelemetry<any, any>>, RouterCtx extends Record<string, any> = CrawleeOneRouteCtx<T>>(router: CrawlerRouter<T["context"]>, routes: Record<T["labels"], CrawleeOneRoute<T, RouterCtx>>, options?: {
-    routerContext?: RouterCtx | undefined;
-    handlerWrappers?: CrawleeOneRouteWrapper<T, RouterCtx>[] | undefined;
-    onSetCtx?: ((ctx: (Omit<T["context"] & RouterCtx, "request"> & {
-        request: CrawlerRequest<import("crawlee").Dictionary>;
-    }) | null) => void) | undefined;
-} | undefined) => Promise<void>;
+export declare const registerHandlers: <T extends CrawleeOneCtx<import("crawlee").CrawlingContext<import("crawlee").JSDOMCrawler | import("crawlee").CheerioCrawler | import("crawlee").PlaywrightCrawler | import("crawlee").PuppeteerCrawler | import("crawlee").BasicCrawler<import("crawlee").BasicCrawlingContext<import("crawlee").Dictionary>> | import("crawlee").HttpCrawler<import("crawlee").InternalHttpCrawlingContext<any, any, import("crawlee").HttpCrawler<any>>>, import("crawlee").Dictionary>, string, Record<string, any>, import("../..").CrawleeOneIO<object, object, object>, import("../..").CrawleeOneTelemetry<any, any>>, RouterCtx extends Record<string, any> = CrawleeOneRouteCtx<T>>(router: CrawlerRouter<T['context']>, routes: Record<T['labels'], CrawleeOneRoute<T, RouterCtx>>, options?: {
+    routerContext?: RouterCtx;
+    handlerWrappers?: CrawleeOneRouteWrapper<T, RouterCtx>[];
+    onSetCtx?: (ctx: Parameters<CrawleeOneRouteHandler<T, RouterCtx>>[0] | null) => void;
+}) => Promise<void>;
 /**
  * Configures the default router handler to redirect URLs to labelled route handlers
  * based on which route the URL matches first.
@@ -82,14 +80,12 @@ export declare const registerHandlers: <T extends CrawleeOneCtx<import("crawlee"
  * // Now set up the labelled routes
  * await router.addHandler(routeLabels.JOB_LISTING, async (ctx) => { ... }
  */
-export declare const setupDefaultHandlers: <T extends CrawleeOneCtx<import("crawlee").CrawlingContext<import("crawlee").BasicCrawler<import("crawlee").BasicCrawlingContext<import("crawlee").Dictionary>> | import("crawlee").PuppeteerCrawler | import("crawlee").PlaywrightCrawler | import("crawlee").JSDOMCrawler | import("crawlee").CheerioCrawler | import("crawlee").HttpCrawler<import("crawlee").InternalHttpCrawlingContext<any, any, import("crawlee").HttpCrawler<any>>>, import("crawlee").Dictionary>, string, Record<string, any>, import("../..").CrawleeOneIO<object, object, object>, import("../..").CrawleeOneTelemetry<any, any>>, RouterCtx extends Record<string, any> = CrawleeOneRouteCtx<T>>({ io, router, routeHandlerWrappers, routerContext, routes, input, onSetCtx, }: {
-    io: T["io"];
-    router: CrawlerRouter<T["context"]>;
-    routeHandlerWrappers?: CrawleeOneRouteWrapper<T, RouterCtx>[] | undefined;
-    routerContext?: RouterCtx | undefined;
-    routes: Record<T["labels"], CrawleeOneRoute<T, RouterCtx>>;
-    input?: T["input"] | null | undefined;
-    onSetCtx?: ((ctx: (Omit<T["context"] & RouterCtx, "request"> & {
-        request: CrawlerRequest<import("crawlee").Dictionary>;
-    }) | null) => void) | undefined;
+export declare const setupDefaultHandlers: <T extends CrawleeOneCtx<import("crawlee").CrawlingContext<import("crawlee").JSDOMCrawler | import("crawlee").CheerioCrawler | import("crawlee").PlaywrightCrawler | import("crawlee").PuppeteerCrawler | import("crawlee").BasicCrawler<import("crawlee").BasicCrawlingContext<import("crawlee").Dictionary>> | import("crawlee").HttpCrawler<import("crawlee").InternalHttpCrawlingContext<any, any, import("crawlee").HttpCrawler<any>>>, import("crawlee").Dictionary>, string, Record<string, any>, import("../..").CrawleeOneIO<object, object, object>, import("../..").CrawleeOneTelemetry<any, any>>, RouterCtx extends Record<string, any> = CrawleeOneRouteCtx<T>>({ io, router, routeHandlerWrappers, routerContext, routes, input, onSetCtx, }: {
+    io: T['io'];
+    router: CrawlerRouter<T['context']>;
+    routeHandlerWrappers?: CrawleeOneRouteWrapper<T, RouterCtx>[];
+    routerContext?: RouterCtx;
+    routes: Record<T['labels'], CrawleeOneRoute<T, RouterCtx>>;
+    input?: T['input'] | null;
+    onSetCtx?: (ctx: Parameters<CrawleeOneRouteHandler<T, RouterCtx>>[0] | null) => void;
 }) => Promise<void>;

package/dist/cjs/lib/router/router.js

@@ -157,7 +157,7 @@ const createDefaultHandler = (input) => {
             const logSuffix = `Batch ${handledRequestsCount + 1} of ${perfBatchSize !== null && perfBatchSize !== void 0 ? perfBatchSize : 1}. URL: ${url}`;
             // Find route handler for given URL
             log.debug(`Searching for a handler for given Request. ${logSuffix}`);
-            const [routeName, route] = (_c = (yield (0, async_1.serialAsyncFind)(Object.entries(resolvedRoutes), ([key, currRoute]) => __awaiter(void 0, void 0, void 0, function* () {
+            const [routeName, route] = (_c = (yield (0, async_1.serialAsyncFind)(Object.entries(resolvedRoutes), (_d) => __awaiter(void 0, [_d], void 0, function* ([key, currRoute]) {
                 const isMatch = yield (0, async_1.serialAsyncFind)(currRoute.match, (matchFn) => __awaiter(void 0, void 0, void 0, function* () {
                     return matchFn(url, ctx, currRoute, routes);
                 }));
@@ -236,7 +236,7 @@ const createDefaultHandler = (input) => {
  * // Now set up the labelled routes
  * await router.addHandler(routeLabels.JOB_LISTING, async (ctx) => { ... }
  */
-const setupDefaultHandlers = ({ io, router, routeHandlerWrappers, routerContext, routes, input, onSetCtx, }) => __awaiter(void 0, void 0, void 0, function* () {
+const setupDefaultHandlers = (_a) => __awaiter(void 0, [_a], void 0, function* ({ io, router, routeHandlerWrappers, routerContext, routes, input, onSetCtx, }) {
     const { perfBatchSize, perfBatchWaitSecs, requestQueueId } = (input || {});
     const defaultHandler = createDefaultHandler({
         io,

package/dist/cjs/lib/router/router.js.map

@@ -1 +1 @@
-{"version":3,"file":"router.js","sourceRoot":"","sources":["../../../../src/lib/router/router.ts"],"names":[],"mappings":";;;;;;;;;;;;AAIA,6CAA0D;AAY1D,wHAAwH;AAExH;;;;;;GAMG;AACH,MAAM,kBAAkB,GAAG,CACzB,EAAO,EACP,WAA+C,EAAE,EACjD,EAAE;IACF,OAAO,QAAQ,CAAC,WAAW,CAAe,CAAO,gBAAgB,EAAE,OAAO,EAAE,EAAE;QAC5E,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC;QACzC,OAAO,OAAO,CAAC,SAAS,CAAC,CAAC;IAC5B,CAAC,CAAA,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;AAC1B,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAIpB,MAA0D,EAC1D,EAAE,CACF,MAAM,CAAC,OAAO,CAAgC,MAAM,CAAC,CAAC,MAAM,CAE1D,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;IACtB,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC1E,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;QACpB,MAAM,KAAK,CAAC,UAAU,GAAG,2EAA2E,CAAC,CAAC,CAAC,kBAAkB;KAC1H;IAED,MAAM,gBAAgB,GAAG,QAAQ,CAAC,GAAG,CAAyC,CAAC,OAAO,EAAE,EAAE;QACxF,IAAI,OAAO,OAAO,KAAK,UAAU;YAAE,OAAO,OAAO,CAAC;QAClD,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,YAAY,MAAM,EAAE;YAC5D,MAAM,YAAY,GAA2C,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC3F,OAAO,YAAY,CAAC;SACrB;QACD,yBAAyB;QACzB,MAAM,KAAK,CAAC,UAAU,GAAG,yFAAyF,OAAO,EAAE,CAAC,CAAC,CAAC,kBAAkB;IAClJ,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,GAAkB,CAAC,mCAAQ,KAAK,KAAE,KAAK,EAAE,gBAAgB,GAAE,CAAC;IAChE,OAAO,GAAG,CAAC;AACb,CAAC,EAAE,EAAS,CAAC,CAAC;AAEhB;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACI,MAAM,gBAAgB,GAAG,CAI9B,MAAmC,EACnC,MAA0D,EAC1D,OAIC,EACD,EAAE;IACF,MAAM,EAAE,aAAa,EAAE,eAAe,EAAE,QAAQ,EAAE,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC;IAEnE,mBAAmB;IACnB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAgC,MAAM,CAAC,EAAE;QAChF,4CAA4C;QAC5C,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,OAAO,EAAE,eAAe,aAAf,eAAe,cAAf,eAAe,GAAI,EAAE,CAAC,CAAC;QAEtF,sCAAsC;QACtC,MAAM,MAAM,CAAC,UAAU,CAAe,GAAG,EAAE,CAAO,GAAG,EAAE,EAAE;YACvD,6FAA6F;YAC7F,gCAAgC;YAChC,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAG,GAAU,CAAC,CAAC;YACvB,IAAI,MAAM,CAAC;YACX,IAAI;gBACF,MAAM,GAAG,MAAM,cAAc,CAAC,gCAAK,aAAa,GAAK,GAAG,CAAS,CAAC,CAAC;aACpE;oBAAS;gBACR,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAG,IAAI,CAAC,CAAC;aAClB;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAA,CAAC,CAAC;KACJ;AACH,CAAC,CAAA,CAAC;AAjCW,QAAA,gBAAgB,oBAiC3B;AAEF,MAAM,oBAAoB,GAAG,CAI3B,KAI2C,EAC3C,EAAE;IACF,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,aAAa,EAAE,iBAAiB,EAAE,GAAG,KAAK,CAAC;IAE/E,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAE7C,sDAAsD;IACtD,iFAAiF;IACjF,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;IAE5D,MAAM,YAAY,GAAG,CAAO,GAA0B,EAAE,EAAE;QACxD,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,QAAQ,GAAG,MAAM,SAAS,EAAE,CAAC;QACnC,MAAM,QAAQ,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC,CAAA,CAAC;IAEF,MAAM,eAAe,GAAG,CAAO,MAAc,EAAE,OAAoC,EAAE,EAAE;;QACrF,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC;QAEpC,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,CAAC,0CAA0C,MAAM,EAAE,CAAC,CAAC;QAE/D,IAAI,iBAAiB;YAAE,MAAM,IAAA,YAAI,EAAC,iBAAiB,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAG,MAAM,SAAS,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,MAAA,CAAC,MAAM,QAAQ,CAAC,gBAAgB,EAAE,CAAC,mCAAI,IAAI,CAAC;QAE3D,IAAI,MAAM,EAAE;YACV,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,CAAC,mCAAmC,MAAM,EAAE,CAAC,CAAC;YAExD,oEAAoE;YACpE,wEAAwE;YACxE,0CAA0C;YAC1C,IAAI,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,IAAI;gBAAE,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;SAC7C;aAAM;YACL,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,CAAC,kCAAkC,MAAM,EAAE,CAAC,CAAC;SACxD;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAA,CAAC;IAEF,MAAM,OAAO,GAAG,CAAO,GAAQ,EAAE,GAA0B,EAAE,GAAQ,EAAE,EAAE;QACvE,GAAG,CAAC,KAAK,CAAC,gEAAgE,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,SAAS,MAAI,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,GAAG,CAAA,GAAG,CAAC,CAAC,CAAC,kBAAkB;QAC5H,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACf,iFAAiF;QACjF,IAAI,GAAG,EAAE;YACP,MAAM,QAAQ,GAAG,MAAM,SAAS,EAAE,CAAC;YACnC,MAAM,QAAQ,CAAC,cAAc,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;SACzD;IACH,CAAC,CAAA,CAAC;IAEF,uEAAuE;IACvE,MAAM,aAAa,GAAG,CACpB,GAAqC,EACrC,GAAW,EACX,SAAiB,EACjB,KAAoC,EACpC,EAAE;QACF,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE;YAClB,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,+BAA+B,SAAS,sCAAsC,GAAG,EAAE,CAAC,CAAC,CAAC,kBAAkB;YACtH,OAAO;SACR;QACD,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,qCAAqC,SAAS,WAAW,GAAG,EAAE,CAAC,CAAC;QAC7E,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC,CAAA,CAAC;IAEF,MAAM,cAAc,GAAG,CACrB,GAAS,EACM,EAAE;;QACjB,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,GAAG,CAAC;QACrC,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;QAErD,IAAI,oBAAoB,GAAG,CAAC,CAAC;QAC7B,IAAI,GAAG,GAA0B,MAAA,GAAG,CAAC,OAAO,mCAAI,IAAI,CAAC;QAErD,MAAM,YAAY,GAAG,GAAG,EAAE,CACxB,aAAa,IAAI,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,oBAAoB,GAAG,aAAa,CAAC;QAE/E,uEAAuE;QACvE,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,IAA0B,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,GAAI,CAAC,SAAS,IAAI,GAAI,CAAC,GAAG,CAAC,CAAC;QAE7F,MAAM,SAAS,GAAG,GAAS,EAAE;;YAC3B,MAAM,GAAG,GAAG,MAAM,MAAM,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,SAAS,oBAAoB,GAAG,CAAC,OAAO,aAAa,aAAb,aAAa,cAAb,aAAa,GAAI,CAAC,UAAU,GAAG,EAAE,CAAC;YAE5F,mCAAmC;YACnC,GAAG,CAAC,KAAK,CAAC,8CAA8C,SAAS,EAAE,CAAC,CAAC;YACrE,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,GACtB,MAAA,CAAC,MAAM,IAAA,uBAAe,EACpB,MAAM,CAAC,OAAO,CAAgC,cAAc,CAAC,EAC7D,CAAO,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE,EAAE;gBACzB,MAAM,OAAO,GAAG,MAAM,IAAA,uBAAe,EACnC,SAAS,CAAC,KAAiD,EAC3D,CAAO,OAAO,EAAE,EAAE;oBAChB,OAAO,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;gBAC9C,CAAC,CAAA,CACF,CAAC;gBACF,OAAO,OAAO,CAAC;YACjB,CAAC,CAAA,CACF,CAAC,mCAAI,EAAE,CAAC;YAEX,kBAAkB;YAClB,IAAI,KAAK,EAAE;gBACT,GAAG,CAAC,IAAI,CAAC,sBAAsB,SAAS,MAAM,SAAS,EAAE,CAAC,CAAC;gBAC3D,MAAM,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,aAAT,SAAS,cAAT,SAAS,GAAI,EAAE,EAAE,KAAK,CAAC,CAAC;aACvD;iBAAM;gBACL,GAAG,CAAC,KAAK,CAAC,oDAAoD,SAAS,EAAE,CAAC,CAAC;aAC5E;YAED,yCAAyC;YACzC,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;YACxB,oBAAoB,EAAE,CAAC;YAEvB,GAAG,GAAG,MAAM,eAAe,CAAC,SAAS,EAAE,EAAE,IAAI,EAAE,IAAY,EAAE,GAAG,EAAE,CAAC,CAAC;QACtE,CAAC,CAAA,CAAC;QAEF,IAAI;YACF,GAAG;gBACD,MAAM,SAAS,EAAE,CAAC;aACnB,QAAQ,YAAY,EAAE,EAAE;SAC1B;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;SAC9B;IACH,CAAC,CAAA,CAAC;IAEF,OAAO,cAAc,CAAC;AACxB,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AACI,MAAM,oBAAoB,GAAG,CAGlC,EACA,EAAE,EACF,MAAM,EACN,oBAAoB,EACpB,aAAa,EACb,MAAM,EACN,KAAK,EACL,QAAQ,GAST,EAAE,EAAE;IACH,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,cAAc,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,CACtD,CAAC;IAEpB,MAAM,cAAc,GAAG,oBAAoB,CAAC;QAC1C,EAAE;QACF,MAAM;QACN,cAAc;QACd,aAAa;QACb,iBAAiB;KAClB,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,cAAc,EAAE,oBAAoB,aAApB,oBAAoB,cAApB,oBAAoB,GAAI,EAAE,CAAC,CAAC;IAC5F,MAAM,MAAM,CAAC,iBAAiB,CAAe,CAAO,GAAG,EAAE,EAAE;QACzD,6FAA6F;QAC7F,gCAAgC;QAChC,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAG,GAAU,CAAC,CAAC;QACvB,IAAI,MAAM,CAAC;QACX,IAAI;YACF,MAAM,GAAG,MAAM,cAAc,CAAC,gCAAK,aAAa,GAAK,GAAG,CAAS,CAAC,CAAC;SACpE;gBAAS;YACR,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAG,IAAI,CAAC,CAAC;SAClB;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAA,CAAC,CAAC;AACL,CAAC,CAAA,CAAC;AA5CW,QAAA,oBAAoB,wBA4C/B","sourcesContent":["import type { RouterHandler as CrawlerRouter, Request as CrawlerRequest, Log } from 'crawlee';\nimport type { CommonPage } from '@crawlee/browser-pool';\nimport type { Page } from 'playwright';\n\nimport { serialAsyncFind, wait } from '../../utils/async';\nimport type { MaybePromise } from '../../utils/types';\nimport type { PerfActorInput, RequestActorInput } from '../input';\nimport type {\n  CrawleeOneRouteWrapper,\n  CrawleeOneRouteHandler,\n  CrawleeOneRouteCtx,\n  CrawleeOneRouteMatcherFn,\n  CrawleeOneRoute,\n} from './types';\nimport type { CrawleeOneCtx } from '../actor/types';\n\n// Read about router on https://docs.apify.com/academy/expert-scraping-with-apify/solutions/using-storage-creating-tasks\n\n/**\n * Given a function `fn` and a list of wrappers [a, b, c, ...],\n * wrap the function to generate composite `a( b( c( fn ) ) )`.\n *\n * That is, the wrappers on the left side of the array (start)\n * wrap those on the right side of the array (end).\n */\nconst applyWrappersRight = <TFn extends (...args: any[]) => any>(\n  fn: TFn,\n  wrappers: ((fn: TFn) => MaybePromise<TFn>)[] = []\n) => {\n  return wrappers.reduceRight<Promise<TFn>>(async (interimFnPromise, wrapper) => {\n    const interimFn = await interimFnPromise;\n    return wrapper(interimFn);\n  }, Promise.resolve(fn));\n};\n\nconst resolveRoutes = <\n  T extends CrawleeOneCtx,\n  RouterCtx extends Record<string, any> = CrawleeOneRouteCtx<T>\n>(\n  routes: Record<T['labels'], CrawleeOneRoute<T, RouterCtx>>\n) =>\n  Object.entries<CrawleeOneRoute<T, RouterCtx>>(routes).reduce<\n    Record<T['labels'], CrawleeOneRoute<T, RouterCtx>>\n  >((agg, [key, route]) => {\n    const matchers = Array.isArray(route.match) ? route.match : [route.match];\n    if (!matchers.length) {\n      throw Error(`Route \"${key}\" has NO \"match\" item. It can be a RegExp, function, or array of the two.`); // prettier-ignore\n    }\n\n    const resolvedMatchers = matchers.map<CrawleeOneRouteMatcherFn<T, RouterCtx>>((matcher) => {\n      if (typeof matcher === 'function') return matcher;\n      if (typeof matcher === 'string' || matcher instanceof RegExp) {\n        const newMatcherFn: CrawleeOneRouteMatcherFn<T, RouterCtx> = (url) => !!url.match(matcher);\n        return newMatcherFn;\n      }\n      // We shouldn't get here!\n      throw Error(`Route \"${key}\" has INVALID \"match\" item. It can be only RegExp, function, or array of the two. Got ${matcher}`); // prettier-ignore\n    });\n\n    agg[key as T['labels']] = { ...route, match: resolvedMatchers };\n    return agg;\n  }, {} as any);\n\n/**\n * Register many handlers at once onto the Crawlee's RouterHandler.\n *\n * The labels under which the handlers are registered are the respective object keys.\n *\n * Example:\n *\n * ```js\n * registerHandlers(router, { labelA: fn1, labelB: fn2 });\n * ```\n *\n * Is similar to:\n * ```js\n * router.addHandler(labelA, fn1)\n * router.addHandler(labelB, fn2)\n * ```\n *\n * You can also specify a list of wrappers to override the behaviour of all handlers\n * all at once.\n *\n * A list of wrappers `[a, b, c]` will be applied to the handlers right-to-left as so\n * `a( b( c( handler ) ) )`.\n *\n * The entries on the `routerContext` object will be made available to all handlers.\n */\nexport const registerHandlers = async <\n  T extends CrawleeOneCtx,\n  RouterCtx extends Record<string, any> = CrawleeOneRouteCtx<T>\n>(\n  router: CrawlerRouter<T['context']>,\n  routes: Record<T['labels'], CrawleeOneRoute<T, RouterCtx>>,\n  options?: {\n    routerContext?: RouterCtx;\n    handlerWrappers?: CrawleeOneRouteWrapper<T, RouterCtx>[];\n    onSetCtx?: (ctx: Parameters<CrawleeOneRouteHandler<T, RouterCtx>>[0] | null) => void;\n  }\n) => {\n  const { routerContext, handlerWrappers, onSetCtx } = options ?? {};\n\n  // For each handler\n  for (const [key, route] of Object.entries<CrawleeOneRoute<T, RouterCtx>>(routes)) {\n    // First apply all wrappers onto the handler\n    const wrappedHandler = await applyWrappersRight(route.handler, handlerWrappers ?? []);\n\n    // Then register the composite handler\n    await router.addHandler<T['context']>(key, async (ctx) => {\n      // For the duration of the handler execution, set the actor.handlerCtx to the value of `ctx`,\n      // and then set it back to null;\n      onSetCtx?.(ctx as any);\n      let result;\n      try {\n        result = await wrappedHandler({ ...routerContext, ...ctx } as any);\n      } finally {\n        onSetCtx?.(null);\n      }\n      return result;\n    });\n  }\n};\n\nconst createDefaultHandler = <\n  T extends CrawleeOneCtx,\n  RouterCtx extends Record<string, any> = CrawleeOneRouteCtx<T>\n>(\n  input: {\n    io: T['io'];\n    routes: Record<T['labels'], CrawleeOneRoute<T, RouterCtx>>;\n  } & PerfActorInput &\n    Pick<RequestActorInput, 'requestQueueId'>\n) => {\n  const { io, routes, requestQueueId, perfBatchSize, perfBatchWaitSecs } = input;\n\n  const resolvedRoutes = resolveRoutes(routes);\n\n  // NOTE: Because we \"clear\" the queue by replacing it,\n  // we need to always call `openRequestQueue` to ensure we use the latest instance\n  const openQueue = () => io.openRequestQueue(requestQueueId);\n\n  const closeRequest = async (req: CrawlerRequest | null) => {\n    if (!req) return;\n    const reqQueue = await openQueue();\n    await reqQueue.markRequestHandled(req);\n  };\n\n  const loadNextRequest = async (suffix: string, options?: { page?: Page; log?: Log }) => {\n    const { page, log } = options ?? {};\n\n    log?.debug(`Checking for new Request in the queue. ${suffix}`);\n\n    if (perfBatchWaitSecs) await wait(perfBatchWaitSecs);\n    const reqQueue = await openQueue();\n    const newReq = (await reqQueue.fetchNextRequest()) ?? null;\n\n    if (newReq) {\n      log?.debug(`Found new Request in the queue. ${suffix}`);\n\n      // WARNING - For each subsequent Request, it must be loaded manually\n      //           Hence, batching is suitable only for browser-based Crawlers\n      //           like Playwright or Puppeteer.\n      if (page?.goto) await page.goto(newReq.url);\n    } else {\n      log?.debug(`No more Requests in the queue. ${suffix}`);\n    }\n    return newReq;\n  };\n\n  const onError = async (err: any, req: CrawlerRequest | null, log: Log) => {\n    log.error(`Failed to process a request, returning it to the queue. URL: ${req?.loadedUrl || req?.url}.`); // prettier-ignore\n    log.error(err);\n    // Reinsert the request into the queue if we failed to process it due to an error\n    if (req) {\n      const reqQueue = await openQueue();\n      await reqQueue.reclaimRequest(req, { forefront: true });\n    }\n  };\n\n  /** Redirect the URL to the labelled route identical to route's name */\n  const defaultAction = async (\n    ctx: CrawleeOneRouteCtx<T, RouterCtx>,\n    url: string,\n    routeName: string,\n    route: CrawleeOneRoute<T, RouterCtx>\n  ) => {\n    if (!route.handler) {\n      ctx.log.error(`No handler found for route \"${routeName}\". URL will not be processed. URL: ${url}`); // prettier-ignore\n      return;\n    }\n    ctx.log.info(`Passing URL to handler for route \"${routeName}\". URL: ${url}`);\n    await route.handler(ctx);\n  };\n\n  const defaultHandler = async <TCtx extends CrawleeOneRouteCtx<T, RouterCtx>>(\n    ctx: TCtx\n  ): Promise<void> => {\n    const { page, log: parentLog } = ctx;\n    const log = parentLog.child({ prefix: '[Router] ' });\n\n    let handledRequestsCount = 0;\n    let req: CrawlerRequest | null = ctx.request ?? null;\n\n    const hasBatchReqs = () =>\n      perfBatchSize != null && req != null && handledRequestsCount < perfBatchSize;\n\n    /* eslint-disable-next-line @typescript-eslint/no-non-null-assertion */\n    const getUrl = () => (page ? (page as any as CommonPage).url() : req!.loadedUrl || req!.url);\n\n    const onRequest = async () => {\n      const url = await getUrl();\n      const logSuffix = `Batch ${handledRequestsCount + 1} of ${perfBatchSize ?? 1}. URL: ${url}`;\n\n      // Find route handler for given URL\n      log.debug(`Searching for a handler for given Request. ${logSuffix}`);\n      const [routeName, route] =\n        (await serialAsyncFind(\n          Object.entries<CrawleeOneRoute<T, RouterCtx>>(resolvedRoutes),\n          async ([key, currRoute]) => {\n            const isMatch = await serialAsyncFind(\n              currRoute.match as CrawleeOneRouteMatcherFn<T, RouterCtx>[],\n              async (matchFn) => {\n                return matchFn(url, ctx, currRoute, routes);\n              }\n            );\n            return isMatch;\n          }\n        )) ?? [];\n\n      // Run the handler\n      if (route) {\n        log.info(`URL matched route \"${routeName}\". ${logSuffix}`);\n        await defaultAction(ctx, url, routeName ?? '', route);\n      } else {\n        log.error(`No route matched URL. URL will not be processed. ${logSuffix}`);\n      }\n\n      // Clean up and move onto another request\n      await closeRequest(req);\n      handledRequestsCount++;\n\n      req = await loadNextRequest(logSuffix, { page: page as Page, log });\n    };\n\n    try {\n      do {\n        await onRequest();\n      } while (hasBatchReqs());\n    } catch (err) {\n      await onError(err, req, log);\n    }\n  };\n\n  return defaultHandler;\n};\n\n/**\n * Configures the default router handler to redirect URLs to labelled route handlers\n * based on which route the URL matches first.\n *\n * NOTE: This does mean that the URLs passed to this default handler will be fetched\n * twice (as the URL will be requeued to the correct handler). We recommend to use this\n * function only in the scenarios where there is a small number of startUrls, yet these\n * may need various ways of processing based on different paths or etc.\n *\n * @example\n *\n * const routeLabels = {\n *   MAIN_PAGE: 'MAIN_PAGE',\n *   JOB_LISTING: 'JOB_LISTING',\n *   JOB_DETAIL: 'JOB_DETAIL',\n *   JOB_RELATED_LIST: 'JOB_RELATED_LIST',\n *   PARTNERS: 'PARTNERS',\n * } as const;\n *\n * const router = createPlaywrightRouter();\n *\n * const routes = createPlaywrightCrawleeOneRouteMatchers<typeof routeLabels>([\n *  // URLs that match this route are redirected to router.addHandler(routeLabels.MAIN_PAGE)\n *  {\n *     route: routeLabels.MAIN_PAGE,\n *     // Check for main page like https://www.profesia.sk/?#\n *     match: (url) => url.match(/[\\W]profesia\\.sk\\/?(?:[?#~]|$)/i),\n *   },\n *\n *  // Optionally override the logic that assigns the URL to the route by specifying the `action` prop\n *  {\n *     route: routeLabels.MAIN_PAGE,\n *     // Check for main page like https://www.profesia.sk/?#\n *     match: (url) => url.match(/[\\W]profesia\\.sk\\/?(?:[?#~]|$)/i),\n *     action: async (ctx) => {\n *       await ctx.crawler.addRequests([{\n *         url: 'https://profesia.sk/praca',\n *         label: routeLabels.JOB_LISTING,\n *       }]);\n *     },\n *   },\n * ]);\n *\n * // Set up default route to redirect to labelled routes\n * setupDefaultHandlers({ router, routes });\n *\n * // Now set up the labelled routes\n * await router.addHandler(routeLabels.JOB_LISTING, async (ctx) => { ... }\n */\nexport const setupDefaultHandlers = async <\n  T extends CrawleeOneCtx,\n  RouterCtx extends Record<string, any> = CrawleeOneRouteCtx<T>\n>({\n  io,\n  router,\n  routeHandlerWrappers,\n  routerContext,\n  routes,\n  input,\n  onSetCtx,\n}: {\n  io: T['io'];\n  router: CrawlerRouter<T['context']>;\n  routeHandlerWrappers?: CrawleeOneRouteWrapper<T, RouterCtx>[];\n  routerContext?: RouterCtx;\n  routes: Record<T['labels'], CrawleeOneRoute<T, RouterCtx>>;\n  input?: T['input'] | null;\n  onSetCtx?: (ctx: Parameters<CrawleeOneRouteHandler<T, RouterCtx>>[0] | null) => void;\n}) => {\n  const { perfBatchSize, perfBatchWaitSecs, requestQueueId } = (input || {}) as PerfActorInput &\n    RequestActorInput;\n\n  const defaultHandler = createDefaultHandler({\n    io,\n    routes,\n    requestQueueId,\n    perfBatchSize,\n    perfBatchWaitSecs,\n  });\n\n  const wrappedHandler = await applyWrappersRight(defaultHandler, routeHandlerWrappers ?? []);\n  await router.addDefaultHandler<T['context']>(async (ctx) => {\n    // For the duration of the handler execution, set the actor.handlerCtx to the value of `ctx`,\n    // and then set it back to null;\n    onSetCtx?.(ctx as any);\n    let result;\n    try {\n      result = await wrappedHandler({ ...routerContext, ...ctx } as any);\n    } finally {\n      onSetCtx?.(null);\n    }\n    return result;\n  });\n};\n"]}
+{"version":3,"file":"router.js","sourceRoot":"","sources":["../../../../src/lib/router/router.ts"],"names":[],"mappings":";;;;;;;;;;;;AAIA,6CAA0D;AAY1D,wHAAwH;AAExH;;;;;;GAMG;AACH,MAAM,kBAAkB,GAAG,CACzB,EAAO,EACP,WAA+C,EAAE,EACjD,EAAE;IACF,OAAO,QAAQ,CAAC,WAAW,CAAe,CAAO,gBAAgB,EAAE,OAAO,EAAE,EAAE;QAC5E,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC;QACzC,OAAO,OAAO,CAAC,SAAS,CAAC,CAAC;IAC5B,CAAC,CAAA,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;AAC1B,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAIpB,MAA0D,EAC1D,EAAE,CACF,MAAM,CAAC,OAAO,CAAgC,MAAM,CAAC,CAAC,MAAM,CAE1D,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;IACtB,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC1E,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACrB,MAAM,KAAK,CAAC,UAAU,GAAG,2EAA2E,CAAC,CAAC,CAAC,kBAAkB;IAC3H,CAAC;IAED,MAAM,gBAAgB,GAAG,QAAQ,CAAC,GAAG,CAAyC,CAAC,OAAO,EAAE,EAAE;QACxF,IAAI,OAAO,OAAO,KAAK,UAAU;YAAE,OAAO,OAAO,CAAC;QAClD,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,YAAY,MAAM,EAAE,CAAC;YAC7D,MAAM,YAAY,GAA2C,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC3F,OAAO,YAAY,CAAC;QACtB,CAAC;QACD,yBAAyB;QACzB,MAAM,KAAK,CAAC,UAAU,GAAG,yFAAyF,OAAO,EAAE,CAAC,CAAC,CAAC,kBAAkB;IAClJ,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,GAAkB,CAAC,mCAAQ,KAAK,KAAE,KAAK,EAAE,gBAAgB,GAAE,CAAC;IAChE,OAAO,GAAG,CAAC;AACb,CAAC,EAAE,EAAS,CAAC,CAAC;AAEhB;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACI,MAAM,gBAAgB,GAAG,CAI9B,MAAmC,EACnC,MAA0D,EAC1D,OAIC,EACD,EAAE;IACF,MAAM,EAAE,aAAa,EAAE,eAAe,EAAE,QAAQ,EAAE,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC;IAEnE,mBAAmB;IACnB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAgC,MAAM,CAAC,EAAE,CAAC;QACjF,4CAA4C;QAC5C,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,OAAO,EAAE,eAAe,aAAf,eAAe,cAAf,eAAe,GAAI,EAAE,CAAC,CAAC;QAEtF,sCAAsC;QACtC,MAAM,MAAM,CAAC,UAAU,CAAe,GAAG,EAAE,CAAO,GAAG,EAAE,EAAE;YACvD,6FAA6F;YAC7F,gCAAgC;YAChC,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAG,GAAU,CAAC,CAAC;YACvB,IAAI,MAAM,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,cAAc,CAAC,gCAAK,aAAa,GAAK,GAAG,CAAS,CAAC,CAAC;YACrE,CAAC;oBAAS,CAAC;gBACT,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAG,IAAI,CAAC,CAAC;YACnB,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAA,CAAC,CAAC;IACL,CAAC;AACH,CAAC,CAAA,CAAC;AAjCW,QAAA,gBAAgB,oBAiC3B;AAEF,MAAM,oBAAoB,GAAG,CAI3B,KAI2C,EAC3C,EAAE;IACF,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,aAAa,EAAE,iBAAiB,EAAE,GAAG,KAAK,CAAC;IAE/E,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAE7C,sDAAsD;IACtD,iFAAiF;IACjF,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;IAE5D,MAAM,YAAY,GAAG,CAAO,GAA0B,EAAE,EAAE;QACxD,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,QAAQ,GAAG,MAAM,SAAS,EAAE,CAAC;QACnC,MAAM,QAAQ,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC,CAAA,CAAC;IAEF,MAAM,eAAe,GAAG,CAAO,MAAc,EAAE,OAAoC,EAAE,EAAE;;QACrF,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC;QAEpC,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,CAAC,0CAA0C,MAAM,EAAE,CAAC,CAAC;QAE/D,IAAI,iBAAiB;YAAE,MAAM,IAAA,YAAI,EAAC,iBAAiB,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAG,MAAM,SAAS,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,MAAA,CAAC,MAAM,QAAQ,CAAC,gBAAgB,EAAE,CAAC,mCAAI,IAAI,CAAC;QAE3D,IAAI,MAAM,EAAE,CAAC;YACX,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,CAAC,mCAAmC,MAAM,EAAE,CAAC,CAAC;YAExD,oEAAoE;YACpE,wEAAwE;YACxE,0CAA0C;YAC1C,IAAI,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,IAAI;gBAAE,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC9C,CAAC;aAAM,CAAC;YACN,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,CAAC,kCAAkC,MAAM,EAAE,CAAC,CAAC;QACzD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAA,CAAC;IAEF,MAAM,OAAO,GAAG,CAAO,GAAQ,EAAE,GAA0B,EAAE,GAAQ,EAAE,EAAE;QACvE,GAAG,CAAC,KAAK,CAAC,gEAAgE,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,SAAS,MAAI,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,GAAG,CAAA,GAAG,CAAC,CAAC,CAAC,kBAAkB;QAC5H,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACf,iFAAiF;QACjF,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,QAAQ,GAAG,MAAM,SAAS,EAAE,CAAC;YACnC,MAAM,QAAQ,CAAC,cAAc,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC,CAAA,CAAC;IAEF,uEAAuE;IACvE,MAAM,aAAa,GAAG,CACpB,GAAqC,EACrC,GAAW,EACX,SAAiB,EACjB,KAAoC,EACpC,EAAE;QACF,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACnB,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,+BAA+B,SAAS,sCAAsC,GAAG,EAAE,CAAC,CAAC,CAAC,kBAAkB;YACtH,OAAO;QACT,CAAC;QACD,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,qCAAqC,SAAS,WAAW,GAAG,EAAE,CAAC,CAAC;QAC7E,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC,CAAA,CAAC;IAEF,MAAM,cAAc,GAAG,CACrB,GAAS,EACM,EAAE;;QACjB,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,GAAG,CAAC;QACrC,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;QAErD,IAAI,oBAAoB,GAAG,CAAC,CAAC;QAC7B,IAAI,GAAG,GAA0B,MAAA,GAAG,CAAC,OAAO,mCAAI,IAAI,CAAC;QAErD,MAAM,YAAY,GAAG,GAAG,EAAE,CACxB,aAAa,IAAI,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,oBAAoB,GAAG,aAAa,CAAC;QAE/E,uEAAuE;QACvE,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,IAA0B,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,GAAI,CAAC,SAAS,IAAI,GAAI,CAAC,GAAG,CAAC,CAAC;QAE7F,MAAM,SAAS,GAAG,GAAS,EAAE;;YAC3B,MAAM,GAAG,GAAG,MAAM,MAAM,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,SAAS,oBAAoB,GAAG,CAAC,OAAO,aAAa,aAAb,aAAa,cAAb,aAAa,GAAI,CAAC,UAAU,GAAG,EAAE,CAAC;YAE5F,mCAAmC;YACnC,GAAG,CAAC,KAAK,CAAC,8CAA8C,SAAS,EAAE,CAAC,CAAC;YACrE,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,GACtB,MAAA,CAAC,MAAM,IAAA,uBAAe,EACpB,MAAM,CAAC,OAAO,CAAgC,cAAc,CAAC,EAC7D,KAAyB,EAAE,4CAApB,CAAC,GAAG,EAAE,SAAS,CAAC;gBACrB,MAAM,OAAO,GAAG,MAAM,IAAA,uBAAe,EACnC,SAAS,CAAC,KAAiD,EAC3D,CAAO,OAAO,EAAE,EAAE;oBAChB,OAAO,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;gBAC9C,CAAC,CAAA,CACF,CAAC;gBACF,OAAO,OAAO,CAAC;YACjB,CAAC,CAAA,CACF,CAAC,mCAAI,EAAE,CAAC;YAEX,kBAAkB;YAClB,IAAI,KAAK,EAAE,CAAC;gBACV,GAAG,CAAC,IAAI,CAAC,sBAAsB,SAAS,MAAM,SAAS,EAAE,CAAC,CAAC;gBAC3D,MAAM,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,aAAT,SAAS,cAAT,SAAS,GAAI,EAAE,EAAE,KAAK,CAAC,CAAC;YACxD,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,KAAK,CAAC,oDAAoD,SAAS,EAAE,CAAC,CAAC;YAC7E,CAAC;YAED,yCAAyC;YACzC,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;YACxB,oBAAoB,EAAE,CAAC;YAEvB,GAAG,GAAG,MAAM,eAAe,CAAC,SAAS,EAAE,EAAE,IAAI,EAAE,IAAY,EAAE,GAAG,EAAE,CAAC,CAAC;QACtE,CAAC,CAAA,CAAC;QAEF,IAAI,CAAC;YACH,GAAG,CAAC;gBACF,MAAM,SAAS,EAAE,CAAC;YACpB,CAAC,QAAQ,YAAY,EAAE,EAAE;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC,CAAA,CAAC;IAEF,OAAO,cAAc,CAAC;AACxB,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AACI,MAAM,oBAAoB,GAAG,KAmBjC,EAAE,4CAhBH,EACA,EAAE,EACF,MAAM,EACN,oBAAoB,EACpB,aAAa,EACb,MAAM,EACN,KAAK,EACL,QAAQ,GAST;IACC,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,cAAc,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,CACtD,CAAC;IAEpB,MAAM,cAAc,GAAG,oBAAoB,CAAC;QAC1C,EAAE;QACF,MAAM;QACN,cAAc;QACd,aAAa;QACb,iBAAiB;KAClB,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,cAAc,EAAE,oBAAoB,aAApB,oBAAoB,cAApB,oBAAoB,GAAI,EAAE,CAAC,CAAC;IAC5F,MAAM,MAAM,CAAC,iBAAiB,CAAe,CAAO,GAAG,EAAE,EAAE;QACzD,6FAA6F;QAC7F,gCAAgC;QAChC,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAG,GAAU,CAAC,CAAC;QACvB,IAAI,MAAM,CAAC;QACX,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,cAAc,CAAC,gCAAK,aAAa,GAAK,GAAG,CAAS,CAAC,CAAC;QACrE,CAAC;gBAAS,CAAC;YACT,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAG,IAAI,CAAC,CAAC;QACnB,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAA,CAAC,CAAC;AACL,CAAC,CAAA,CAAC;AA5CW,QAAA,oBAAoB,wBA4C/B","sourcesContent":["import type { RouterHandler as CrawlerRouter, Request as CrawlerRequest, Log } from 'crawlee';\nimport type { CommonPage } from '@crawlee/browser-pool';\nimport type { Page } from 'playwright';\n\nimport { serialAsyncFind, wait } from '../../utils/async';\nimport type { MaybePromise } from '../../utils/types';\nimport type { PerfActorInput, RequestActorInput } from '../input';\nimport type {\n  CrawleeOneRouteWrapper,\n  CrawleeOneRouteHandler,\n  CrawleeOneRouteCtx,\n  CrawleeOneRouteMatcherFn,\n  CrawleeOneRoute,\n} from './types';\nimport type { CrawleeOneCtx } from '../actor/types';\n\n// Read about router on https://docs.apify.com/academy/expert-scraping-with-apify/solutions/using-storage-creating-tasks\n\n/**\n * Given a function `fn` and a list of wrappers [a, b, c, ...],\n * wrap the function to generate composite `a( b( c( fn ) ) )`.\n *\n * That is, the wrappers on the left side of the array (start)\n * wrap those on the right side of the array (end).\n */\nconst applyWrappersRight = <TFn extends (...args: any[]) => any>(\n  fn: TFn,\n  wrappers: ((fn: TFn) => MaybePromise<TFn>)[] = []\n) => {\n  return wrappers.reduceRight<Promise<TFn>>(async (interimFnPromise, wrapper) => {\n    const interimFn = await interimFnPromise;\n    return wrapper(interimFn);\n  }, Promise.resolve(fn));\n};\n\nconst resolveRoutes = <\n  T extends CrawleeOneCtx,\n  RouterCtx extends Record<string, any> = CrawleeOneRouteCtx<T>\n>(\n  routes: Record<T['labels'], CrawleeOneRoute<T, RouterCtx>>\n) =>\n  Object.entries<CrawleeOneRoute<T, RouterCtx>>(routes).reduce<\n    Record<T['labels'], CrawleeOneRoute<T, RouterCtx>>\n  >((agg, [key, route]) => {\n    const matchers = Array.isArray(route.match) ? route.match : [route.match];\n    if (!matchers.length) {\n      throw Error(`Route \"${key}\" has NO \"match\" item. It can be a RegExp, function, or array of the two.`); // prettier-ignore\n    }\n\n    const resolvedMatchers = matchers.map<CrawleeOneRouteMatcherFn<T, RouterCtx>>((matcher) => {\n      if (typeof matcher === 'function') return matcher;\n      if (typeof matcher === 'string' || matcher instanceof RegExp) {\n        const newMatcherFn: CrawleeOneRouteMatcherFn<T, RouterCtx> = (url) => !!url.match(matcher);\n        return newMatcherFn;\n      }\n      // We shouldn't get here!\n      throw Error(`Route \"${key}\" has INVALID \"match\" item. It can be only RegExp, function, or array of the two. Got ${matcher}`); // prettier-ignore\n    });\n\n    agg[key as T['labels']] = { ...route, match: resolvedMatchers };\n    return agg;\n  }, {} as any);\n\n/**\n * Register many handlers at once onto the Crawlee's RouterHandler.\n *\n * The labels under which the handlers are registered are the respective object keys.\n *\n * Example:\n *\n * ```js\n * registerHandlers(router, { labelA: fn1, labelB: fn2 });\n * ```\n *\n * Is similar to:\n * ```js\n * router.addHandler(labelA, fn1)\n * router.addHandler(labelB, fn2)\n * ```\n *\n * You can also specify a list of wrappers to override the behaviour of all handlers\n * all at once.\n *\n * A list of wrappers `[a, b, c]` will be applied to the handlers right-to-left as so\n * `a( b( c( handler ) ) )`.\n *\n * The entries on the `routerContext` object will be made available to all handlers.\n */\nexport const registerHandlers = async <\n  T extends CrawleeOneCtx,\n  RouterCtx extends Record<string, any> = CrawleeOneRouteCtx<T>\n>(\n  router: CrawlerRouter<T['context']>,\n  routes: Record<T['labels'], CrawleeOneRoute<T, RouterCtx>>,\n  options?: {\n    routerContext?: RouterCtx;\n    handlerWrappers?: CrawleeOneRouteWrapper<T, RouterCtx>[];\n    onSetCtx?: (ctx: Parameters<CrawleeOneRouteHandler<T, RouterCtx>>[0] | null) => void;\n  }\n) => {\n  const { routerContext, handlerWrappers, onSetCtx } = options ?? {};\n\n  // For each handler\n  for (const [key, route] of Object.entries<CrawleeOneRoute<T, RouterCtx>>(routes)) {\n    // First apply all wrappers onto the handler\n    const wrappedHandler = await applyWrappersRight(route.handler, handlerWrappers ?? []);\n\n    // Then register the composite handler\n    await router.addHandler<T['context']>(key, async (ctx) => {\n      // For the duration of the handler execution, set the actor.handlerCtx to the value of `ctx`,\n      // and then set it back to null;\n      onSetCtx?.(ctx as any);\n      let result;\n      try {\n        result = await wrappedHandler({ ...routerContext, ...ctx } as any);\n      } finally {\n        onSetCtx?.(null);\n      }\n      return result;\n    });\n  }\n};\n\nconst createDefaultHandler = <\n  T extends CrawleeOneCtx,\n  RouterCtx extends Record<string, any> = CrawleeOneRouteCtx<T>\n>(\n  input: {\n    io: T['io'];\n    routes: Record<T['labels'], CrawleeOneRoute<T, RouterCtx>>;\n  } & PerfActorInput &\n    Pick<RequestActorInput, 'requestQueueId'>\n) => {\n  const { io, routes, requestQueueId, perfBatchSize, perfBatchWaitSecs } = input;\n\n  const resolvedRoutes = resolveRoutes(routes);\n\n  // NOTE: Because we \"clear\" the queue by replacing it,\n  // we need to always call `openRequestQueue` to ensure we use the latest instance\n  const openQueue = () => io.openRequestQueue(requestQueueId);\n\n  const closeRequest = async (req: CrawlerRequest | null) => {\n    if (!req) return;\n    const reqQueue = await openQueue();\n    await reqQueue.markRequestHandled(req);\n  };\n\n  const loadNextRequest = async (suffix: string, options?: { page?: Page; log?: Log }) => {\n    const { page, log } = options ?? {};\n\n    log?.debug(`Checking for new Request in the queue. ${suffix}`);\n\n    if (perfBatchWaitSecs) await wait(perfBatchWaitSecs);\n    const reqQueue = await openQueue();\n    const newReq = (await reqQueue.fetchNextRequest()) ?? null;\n\n    if (newReq) {\n      log?.debug(`Found new Request in the queue. ${suffix}`);\n\n      // WARNING - For each subsequent Request, it must be loaded manually\n      //           Hence, batching is suitable only for browser-based Crawlers\n      //           like Playwright or Puppeteer.\n      if (page?.goto) await page.goto(newReq.url);\n    } else {\n      log?.debug(`No more Requests in the queue. ${suffix}`);\n    }\n    return newReq;\n  };\n\n  const onError = async (err: any, req: CrawlerRequest | null, log: Log) => {\n    log.error(`Failed to process a request, returning it to the queue. URL: ${req?.loadedUrl || req?.url}.`); // prettier-ignore\n    log.error(err);\n    // Reinsert the request into the queue if we failed to process it due to an error\n    if (req) {\n      const reqQueue = await openQueue();\n      await reqQueue.reclaimRequest(req, { forefront: true });\n    }\n  };\n\n  /** Redirect the URL to the labelled route identical to route's name */\n  const defaultAction = async (\n    ctx: CrawleeOneRouteCtx<T, RouterCtx>,\n    url: string,\n    routeName: string,\n    route: CrawleeOneRoute<T, RouterCtx>\n  ) => {\n    if (!route.handler) {\n      ctx.log.error(`No handler found for route \"${routeName}\". URL will not be processed. URL: ${url}`); // prettier-ignore\n      return;\n    }\n    ctx.log.info(`Passing URL to handler for route \"${routeName}\". URL: ${url}`);\n    await route.handler(ctx);\n  };\n\n  const defaultHandler = async <TCtx extends CrawleeOneRouteCtx<T, RouterCtx>>(\n    ctx: TCtx\n  ): Promise<void> => {\n    const { page, log: parentLog } = ctx;\n    const log = parentLog.child({ prefix: '[Router] ' });\n\n    let handledRequestsCount = 0;\n    let req: CrawlerRequest | null = ctx.request ?? null;\n\n    const hasBatchReqs = () =>\n      perfBatchSize != null && req != null && handledRequestsCount < perfBatchSize;\n\n    /* eslint-disable-next-line @typescript-eslint/no-non-null-assertion */\n    const getUrl = () => (page ? (page as any as CommonPage).url() : req!.loadedUrl || req!.url);\n\n    const onRequest = async () => {\n      const url = await getUrl();\n      const logSuffix = `Batch ${handledRequestsCount + 1} of ${perfBatchSize ?? 1}. URL: ${url}`;\n\n      // Find route handler for given URL\n      log.debug(`Searching for a handler for given Request. ${logSuffix}`);\n      const [routeName, route] =\n        (await serialAsyncFind(\n          Object.entries<CrawleeOneRoute<T, RouterCtx>>(resolvedRoutes),\n          async ([key, currRoute]) => {\n            const isMatch = await serialAsyncFind(\n              currRoute.match as CrawleeOneRouteMatcherFn<T, RouterCtx>[],\n              async (matchFn) => {\n                return matchFn(url, ctx, currRoute, routes);\n              }\n            );\n            return isMatch;\n          }\n        )) ?? [];\n\n      // Run the handler\n      if (route) {\n        log.info(`URL matched route \"${routeName}\". ${logSuffix}`);\n        await defaultAction(ctx, url, routeName ?? '', route);\n      } else {\n        log.error(`No route matched URL. URL will not be processed. ${logSuffix}`);\n      }\n\n      // Clean up and move onto another request\n      await closeRequest(req);\n      handledRequestsCount++;\n\n      req = await loadNextRequest(logSuffix, { page: page as Page, log });\n    };\n\n    try {\n      do {\n        await onRequest();\n      } while (hasBatchReqs());\n    } catch (err) {\n      await onError(err, req, log);\n    }\n  };\n\n  return defaultHandler;\n};\n\n/**\n * Configures the default router handler to redirect URLs to labelled route handlers\n * based on which route the URL matches first.\n *\n * NOTE: This does mean that the URLs passed to this default handler will be fetched\n * twice (as the URL will be requeued to the correct handler). We recommend to use this\n * function only in the scenarios where there is a small number of startUrls, yet these\n * may need various ways of processing based on different paths or etc.\n *\n * @example\n *\n * const routeLabels = {\n *   MAIN_PAGE: 'MAIN_PAGE',\n *   JOB_LISTING: 'JOB_LISTING',\n *   JOB_DETAIL: 'JOB_DETAIL',\n *   JOB_RELATED_LIST: 'JOB_RELATED_LIST',\n *   PARTNERS: 'PARTNERS',\n * } as const;\n *\n * const router = createPlaywrightRouter();\n *\n * const routes = createPlaywrightCrawleeOneRouteMatchers<typeof routeLabels>([\n *  // URLs that match this route are redirected to router.addHandler(routeLabels.MAIN_PAGE)\n *  {\n *     route: routeLabels.MAIN_PAGE,\n *     // Check for main page like https://www.profesia.sk/?#\n *     match: (url) => url.match(/[\\W]profesia\\.sk\\/?(?:[?#~]|$)/i),\n *   },\n *\n *  // Optionally override the logic that assigns the URL to the route by specifying the `action` prop\n *  {\n *     route: routeLabels.MAIN_PAGE,\n *     // Check for main page like https://www.profesia.sk/?#\n *     match: (url) => url.match(/[\\W]profesia\\.sk\\/?(?:[?#~]|$)/i),\n *     action: async (ctx) => {\n *       await ctx.crawler.addRequests([{\n *         url: 'https://profesia.sk/praca',\n *         label: routeLabels.JOB_LISTING,\n *       }]);\n *     },\n *   },\n * ]);\n *\n * // Set up default route to redirect to labelled routes\n * setupDefaultHandlers({ router, routes });\n *\n * // Now set up the labelled routes\n * await router.addHandler(routeLabels.JOB_LISTING, async (ctx) => { ... }\n */\nexport const setupDefaultHandlers = async <\n  T extends CrawleeOneCtx,\n  RouterCtx extends Record<string, any> = CrawleeOneRouteCtx<T>\n>({\n  io,\n  router,\n  routeHandlerWrappers,\n  routerContext,\n  routes,\n  input,\n  onSetCtx,\n}: {\n  io: T['io'];\n  router: CrawlerRouter<T['context']>;\n  routeHandlerWrappers?: CrawleeOneRouteWrapper<T, RouterCtx>[];\n  routerContext?: RouterCtx;\n  routes: Record<T['labels'], CrawleeOneRoute<T, RouterCtx>>;\n  input?: T['input'] | null;\n  onSetCtx?: (ctx: Parameters<CrawleeOneRouteHandler<T, RouterCtx>>[0] | null) => void;\n}) => {\n  const { perfBatchSize, perfBatchWaitSecs, requestQueueId } = (input || {}) as PerfActorInput &\n    RequestActorInput;\n\n  const defaultHandler = createDefaultHandler({\n    io,\n    routes,\n    requestQueueId,\n    perfBatchSize,\n    perfBatchWaitSecs,\n  });\n\n  const wrappedHandler = await applyWrappersRight(defaultHandler, routeHandlerWrappers ?? []);\n  await router.addDefaultHandler<T['context']>(async (ctx) => {\n    // For the duration of the handler execution, set the actor.handlerCtx to the value of `ctx`,\n    // and then set it back to null;\n    onSetCtx?.(ctx as any);\n    let result;\n    try {\n      result = await wrappedHandler({ ...routerContext, ...ctx } as any);\n    } finally {\n      onSetCtx?.(null);\n    }\n    return result;\n  });\n};\n"]}

package/dist/cjs/lib/telemetry/sentry.d.ts

@@ -1,4 +1,4 @@
 import * as Sentry from '@sentry/node';
 import type { CrawleeOneTelemetry } from './types';
 import type { CrawleeOneCtx } from '../actor/types';
-export declare const createSentryTelemetry: <T extends CrawleeOneTelemetry<CrawleeOneCtx<import("crawlee").CrawlingContext<import("crawlee").BasicCrawler<import("crawlee").BasicCrawlingContext<import("crawlee").Dictionary>> | import("crawlee").PuppeteerCrawler | import("crawlee").PlaywrightCrawler | import("crawlee").JSDOMCrawler | import("crawlee").CheerioCrawler | import("crawlee").HttpCrawler<import("crawlee").InternalHttpCrawlingContext<any, any, import("crawlee").HttpCrawler<any>>>, import("crawlee").Dictionary>, string, Record<string, any>, import("../..").CrawleeOneIO<object, object, object>, CrawleeOneTelemetry<any, any>>, import("../..").CrawleeOneErrorHandlerOptions<import("../..").CrawleeOneIO<object, object, object>>>>(sentryOptions?: Sentry.NodeOptions) => T;
+export declare const createSentryTelemetry: <T extends CrawleeOneTelemetry<CrawleeOneCtx<import("crawlee").CrawlingContext<import("crawlee").JSDOMCrawler | import("crawlee").CheerioCrawler | import("crawlee").PlaywrightCrawler | import("crawlee").PuppeteerCrawler | import("crawlee").BasicCrawler<import("crawlee").BasicCrawlingContext<import("crawlee").Dictionary>> | import("crawlee").HttpCrawler<import("crawlee").InternalHttpCrawlingContext<any, any, import("crawlee").HttpCrawler<any>>>, import("crawlee").Dictionary>, string, Record<string, any>, import("../..").CrawleeOneIO<object, object, object>, CrawleeOneTelemetry<any, any>>, import("../..").CrawleeOneErrorHandlerOptions<import("../..").CrawleeOneIO<object, object, object>>>>(sentryOptions?: Sentry.NodeOptions) => T;

package/dist/cjs/lib/test/actor.d.ts

@@ -4,18 +4,18 @@ import type { MaybeArray, MaybePromise } from '../../utils/types';
 import { OnBatchAddRequests } from './mockApifyClient';
 export declare const setupMockApifyActor: <TInput, TData extends MaybeArray<Dictionary> = MaybeArray<Dictionary>>({ vi: viInstance, actorInput, log, onPushData, onBatchAddRequests, onGetInfo, }: {
     vi: typeof vi;
-    actorInput?: TInput | undefined;
-    log?: ((...args: any[]) => void) | undefined;
-    onPushData?: ((data: TData) => MaybePromise<void>) | undefined;
-    onBatchAddRequests?: OnBatchAddRequests | undefined;
-    onGetInfo?: ((...args: any[]) => MaybePromise<void>) | undefined;
+    actorInput?: TInput;
+    log?: (...args: any[]) => void;
+    onPushData?: (data: TData) => MaybePromise<void>;
+    onBatchAddRequests?: OnBatchAddRequests;
+    onGetInfo?: (...args: any[]) => MaybePromise<void>;
 }) => Promise<void>;
 export declare const runCrawlerTest: <TData extends MaybeArray<Dictionary>, TInput>({ vi: viInstance, input, runCrawler, log, onPushData, onBatchAddRequests, onDone, }: {
     vi: typeof vi;
     input: TInput;
     runCrawler: () => MaybePromise<void>;
-    log?: ((...args: any[]) => void) | undefined;
-    onPushData?: ((data: any, done: () => void) => MaybePromise<void>) | undefined;
-    onBatchAddRequests?: OnBatchAddRequests | undefined;
-    onDone?: ((done: () => void) => MaybePromise<void>) | undefined;
+    log?: (...args: any[]) => void;
+    onPushData?: (data: any, done: () => void) => MaybePromise<void>;
+    onBatchAddRequests?: OnBatchAddRequests;
+    onDone?: (done: () => void) => MaybePromise<void>;
 }) => Promise<void>;

package/dist/cjs/lib/test/actor.js

@@ -13,7 +13,7 @@ exports.runCrawlerTest = exports.setupMockApifyActor = void 0;
 const apify_1 = require("apify");
 const crawlee_1 = require("crawlee");
 const mockApifyClient_1 = require("./mockApifyClient");
-const setupMockApifyActor = ({ vi: viInstance, actorInput, log, onPushData, onBatchAddRequests, onGetInfo, }) => __awaiter(void 0, void 0, void 0, function* () {
+const setupMockApifyActor = (_a) => __awaiter(void 0, [_a], void 0, function* ({ vi: viInstance, actorInput, log, onPushData, onBatchAddRequests, onGetInfo, }) {
     const mockStorageClient = (0, mockApifyClient_1.createMockStorageClient)({ log, onBatchAddRequests });
     viInstance.spyOn(apify_1.Actor, 'main').mockImplementation((fn) => __awaiter(void 0, void 0, void 0, function* () { return fn(); }));
     viInstance.spyOn(apify_1.Actor, 'getInput').mockImplementation(() => Promise.resolve(actorInput));
@@ -39,7 +39,7 @@ const setupMockApifyActor = ({ vi: viInstance, actorInput, log, onPushData, onBa
     yield apify_1.Actor.init();
 });
 exports.setupMockApifyActor = setupMockApifyActor;
-const runCrawlerTest = ({ vi: viInstance, input, runCrawler, log, onPushData, onBatchAddRequests, onDone = (done) => done(), }) => __awaiter(void 0, void 0, void 0, function* () {
+const runCrawlerTest = (_b) => __awaiter(void 0, [_b], void 0, function* ({ vi: viInstance, input, runCrawler, log, onPushData, onBatchAddRequests, onDone = (done) => done(), }) {
     yield new Promise((done) => __awaiter(void 0, void 0, void 0, function* () {
         yield (0, exports.setupMockApifyActor)({
             vi: viInstance,

package/dist/cjs/lib/test/actor.js.map

@@ -1 +1 @@
-{"version":3,"file":"actor.js","sourceRoot":"","sources":["../../../../src/lib/test/actor.ts"],"names":[],"mappings":";;;;;;;;;;;;AACA,iCAA4C;AAC5C,qCAAoD;AAGpD,uDAI2B;AAEpB,MAAM,mBAAmB,GAAG,CAGjC,EACA,EAAE,EAAE,UAAU,EACd,UAAU,EACV,GAAG,EACH,UAAU,EACV,kBAAkB,EAClB,SAAS,GAQV,EAAE,EAAE;IACH,MAAM,iBAAiB,GAAG,IAAA,yCAAuB,EAAC,EAAE,GAAG,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAE/E,UAAU,CAAC,KAAK,CAAC,aAAK,EAAE,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAO,EAAE,EAAE,EAAE,kDAAC,OAAA,EAAE,EAAE,CAAA,GAAA,CAAC,CAAC;IACvE,UAAU,CAAC,KAAK,CAAC,aAAK,EAAE,UAAU,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IAE1F,UAAU,CAAC,KAAK,CAAC,aAAK,EAAE,aAAa,CAAC,CAAC,kBAAkB,CAAC,CAAO,SAAS,EAAE,OAAO,EAAE,EAAE;QACrF,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE,SAAS,CAAC,CAAC;QACnD,OAAO,IAAA,0CAAwB,EAAC,SAAS,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC;IACtF,CAAC,CAAA,CAAC,CAAC;IACH,UAAU,CAAC,KAAK,CAAC,aAAK,EAAE,UAAU,CAAC,CAAC,kBAAkB,CAAC,CAAO,IAAI,EAAE,EAAE;QACpE,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,IAAI,UAAU;YAAE,MAAM,UAAU,CAAC,IAAW,CAAC,CAAC;IAChD,CAAC,CAAA,CAAC,CAAC;IAEH,UAAU,CAAC,KAAK,CAAC,oBAAY,EAAE,MAAM,CAAC,CAAC,kBAAkB,CAAC,GAAS,EAAE;QACnE,MAAM,QAAQ,GAAG,IAAI,oBAAY,CAAC;YAChC,EAAE,EAAE,MAAM;YACV,MAAM,EAAE,iBAAiB;SAC1B,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAA,CAAC,CAAC;IAEH,UAAU;SACP,KAAK,CAAC,uBAAa,EAAE,MAAM,CAAC;SAC5B,kBAAkB,CACjB,GAAS,EAAE,kDAAC,OAAA,IAAI,uBAAa,CAAC,EAAE,EAAE,EAAE,aAAa,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAA,GAAA,CAChF,CAAC;IAEJ,MAAM,aAAK,CAAC,IAAI,EAAE,CAAC;AACrB,CAAC,CAAA,CAAC;AA/CW,QAAA,mBAAmB,uBA+C9B;AAEK,MAAM,cAAc,GAAG,CAAqD,EACjF,EAAE,EAAE,UAAU,EACd,KAAK,EACL,UAAU,EACV,GAAG,EACH,UAAU,EACV,kBAAkB,EAClB,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,GAS1B,EAAE,EAAE;IACH,MAAM,IAAI,OAAO,CAAO,CAAO,IAAI,EAAE,EAAE;QACrC,MAAM,IAAA,2BAAmB,EAAgB;YACvC,EAAE,EAAE,UAAU;YACd,UAAU,oBAAO,KAAK,CAAE;YACxB,GAAG;YACH,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAG,IAAI,EAAE,IAAI,CAAC;YAC9C,kBAAkB;SACnB,CAAC,CAAC;QAEH,MAAM,UAAU,EAAE,CAAC;QACnB,MAAM,CAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAG,IAAI,CAAC,CAAA,CAAC;IACvB,CAAC,CAAA,CAAC,CAAC;AACL,CAAC,CAAA,CAAC;AA7BW,QAAA,cAAc,kBA6BzB","sourcesContent":["import type { vi } from 'vitest';\nimport { Actor, RequestQueue } from 'apify';\nimport { Dictionary, KeyValueStore } from 'crawlee';\n\nimport type { MaybeArray, MaybePromise } from '../../utils/types';\nimport {\n  OnBatchAddRequests,\n  createMockStorageClient,\n  createMockStorageDataset,\n} from './mockApifyClient';\n\nexport const setupMockApifyActor = async <\n  TInput,\n  TData extends MaybeArray<Dictionary> = MaybeArray<Dictionary>\n>({\n  vi: viInstance,\n  actorInput,\n  log,\n  onPushData,\n  onBatchAddRequests,\n  onGetInfo,\n}: {\n  vi: typeof vi;\n  actorInput?: TInput;\n  log?: (...args: any[]) => void;\n  onPushData?: (data: TData) => MaybePromise<void>;\n  onBatchAddRequests?: OnBatchAddRequests;\n  onGetInfo?: (...args: any[]) => MaybePromise<void>;\n}) => {\n  const mockStorageClient = createMockStorageClient({ log, onBatchAddRequests });\n\n  viInstance.spyOn(Actor, 'main').mockImplementation(async (fn) => fn());\n  viInstance.spyOn(Actor, 'getInput').mockImplementation(() => Promise.resolve(actorInput));\n\n  viInstance.spyOn(Actor, 'openDataset').mockImplementation(async (datasetId, options) => {\n    console.log('Mock Actor.openDataset: ', datasetId);\n    return createMockStorageDataset(datasetId, options, { log, onPushData, onGetInfo });\n  });\n  viInstance.spyOn(Actor, 'pushData').mockImplementation(async (data) => {\n    console.log('Mock Actor.pushData');\n    if (onPushData) await onPushData(data as any);\n  });\n\n  viInstance.spyOn(RequestQueue, 'open').mockImplementation(async () => {\n    const reqQueue = new RequestQueue({\n      id: 'test',\n      client: mockStorageClient,\n    });\n    return reqQueue;\n  });\n\n  viInstance\n    .spyOn(KeyValueStore, 'open')\n    .mockImplementation(\n      async () => new KeyValueStore({ id: 'keyvalstore', client: mockStorageClient })\n    );\n\n  await Actor.init();\n};\n\nexport const runCrawlerTest = async <TData extends MaybeArray<Dictionary>, TInput>({\n  vi: viInstance,\n  input,\n  runCrawler,\n  log,\n  onPushData,\n  onBatchAddRequests,\n  onDone = (done) => done(),\n}: {\n  vi: typeof vi;\n  input: TInput;\n  runCrawler: () => MaybePromise<void>;\n  log?: (...args: any[]) => void;\n  onPushData?: (data: any, done: () => void) => MaybePromise<void>;\n  onBatchAddRequests?: OnBatchAddRequests;\n  onDone?: (done: () => void) => MaybePromise<void>;\n}) => {\n  await new Promise<void>(async (done) => {\n    await setupMockApifyActor<TInput, TData>({\n      vi: viInstance,\n      actorInput: { ...input },\n      log,\n      onPushData: (data) => onPushData?.(data, done),\n      onBatchAddRequests,\n    });\n\n    await runCrawler();\n    await onDone?.(done);\n  });\n};\n"]}
+{"version":3,"file":"actor.js","sourceRoot":"","sources":["../../../../src/lib/test/actor.ts"],"names":[],"mappings":";;;;;;;;;;;;AACA,iCAA4C;AAC5C,qCAAoD;AAGpD,uDAI2B;AAEpB,MAAM,mBAAmB,GAAG,KAiBhC,EAAE,4CAdH,EACA,EAAE,EAAE,UAAU,EACd,UAAU,EACV,GAAG,EACH,UAAU,EACV,kBAAkB,EAClB,SAAS,GAQV;IACC,MAAM,iBAAiB,GAAG,IAAA,yCAAuB,EAAC,EAAE,GAAG,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAE/E,UAAU,CAAC,KAAK,CAAC,aAAK,EAAE,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAO,EAAE,EAAE,EAAE,kDAAC,OAAA,EAAE,EAAE,CAAA,GAAA,CAAC,CAAC;IACvE,UAAU,CAAC,KAAK,CAAC,aAAK,EAAE,UAAU,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IAE1F,UAAU,CAAC,KAAK,CAAC,aAAK,EAAE,aAAa,CAAC,CAAC,kBAAkB,CAAC,CAAO,SAAS,EAAE,OAAO,EAAE,EAAE;QACrF,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE,SAAS,CAAC,CAAC;QACnD,OAAO,IAAA,0CAAwB,EAAC,SAAS,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC;IACtF,CAAC,CAAA,CAAC,CAAC;IACH,UAAU,CAAC,KAAK,CAAC,aAAK,EAAE,UAAU,CAAC,CAAC,kBAAkB,CAAC,CAAO,IAAI,EAAE,EAAE;QACpE,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,IAAI,UAAU;YAAE,MAAM,UAAU,CAAC,IAAW,CAAC,CAAC;IAChD,CAAC,CAAA,CAAC,CAAC;IAEH,UAAU,CAAC,KAAK,CAAC,oBAAY,EAAE,MAAM,CAAC,CAAC,kBAAkB,CAAC,GAAS,EAAE;QACnE,MAAM,QAAQ,GAAG,IAAI,oBAAY,CAAC;YAChC,EAAE,EAAE,MAAM;YACV,MAAM,EAAE,iBAAiB;SAC1B,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAA,CAAC,CAAC;IAEH,UAAU;SACP,KAAK,CAAC,uBAAa,EAAE,MAAM,CAAC;SAC5B,kBAAkB,CACjB,GAAS,EAAE,kDAAC,OAAA,IAAI,uBAAa,CAAC,EAAE,EAAE,EAAE,aAAa,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAA,GAAA,CAChF,CAAC;IAEJ,MAAM,aAAK,CAAC,IAAI,EAAE,CAAC;AACrB,CAAC,CAAA,CAAC;AA/CW,QAAA,mBAAmB,uBA+C9B;AAEK,MAAM,cAAc,GAAG,KAgB3B,EAAE,4CAhB8E,EACjF,EAAE,EAAE,UAAU,EACd,KAAK,EACL,UAAU,EACV,GAAG,EACH,UAAU,EACV,kBAAkB,EAClB,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,GAS1B;IACC,MAAM,IAAI,OAAO,CAAO,CAAO,IAAI,EAAE,EAAE;QACrC,MAAM,IAAA,2BAAmB,EAAgB;YACvC,EAAE,EAAE,UAAU;YACd,UAAU,oBAAO,KAAK,CAAE;YACxB,GAAG;YACH,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAG,IAAI,EAAE,IAAI,CAAC;YAC9C,kBAAkB;SACnB,CAAC,CAAC;QAEH,MAAM,UAAU,EAAE,CAAC;QACnB,MAAM,CAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAG,IAAI,CAAC,CAAA,CAAC;IACvB,CAAC,CAAA,CAAC,CAAC;AACL,CAAC,CAAA,CAAC;AA7BW,QAAA,cAAc,kBA6BzB","sourcesContent":["import type { vi } from 'vitest';\nimport { Actor, RequestQueue } from 'apify';\nimport { Dictionary, KeyValueStore } from 'crawlee';\n\nimport type { MaybeArray, MaybePromise } from '../../utils/types';\nimport {\n  OnBatchAddRequests,\n  createMockStorageClient,\n  createMockStorageDataset,\n} from './mockApifyClient';\n\nexport const setupMockApifyActor = async <\n  TInput,\n  TData extends MaybeArray<Dictionary> = MaybeArray<Dictionary>\n>({\n  vi: viInstance,\n  actorInput,\n  log,\n  onPushData,\n  onBatchAddRequests,\n  onGetInfo,\n}: {\n  vi: typeof vi;\n  actorInput?: TInput;\n  log?: (...args: any[]) => void;\n  onPushData?: (data: TData) => MaybePromise<void>;\n  onBatchAddRequests?: OnBatchAddRequests;\n  onGetInfo?: (...args: any[]) => MaybePromise<void>;\n}) => {\n  const mockStorageClient = createMockStorageClient({ log, onBatchAddRequests });\n\n  viInstance.spyOn(Actor, 'main').mockImplementation(async (fn) => fn());\n  viInstance.spyOn(Actor, 'getInput').mockImplementation(() => Promise.resolve(actorInput));\n\n  viInstance.spyOn(Actor, 'openDataset').mockImplementation(async (datasetId, options) => {\n    console.log('Mock Actor.openDataset: ', datasetId);\n    return createMockStorageDataset(datasetId, options, { log, onPushData, onGetInfo });\n  });\n  viInstance.spyOn(Actor, 'pushData').mockImplementation(async (data) => {\n    console.log('Mock Actor.pushData');\n    if (onPushData) await onPushData(data as any);\n  });\n\n  viInstance.spyOn(RequestQueue, 'open').mockImplementation(async () => {\n    const reqQueue = new RequestQueue({\n      id: 'test',\n      client: mockStorageClient,\n    });\n    return reqQueue;\n  });\n\n  viInstance\n    .spyOn(KeyValueStore, 'open')\n    .mockImplementation(\n      async () => new KeyValueStore({ id: 'keyvalstore', client: mockStorageClient })\n    );\n\n  await Actor.init();\n};\n\nexport const runCrawlerTest = async <TData extends MaybeArray<Dictionary>, TInput>({\n  vi: viInstance,\n  input,\n  runCrawler,\n  log,\n  onPushData,\n  onBatchAddRequests,\n  onDone = (done) => done(),\n}: {\n  vi: typeof vi;\n  input: TInput;\n  runCrawler: () => MaybePromise<void>;\n  log?: (...args: any[]) => void;\n  onPushData?: (data: any, done: () => void) => MaybePromise<void>;\n  onBatchAddRequests?: OnBatchAddRequests;\n  onDone?: (done: () => void) => MaybePromise<void>;\n}) => {\n  await new Promise<void>(async (done) => {\n    await setupMockApifyActor<TInput, TData>({\n      vi: viInstance,\n      actorInput: { ...input },\n      log,\n      onPushData: (data) => onPushData?.(data, done),\n      onBatchAddRequests,\n    });\n\n    await runCrawler();\n    await onDone?.(done);\n  });\n};\n"]}

package/dist/cjs/lib/test/mockApifyClient.d.ts

@@ -11,18 +11,18 @@ export type OnBatchAddRequests = (...args: OnBatchAddRequestsArgs) => MaybePromi
 export declare const createMockClientDataset: (overrides?: ClientDataset) => ClientDataset;
 export declare const createMockClientRequestQueue: (overrides?: ClientRequestQueue) => ClientRequestQueue;
 export declare const createMockKeyValueStoreClient: ({ log, }?: {
-    log?: ((args: any) => void) | undefined;
+    log?: (args: any) => void;
 }) => KeyValueStoreClient;
 export declare const createMockRequestQueueClient: ({ log, onBatchAddRequests, }?: {
-    log?: ((args: any) => void) | undefined;
-    onBatchAddRequests?: OnBatchAddRequests | undefined;
+    log?: (args: any) => void;
+    onBatchAddRequests?: OnBatchAddRequests;
 }) => RequestQueueClient;
 export declare const createMockDatasetCollectionClient: ({ log, }?: {
-    log?: ((args: any) => void) | undefined;
+    log?: (args: any) => void;
 }) => DatasetCollectionClient;
 export declare const createMockStorageClient: ({ log, onBatchAddRequests, }?: {
-    log?: ((args: any) => void) | undefined;
-    onBatchAddRequests?: OnBatchAddRequests | undefined;
+    log?: (args: any) => void;
+    onBatchAddRequests?: OnBatchAddRequests;
 }) => StorageClient;
 export declare const createMockStorageDataset: (datasetId?: string | null | undefined, options?: OpenStorageOptions | undefined, custom?: {
     log?: ((...args: any[]) => void) | undefined;

package/dist/cjs/lib/test/mockApifyClient.js.map

@@ -1 +1 @@
-{"version":3,"file":"mockApifyClient.js","sourceRoot":"","sources":["../../../../src/lib/test/mockApifyClient.ts"],"names":[],"mappings":";;;;;;;;;;;;AAgCO,MAAM,uBAAuB,GAAG,CAAC,SAAyB,EAAiB,EAAE,CAAC,iBACnF,EAAE,EAAE,sBAAsB,EAC1B,IAAI,EAAE,wBAAwB,EAC9B,KAAK,EAAE,yBAAyB,EAChC,MAAM,EAAE,6BAA6B,EACrC,SAAS,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,EACjC,UAAU,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,EAClC,UAAU,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,EAClC,SAAS,EAAE,EAAE,EACb,cAAc,EAAE,EAAE,EAClB,KAAK,EAAE,yBAAyB,EAChC,QAAQ,EAAE,4BAA4B,EACtC,KAAK,EAAE,EAAE,EACT,MAAM,EAAE,EAAE,IACP,SAAS,EACZ,CAAC;AAfU,QAAA,uBAAuB,2BAejC;AAEI,MAAM,4BAA4B,GAAG,CAC1C,SAA8B,EACV,EAAE,CAAC,iBACvB,EAAE,EAAE,2BAA2B,EAC/B,IAAI,EAAE,6BAA6B,EACnC,KAAK,EAAE,8BAA8B,EACrC,MAAM,EAAE,+BAA+B,EACvC,SAAS,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,EACjC,UAAU,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,EAClC,UAAU,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC;IAClC,qBAAqB;IACrB,iBAAiB,EAAE,CAAC,EACpB,mBAAmB,EAAE,CAAC,EACtB,mBAAmB,EAAE,CAAC;IACtB,kBAAkB;IAClB,qBAAqB;IACrB,kBAAkB,EAAE,KAAK,EACzB,KAAK,EAAE,EAAE,IACN,SAAS,EACZ,CAAC;AAnBU,QAAA,4BAA4B,gCAmBtC;AAEI,MAAM,6BAA6B,GAAG,CAAC,EAC5C,GAAG,MAC8B,EAAE,EAAuB,EAAE;IAC5D,OAAO;QACL,SAAS,EAAE,CAAC,GAAG,IAAc,EAAiD,EAAE;YAC9E,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;YACnB,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,iDAAiD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/E,OAAO,OAAO,CAAC,OAAO,CAAC;gBACrB,GAAG;gBACH,KAAK,EAAE,IAAI;gBACX,wBAAwB;aACzB,CAAC,CAAC;QACL,CAAC;QACD,SAAS,EAAE,CAAC,GAAG,IAAwC,EAAiB,EAAE;YACxE,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,iDAAiD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/E,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;QACD,iDAAiD;QACjD,4FAA4F;QAC5F,2BAA2B;QAC3B,4FAA4F;QAC5F,mHAAmH;QACnH,yDAAyD;QACzD,4CAA4C;KACtC,CAAC;AACX,CAAC,CAAC;AAzBW,QAAA,6BAA6B,iCAyBxC;AAEK,MAAM,4BAA4B,GAAG,CAAC,EAC3C,GAAG,EACH,kBAAkB,MAIhB,EAAE,EAAsB,EAAE;IAC5B,MAAM,YAAY,GAAU,EAAE,CAAC;IAE/B,OAAO;QACL,gBAAgB,EAAE,CAChB,GAAG,IAA4B,EAC0B,EAAE;YAC3D,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;YACjC,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,uDAAuD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACrF,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iCAAM,CAAC,KAAE,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAG,CAAC,CAAC;YAC5E,YAAY,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;YAElC,MAAM,gBAAgB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/C,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,SAAS,EAAE,CAAC,CAAC,EAAE;gBACf,iBAAiB,EAAE,KAAK;gBACxB,iBAAiB,EAAE,KAAK;aACzB,CAAC,CAAC,CAAC;YAEJ,MAAM,CAAA,kBAAkB,aAAlB,kBAAkB,uBAAlB,kBAAkB,CAAG,QAAQ,EAAE,OAAO,CAAC,CAAA,CAAC;YAE9C,OAAO,OAAO,CAAC,OAAO,CAAC;gBACrB,mBAAmB,EAAE,EAAE;gBACvB,iBAAiB,EAAE,gBAAgB;aACpC,CAAC,CAAC;QACL,CAAC,CAAA;QAED,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE;YACf,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,0CAA0C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxE,OAAO,OAAO,CAAC,OAAO,CAAC,IAAA,oCAA4B,GAAE,CAAC,CAAC;QACzD,CAAC;QAED,UAAU,EAAE,CACV,GAAG,IAAkB,EACoC,EAAE;YAC3D,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC;YAClB,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,iDAAiD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/E,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;YAExE,MAAM,kBAAkB,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YACtE,IAAI,kBAAkB,KAAK,CAAC,CAAC;gBAAE,OAAO,SAAS,CAAC;YAEhD,MAAM,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC;YACzD,OAAO,GAAG,CAAC;QACb,CAAC,CAAA;QAED,aAAa,EAAE,CACb,GAAG,IAGF,EAC4C,EAAE;YAC/C,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;YAChC,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,oDAAoD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClF,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;YAEvD,MAAM,qBAAqB,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,EAAE,CAAC,CAAC;YACjF,MAAM,iBAAiB,GAAG,qBAAqB,KAAK,CAAC,CAAC,CAAC;YACvD,IAAI,iBAAiB,EAAE;gBACrB,MAAM,GAAG,GAAG,YAAY,CAAC,qBAAqB,CAAC,CAAC;gBAChD,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;aAC7B;iBAAM;gBACL,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;gBACvC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;aACxB;YAED,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;YAEvD,yTAAyT;YACzT,OAAO,OAAO,CAAC,OAAO,CAAC;gBACrB,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,iBAAiB;gBACjB,iBAAiB,EAAE,KAAK;aACzB,CAAC,CAAC;QACL,CAAC;QAED,QAAQ,EAAE,CACR,GAAG,IAAmD,EACX,EAAE;;YAC7C,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;YACvB,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,+CAA+C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC7E,OAAO,OAAO,CAAC,OAAO,CAAC;gBACrB,KAAK,EAAE,EAAE;gBACT,eAAe,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC;gBACvC,kBAAkB,EAAE,KAAK;gBACzB,KAAK,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,KAAK,mCAAI,GAAG;aAC7B,CAAC,CAAC;QACL,CAAC;KACK,CAAC;AACX,CAAC,CAAC;AA/FW,QAAA,4BAA4B,gCA+FvC;AAEK,MAAM,iCAAiC,GAAG,CAAC,EAChD,GAAG,MAC8B,EAAE,EAA2B,EAAE;IAChE,MAAM,OAAO,GAAG,EAAE,CAAC;IAEnB,OAAO;QACL,IAAI,EAAE,CACJ,OAA4C,EACA,EAAE;;YAC9C,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;YAC5B,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI;gBAAE,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;YAC3C,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM;gBAAE,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAEzD,OAAO,OAAO,CAAC,OAAO,CAAC;gBACrB,KAAK,EAAE,OAAO,CAAC,MAAM;gBACrB,KAAK,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,KAAK,mCAAI,KAAK,CAAC,MAAM;gBACrC,MAAM,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM,mCAAI,CAAC;gBAC5B,KAAK,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,KAAK,mCAAI,KAAK,CAAC,MAAM;gBACrC,IAAI,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,mCAAI,KAAK;gBAC5B,KAAK;aACN,CAAC,CAAC;QACL,CAAC;QAED,WAAW,EAAE,CACX,IAAa,EACb,OAAmD,EAC3B,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAA,+BAAuB,GAAE,CAAC;QAEvE,wCAAwC;KAClC,CAAC;AACX,CAAC,CAAC;AA9BW,QAAA,iCAAiC,qCA8B5C;AAEK,MAAM,uBAAuB,GAAG,CAAC,EACtC,GAAG,EACH,kBAAkB,MAIhB,EAAE,EAAiB,EAAE;IACvB,OAAO;QACL,QAAQ,EAAE,CAAC,GAAG,IAAI,EAA2B,EAAE;YAC7C,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,0CAA0C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxE,OAAO,IAAA,yCAAiC,EAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,YAAY,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE;YACxB,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,8CAA8C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC5E,OAAO,IAAA,oCAA4B,EAAC,EAAE,GAAG,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACnE,CAAC;QACD,aAAa,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE;YACzB,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,+CAA+C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC7E,OAAO,IAAA,qCAA6B,EAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QAChD,CAAC;QAED,yCAAyC;KACnC,CAAC;AACX,CAAC,CAAC;AAvBW,QAAA,uBAAuB,2BAuBlC;AAEK,MAAM,wBAAwB,GAAG,CACtC,GAAG,IAQF,EACsB,EAAE;IACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1C,MAAM,CAAC,SAAS,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IACrC,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,MAAM,IAAI,EAAE,CAAC;IACpD,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,kCAAkC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAEpE,OAAO,OAAO,CAAC,OAAO,CAAC;QACrB,QAAQ,EAAE,CAAO,GAAG,IAAI,EAAE,EAAE;YAC1B,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,gDAAgD,SAAS,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,kBAAkB;YACpH,MAAM,CAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAG,GAAG,IAAI,CAAC,CAAA,CAAC;QAC9B,CAAC,CAAA;QACD,OAAO,EAAE,CAAO,GAAG,IAAI,EAAE,EAAE;YACzB,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,+CAA+C,SAAS,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,kBAAkB;YACnH,OAAO,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;QAC5D,CAAC,CAAA;KACS,CAAC,CAAC;AAChB,CAAC,CAAC;AA1BW,QAAA,wBAAwB,4BA0BnC","sourcesContent":["import type { Dataset, OpenStorageOptions } from 'apify';\nimport type {\n  RequestQueue as ClientRequestQueue,\n  RequestQueueClientAddRequestOptions,\n  RequestQueueClientAddRequestResult,\n  RequestQueueClientBatchAddRequestWithRetriesOptions,\n  RequestQueueClientBatchRequestsOperationResult,\n  RequestQueueClientGetRequestResult,\n  RequestQueueClientListHeadOptions,\n  RequestQueueClientListHeadResult,\n  RequestQueueClientRequestSchema,\n} from 'apify-client';\nimport type {\n  Dataset as ClientDataset,\n  DatasetCollectionClient,\n  DatasetCollectionClientGetOrCreateOptions,\n  DatasetCollectionClientListOptions,\n  DatasetCollectionClientListResult,\n  KeyValueStoreClient,\n  KeyValueStoreRecord,\n  RequestQueueClient,\n} from 'apify-client';\nimport type { StorageClient } from 'crawlee';\n\nimport type { MaybePromise } from '../../utils/types';\n\nexport type OnBatchAddRequestsArgs = [\n  requests: Omit<RequestQueueClientRequestSchema, 'id'>[],\n  options?: RequestQueueClientBatchAddRequestWithRetriesOptions\n];\nexport type OnBatchAddRequests = (...args: OnBatchAddRequestsArgs) => MaybePromise<void>;\n\nexport const createMockClientDataset = (overrides?: ClientDataset): ClientDataset => ({\n  id: 'MockClientDataset.id',\n  name: 'MockClientDataset.name',\n  title: 'MockClientDataset.title',\n  userId: 'MockClientDataset.id.userId',\n  createdAt: new Date('2023-04-17'),\n  modifiedAt: new Date('2023-04-17'),\n  accessedAt: new Date('2023-04-17'),\n  itemCount: 10,\n  cleanItemCount: 10,\n  actId: 'MockClientDataset.actId',\n  actRunId: 'MockClientDataset.actRunId',\n  stats: {},\n  fields: [],\n  ...overrides,\n});\n\nexport const createMockClientRequestQueue = (\n  overrides?: ClientRequestQueue\n): ClientRequestQueue => ({\n  id: 'MockClientRequestQueue.id',\n  name: 'MockClientRequestQueue.name',\n  title: 'MockClientRequestQueue.title',\n  userId: 'MockClientRequestQueue.userId',\n  createdAt: new Date('2023-04-17'),\n  modifiedAt: new Date('2023-04-17'),\n  accessedAt: new Date('2023-04-17'),\n  // expireAt?: string;\n  totalRequestCount: 0,\n  handledRequestCount: 0,\n  pendingRequestCount: 0,\n  // actId?: string;\n  // actRunId?: string;\n  hadMultipleClients: false,\n  stats: {},\n  ...overrides,\n});\n\nexport const createMockKeyValueStoreClient = ({\n  log,\n}: { log?: (args: any) => void } = {}): KeyValueStoreClient => {\n  return {\n    getRecord: (...args: [string]): Promise<KeyValueStoreRecord<any> | undefined> => {\n      const [key] = args;\n      log?.(`Called MockKeyValueStoreClient.getRecord with ${JSON.stringify(args)}`);\n      return Promise.resolve({\n        key,\n        value: null,\n        // contentType?: string;\n      });\n    },\n    setRecord: (...args: [record: KeyValueStoreRecord<any>]): Promise<void> => {\n      log?.(`Called MockKeyValueStoreClient.setRecord with ${JSON.stringify(args)}`);\n      return Promise.resolve();\n    },\n    // get(): Promise<KeyValueStoreInfo | undefined>;\n    // update(newFields: KeyValueStoreClientUpdateOptions): Promise<Partial<KeyValueStoreInfo>>;\n    // delete(): Promise<void>;\n    // listKeys(options?: KeyValueStoreClientListOptions): Promise<KeyValueStoreClientListData>;\n    // getRecord(key: string, options?: KeyValueStoreClientGetRecordOptions): Promise<KeyValueStoreRecord | undefined>;\n    // setRecord(record: KeyValueStoreRecord): Promise<void>;\n    // deleteRecord(key: string): Promise<void>;\n  } as any;\n};\n\nexport const createMockRequestQueueClient = ({\n  log,\n  onBatchAddRequests,\n}: {\n  log?: (args: any) => void;\n  onBatchAddRequests?: OnBatchAddRequests;\n} = {}): RequestQueueClient => {\n  const requestQueue: any[] = [];\n\n  return {\n    batchAddRequests: async (\n      ...args: OnBatchAddRequestsArgs\n    ): Promise<RequestQueueClientBatchRequestsOperationResult> => {\n      const [requests, options] = args;\n      log?.(`Called MockRequestQueueClient.batchAddRequests with ${JSON.stringify(args)}`);\n      const reqsWithIds = requests.map((r) => ({ ...r, id: Math.random() + '' }));\n      requestQueue.push(...reqsWithIds);\n\n      const reqsAddedToQueue = reqsWithIds.map((r) => ({\n        uniqueKey: r.uniqueKey,\n        requestId: r.id,\n        wasAlreadyPresent: false,\n        wasAlreadyHandled: false,\n      }));\n\n      await onBatchAddRequests?.(requests, options);\n\n      return Promise.resolve({\n        unprocessedRequests: [],\n        processedRequests: reqsAddedToQueue,\n      });\n    },\n\n    get: (...args) => {\n      log?.(`Called MockRequestQueueClient.get with ${JSON.stringify(args)}`);\n      return Promise.resolve(createMockClientRequestQueue());\n    },\n\n    getRequest: async (\n      ...args: [id: string]\n    ): Promise<RequestQueueClientGetRequestResult | undefined> => {\n      const [id] = args;\n      log?.(`Called MockRequestQueueClient.getRequest with ${JSON.stringify(args)}`);\n      log?.(`requestQueue: ${JSON.stringify(requestQueue.map((r) => r.id))}`);\n\n      const indexOfNextRequest = requestQueue.findIndex((r) => r.id === id);\n      if (indexOfNextRequest === -1) return undefined;\n\n      const [req] = requestQueue.splice(indexOfNextRequest, 1);\n      return req;\n    },\n\n    updateRequest: (\n      ...args: [\n        request: RequestQueueClientRequestSchema,\n        options?: RequestQueueClientAddRequestOptions\n      ]\n    ): Promise<RequestQueueClientAddRequestResult> => {\n      const [request, options] = args;\n      log?.(`Called MockRequestQueueClient.updateRequest with ${JSON.stringify(args)}`);\n      log?.(`requestQueue: ${JSON.stringify(requestQueue)}`);\n\n      const indexOfUpdatedRequest = requestQueue.findIndex((r) => r.id === request.id);\n      const wasAlreadyPresent = indexOfUpdatedRequest !== -1;\n      if (wasAlreadyPresent) {\n        const req = requestQueue[indexOfUpdatedRequest];\n        Object.assign(req, request);\n      } else {\n        const req = Object.assign({}, request);\n        requestQueue.push(req);\n      }\n\n      log?.(`requestQueue: ${JSON.stringify(requestQueue)}`);\n\n      // [{\"id\":\"0.9121846955138175\",\"url\":\"https://profesia.sk/partneri\",\"loadedUrl\":\"https://www.profesia.sk/partneri\",\"uniqueKey\":\"https://profesia.sk/partneri\",\"method\":\"GET\",\"noRetry\":false,\"retryCount\":0,\"errorMessages\":[],\"headers\":{},\"userData\":{\"__crawlee\":{\"state\":4}},\"handledAt\":\"2023-04-17T12:15:57.350Z\"}]\n      return Promise.resolve({\n        requestId: request.id,\n        wasAlreadyPresent,\n        wasAlreadyHandled: false,\n      });\n    },\n\n    listHead: (\n      ...args: [options?: RequestQueueClientListHeadOptions]\n    ): Promise<RequestQueueClientListHeadResult> => {\n      const [options] = args;\n      log?.(`Called MockRequestQueueClient.listHead with ${JSON.stringify(args)}`);\n      return Promise.resolve({\n        items: [],\n        queueModifiedAt: new Date('2023-04-17'),\n        hadMultipleClients: false,\n        limit: options?.limit ?? 100,\n      });\n    },\n  } as any;\n};\n\nexport const createMockDatasetCollectionClient = ({\n  log,\n}: { log?: (args: any) => void } = {}): DatasetCollectionClient => {\n  const dataset = [];\n\n  return {\n    list: (\n      options?: DatasetCollectionClientListOptions\n    ): Promise<DatasetCollectionClientListResult> => {\n      let items = dataset.slice();\n      if (options?.desc) items = items.reverse();\n      if (options?.offset) items = items.slice(options.offset);\n\n      return Promise.resolve({\n        total: dataset.length,\n        count: options?.limit ?? items.length,\n        offset: options?.offset ?? 0,\n        limit: options?.limit ?? items.length,\n        desc: options?.desc ?? false,\n        items,\n      });\n    },\n\n    getOrCreate: (\n      name?: string,\n      options?: DatasetCollectionClientGetOrCreateOptions\n    ): Promise<ClientDataset> => Promise.resolve(createMockClientDataset()),\n\n    // _list, _create, _getOrCreate, baseUrl\n  } as any;\n};\n\nexport const createMockStorageClient = ({\n  log,\n  onBatchAddRequests,\n}: {\n  log?: (args: any) => void;\n  onBatchAddRequests?: OnBatchAddRequests;\n} = {}): StorageClient => {\n  return {\n    datasets: (...args): DatasetCollectionClient => {\n      log?.(`Called MockStorageClient.datasets with ${JSON.stringify(args)}`);\n      return createMockDatasetCollectionClient({ log });\n    },\n    requestQueue: (...args) => {\n      log?.(`Called MockStorageClient.requestQueue with ${JSON.stringify(args)}`);\n      return createMockRequestQueueClient({ log, onBatchAddRequests });\n    },\n    keyValueStore: (...args) => {\n      log?.(`Called MockStorageClient.keyValueStore with ${JSON.stringify(args)}`);\n      return createMockKeyValueStoreClient({ log });\n    },\n\n    // dataset, keyValueStores, requestQueues\n  } as any;\n};\n\nexport const createMockStorageDataset = (\n  ...args: [\n    datasetId?: string | null,\n    options?: OpenStorageOptions,\n    custom?: {\n      log?: (...args: any[]) => void;\n      onPushData?: (...args: any[]) => MaybePromise<void>;\n      onGetInfo?: (...args: any[]) => MaybePromise<void>;\n    }\n  ]\n): Promise<Dataset<any>> => {\n  const origArgs = args.slice().slice(0, 2);\n  const [datasetId, __, custom] = args;\n  const { log, onPushData, onGetInfo } = custom || {};\n  log?.(`Called MockStorageDataset with ${JSON.stringify(origArgs)}`);\n\n  return Promise.resolve({\n    pushData: async (...args) => {\n      log?.(`Called MockStorageDataset.pushData (instance ${datasetId}) with ${JSON.stringify(args)}`); // prettier-ignore\n      await onPushData?.(...args);\n    },\n    getInfo: async (...args) => {\n      log?.(`Called MockStorageDataset.getInfo (instance ${datasetId}) with ${JSON.stringify(args)}`); // prettier-ignore\n      return onGetInfo ? onGetInfo(...args) : { itemCount: 10 };\n    },\n  } as Dataset);\n};\n"]}
+{"version":3,"file":"mockApifyClient.js","sourceRoot":"","sources":["../../../../src/lib/test/mockApifyClient.ts"],"names":[],"mappings":";;;;;;;;;;;;AAgCO,MAAM,uBAAuB,GAAG,CAAC,SAAyB,EAAiB,EAAE,CAAC,iBACnF,EAAE,EAAE,sBAAsB,EAC1B,IAAI,EAAE,wBAAwB,EAC9B,KAAK,EAAE,yBAAyB,EAChC,MAAM,EAAE,6BAA6B,EACrC,SAAS,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,EACjC,UAAU,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,EAClC,UAAU,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,EAClC,SAAS,EAAE,EAAE,EACb,cAAc,EAAE,EAAE,EAClB,KAAK,EAAE,yBAAyB,EAChC,QAAQ,EAAE,4BAA4B,EACtC,KAAK,EAAE,EAAE,EACT,MAAM,EAAE,EAAE,IACP,SAAS,EACZ,CAAC;AAfU,QAAA,uBAAuB,2BAejC;AAEI,MAAM,4BAA4B,GAAG,CAC1C,SAA8B,EACV,EAAE,CAAC,iBACvB,EAAE,EAAE,2BAA2B,EAC/B,IAAI,EAAE,6BAA6B,EACnC,KAAK,EAAE,8BAA8B,EACrC,MAAM,EAAE,+BAA+B,EACvC,SAAS,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,EACjC,UAAU,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,EAClC,UAAU,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC;IAClC,qBAAqB;IACrB,iBAAiB,EAAE,CAAC,EACpB,mBAAmB,EAAE,CAAC,EACtB,mBAAmB,EAAE,CAAC;IACtB,kBAAkB;IAClB,qBAAqB;IACrB,kBAAkB,EAAE,KAAK,EACzB,KAAK,EAAE,EAAE,IACN,SAAS,EACZ,CAAC;AAnBU,QAAA,4BAA4B,gCAmBtC;AAEI,MAAM,6BAA6B,GAAG,CAAC,EAC5C,GAAG,MAC8B,EAAE,EAAuB,EAAE;IAC5D,OAAO;QACL,SAAS,EAAE,CAAC,GAAG,IAAc,EAAiD,EAAE;YAC9E,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;YACnB,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,iDAAiD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/E,OAAO,OAAO,CAAC,OAAO,CAAC;gBACrB,GAAG;gBACH,KAAK,EAAE,IAAI;gBACX,wBAAwB;aACzB,CAAC,CAAC;QACL,CAAC;QACD,SAAS,EAAE,CAAC,GAAG,IAAwC,EAAiB,EAAE;YACxE,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,iDAAiD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/E,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;QACD,iDAAiD;QACjD,4FAA4F;QAC5F,2BAA2B;QAC3B,4FAA4F;QAC5F,mHAAmH;QACnH,yDAAyD;QACzD,4CAA4C;KACtC,CAAC;AACX,CAAC,CAAC;AAzBW,QAAA,6BAA6B,iCAyBxC;AAEK,MAAM,4BAA4B,GAAG,CAAC,EAC3C,GAAG,EACH,kBAAkB,MAIhB,EAAE,EAAsB,EAAE;IAC5B,MAAM,YAAY,GAAU,EAAE,CAAC;IAE/B,OAAO;QACL,gBAAgB,EAAE,CAChB,GAAG,IAA4B,EAC0B,EAAE;YAC3D,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;YACjC,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,uDAAuD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACrF,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iCAAM,CAAC,KAAE,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAG,CAAC,CAAC;YAC5E,YAAY,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;YAElC,MAAM,gBAAgB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/C,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,SAAS,EAAE,CAAC,CAAC,EAAE;gBACf,iBAAiB,EAAE,KAAK;gBACxB,iBAAiB,EAAE,KAAK;aACzB,CAAC,CAAC,CAAC;YAEJ,MAAM,CAAA,kBAAkB,aAAlB,kBAAkB,uBAAlB,kBAAkB,CAAG,QAAQ,EAAE,OAAO,CAAC,CAAA,CAAC;YAE9C,OAAO,OAAO,CAAC,OAAO,CAAC;gBACrB,mBAAmB,EAAE,EAAE;gBACvB,iBAAiB,EAAE,gBAAgB;aACpC,CAAC,CAAC;QACL,CAAC,CAAA;QAED,GAAG,EAAE,CAAC,GAAG,IAAW,EAAE,EAAE;YACtB,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,0CAA0C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxE,OAAO,OAAO,CAAC,OAAO,CAAC,IAAA,oCAA4B,GAAE,CAAC,CAAC;QACzD,CAAC;QAED,UAAU,EAAE,CACV,GAAG,IAAkB,EACoC,EAAE;YAC3D,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC;YAClB,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,iDAAiD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/E,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;YAExE,MAAM,kBAAkB,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YACtE,IAAI,kBAAkB,KAAK,CAAC,CAAC;gBAAE,OAAO,SAAS,CAAC;YAEhD,MAAM,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC;YACzD,OAAO,GAAG,CAAC;QACb,CAAC,CAAA;QAED,aAAa,EAAE,CACb,GAAG,IAGF,EAC4C,EAAE;YAC/C,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;YAChC,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,oDAAoD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClF,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;YAEvD,MAAM,qBAAqB,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,EAAE,CAAC,CAAC;YACjF,MAAM,iBAAiB,GAAG,qBAAqB,KAAK,CAAC,CAAC,CAAC;YACvD,IAAI,iBAAiB,EAAE,CAAC;gBACtB,MAAM,GAAG,GAAG,YAAY,CAAC,qBAAqB,CAAC,CAAC;gBAChD,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;gBACvC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;YAED,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;YAEvD,yTAAyT;YACzT,OAAO,OAAO,CAAC,OAAO,CAAC;gBACrB,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,iBAAiB;gBACjB,iBAAiB,EAAE,KAAK;aACzB,CAAC,CAAC;QACL,CAAC;QAED,QAAQ,EAAE,CACR,GAAG,IAAmD,EACX,EAAE;;YAC7C,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;YACvB,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,+CAA+C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC7E,OAAO,OAAO,CAAC,OAAO,CAAC;gBACrB,KAAK,EAAE,EAAE;gBACT,eAAe,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC;gBACvC,kBAAkB,EAAE,KAAK;gBACzB,KAAK,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,KAAK,mCAAI,GAAG;aAC7B,CAAC,CAAC;QACL,CAAC;KACK,CAAC;AACX,CAAC,CAAC;AA/FW,QAAA,4BAA4B,gCA+FvC;AAEK,MAAM,iCAAiC,GAAG,CAAC,EAChD,GAAG,MAC8B,EAAE,EAA2B,EAAE;IAChE,MAAM,OAAO,GAAU,EAAE,CAAC;IAE1B,OAAO;QACL,IAAI,EAAE,CACJ,OAA4C,EACA,EAAE;;YAC9C,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;YAC5B,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI;gBAAE,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;YAC3C,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM;gBAAE,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAEzD,OAAO,OAAO,CAAC,OAAO,CAAC;gBACrB,KAAK,EAAE,OAAO,CAAC,MAAM;gBACrB,KAAK,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,KAAK,mCAAI,KAAK,CAAC,MAAM;gBACrC,MAAM,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM,mCAAI,CAAC;gBAC5B,KAAK,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,KAAK,mCAAI,KAAK,CAAC,MAAM;gBACrC,IAAI,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,mCAAI,KAAK;gBAC5B,KAAK;aACN,CAAC,CAAC;QACL,CAAC;QAED,WAAW,EAAE,CACX,IAAa,EACb,OAAmD,EAC3B,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAA,+BAAuB,GAAE,CAAC;QAEvE,wCAAwC;KAClC,CAAC;AACX,CAAC,CAAC;AA9BW,QAAA,iCAAiC,qCA8B5C;AAEK,MAAM,uBAAuB,GAAG,CAAC,EACtC,GAAG,EACH,kBAAkB,MAIhB,EAAE,EAAiB,EAAE;IACvB,OAAO;QACL,QAAQ,EAAE,CAAC,GAAG,IAAW,EAA2B,EAAE;YACpD,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,0CAA0C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxE,OAAO,IAAA,yCAAiC,EAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,YAAY,EAAE,CAAC,GAAG,IAAW,EAAE,EAAE;YAC/B,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,8CAA8C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC5E,OAAO,IAAA,oCAA4B,EAAC,EAAE,GAAG,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACnE,CAAC;QACD,aAAa,EAAE,CAAC,GAAG,IAAW,EAAE,EAAE;YAChC,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,+CAA+C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC7E,OAAO,IAAA,qCAA6B,EAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QAChD,CAAC;QAED,yCAAyC;KACnC,CAAC;AACX,CAAC,CAAC;AAvBW,QAAA,uBAAuB,2BAuBlC;AAEK,MAAM,wBAAwB,GAAG,CACtC,GAAG,IAQF,EACsB,EAAE;IACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1C,MAAM,CAAC,SAAS,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IACrC,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,MAAM,IAAI,EAAE,CAAC;IACpD,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,kCAAkC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAEpE,OAAO,OAAO,CAAC,OAAO,CAAC;QACrB,QAAQ,EAAE,CAAO,GAAG,IAAI,EAAE,EAAE;YAC1B,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,gDAAgD,SAAS,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,kBAAkB;YACpH,MAAM,CAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAG,GAAG,IAAI,CAAC,CAAA,CAAC;QAC9B,CAAC,CAAA;QACD,OAAO,EAAE,CAAO,GAAG,IAAI,EAAE,EAAE;YACzB,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAG,+CAA+C,SAAS,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,kBAAkB;YACnH,OAAO,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;QAC5D,CAAC,CAAA;KACS,CAAC,CAAC;AAChB,CAAC,CAAC;AA1BW,QAAA,wBAAwB,4BA0BnC","sourcesContent":["import type { Dataset, OpenStorageOptions } from 'apify';\nimport type {\n  RequestQueue as ClientRequestQueue,\n  RequestQueueClientAddRequestOptions,\n  RequestQueueClientAddRequestResult,\n  RequestQueueClientBatchAddRequestWithRetriesOptions,\n  RequestQueueClientBatchRequestsOperationResult,\n  RequestQueueClientGetRequestResult,\n  RequestQueueClientListHeadOptions,\n  RequestQueueClientListHeadResult,\n  RequestQueueClientRequestSchema,\n} from 'apify-client';\nimport type {\n  Dataset as ClientDataset,\n  DatasetCollectionClient,\n  DatasetCollectionClientGetOrCreateOptions,\n  DatasetCollectionClientListOptions,\n  DatasetCollectionClientListResult,\n  KeyValueStoreClient,\n  KeyValueStoreRecord,\n  RequestQueueClient,\n} from 'apify-client';\nimport type { StorageClient } from 'crawlee';\n\nimport type { MaybePromise } from '../../utils/types';\n\nexport type OnBatchAddRequestsArgs = [\n  requests: Omit<RequestQueueClientRequestSchema, 'id'>[],\n  options?: RequestQueueClientBatchAddRequestWithRetriesOptions\n];\nexport type OnBatchAddRequests = (...args: OnBatchAddRequestsArgs) => MaybePromise<void>;\n\nexport const createMockClientDataset = (overrides?: ClientDataset): ClientDataset => ({\n  id: 'MockClientDataset.id',\n  name: 'MockClientDataset.name',\n  title: 'MockClientDataset.title',\n  userId: 'MockClientDataset.id.userId',\n  createdAt: new Date('2023-04-17'),\n  modifiedAt: new Date('2023-04-17'),\n  accessedAt: new Date('2023-04-17'),\n  itemCount: 10,\n  cleanItemCount: 10,\n  actId: 'MockClientDataset.actId',\n  actRunId: 'MockClientDataset.actRunId',\n  stats: {},\n  fields: [],\n  ...overrides,\n});\n\nexport const createMockClientRequestQueue = (\n  overrides?: ClientRequestQueue\n): ClientRequestQueue => ({\n  id: 'MockClientRequestQueue.id',\n  name: 'MockClientRequestQueue.name',\n  title: 'MockClientRequestQueue.title',\n  userId: 'MockClientRequestQueue.userId',\n  createdAt: new Date('2023-04-17'),\n  modifiedAt: new Date('2023-04-17'),\n  accessedAt: new Date('2023-04-17'),\n  // expireAt?: string;\n  totalRequestCount: 0,\n  handledRequestCount: 0,\n  pendingRequestCount: 0,\n  // actId?: string;\n  // actRunId?: string;\n  hadMultipleClients: false,\n  stats: {},\n  ...overrides,\n});\n\nexport const createMockKeyValueStoreClient = ({\n  log,\n}: { log?: (args: any) => void } = {}): KeyValueStoreClient => {\n  return {\n    getRecord: (...args: [string]): Promise<KeyValueStoreRecord<any> | undefined> => {\n      const [key] = args;\n      log?.(`Called MockKeyValueStoreClient.getRecord with ${JSON.stringify(args)}`);\n      return Promise.resolve({\n        key,\n        value: null,\n        // contentType?: string;\n      });\n    },\n    setRecord: (...args: [record: KeyValueStoreRecord<any>]): Promise<void> => {\n      log?.(`Called MockKeyValueStoreClient.setRecord with ${JSON.stringify(args)}`);\n      return Promise.resolve();\n    },\n    // get(): Promise<KeyValueStoreInfo | undefined>;\n    // update(newFields: KeyValueStoreClientUpdateOptions): Promise<Partial<KeyValueStoreInfo>>;\n    // delete(): Promise<void>;\n    // listKeys(options?: KeyValueStoreClientListOptions): Promise<KeyValueStoreClientListData>;\n    // getRecord(key: string, options?: KeyValueStoreClientGetRecordOptions): Promise<KeyValueStoreRecord | undefined>;\n    // setRecord(record: KeyValueStoreRecord): Promise<void>;\n    // deleteRecord(key: string): Promise<void>;\n  } as any;\n};\n\nexport const createMockRequestQueueClient = ({\n  log,\n  onBatchAddRequests,\n}: {\n  log?: (args: any) => void;\n  onBatchAddRequests?: OnBatchAddRequests;\n} = {}): RequestQueueClient => {\n  const requestQueue: any[] = [];\n\n  return {\n    batchAddRequests: async (\n      ...args: OnBatchAddRequestsArgs\n    ): Promise<RequestQueueClientBatchRequestsOperationResult> => {\n      const [requests, options] = args;\n      log?.(`Called MockRequestQueueClient.batchAddRequests with ${JSON.stringify(args)}`);\n      const reqsWithIds = requests.map((r) => ({ ...r, id: Math.random() + '' }));\n      requestQueue.push(...reqsWithIds);\n\n      const reqsAddedToQueue = reqsWithIds.map((r) => ({\n        uniqueKey: r.uniqueKey,\n        requestId: r.id,\n        wasAlreadyPresent: false,\n        wasAlreadyHandled: false,\n      }));\n\n      await onBatchAddRequests?.(requests, options);\n\n      return Promise.resolve({\n        unprocessedRequests: [],\n        processedRequests: reqsAddedToQueue,\n      });\n    },\n\n    get: (...args: any[]) => {\n      log?.(`Called MockRequestQueueClient.get with ${JSON.stringify(args)}`);\n      return Promise.resolve(createMockClientRequestQueue());\n    },\n\n    getRequest: async (\n      ...args: [id: string]\n    ): Promise<RequestQueueClientGetRequestResult | undefined> => {\n      const [id] = args;\n      log?.(`Called MockRequestQueueClient.getRequest with ${JSON.stringify(args)}`);\n      log?.(`requestQueue: ${JSON.stringify(requestQueue.map((r) => r.id))}`);\n\n      const indexOfNextRequest = requestQueue.findIndex((r) => r.id === id);\n      if (indexOfNextRequest === -1) return undefined;\n\n      const [req] = requestQueue.splice(indexOfNextRequest, 1);\n      return req;\n    },\n\n    updateRequest: (\n      ...args: [\n        request: RequestQueueClientRequestSchema,\n        options?: RequestQueueClientAddRequestOptions\n      ]\n    ): Promise<RequestQueueClientAddRequestResult> => {\n      const [request, options] = args;\n      log?.(`Called MockRequestQueueClient.updateRequest with ${JSON.stringify(args)}`);\n      log?.(`requestQueue: ${JSON.stringify(requestQueue)}`);\n\n      const indexOfUpdatedRequest = requestQueue.findIndex((r) => r.id === request.id);\n      const wasAlreadyPresent = indexOfUpdatedRequest !== -1;\n      if (wasAlreadyPresent) {\n        const req = requestQueue[indexOfUpdatedRequest];\n        Object.assign(req, request);\n      } else {\n        const req = Object.assign({}, request);\n        requestQueue.push(req);\n      }\n\n      log?.(`requestQueue: ${JSON.stringify(requestQueue)}`);\n\n      // [{\"id\":\"0.9121846955138175\",\"url\":\"https://profesia.sk/partneri\",\"loadedUrl\":\"https://www.profesia.sk/partneri\",\"uniqueKey\":\"https://profesia.sk/partneri\",\"method\":\"GET\",\"noRetry\":false,\"retryCount\":0,\"errorMessages\":[],\"headers\":{},\"userData\":{\"__crawlee\":{\"state\":4}},\"handledAt\":\"2023-04-17T12:15:57.350Z\"}]\n      return Promise.resolve({\n        requestId: request.id,\n        wasAlreadyPresent,\n        wasAlreadyHandled: false,\n      });\n    },\n\n    listHead: (\n      ...args: [options?: RequestQueueClientListHeadOptions]\n    ): Promise<RequestQueueClientListHeadResult> => {\n      const [options] = args;\n      log?.(`Called MockRequestQueueClient.listHead with ${JSON.stringify(args)}`);\n      return Promise.resolve({\n        items: [],\n        queueModifiedAt: new Date('2023-04-17'),\n        hadMultipleClients: false,\n        limit: options?.limit ?? 100,\n      });\n    },\n  } as any;\n};\n\nexport const createMockDatasetCollectionClient = ({\n  log,\n}: { log?: (args: any) => void } = {}): DatasetCollectionClient => {\n  const dataset: any[] = [];\n\n  return {\n    list: (\n      options?: DatasetCollectionClientListOptions\n    ): Promise<DatasetCollectionClientListResult> => {\n      let items = dataset.slice();\n      if (options?.desc) items = items.reverse();\n      if (options?.offset) items = items.slice(options.offset);\n\n      return Promise.resolve({\n        total: dataset.length,\n        count: options?.limit ?? items.length,\n        offset: options?.offset ?? 0,\n        limit: options?.limit ?? items.length,\n        desc: options?.desc ?? false,\n        items,\n      });\n    },\n\n    getOrCreate: (\n      name?: string,\n      options?: DatasetCollectionClientGetOrCreateOptions\n    ): Promise<ClientDataset> => Promise.resolve(createMockClientDataset()),\n\n    // _list, _create, _getOrCreate, baseUrl\n  } as any;\n};\n\nexport const createMockStorageClient = ({\n  log,\n  onBatchAddRequests,\n}: {\n  log?: (args: any) => void;\n  onBatchAddRequests?: OnBatchAddRequests;\n} = {}): StorageClient => {\n  return {\n    datasets: (...args: any[]): DatasetCollectionClient => {\n      log?.(`Called MockStorageClient.datasets with ${JSON.stringify(args)}`);\n      return createMockDatasetCollectionClient({ log });\n    },\n    requestQueue: (...args: any[]) => {\n      log?.(`Called MockStorageClient.requestQueue with ${JSON.stringify(args)}`);\n      return createMockRequestQueueClient({ log, onBatchAddRequests });\n    },\n    keyValueStore: (...args: any[]) => {\n      log?.(`Called MockStorageClient.keyValueStore with ${JSON.stringify(args)}`);\n      return createMockKeyValueStoreClient({ log });\n    },\n\n    // dataset, keyValueStores, requestQueues\n  } as any;\n};\n\nexport const createMockStorageDataset = (\n  ...args: [\n    datasetId?: string | null,\n    options?: OpenStorageOptions,\n    custom?: {\n      log?: (...args: any[]) => void;\n      onPushData?: (...args: any[]) => MaybePromise<void>;\n      onGetInfo?: (...args: any[]) => MaybePromise<void>;\n    }\n  ]\n): Promise<Dataset<any>> => {\n  const origArgs = args.slice().slice(0, 2);\n  const [datasetId, __, custom] = args;\n  const { log, onPushData, onGetInfo } = custom || {};\n  log?.(`Called MockStorageDataset with ${JSON.stringify(origArgs)}`);\n\n  return Promise.resolve({\n    pushData: async (...args) => {\n      log?.(`Called MockStorageDataset.pushData (instance ${datasetId}) with ${JSON.stringify(args)}`); // prettier-ignore\n      await onPushData?.(...args);\n    },\n    getInfo: async (...args) => {\n      log?.(`Called MockStorageDataset.getInfo (instance ${datasetId}) with ${JSON.stringify(args)}`); // prettier-ignore\n      return onGetInfo ? onGetInfo(...args) : { itemCount: 10 };\n    },\n  } as Dataset);\n};\n"]}

package/dist/cjs/utils/async.js

@@ -45,7 +45,7 @@ exports.serialAsyncFind = serialAsyncFind;
 const wait = (ms) => new Promise((res) => setTimeout(res, ms));
 exports.wait = wait;
 /** Call async function, and retry to call it `maxRetries` times if it fails. */
-const retryAsync = (fn, { maxRetries = 1, delay = 0, onError = () => { } } = {}) => __awaiter(void 0, void 0, void 0, function* () {
+const retryAsync = (fn_1, ...args_1) => __awaiter(void 0, [fn_1, ...args_1], void 0, function* (fn, { maxRetries = 1, delay = 0, onError = () => { } } = {}) {
     if (typeof maxRetries !== 'number' || maxRetries < 0) {
         throw Error(`Invalid input for maxRetries in retryAsync(fn, maxRetries). maxRetries must be a non-negative number. Got ${maxRetries}`);
     }

package/dist/cjs/utils/async.js.map

@@ -1 +1 @@
-{"version":3,"file":"async.js","sourceRoot":"","sources":["../../../src/utils/async.ts"],"names":[],"mappings":";;;;;;;;;;;;AAEO,MAAM,cAAc,GAAG,CAC5B,QAAa,EACb,EAA+C,EAC/C,EAAE;IACF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAO,gBAAgB,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;QAC7E,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjB,OAAO,GAAG,CAAC;IACb,CAAC,CAAA,EAAE,OAAO,CAAC,OAAO,CAAC,EAAS,CAAC,CAAC,CAAC;IAE/B,OAAO,OAAO,CAAC;AACjB,CAAC,CAAA,CAAC;AAZW,QAAA,cAAc,kBAYzB;AAEK,MAAM,iBAAiB,GAAG,CAC/B,QAAa,EACb,EAAiD,EACjD,EAAE;IACF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAO,gBAAgB,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;QAC7E,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,IAAI,MAAM;YAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,GAAG,CAAC;IACb,CAAC,CAAA,EAAE,OAAO,CAAC,OAAO,CAAC,EAAS,CAAC,CAAC,CAAC;IAE/B,OAAO,OAAO,CAAC;AACjB,CAAC,CAAA,CAAC;AAZW,QAAA,iBAAiB,qBAY5B;AAEK,MAAM,eAAe,GAAG,CAC7B,QAAa,EACb,EAAiD,EACjD,EAAE;IACF,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE;QAC5B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,IAAI,MAAM;YAAE,OAAO,KAAK,CAAC;QACzB,KAAK,EAAE,CAAC;KACT;AACH,CAAC,CAAA,CAAC;AAVW,QAAA,eAAe,mBAU1B;AAWF,8CAA8C;AACvC,MAAM,IAAI,GAAG,CAAC,EAAW,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;AAAlE,QAAA,IAAI,QAA8D;AAE/E,gFAAgF;AACzE,MAAM,UAAU,GAAG,CACxB,EAAmC,EACnC,EAAE,UAAU,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,GAAG,GAAG,EAAE,GAAE,CAAC,KAAwB,EAAE,EACzE,EAAE;IACF,IAAI,OAAO,UAAU,KAAK,QAAQ,IAAI,UAAU,GAAG,CAAC,EAAE;QACpD,MAAM,KAAK,CACT,6GAA6G,UAAU,EAAE,CAC1H,CAAC;KACH;IAED,IAAI,MAAM,GAAa,IAAI,CAAC;IAC5B,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,OAAO,OAAO,IAAI,UAAU,EAAE;QAC5B,IAAI;YACF,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC;SAC5B;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjB,MAAM,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC5B,MAAM,IAAA,YAAI,EAAC,KAAK,CAAC,CAAC;YAClB,OAAO,EAAE,CAAC;YACV,SAAS,CAAC,kBAAkB;SAC7B;QACD,MAAM,CAAC,yCAAyC;KACjD;IACD,OAAO;QACL,MAAM;QACN,MAAM;KACP,CAAC;AACJ,CAAC,CAAA,CAAC;AA7BW,QAAA,UAAU,cA6BrB","sourcesContent":["import type { MaybePromise } from '../utils/types';\n\nexport const serialAsyncMap = async <T, R>(\n  inputArr: T[],\n  fn: (item: T, index: number) => MaybePromise<R>\n) => {\n  const results = await inputArr.reduce(async (aggResultPromise, input, index) => {\n    const agg = await aggResultPromise;\n    const result = await fn(input, index);\n    agg.push(result);\n    return agg;\n  }, Promise.resolve([] as R[]));\n\n  return results;\n};\n\nexport const serialAsyncFilter = async <T>(\n  inputArr: T[],\n  fn: (item: T, index: number) => MaybePromise<any>\n) => {\n  const results = await inputArr.reduce(async (aggResultPromise, input, index) => {\n    const agg = await aggResultPromise;\n    const result = await fn(input, index);\n    if (result) agg.push(input);\n    return agg;\n  }, Promise.resolve([] as T[]));\n\n  return results;\n};\n\nexport const serialAsyncFind = async <T>(\n  inputArr: T[],\n  fn: (item: T, index: number) => MaybePromise<any>\n) => {\n  let index = 0;\n  for (const input of inputArr) {\n    const result = await fn(input, index);\n    if (result) return input;\n    index++;\n  }\n};\n\nexport interface RetryAsyncOptions {\n  /** Number of retries after the function call fails */\n  maxRetries?: number;\n  /** Delay between retries in milliseconds */\n  delay?: number;\n  /** Callback called with error if the function call errors */\n  onError?: (error: unknown, retryIndex: number) => MaybePromise<void>;\n}\n\n/** @param {number} ms Number of ms to wait */\nexport const wait = (ms?: number) => new Promise((res) => setTimeout(res, ms));\n\n/** Call async function, and retry to call it `maxRetries` times if it fails. */\nexport const retryAsync = async <T>(\n  fn: (retries: number) => Promise<T>,\n  { maxRetries = 1, delay = 0, onError = () => {} }: RetryAsyncOptions = {}\n) => {\n  if (typeof maxRetries !== 'number' || maxRetries < 0) {\n    throw Error(\n      `Invalid input for maxRetries in retryAsync(fn, maxRetries). maxRetries must be a non-negative number. Got ${maxRetries}`\n    );\n  }\n\n  let result: T | null = null;\n  const errors: unknown[] = [];\n  let retries = 0;\n  while (retries <= maxRetries) {\n    try {\n      result = await fn(retries);\n    } catch (err) {\n      errors.push(err);\n      await onError(err, retries);\n      await wait(delay);\n      retries++;\n      continue; // Retry if failed\n    }\n    break; // Exit loop and continue flow if success\n  }\n  return {\n    result,\n    errors,\n  };\n};\n"]}
+{"version":3,"file":"async.js","sourceRoot":"","sources":["../../../src/utils/async.ts"],"names":[],"mappings":";;;;;;;;;;;;AAEO,MAAM,cAAc,GAAG,CAC5B,QAAa,EACb,EAA+C,EAC/C,EAAE;IACF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAO,gBAAgB,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;QAC7E,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjB,OAAO,GAAG,CAAC;IACb,CAAC,CAAA,EAAE,OAAO,CAAC,OAAO,CAAC,EAAS,CAAC,CAAC,CAAC;IAE/B,OAAO,OAAO,CAAC;AACjB,CAAC,CAAA,CAAC;AAZW,QAAA,cAAc,kBAYzB;AAEK,MAAM,iBAAiB,GAAG,CAC/B,QAAa,EACb,EAAiD,EACjD,EAAE;IACF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAO,gBAAgB,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;QAC7E,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,IAAI,MAAM;YAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,GAAG,CAAC;IACb,CAAC,CAAA,EAAE,OAAO,CAAC,OAAO,CAAC,EAAS,CAAC,CAAC,CAAC;IAE/B,OAAO,OAAO,CAAC;AACjB,CAAC,CAAA,CAAC;AAZW,QAAA,iBAAiB,qBAY5B;AAEK,MAAM,eAAe,GAAG,CAC7B,QAAa,EACb,EAAiD,EACjD,EAAE;IACF,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,IAAI,MAAM;YAAE,OAAO,KAAK,CAAC;QACzB,KAAK,EAAE,CAAC;IACV,CAAC;AACH,CAAC,CAAA,CAAC;AAVW,QAAA,eAAe,mBAU1B;AAWF,8CAA8C;AACvC,MAAM,IAAI,GAAG,CAAC,EAAW,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;AAAlE,QAAA,IAAI,QAA8D;AAE/E,gFAAgF;AACzE,MAAM,UAAU,GAAG,kBAGxB,EAAE,yDAFF,EAAmC,EACnC,EAAE,UAAU,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,GAAG,GAAG,EAAE,GAAE,CAAC,KAAwB,EAAE;IAEzE,IAAI,OAAO,UAAU,KAAK,QAAQ,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACrD,MAAM,KAAK,CACT,6GAA6G,UAAU,EAAE,CAC1H,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,GAAa,IAAI,CAAC;IAC5B,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,OAAO,OAAO,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjB,MAAM,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC5B,MAAM,IAAA,YAAI,EAAC,KAAK,CAAC,CAAC;YAClB,OAAO,EAAE,CAAC;YACV,SAAS,CAAC,kBAAkB;QAC9B,CAAC;QACD,MAAM,CAAC,yCAAyC;IAClD,CAAC;IACD,OAAO;QACL,MAAM;QACN,MAAM;KACP,CAAC;AACJ,CAAC,CAAA,CAAC;AA7BW,QAAA,UAAU,cA6BrB","sourcesContent":["import type { MaybePromise } from '../utils/types';\n\nexport const serialAsyncMap = async <T, R>(\n  inputArr: T[],\n  fn: (item: T, index: number) => MaybePromise<R>\n) => {\n  const results = await inputArr.reduce(async (aggResultPromise, input, index) => {\n    const agg = await aggResultPromise;\n    const result = await fn(input, index);\n    agg.push(result);\n    return agg;\n  }, Promise.resolve([] as R[]));\n\n  return results;\n};\n\nexport const serialAsyncFilter = async <T>(\n  inputArr: T[],\n  fn: (item: T, index: number) => MaybePromise<any>\n) => {\n  const results = await inputArr.reduce(async (aggResultPromise, input, index) => {\n    const agg = await aggResultPromise;\n    const result = await fn(input, index);\n    if (result) agg.push(input);\n    return agg;\n  }, Promise.resolve([] as T[]));\n\n  return results;\n};\n\nexport const serialAsyncFind = async <T>(\n  inputArr: T[],\n  fn: (item: T, index: number) => MaybePromise<any>\n) => {\n  let index = 0;\n  for (const input of inputArr) {\n    const result = await fn(input, index);\n    if (result) return input;\n    index++;\n  }\n};\n\nexport interface RetryAsyncOptions {\n  /** Number of retries after the function call fails */\n  maxRetries?: number;\n  /** Delay between retries in milliseconds */\n  delay?: number;\n  /** Callback called with error if the function call errors */\n  onError?: (error: unknown, retryIndex: number) => MaybePromise<void>;\n}\n\n/** @param {number} ms Number of ms to wait */\nexport const wait = (ms?: number) => new Promise((res) => setTimeout(res, ms));\n\n/** Call async function, and retry to call it `maxRetries` times if it fails. */\nexport const retryAsync = async <T>(\n  fn: (retries: number) => Promise<T>,\n  { maxRetries = 1, delay = 0, onError = () => {} }: RetryAsyncOptions = {}\n) => {\n  if (typeof maxRetries !== 'number' || maxRetries < 0) {\n    throw Error(\n      `Invalid input for maxRetries in retryAsync(fn, maxRetries). maxRetries must be a non-negative number. Got ${maxRetries}`\n    );\n  }\n\n  let result: T | null = null;\n  const errors: unknown[] = [];\n  let retries = 0;\n  while (retries <= maxRetries) {\n    try {\n      result = await fn(retries);\n    } catch (err) {\n      errors.push(err);\n      await onError(err, retries);\n      await wait(delay);\n      retries++;\n      continue; // Retry if failed\n    }\n    break; // Exit loop and continue flow if success\n  }\n  return {\n    result,\n    errors,\n  };\n};\n"]}

package/dist/cjs/utils/error.d.ts

@@ -1 +1 @@
-export declare const logAndRethrow: (e: any, log?: ((e: any) => any) | undefined) => never;
+export declare const logAndRethrow: (e: any, log?: (e: any) => any) => never;

package/dist/cjs/utils/package.js.map

@@ -1 +1 @@
-{"version":3,"file":"package.js","sourceRoot":"","sources":["../../../src/utils/package.ts"],"names":[],"mappings":";;;;;;AAAA,sDAA8B;AAE9B;;;;;;KAMK;AACE,MAAM,kBAAkB,GAAG,CAChC,gBAGc,EACd,MAAiB,EACjB,EAAE;IACF,MAAM,GAAG,GAAG;QACV,QAAQ,EAAE,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,QAAQ;QACpC,EAAE,EAAE,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,EAAE;QACxB,OAAO,EAAE,EAAE,EAAE,2CAA2C;KACzD,CAAC;IACF,IAAA,iBAAO,EAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAClC,OAAO,GAAG,CAAC,OAA+B,CAAC;AAC7C,CAAC,CAAC;AAdW,QAAA,kBAAkB,sBAc7B","sourcesContent":["import pkginfo from 'pkginfo';\n\n/**\n * `pkginfo` is an npm package that finds closest package.json\n * and returns only the fields that we request. While it works\n * well, the usage is strange. Hence we wrap it to contain the magic.\n *\n * See https://www.npmjs.com/package/pkginfo\n * */\nexport const getPackageJsonInfo = <TFields extends string = string>(\n  filenameOrModule:\n    | { filename: string; id?: string }\n    | { id: string; filename?: string }\n    | NodeModule,\n  fields: TFields[]\n) => {\n  const obj = {\n    filename: filenameOrModule?.filename,\n    id: filenameOrModule?.id,\n    exports: {}, // This is where the fields will be written\n  };\n  pkginfo(obj, { include: fields });\n  return obj.exports as Record<TFields, any>;\n};\n"]}
+{"version":3,"file":"package.js","sourceRoot":"","sources":["../../../src/utils/package.ts"],"names":[],"mappings":";;;;;;AAAA,sDAA8B;AAE9B;;;;;;KAMK;AACE,MAAM,kBAAkB,GAAG,CAChC,gBAGc,EACd,MAAiB,EACjB,EAAE;IACF,MAAM,GAAG,GAAG;QACV,QAAQ,EAAE,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,QAAQ;QACpC,EAAE,EAAE,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,EAAE;QACxB,OAAO,EAAE,EAAE,EAAE,2CAA2C;KACzD,CAAC;IACF,IAAA,iBAAO,EAAC,GAAU,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IACzC,OAAO,GAAG,CAAC,OAA+B,CAAC;AAC7C,CAAC,CAAC;AAdW,QAAA,kBAAkB,sBAc7B","sourcesContent":["import pkginfo from 'pkginfo';\n\n/**\n * `pkginfo` is an npm package that finds closest package.json\n * and returns only the fields that we request. While it works\n * well, the usage is strange. Hence we wrap it to contain the magic.\n *\n * See https://www.npmjs.com/package/pkginfo\n * */\nexport const getPackageJsonInfo = <TFields extends string = string>(\n  filenameOrModule:\n    | { filename: string; id?: string }\n    | { id: string; filename?: string }\n    | NodeModule,\n  fields: TFields[]\n) => {\n  const obj = {\n    filename: filenameOrModule?.filename,\n    id: filenameOrModule?.id,\n    exports: {}, // This is where the fields will be written\n  };\n  pkginfo(obj as any, { include: fields });\n  return obj.exports as Record<TFields, any>;\n};\n"]}

package/dist/cjs/utils/url.js.map

@@ -1 +1 @@
-{"version":3,"file":"url.js","sourceRoot":"","sources":["../../../src/utils/url.ts"],"names":[],"mappings":";;;AAAA,oCAAoC;AAC7B,MAAM,WAAW,GAAG,CAAC,GAAW,EAAE,EAAE;IACzC,IAAI;QACF,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;KACd;IAAC,OAAO,GAAG,EAAE;QACX,GAAa,CAAC,OAAO,IAAI,WAAW,GAAG,GAAG,CAAC;QAC5C,MAAM,GAAG,CAAC;KACX;AACH,CAAC,CAAC;AAPW,QAAA,WAAW,eAOtB;AAEF,oDAAoD;AAC7C,MAAM,cAAc,GAAG,CAAC,OAAe,EAAE,OAAe,EAAE,EAAE;IACjE,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC7B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC;IACvB,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC,CAAC;AAJW,QAAA,cAAc,kBAIzB;AAOK,MAAM,SAAS,GAAG,CACvB,QAAoB,EACpB,EAAE,aAAa,EAAE,OAAO,KAAuB,EAAE,EACjD,EAAE;IACF,IAAI,OAAO,QAAQ,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAClD,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,aAAa;QAAE,OAAO,QAAQ,CAAC;IAEhE,IAAI,CAAC,OAAO;QAAE,MAAM,KAAK,CAAC,wEAAwE,CAAC,CAAC,CAAC,kBAAkB;IACvH,OAAO,IAAA,sBAAc,EAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC3C,CAAC,CAAC;AATW,QAAA,SAAS,aASpB","sourcesContent":["/** Validate correctness of a URL */\nexport const validateUrl = (url: string) => {\n  try {\n    new URL(url);\n  } catch (err) {\n    (err as Error).message += `\\nURL: \"${url}\"`;\n    throw err;\n  }\n};\n\n/** Transform relative URL paths to absolute URLs */\nexport const resolveUrlPath = (urlBase: string, urlPath: string) => {\n  const url = new URL(urlBase);\n  url.pathname = urlPath;\n  return url.href;\n};\n\nexport interface FormatUrlOptions {\n  allowRelative?: boolean;\n  baseUrl?: string | null;\n}\n\nexport const formatUrl = <T>(\n  maybeUrl: string | T,\n  { allowRelative, baseUrl }: FormatUrlOptions = {}\n) => {\n  if (typeof maybeUrl !== 'string') return maybeUrl;\n  if (!maybeUrl.startsWith('/') || allowRelative) return maybeUrl;\n\n  if (!baseUrl) throw Error('Cannot convert URL from relative to absolute path - baseUrl is missing'); // prettier-ignore\n  return resolveUrlPath(baseUrl, maybeUrl);\n};\n"]}
+{"version":3,"file":"url.js","sourceRoot":"","sources":["../../../src/utils/url.ts"],"names":[],"mappings":";;;AAAA,oCAAoC;AAC7B,MAAM,WAAW,GAAG,CAAC,GAAW,EAAE,EAAE;IACzC,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACZ,GAAa,CAAC,OAAO,IAAI,WAAW,GAAG,GAAG,CAAC;QAC5C,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC,CAAC;AAPW,QAAA,WAAW,eAOtB;AAEF,oDAAoD;AAC7C,MAAM,cAAc,GAAG,CAAC,OAAe,EAAE,OAAe,EAAE,EAAE;IACjE,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC7B,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC;IACvB,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC,CAAC;AAJW,QAAA,cAAc,kBAIzB;AAOK,MAAM,SAAS,GAAG,CACvB,QAAoB,EACpB,EAAE,aAAa,EAAE,OAAO,KAAuB,EAAE,EACjD,EAAE;IACF,IAAI,OAAO,QAAQ,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAClD,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,aAAa;QAAE,OAAO,QAAQ,CAAC;IAEhE,IAAI,CAAC,OAAO;QAAE,MAAM,KAAK,CAAC,wEAAwE,CAAC,CAAC,CAAC,kBAAkB;IACvH,OAAO,IAAA,sBAAc,EAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC3C,CAAC,CAAC;AATW,QAAA,SAAS,aASpB","sourcesContent":["/** Validate correctness of a URL */\nexport const validateUrl = (url: string) => {\n  try {\n    new URL(url);\n  } catch (err) {\n    (err as Error).message += `\\nURL: \"${url}\"`;\n    throw err;\n  }\n};\n\n/** Transform relative URL paths to absolute URLs */\nexport const resolveUrlPath = (urlBase: string, urlPath: string) => {\n  const url = new URL(urlBase);\n  url.pathname = urlPath;\n  return url.href;\n};\n\nexport interface FormatUrlOptions {\n  allowRelative?: boolean;\n  baseUrl?: string | null;\n}\n\nexport const formatUrl = <T>(\n  maybeUrl: string | T,\n  { allowRelative, baseUrl }: FormatUrlOptions = {}\n) => {\n  if (typeof maybeUrl !== 'string') return maybeUrl;\n  if (!maybeUrl.startsWith('/') || allowRelative) return maybeUrl;\n\n  if (!baseUrl) throw Error('Cannot convert URL from relative to absolute path - baseUrl is missing'); // prettier-ignore\n  return resolveUrlPath(baseUrl, maybeUrl);\n};\n"]}

package/dist/cjs/utils/valueMonitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"valueMonitor.js","sourceRoot":"","sources":["../../../src/utils/valueMonitor.ts"],"names":[],"mappings":";;;;;;;;;;;;AAQA;;;;;GAKG;AACI,MAAM,kBAAkB,GAAG,CAChC,YAAqD,EACrD,OAA6B,EAC7B,EAAE;;IACF,MAAM,OAAO,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,OAAO,mCAAI,IAAI,CAAC;IAEzC,IAAI,SAAS,GAAa,IAAI,CAAC;IAC/B,IAAI,kBAAkB,GAAkB,IAAI,CAAC;IAC7C,MAAM,SAAS,GAAuB,EAAE,CAAC;IAEzC,MAAM,gBAAgB,GAAG,CAAC,QAA0B,EAAE,EAAE;QACtD,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,OAAO,GAAG,EAAE,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC;IAEF,MAAM,kBAAkB,GAAG,CAAC,QAA0B,EAAE,EAAE;QACxD,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC1C,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC7B,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,GAAG,EAAE;QACnB,IAAI,kBAAkB,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QAE5C,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,kBAAkB,GAAG,OAAO,CAAC;QAC5C,OAAO,GAAG,IAAI,MAAM,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,GAAS,EAAE;QAC9B,MAAM,QAAQ,GAAG,SAAS,CAAC;QAC3B,MAAM,QAAQ,GAAG,CAAC,SAAS,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC5D,kBAAkB,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;QAE1C,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE;YAC1B,MAAM,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;SAC9B;QAED,OAAO,SAAc,CAAC;IACxB,CAAC,CAAA,CAAC;IAEF,MAAM,QAAQ,GAAG,GAAG,EAAE;QACpB,OAAO,OAAO,EAAE,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAChD,CAAC,CAAC;IAEF,OAAO;QACL,KAAK,EAAE,QAAQ;QACf,OAAO;QACP,OAAO,EAAE,YAAY;QACrB,OAAO,EAAE,gBAAgB;KAC1B,CAAC;AACJ,CAAC,CAAC;AAlDW,QAAA,kBAAkB,sBAkD7B;AAEF;;;;GAIG;AACI,MAAM,iBAAiB,GAAG,CAC/B,OAAe,EACf,UAAsC,EACtC,gBAA0C,EAC1C,OAA6B,EAC7B,EAAE;IACF,MAAM,YAAY,GAAG,IAAA,0BAAkB,EAAS,UAAU,EAAE,OAAO,CAAC,CAAC;IAErE,MAAM,WAAW,GAAG,GAAS,EAAE;QAC7B,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC;QACxC,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE;YAAE,OAAO,IAAI,CAAC;QAClF,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC;QAC7C,OAAO,OAAO,CAAC;IACjB,CAAC,CAAA,CAAC;IAEF,MAAM,aAAa,GAAG,CAAU,GAAQ,EAAE,EAAE;QAC1C,MAAM,QAAQ,GAAG,MAAM,WAAW,EAAE,CAAC;QACrC,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC,CAAC;IAC1C,CAAC,CAAA,CAAC;IAEF,gDAAgD;IAChD,YAAY,CAAC,OAAO,CAAC,CAAO,OAAO,EAAE,EAAE;QACrC,IAAI,OAAO,IAAI,OAAO;YAAE,MAAM,gBAAgB,EAAE,CAAC;IACnD,CAAC,CAAA,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,GAAS,EAAE;QACxB,MAAM,QAAQ,GAAG,MAAM,WAAW,EAAE,CAAC;QACrC,OAAO,QAAQ,IAAI,OAAO,CAAC;IAC7B,CAAC,CAAA,CAAC;IAEF,uCACK,YAAY,KACf,aAAa;QACb,MAAM,IACN;AACJ,CAAC,CAAC;AAnCW,QAAA,iBAAiB,qBAmC5B","sourcesContent":["import type { MaybePromise } from './types';\n\nexport type ValueCallback<T> = (value: T, oldValue: T | null) => MaybePromise<any>;\nexport interface ValueMonitorOptions {\n  /** How long (in milliseconds) after fetching the value can we use it before we have to re-fetch it. */\n  ttlInMs?: number;\n}\n\n/**\n * ValueMonitor checks and caches a (maybe remote) value, and triggers callbacks\n * based on the value.\n *\n * You can think of it as a light replacement for reactivity (Vue) feature.\n */\nexport const createValueMonitor = <T>(\n  fetchValueFn: (oldValue: T | null) => MaybePromise<T>,\n  options?: ValueMonitorOptions\n) => {\n  const ttlInMs = options?.ttlInMs ?? 5000;\n\n  let currValue: T | null = null;\n  let lastFetchTimestamp: number | null = null;\n  const callbacks: ValueCallback<T>[] = [];\n\n  const registerCallback = (callback: ValueCallback<T>) => {\n    callbacks.push(callback);\n    return () => deregisterCallback(callback);\n  };\n\n  const deregisterCallback = (callback: ValueCallback<T>) => {\n    const index = callbacks.indexOf(callback);\n    callbacks.splice(index, 1);\n  };\n\n  const isStale = () => {\n    if (lastFetchTimestamp == null) return true;\n\n    const now = new Date().getTime();\n    const expiry = lastFetchTimestamp + ttlInMs;\n    return now >= expiry;\n  };\n\n  const refreshValue = async () => {\n    const oldValue = currValue;\n    const newValue = (currValue = await fetchValueFn(oldValue));\n    lastFetchTimestamp = new Date().getTime();\n\n    for (const cb of callbacks) {\n      await cb(newValue, oldValue);\n    }\n\n    return currValue as T;\n  };\n\n  const getValue = () => {\n    return isStale() ? refreshValue() : currValue;\n  };\n\n  return {\n    value: getValue,\n    isStale,\n    refresh: refreshValue,\n    onValue: registerCallback,\n  };\n};\n\n/**\n * Variant of ValueMonitor that is used for monitoring size of stores like Apify Dataset or RequestQueue.\n * - Pass an array of items to `shortenToSize` to shorten the array to the size\n *   that still fits the store.\n */\nexport const createSizeMonitor = (\n  maxSize: number,\n  sizeGetter: () => MaybePromise<number>,\n  onMaxSizeReached: () => MaybePromise<void>,\n  options?: ValueMonitorOptions\n) => {\n  const valueMonitor = createValueMonitor<number>(sizeGetter, options);\n\n  const prepareSize = async () => {\n    const size = await valueMonitor.value();\n    if (typeof size === 'number' && size >= 0 && !valueMonitor.isStale()) return size;\n    const newSize = await valueMonitor.refresh();\n    return newSize;\n  };\n\n  const shortenToSize = async <T>(arr: T[]) => {\n    const currSize = await prepareSize();\n    return arr.slice(0, maxSize - currSize);\n  };\n\n  // Define handlers for when the value is updated\n  valueMonitor.onValue(async (newSize) => {\n    if (newSize >= maxSize) await onMaxSizeReached();\n  });\n\n  const isFull = async () => {\n    const currSize = await prepareSize();\n    return currSize >= maxSize;\n  };\n\n  return {\n    ...valueMonitor,\n    shortenToSize,\n    isFull,\n  };\n};\n"]}
+{"version":3,"file":"valueMonitor.js","sourceRoot":"","sources":["../../../src/utils/valueMonitor.ts"],"names":[],"mappings":";;;;;;;;;;;;AAQA;;;;;GAKG;AACI,MAAM,kBAAkB,GAAG,CAChC,YAAqD,EACrD,OAA6B,EAC7B,EAAE;;IACF,MAAM,OAAO,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,OAAO,mCAAI,IAAI,CAAC;IAEzC,IAAI,SAAS,GAAa,IAAI,CAAC;IAC/B,IAAI,kBAAkB,GAAkB,IAAI,CAAC;IAC7C,MAAM,SAAS,GAAuB,EAAE,CAAC;IAEzC,MAAM,gBAAgB,GAAG,CAAC,QAA0B,EAAE,EAAE;QACtD,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,OAAO,GAAG,EAAE,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC,CAAC;IAEF,MAAM,kBAAkB,GAAG,CAAC,QAA0B,EAAE,EAAE;QACxD,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC1C,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC7B,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,GAAG,EAAE;QACnB,IAAI,kBAAkB,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QAE5C,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,kBAAkB,GAAG,OAAO,CAAC;QAC5C,OAAO,GAAG,IAAI,MAAM,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,GAAS,EAAE;QAC9B,MAAM,QAAQ,GAAG,SAAS,CAAC;QAC3B,MAAM,QAAQ,GAAG,CAAC,SAAS,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC5D,kBAAkB,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;QAE1C,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;YAC3B,MAAM,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC/B,CAAC;QAED,OAAO,SAAc,CAAC;IACxB,CAAC,CAAA,CAAC;IAEF,MAAM,QAAQ,GAAG,GAAG,EAAE;QACpB,OAAO,OAAO,EAAE,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAChD,CAAC,CAAC;IAEF,OAAO;QACL,KAAK,EAAE,QAAQ;QACf,OAAO;QACP,OAAO,EAAE,YAAY;QACrB,OAAO,EAAE,gBAAgB;KAC1B,CAAC;AACJ,CAAC,CAAC;AAlDW,QAAA,kBAAkB,sBAkD7B;AAEF;;;;GAIG;AACI,MAAM,iBAAiB,GAAG,CAC/B,OAAe,EACf,UAAsC,EACtC,gBAA0C,EAC1C,OAA6B,EAC7B,EAAE;IACF,MAAM,YAAY,GAAG,IAAA,0BAAkB,EAAS,UAAU,EAAE,OAAO,CAAC,CAAC;IAErE,MAAM,WAAW,GAAG,GAAS,EAAE;QAC7B,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC;QACxC,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE;YAAE,OAAO,IAAI,CAAC;QAClF,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC;QAC7C,OAAO,OAAO,CAAC;IACjB,CAAC,CAAA,CAAC;IAEF,MAAM,aAAa,GAAG,CAAU,GAAQ,EAAE,EAAE;QAC1C,MAAM,QAAQ,GAAG,MAAM,WAAW,EAAE,CAAC;QACrC,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC,CAAC;IAC1C,CAAC,CAAA,CAAC;IAEF,gDAAgD;IAChD,YAAY,CAAC,OAAO,CAAC,CAAO,OAAO,EAAE,EAAE;QACrC,IAAI,OAAO,IAAI,OAAO;YAAE,MAAM,gBAAgB,EAAE,CAAC;IACnD,CAAC,CAAA,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,GAAS,EAAE;QACxB,MAAM,QAAQ,GAAG,MAAM,WAAW,EAAE,CAAC;QACrC,OAAO,QAAQ,IAAI,OAAO,CAAC;IAC7B,CAAC,CAAA,CAAC;IAEF,uCACK,YAAY,KACf,aAAa;QACb,MAAM,IACN;AACJ,CAAC,CAAC;AAnCW,QAAA,iBAAiB,qBAmC5B","sourcesContent":["import type { MaybePromise } from './types';\n\nexport type ValueCallback<T> = (value: T, oldValue: T | null) => MaybePromise<any>;\nexport interface ValueMonitorOptions {\n  /** How long (in milliseconds) after fetching the value can we use it before we have to re-fetch it. */\n  ttlInMs?: number;\n}\n\n/**\n * ValueMonitor checks and caches a (maybe remote) value, and triggers callbacks\n * based on the value.\n *\n * You can think of it as a light replacement for reactivity (Vue) feature.\n */\nexport const createValueMonitor = <T>(\n  fetchValueFn: (oldValue: T | null) => MaybePromise<T>,\n  options?: ValueMonitorOptions\n) => {\n  const ttlInMs = options?.ttlInMs ?? 5000;\n\n  let currValue: T | null = null;\n  let lastFetchTimestamp: number | null = null;\n  const callbacks: ValueCallback<T>[] = [];\n\n  const registerCallback = (callback: ValueCallback<T>) => {\n    callbacks.push(callback);\n    return () => deregisterCallback(callback);\n  };\n\n  const deregisterCallback = (callback: ValueCallback<T>) => {\n    const index = callbacks.indexOf(callback);\n    callbacks.splice(index, 1);\n  };\n\n  const isStale = () => {\n    if (lastFetchTimestamp == null) return true;\n\n    const now = new Date().getTime();\n    const expiry = lastFetchTimestamp + ttlInMs;\n    return now >= expiry;\n  };\n\n  const refreshValue = async () => {\n    const oldValue = currValue;\n    const newValue = (currValue = await fetchValueFn(oldValue));\n    lastFetchTimestamp = new Date().getTime();\n\n    for (const cb of callbacks) {\n      await cb(newValue, oldValue);\n    }\n\n    return currValue as T;\n  };\n\n  const getValue = () => {\n    return isStale() ? refreshValue() : currValue;\n  };\n\n  return {\n    value: getValue,\n    isStale,\n    refresh: refreshValue,\n    onValue: registerCallback,\n  };\n};\n\n/**\n * Variant of ValueMonitor that is used for monitoring size of stores like Apify Dataset or RequestQueue.\n * - Pass an array of items to `shortenToSize` to shorten the array to the size\n *   that still fits the store.\n */\nexport const createSizeMonitor = (\n  maxSize: number,\n  sizeGetter: () => MaybePromise<number>,\n  onMaxSizeReached: () => MaybePromise<void>,\n  options?: ValueMonitorOptions\n) => {\n  const valueMonitor = createValueMonitor<number>(sizeGetter, options);\n\n  const prepareSize = async () => {\n    const size = await valueMonitor.value();\n    if (typeof size === 'number' && size >= 0 && !valueMonitor.isStale()) return size;\n    const newSize = await valueMonitor.refresh();\n    return newSize;\n  };\n\n  const shortenToSize = async <T>(arr: T[]) => {\n    const currSize = await prepareSize();\n    return arr.slice(0, maxSize - currSize);\n  };\n\n  // Define handlers for when the value is updated\n  valueMonitor.onValue(async (newSize) => {\n    if (newSize >= maxSize) await onMaxSizeReached();\n  });\n\n  const isFull = async () => {\n    const currSize = await prepareSize();\n    return currSize >= maxSize;\n  };\n\n  return {\n    ...valueMonitor,\n    shortenToSize,\n    isFull,\n  };\n};\n"]}