crawlee-one 1.0.0

package/dist/cjs/lib/router.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"router.js","sourceRoot":"","sources":["../../../src/lib/router.ts"],"names":[],"mappings":";;;;;;;;;;;;AAgBA,0CAAuE;AAoFhE,MAAM,mBAAmB,GAAG,CAIjC,QAAuD,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,kBAAkB;AAJ7E,QAAA,mBAAmB,uBAIuC;AAEvE,4BAA4B;AACrB,MAAM,wBAAwB,GAAG,CAItC,QAAuD,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,kBAAkB;AAJ7E,QAAA,wBAAwB,4BAIkC;AAChE,MAAM,uBAAuB,GAAG,CAIrC,QAAuD,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,kBAAkB;AAJ7E,QAAA,uBAAuB,2BAImC;AAChE,MAAM,wBAAwB,GAAG,CAItC,QAAuD,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,kBAAkB;AAJ7E,QAAA,wBAAwB,4BAIkC;AAChE,MAAM,0BAA0B,GAAG,CAIxC,QAAuD,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,kBAAkB;AAJ7E,QAAA,0BAA0B,8BAIgC;AAChE,MAAM,6BAA6B,GAAG,CAI3C,QAAuD,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,kBAAkB;AAJ7E,QAAA,6BAA6B,iCAI6B;AAChE,MAAM,4BAA4B,GAAG,CAI1C,QAAuD,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,kBAAkB;AAJ7E,QAAA,4BAA4B,gCAI8B;AAEhE,MAAM,gBAAgB,GAAG,CAI9B,EACA,MAAM,EACN,cAAc,EACd,aAAa,EACb,aAAa,GAMd,EAAE,EAAE;IACH,MAAM,IAAA,sBAAc,EAAC,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAO,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE;QAC3E,MAAM,cAAc,GAAG,CAAC,cAAc,aAAd,cAAc,cAAd,cAAc,GAAI,EAAE,CAAC,CAAC,WAAW,CACvD,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,EAC1C,OAA2E,CAC5E,CAAC;QACF,MAAM,MAAM,CAAC,UAAU,CAAa,GAAG,EAAE,CAAO,GAAG,EAAE,EAAE,kDACrD,OAAA,cAAc,CAAC,gCAAK,aAAa,GAAK,GAAG,CAAS,CAAC,CAAA,GAAA,CACpD,CAAC;IACJ,CAAC,CAAA,CAAC,CAAC;AACL,CAAC,CAAA,CAAC;AAxBW,QAAA,gBAAgB,oBAwB3B;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AACI,MAAM,iBAAiB,GAAG,CAK/B,EACA,EAAE,EACF,MAAM,EACN,cAAc,EACd,aAAa,EACb,MAAM,EACN,aAAa,EACb,KAAK,GASN,EAAE,EAAE;IACH,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,cAAc,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,CACtD,CAAC;IAEpB,uEAAuE;IACvE,kBAAkB;IAClB,MAAM,aAAa,GAA0D,CAAO,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACrG,MAAM,OAAO,GAAG,KAAK,CAAC,YAAY,IAAI,IAAI,IAAI,aAAa,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAChF,IAAI,CAAC,OAAO,EAAE;YACZ,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,8BAA8B,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,YAAY,sCAAsC,GAAG,EAAE,CAAC,CAAC,CAAC,kBAAkB;YAC7I,OAAO;SACR;QACD,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,0BAA0B,KAAK,CAAC,YAAY,UAAU,GAAG,EAAE,CAAC,CAAC;QAC1E,MAAM,OAAO,CAAC,GAAU,CAAC,CAAC;IAC5B,CAAC,CAAA,CAAC;IAEF,MAAM,cAAc,GAAG,CAAO,GAA6C,EAAiB,EAAE;;QAC5F,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,GAAG,CAAC;QACrC,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;QAErD,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;QAE3D,IAAI,oBAAoB,GAAG,CAAC,CAAC;QAC7B,IAAI,GAAG,GAA0B,GAAG,CAAC,OAAO,CAAC;QAE7C,MAAM,YAAY,GAAG,GAAS,EAAE;YAC9B,IAAI,CAAC,GAAG;gBAAE,OAAO;YACjB,MAAM,QAAQ,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;YACvC,oBAAoB,EAAE,CAAC;QACzB,CAAC,CAAA,CAAC;QAEF,MAAM,eAAe,GAAG,CAAO,MAAc,EAAE,EAAE;YAC/C,GAAG,CAAC,KAAK,CAAC,0CAA0C,MAAM,EAAE,CAAC,CAAC;YAE9D,IAAI,iBAAiB;gBAAE,MAAM,IAAA,YAAI,EAAC,iBAAiB,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,gBAAgB,EAAE,CAAC;YACjD,GAAG,GAAG,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,IAAI,CAAC;YAErB,IAAI,GAAG,EAAE;gBACP,GAAG,CAAC,KAAK,CAAC,mCAAmC,MAAM,EAAE,CAAC,CAAC;gBAEvD,oEAAoE;gBACpE,wEAAwE;gBACxE,0CAA0C;gBAC1C,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI;oBAAE,MAAO,IAAa,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;aAC3D;iBAAM;gBACL,GAAG,CAAC,KAAK,CAAC,kCAAkC,MAAM,EAAE,CAAC,CAAC;aACvD;QACH,CAAC,CAAA,CAAC;QAEF,MAAM,YAAY,GAAG,GAAG,EAAE,CACxB,aAAa,IAAI,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,oBAAoB,GAAG,aAAa,CAAC;QAE/E,IAAI;YACF,GAAG;gBACD,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,MAAO,IAA0B,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,SAAS,MAAI,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,GAAG,CAAA,CAAC;gBACxF,MAAM,SAAS,GAAG,SAAS,oBAAoB,GAAG,CAAC,OAAO,aAAa,aAAb,aAAa,cAAb,aAAa,GAAI,CAAC,UAAU,GAAG,EAAE,CAAC;gBAE5F,mCAAmC;gBACnC,GAAG,CAAC,KAAK,CAAC,8CAA8C,SAAS,EAAE,CAAC,CAAC;gBACrE,MAAM,KAAK,GAAG,MAAM,IAAA,uBAAe,EAAC,MAAM,EAAE,CAAO,SAAS,EAAE,EAAE;oBAC9D,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;oBAC1E,OAAO,OAAO,CAAC;gBACjB,CAAC,CAAA,CAAC,CAAC;gBAEH,kBAAkB;gBAClB,IAAI,KAAK,EAAE;oBACT,GAAG,CAAC,IAAI,CAAC,qBAAqB,KAAK,CAAC,IAAI,mBAAmB,KAAK,CAAC,YAAY,MAAM,SAAS,EAAE,CAAC,CAAC,CAAC,kBAAkB;oBACnH,MAAM,CAAC,MAAA,KAAK,CAAC,MAAM,mCAAI,aAAa,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;iBACvE;qBAAM;oBACL,GAAG,CAAC,KAAK,CAAC,oDAAoD,SAAS,EAAE,CAAC,CAAC;iBAC5E;gBAED,yCAAyC;gBACzC,MAAM,YAAY,EAAE,CAAC;gBACrB,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;aAClC,QAAQ,YAAY,EAAE,EAAE;SAC1B;QAAC,OAAO,GAAG,EAAE;YACZ,GAAG,CAAC,KAAK,CAAC,gEAAgE,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,SAAS,MAAI,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,GAAG,CAAA,GAAG,CAAC,CAAC,CAAC,kBAAkB;YAC5H,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACf,iFAAiF;YACjF,IAAI,GAAG;gBAAE,MAAM,QAAQ,CAAC,cAAc,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;SAClE;IACH,CAAC,CAAA,CAAC;IAEF,MAAM,cAAc,GAAG,CAAC,cAAc,aAAd,cAAc,cAAd,cAAc,GAAI,EAAE,CAAC,CAAC,WAAW,CACvD,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,EAC5B,cAAc,CACf,CAAC;IACF,MAAM,MAAM,CAAC,iBAAiB,CAAa,CAAC,GAAG,EAAE,EAAE,CACjD,cAAc,CAAC,gCAAK,aAAa,GAAK,GAAG,CAAS,CAAC,CACpD,CAAC;AACJ,CAAC,CAAA,CAAC;AAjHW,QAAA,iBAAiB,qBAiH5B","sourcesContent":["import type {\n  BasicCrawler,\n  BasicCrawlingContext,\n  CheerioCrawlingContext,\n  CrawlingContext,\n  HttpCrawlingContext,\n  JSDOMCrawlingContext,\n  PlaywrightCrawlingContext,\n  PuppeteerCrawlingContext,\n  RouterHandler as CrawlerRouter,\n  Request as CrawlerRequest,\n} from 'crawlee';\nimport type { CommonPage } from '@crawlee/browser-pool';\nimport type { Page } from 'playwright';\n\nimport type { MaybePromise } from '../utils/types';\nimport { serialAsyncFind, serialAsyncMap, wait } from '../utils/async';\nimport type { PerfActorInput, RequestActorInput } from './config';\nimport type { CrawleeOneIO } from './integrations/types';\n\n// Read about router on https://docs.apify.com/academy/expert-scraping-with-apify/solutions/using-storage-creating-tasks\n\n/** Context object provided in CrawlerRouter */\nexport type RouterHandlerCtx<CrawlerCtx extends CrawlingContext> = Parameters<\n  Parameters<CrawlerRouter<CrawlerCtx>['addHandler']>[1]\n>[0];\n\n/** Function that's passed to `router.addHandler(label, handler)` */\nexport type RouteHandler<\n  CrawlerCtx extends CrawlingContext = CrawlingContext<BasicCrawler>,\n  RouterCtx extends Record<string, any> = Record<string, any>,\n> = Parameters<CrawlerRouter<RouterHandlerCtx<CrawlerCtx & RouterCtx>>['addHandler']>[1]; // prettier-ignore\n\n/** Wrapper that modifies behavior of RouteHandler */\nexport type CrawlerRouterWrapper<\n  CrawlerCtx extends CrawlingContext = CrawlingContext<BasicCrawler>,\n  RouterCtx extends Record<string, any> = Record<string, any>\n> = (\n  handler: (ctx: RouterHandlerCtx<CrawlerCtx & RouterCtx>) => Promise<void>\n) => (ctx: RouterHandlerCtx<CrawlerCtx & RouterCtx>) => Promise<void>;\n\n/**\n * Criteria that un-labelled requests are matched against.\n *\n * E.g. If `match` function returns truthy value,\n * the request is passed to the `action` function for processing.\n */\nexport interface RouteMatcher<\n  CrawlerCtx extends CrawlingContext = CrawlingContext<BasicCrawler>,\n  RouterCtx extends Record<string, any> = Record<string, any>,\n  Labels extends string = string\n> {\n  /** Human readable name */\n  name: string;\n  /**\n   * Label of the handler registered with `router.addHandler(label, handler)`\n   * that will process this request.\n   *\n   * NOTE: This value is used by the default `action` function. If you override\n   * the `action` function, `handlerLabel` is ignored and you have to process it yourself.\n   */\n  handlerLabel: Labels | null;\n  /**\n   * Function that decides whether the request will processed by this `action` function.\n   *\n   * @example\n   * [{\n   *   // If match returns true, the request is forwarded to handler\n   *   // with label JOB_DETAIL.\n   *   name: 'Job detail',\n   *   match: (url, ctx, route, handlers) => isUrlOfJobOffer(url),\n   *   handlerLabel: routeLabels.JOB_DETAIL,\n   * }]\n   */\n  match: (\n    url: string,\n    ctx: RouterHandlerCtx<CrawlerCtx & RouterCtx>,\n    route: RouteMatcher<CrawlerCtx, RouterCtx, Labels>,\n    handlers: Record<Labels, RouteHandler<CrawlerCtx, RouterCtx>>\n  ) => unknown;\n  /**\n   * Request is passed to this function if `match` returned truthy value.\n   *\n   * @example\n   * [{\n   *   // If match returns true, the request is forwarded to handler\n   *   // with label JOB_DETAIL.\n   *   name: 'Job detail',\n   *   match: (url, ctx, route, handlers) => isUrlOfJobOffer(url),\n   *   handlerLabel: routeLabels.JOB_DETAIL,\n   * }]\n   */\n  action?: (\n    url: string,\n    ctx: RouterHandlerCtx<CrawlerCtx>,\n    route: RouteMatcher<CrawlerCtx, RouterCtx, Labels>,\n    handlers: Record<Labels, RouteHandler<CrawlerCtx, RouterCtx>>\n  ) => MaybePromise<void>;\n}\n\nexport const createRouteMatchers = <\n  CrawlerCtx extends CrawlingContext = CrawlingContext<BasicCrawler>,\n  RouterCtx extends Record<string, any> = Record<string, any>,\n  Labels extends string = string\n>(matchers: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[]) => matchers; // prettier-ignore\n\n// Context-specific variants\nexport const createBasicRouteMatchers = <\n  CrawlerCtx extends BasicCrawlingContext = BasicCrawlingContext,\n  RouterCtx extends Record<string, any> = Record<string, any>,\n  Labels extends string = string\n>(matchers: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[]) => matchers; // prettier-ignore\nexport const createHttpRouteMatchers = <\n  CrawlerCtx extends HttpCrawlingContext = HttpCrawlingContext,\n  RouterCtx extends Record<string, any> = Record<string, any>,\n  Labels extends string = string\n>(matchers: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[]) => matchers; // prettier-ignore\nexport const createJsdomRouteMatchers = <\n  CrawlerCtx extends JSDOMCrawlingContext = JSDOMCrawlingContext,\n  RouterCtx extends Record<string, any> = Record<string, any>,\n  Labels extends string = string\n>(matchers: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[]) => matchers; // prettier-ignore\nexport const createCheerioRouteMatchers = <\nCrawlerCtx extends CheerioCrawlingContext = CheerioCrawlingContext,\n  RouterCtx extends Record<string, any> = Record<string, any>,\n  Labels extends string = string\n>(matchers: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[]) => matchers; // prettier-ignore\nexport const createPlaywrightRouteMatchers = <\n  CrawlerCtx extends PlaywrightCrawlingContext = PlaywrightCrawlingContext,\n  RouterCtx extends Record<string, any> = Record<string, any>,\n  Labels extends string = string\n>(matchers: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[]) => matchers; // prettier-ignore\nexport const createPuppeteerRouteMatchers = <\n  CrawlerCtx extends PuppeteerCrawlingContext = PuppeteerCrawlingContext,\n  RouterCtx extends Record<string, any> = Record<string, any>,\n  Labels extends string = string\n>(matchers: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[]) => matchers; // prettier-ignore\n\nexport const registerHandlers = async <\n  CrawlerCtx extends CrawlingContext,\n  RouterCtx extends Record<string, any> = Record<string, any>,\n  Labels extends string = string\n>({\n  router,\n  routerWrappers,\n  routerContext,\n  routeHandlers,\n}: {\n  router: CrawlerRouter<CrawlerCtx>;\n  routerWrappers?: CrawlerRouterWrapper<CrawlerCtx, RouterCtx>[];\n  routerContext?: RouterCtx;\n  routeHandlers: Record<Labels, RouteHandler<CrawlerCtx, RouterCtx>>;\n}) => {\n  await serialAsyncMap(Object.entries(routeHandlers), async ([key, handler]) => {\n    const wrappedHandler = (routerWrappers ?? []).reduceRight(\n      (fn, wrapper) => wrapper((ctx) => fn(ctx)),\n      handler as (ctx: RouterHandlerCtx<CrawlerCtx & RouterCtx>) => Promise<void>\n    );\n    await router.addHandler<CrawlerCtx>(key, async (ctx) =>\n      wrappedHandler({ ...routerContext, ...ctx } as any)\n    );\n  });\n};\n\n/**\n * Configures the default router handler to redirect URLs to labelled route handlers\n * based on which route the URL matches first.\n *\n * NOTE: This does mean that the URLs passed to this default handler will be fetched\n * twice (as the URL will be requeued to the correct handler). We recommend to use this\n * function only in the scenarios where there is a small number of startUrls, yet these\n * may need various ways of processing based on different paths or etc.\n *\n * @example\n *\n * const routeLabels = {\n *   MAIN_PAGE: 'MAIN_PAGE',\n *   JOB_LISTING: 'JOB_LISTING',\n *   JOB_DETAIL: 'JOB_DETAIL',\n *   JOB_RELATED_LIST: 'JOB_RELATED_LIST',\n *   PARTNERS: 'PARTNERS',\n * } as const;\n *\n * const router = createPlaywrightRouter();\n *\n * const routes = createPlaywrightRouteMatchers<typeof routeLabels>([\n *  // URLs that match this route are redirected to router.addHandler(routeLabels.MAIN_PAGE)\n *  {\n *     route: routeLabels.MAIN_PAGE,\n *     // Check for main page like https://www.profesia.sk/?#\n *     match: (url) => url.match(/[\\W]profesia\\.sk\\/?(?:[?#~]|$)/i),\n *   },\n *\n *  // Optionally override the logic that assigns the URL to the route by specifying the `action` prop\n *  {\n *     route: routeLabels.MAIN_PAGE,\n *     // Check for main page like https://www.profesia.sk/?#\n *     match: (url) => url.match(/[\\W]profesia\\.sk\\/?(?:[?#~]|$)/i),\n *     action: async (ctx) => {\n *       await ctx.crawler.addRequests([{\n *         url: 'https://profesia.sk/praca',\n *         label: routeLabels.JOB_LISTING,\n *       }]);\n *     },\n *   },\n * ]);\n *\n * // Set up default route to redirect to labelled routes\n * setupDefaultRoute({ router, routes });\n *\n * // Now set up the labelled routes\n * await router.addHandler(routeLabels.JOB_LISTING, async (ctx) => { ... }\n */\nexport const setupDefaultRoute = async <\n  CrawlerCtx extends CrawlingContext,\n  RouterCtx extends Record<string, any> = Record<string, any>,\n  Labels extends string = string,\n  Input extends Record<string, any> = Record<string, any>\n>({\n  io,\n  router,\n  routerWrappers,\n  routerContext,\n  routes,\n  routeHandlers,\n  input,\n}: {\n  io: CrawleeOneIO;\n  router: CrawlerRouter<CrawlerCtx>;\n  routerWrappers?: CrawlerRouterWrapper<CrawlerCtx, RouterCtx>[];\n  routerContext?: RouterCtx;\n  routes: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[];\n  routeHandlers: Record<Labels, RouteHandler<CrawlerCtx, RouterCtx>>;\n  input?: Input | null;\n}) => {\n  const { perfBatchSize, perfBatchWaitSecs, requestQueueId } = (input || {}) as PerfActorInput &\n    RequestActorInput;\n\n  /** Redirect the URL to the labelled route identical to route's name */\n  // prettier-ignore\n  const defaultAction: RouteMatcher<CrawlerCtx, RouterCtx, Labels>['action'] = async (url, ctx, route) => {\n    const handler = route.handlerLabel != null && routeHandlers[route.handlerLabel];\n    if (!handler) {\n      ctx.log.error(`No handler found for route ${route.name} (${route.handlerLabel}). URL will not be processed. URL: ${url}`); // prettier-ignore\n      return;\n    }\n    ctx.log.info(`Passing URL to handler ${route.handlerLabel}. URL: ${url}`);\n    await handler(ctx as any);\n  };\n\n  const defaultHandler = async (ctx: RouterHandlerCtx<CrawlerCtx & RouterCtx>): Promise<void> => {\n    const { page, log: parentLog } = ctx;\n    const log = parentLog.child({ prefix: '[Router] ' });\n\n    const reqQueue = await io.openRequestQueue(requestQueueId);\n\n    let handledRequestsCount = 0;\n    let req: CrawlerRequest | null = ctx.request;\n\n    const closeRequest = async () => {\n      if (!req) return;\n      await reqQueue.markRequestHandled(req);\n      handledRequestsCount++;\n    };\n\n    const loadNextRequest = async (suffix: string) => {\n      log.debug(`Checking for new Request in the queue. ${suffix}`);\n\n      if (perfBatchWaitSecs) await wait(perfBatchWaitSecs);\n      const newReq = await reqQueue.fetchNextRequest();\n      req = newReq ?? null;\n\n      if (req) {\n        log.debug(`Found new Request in the queue. ${suffix}`);\n\n        // WARNING - For each subsequent Request, it must be loaded manually\n        //           Hence, batching is suitable only for browser-based Crawlers\n        //           like Playwright or Puppeteer.\n        if (page && page.goto) await (page as Page).goto(req.url);\n      } else {\n        log.debug(`No more Requests in the queue. ${suffix}`);\n      }\n    };\n\n    const hasBatchReqs = () =>\n      perfBatchSize != null && req != null && handledRequestsCount < perfBatchSize;\n\n    try {\n      do {\n        const url = page ? await (page as any as CommonPage).url() : req?.loadedUrl || req?.url;\n        const logSuffix = `Batch ${handledRequestsCount + 1} of ${perfBatchSize ?? 1}. URL: ${url}`;\n\n        // Find route handler for given URL\n        log.debug(`Searching for a handler for given Request. ${logSuffix}`);\n        const route = await serialAsyncFind(routes, async (currRoute) => {\n          const isMatch = await currRoute.match(url, ctx, currRoute, routeHandlers);\n          return isMatch;\n        });\n\n        // Run the handler\n        if (route) {\n          log.info(`URL matched route ${route.name} (handlerLabel: ${route.handlerLabel}). ${logSuffix}`); // prettier-ignore\n          await (route.action ?? defaultAction)(url, ctx, route, routeHandlers);\n        } else {\n          log.error(`No route matched URL. URL will not be processed. ${logSuffix}`);\n        }\n\n        // Clean up and move onto another request\n        await closeRequest();\n        await loadNextRequest(logSuffix);\n      } while (hasBatchReqs());\n    } catch (err) {\n      log.error(`Failed to process a request, returning it to the queue. URL: ${req?.loadedUrl || req?.url}.`); // prettier-ignore\n      log.error(err);\n      // Reinsert the request into the queue if we failed to process it due to an error\n      if (req) await reqQueue.reclaimRequest(req, { forefront: true });\n    }\n  };\n\n  const wrappedHandler = (routerWrappers ?? []).reduceRight(\n    (fn, wrapper) => wrapper(fn),\n    defaultHandler\n  );\n  await router.addDefaultHandler<CrawlerCtx>((ctx) =>\n    wrappedHandler({ ...routerContext, ...ctx } as any)\n  );\n};\n"]}

package/dist/cjs/lib/scraper/scrapeListing.d.ts

@@ -0,0 +1,78 @@
+import type { MaybePromise } from '../../utils/types';
+export interface ListingLogger {
+    debug: (msg: string, data?: any) => void;
+    info: (msg: string, data?: any) => void;
+    warning: (msg: string, data?: any) => void;
+    error: (msg: string, data?: any) => void;
+}
+export interface ListingPageFilter {
+    name: string;
+    disabled?: boolean;
+    initState: () => MaybePromise<boolean>;
+    resetState: () => MaybePromise<void>;
+    nextState: () => MaybePromise<void>;
+    hasNextState: () => MaybePromise<boolean>;
+    hasState: () => MaybePromise<boolean>;
+    loadState: () => MaybePromise<void>;
+}
+export interface ListingFiltersSetupOptions<Ctx extends object, UrlType> {
+    context: ListingPageScraperContext<Ctx, UrlType>;
+    filters?: ListingPageFilter[];
+    shouldApplyFilter?: (context: ListingPageScraperContext<Ctx, UrlType>, filter: ListingPageFilter, filters: ListingPageFilter[]) => MaybePromise<boolean>;
+    onResetFilters?: (context: ListingPageScraperContext<Ctx, UrlType>) => MaybePromise<void>;
+    onFiltersLoaded?: (context: ListingPageScraperContext<Ctx, UrlType>) => MaybePromise<void>;
+    log: ListingLogger;
+}
+export interface ListingPageScraperContext<Ctx extends object, UrlType> {
+    context: Ctx;
+    log: ListingLogger;
+    startUrl: UrlType;
+    filters: ListingPageFilter[];
+    /** Use this if you need to load filters again (eg after reloading page manually) */
+    loadFilterState: () => MaybePromise<void>;
+    /** Call this function from any callback to stop scraping */
+    abort: () => void;
+}
+export interface ListingPageScraperOptions<Ctx extends object, UrlType> extends Omit<ListingFiltersSetupOptions<Ctx, UrlType>, 'context'> {
+    context: Ctx;
+    startUrls: UrlType[];
+    listingCountOnly?: boolean;
+    /** Get ID of the current page in the pagination, so it can be logged */
+    pageId?: (context: ListingPageScraperContext<Ctx, UrlType>) => MaybePromise<string>;
+    log: ListingLogger;
+    onNavigate?: (context: ListingPageScraperContext<Ctx, UrlType>, url: UrlType) => MaybePromise<void>;
+    /**
+     * Hook triggered after navigating to the url using Page.goto().
+     *
+     * One use of this hook is to conditionally disable/enable filters based on the page content.
+     **/
+    onAfterNavigation?: (context: ListingPageScraperContext<Ctx, UrlType>) => MaybePromise<void>;
+    /** How many attempts are retried after filters failed to load. Defaults to 3 */
+    loadFiltersRetries?: number;
+    /**
+     * Hook triggered after a failed attempt at loading listings page filters.
+     *
+     * One use of this hook is to reload the page on failed attemp in case something didn't load correctly.
+     **/
+    onLoadFiltersError?: (context: ListingPageScraperContext<Ctx, UrlType>, error: any, retryIndex: number) => MaybePromise<void>;
+    /** Main logic to extract entries from a page */
+    extractEntries: (context: ListingPageScraperContext<Ctx, UrlType>, retryIndex: number) => MaybePromise<UrlType[]>;
+    /** How many attempts are retried after failed to scrape entries from a listing. Defaults to 3 */
+    extractEntriesRetries?: number;
+    /**
+     * Hook triggered after a failed attempt at scraping entries from a listing.
+     *
+     * One use of this hook is to reload the page on failed attemp in case something didn't load correctly.
+     **/
+    onExtractEntriesError?: (context: ListingPageScraperContext<Ctx, UrlType>, error: any, retryIndex: number) => MaybePromise<void>;
+    onExtractEntriesDone?: (context: ListingPageScraperContext<Ctx, UrlType>, entries: UrlType[] | null) => MaybePromise<void>;
+    /**
+     * If goToNextPage hook is defined, it will be called after each page. To indicate that there's no more
+     * pages left, throw an error.
+     **/
+    onGoToNextPage?: (context: ListingPageScraperContext<Ctx, UrlType>, entries: UrlType[] | null) => MaybePromise<void>;
+    /** How long to wait after we've navigated to the next page and before we start extracting? */
+    nextPageWait?: number;
+}
+/** Get entries from a listing page (eg URLs to profiles that should be scraped later) */
+export declare const scrapeListingEntries: <Ctx extends object, UrlType>(options: ListingPageScraperOptions<Ctx, UrlType>) => Promise<UrlType[]>;

package/dist/cjs/lib/scraper/scrapeListing.js

@@ -0,0 +1,242 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scrapeListingEntries = void 0;
+const lodash_1 = require("lodash");
+const async_1 = require("../../utils/async");
+const url_1 = require("../../utils/url");
+/**
+ * Given configuration for listing page filters, set up functions to
+ * navigate through the different states of filters, to allow to paginate
+ * through all states.
+ */
+const setupListingFilters = ({ context, filters = [], shouldApplyFilter, onResetFilters, onFiltersLoaded, log, }) => {
+    let filtersStack = filters;
+    const getNextFilterStateChangeIndex = () => __awaiter(void 0, void 0, void 0, function* () {
+        const hasNextStates = yield (0, async_1.serialAsyncMap)(filtersStack, (filter) => filter.hasNextState());
+        return (0, lodash_1.findLastIndex)(hasNextStates, (x) => x);
+    });
+    const hasState = () => __awaiter(void 0, void 0, void 0, function* () {
+        const hasStates = yield (0, async_1.serialAsyncMap)(filtersStack, (filter) => filter.hasState());
+        return hasStates.some(Boolean);
+    });
+    const hasNextState = () => __awaiter(void 0, void 0, void 0, function* () {
+        const nextFilterStateChangeIndex = yield getNextFilterStateChangeIndex();
+        return nextFilterStateChangeIndex > -1;
+    });
+    const nextState = () => __awaiter(void 0, void 0, void 0, function* () {
+        // Imagine we have 4 filters, each has 3 states (eg 3 options to select from)
+        // We start with all filters in the first state:
+        //   State 1: F1(1), F2(1), F3(1), F4(1)
+        // As we progress, we increment it akin to numbers:
+        //   State 2: F1(1), F2(1), F3(1), F4(2)
+        //   State 3: F1(1), F2(1), F3(1), F4(3)
+        //   State 4: F1(1), F2(1), F3(2), F4(1)
+        // All the way to the last state:
+        //   State n: F1(3), F2(3), F3(3), F4(3)
+        //
+        // When we want move to a next state, we identify the RIGHT-most filter
+        // whose state can be incremented (in this case we select F2):
+        //             YES    YES     NO    NO
+        //   State x: F1(1), F2(2), F3(3), F4(3)
+        //
+        // When we increment a filter state, all the other filter to the RIGHT
+        // will be reset:
+        //   State x:   F1(1), F2(2), F3(3), F4(3)
+        //   State x+1: F1(1), F2(3), F3(1), F4(1)
+        const initStates = yield (0, async_1.serialAsyncMap)(filtersStack, (filter) => filter.initState());
+        if (initStates.some(Boolean))
+            return log.info('Initialised filters');
+        const nextFilterStateChangeIndex = yield getNextFilterStateChangeIndex();
+        if (nextFilterStateChangeIndex === -1)
+            throw Error('Cannot select next filter state - reached end of list');
+        const filterToNextState = filtersStack[nextFilterStateChangeIndex];
+        const filtersToReset = filtersStack.slice(nextFilterStateChangeIndex + 1);
+        log.info('Setting filters to next state');
+        yield filterToNextState.nextState();
+        for (const filter of filtersToReset) {
+            yield filter.resetState();
+            yield filter.nextState();
+        }
+    });
+    /** Load current filter state in the webpage */
+    const loadState = () => __awaiter(void 0, void 0, void 0, function* () {
+        yield resetState();
+        // Load filters one by one, and only if needed
+        filtersStack = [];
+        for (const filter of filters) {
+            const shouldUseFilter = shouldApplyFilter
+                ? yield shouldApplyFilter(context, filter, filters)
+                : true;
+            if (!shouldUseFilter) {
+                log.info(`Not applying filter "${filter.name}" or further filters`);
+                break;
+            }
+            if (!filter.disabled) {
+                log.info(`Applying filter "${filter.name}"`);
+                yield filter.loadState();
+            }
+            else {
+                log.info(`Filter "${filter.name}" recognised but not applied because it is disabled`);
+            }
+            filtersStack.push(filter);
+        }
+        log.info(`Done loading filters`);
+        yield (onFiltersLoaded === null || onFiltersLoaded === void 0 ? void 0 : onFiltersLoaded(context));
+    });
+    /** Reset filter state */
+    const resetState = () => __awaiter(void 0, void 0, void 0, function* () {
+        log.info(`Resetting filter state`);
+        yield (onResetFilters === null || onResetFilters === void 0 ? void 0 : onResetFilters(context));
+        filtersStack = filters;
+        log.info(`Resetting filter state done`);
+    });
+    return {
+        loadState,
+        nextState,
+        hasNextState,
+        hasState,
+    };
+};
+/** Get entries from a listing page (eg URLs to profiles that should be scraped later) */
+const scrapeListingEntries = (options) => __awaiter(void 0, void 0, void 0, function* () {
+    const { context, startUrls, listingCountOnly = false, log, pageId, onNavigate, onAfterNavigation, filters = [], shouldApplyFilter, loadFiltersRetries = 3, onLoadFiltersError = (_, err) => console.error(err), onFiltersLoaded, onResetFilters, extractEntries, extractEntriesRetries = 3, onExtractEntriesError = (_, err) => console.error(err), onExtractEntriesDone, onGoToNextPage, nextPageWait = 500, } = options;
+    /** Collection of ALL urls across all pages and startUrls */
+    const links = [];
+    yield (0, async_1.serialAsyncMap)(startUrls, (startUrl, index) => __awaiter(void 0, void 0, void 0, function* () {
+        if (listingCountOnly && index > 0)
+            return;
+        const logId = `${startUrl} (${index + 1}/${startUrls.length})`;
+        let userAskedToStop = false;
+        const abort = () => { userAskedToStop = true; }; // prettier-ignore
+        // Prepare context shared across all hooks
+        let filterObj = null;
+        const genCtxArg = () => {
+            var _a;
+            return ({
+                context,
+                log,
+                startUrl,
+                filters,
+                loadFilterState: (_a = filterObj === null || filterObj === void 0 ? void 0 : filterObj.loadState) !== null && _a !== void 0 ? _a : (() => { }),
+                abort,
+            });
+        };
+        log.debug(`Validating URL ${logId}`);
+        (0, url_1.validateUrl)(startUrl);
+        log.info(`Navigating URL ${logId}`);
+        yield (onNavigate === null || onNavigate === void 0 ? void 0 : onNavigate(genCtxArg(), startUrl));
+        log.debug(`Done navigating to URL ${logId}`);
+        filterObj = setupListingFilters({
+            context: genCtxArg(),
+            filters,
+            shouldApplyFilter,
+            onFiltersLoaded,
+            onResetFilters,
+            log,
+        });
+        log.debug(`Calling onAfterNavigation callback. URL ${logId}`); // prettier-ignore
+        yield (onAfterNavigation === null || onAfterNavigation === void 0 ? void 0 : onAfterNavigation(genCtxArg()));
+        log.debug(`Done calling onAfterNavigation callback. URL ${logId})`); // prettier-ignore
+        const isUsingFilters = filters.some((filter) => !filter.disabled);
+        let hasFilterStatesToProcess = true;
+        while (hasFilterStatesToProcess && !userAskedToStop) {
+            // Filter loop
+            // Load filters before we start paginating
+            log.info(`Setting up filters for URL ${logId}`);
+            yield (0, async_1.retryAsync)(() => __awaiter(void 0, void 0, void 0, function* () {
+                if (!filterObj)
+                    throw Error(`Filter controller is missing. This should never happen. URL ${logId}`); // prettier-ignore
+                const filterHasState = yield filterObj.hasState();
+                if (!isUsingFilters || !filterHasState) {
+                    log.info(`Not loading filters for URL ${logId}`);
+                    return;
+                }
+                log.debug(`Loading filters for URL ${logId}`);
+                yield filterObj.nextState();
+                yield filterObj.loadState();
+                log.debug(`Done loading filters for URL ${logId}`);
+            }), {
+                maxRetries: loadFiltersRetries,
+                onError: (err, retryIndex) => onLoadFiltersError(genCtxArg(), err, retryIndex),
+            });
+            let nextPageAvailable = true;
+            while (nextPageAvailable && !userAskedToStop) {
+                // Pagination loop
+                let currPageId = 'next page';
+                if (pageId) {
+                    log.debug(`Loading pageId for URL ${logId}`);
+                    currPageId = yield pageId(genCtxArg());
+                    log.debug(`Done loading pageId for URL ${logId}`);
+                }
+                const pageLogId = `${logId} (${currPageId})`;
+                // Extract page links
+                log.info(`Extracting links from page ${pageLogId}`);
+                const { result } = yield (0, async_1.retryAsync)((retryIndex) => __awaiter(void 0, void 0, void 0, function* () { return extractEntries(genCtxArg(), retryIndex); }), {
+                    maxRetries: extractEntriesRetries,
+                    onError: (err, retryIndex) => onExtractEntriesError(genCtxArg(), err, retryIndex),
+                });
+                log.debug(`Done extracting links from page ${pageLogId}`);
+                const pageLinks = result !== null && result !== void 0 ? result : [];
+                links.push(...pageLinks);
+                log.info(`Found ${pageLinks.length} links on page ${pageLogId}`);
+                // Leave after printing the count or on abort
+                if (listingCountOnly || userAskedToStop) {
+                    nextPageAvailable = false;
+                    if (listingCountOnly)
+                        log.info(`Debugging mode. Entries are not scraped. Leaving now. URL ${pageLogId}`); // prettier-ignore
+                    else if (userAskedToStop)
+                        log.info(`Aborting. URL ${pageLogId}`);
+                    continue;
+                }
+                log.debug(`Calling onExtractEntriesDone callback. URL ${pageLogId}`); // prettier-ignore
+                yield (onExtractEntriesDone === null || onExtractEntriesDone === void 0 ? void 0 : onExtractEntriesDone(genCtxArg(), pageLinks));
+                log.debug(`Done calling onExtractEntriesDone callback. URL ${pageLogId}`); // prettier-ignore
+                if (onGoToNextPage && !userAskedToStop) {
+                    // If goToNextPage hook is defined, this will be called after each page, until it errors
+                    try {
+                        log.info(`Navigating to next page from URL ${pageLogId}`);
+                        yield onGoToNextPage(genCtxArg(), pageLinks);
+                        log.debug(`Done navigating to next page from URL ${pageLogId}`); // prettier-ignore
+                    }
+                    catch (e) {
+                        log.info(`Failed navigating to next page from URL ${pageLogId}`); // prettier-ignore
+                        log.error(e.toString());
+                        nextPageAvailable = false;
+                    }
+                }
+                else {
+                    if (userAskedToStop)
+                        log.info(`Aborting. URL ${pageLogId}`);
+                    nextPageAvailable = false;
+                }
+                // Wait before we start scraping the next page
+                yield new Promise((res) => setTimeout(res, nextPageWait));
+            }
+            // Break out if we're not using filters or we've gone through them all
+            log.debug(`Checking if there are more filter states available for URL ${logId}`);
+            hasFilterStatesToProcess = isUsingFilters && (yield filterObj.hasNextState());
+            log.debug(`Done checking if there are more filter states available for URL ${logId}`);
+            if (hasFilterStatesToProcess) {
+                if (!userAskedToStop)
+                    log.info(`Will repeat scraping this URL with different filter setting. URL ${logId}`); // prettier-ignore
+                else
+                    log.info(`There are unprocessed filter setting remaining for this URL, but stopping due to abort. URL ${logId}`); // prettier-ignore
+            }
+            else
+                log.info(`No filter setting remain for scraping this URL. URL ${logId}`); // prettier-ignore
+        }
+        log.info(`Finished URL ${logId}`);
+    }));
+    return links;
+});
+exports.scrapeListingEntries = scrapeListingEntries;
+//# sourceMappingURL=scrapeListing.js.map

package/dist/cjs/lib/scraper/scrapeListing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapeListing.js","sourceRoot":"","sources":["../../../../src/lib/scraper/scrapeListing.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,mCAAuC;AAEvC,6CAA+D;AAC/D,yCAA8C;AA4G9C;;;;GAIG;AACH,MAAM,mBAAmB,GAAG,CAA8B,EACxD,OAAO,EACP,OAAO,GAAG,EAAE,EACZ,iBAAiB,EACjB,cAAc,EACd,eAAe,EACf,GAAG,GACsC,EAA2B,EAAE;IACtE,IAAI,YAAY,GAAwB,OAAO,CAAC;IAEhD,MAAM,6BAA6B,GAAG,GAAS,EAAE;QAC/C,MAAM,aAAa,GAAG,MAAM,IAAA,sBAAc,EAAC,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;QAC5F,OAAO,IAAA,sBAAa,EAAC,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC,CAAA,CAAC;IAEF,MAAM,QAAQ,GAAG,GAAS,EAAE;QAC1B,MAAM,SAAS,GAAG,MAAM,IAAA,sBAAc,EAAC,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QACpF,OAAO,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC,CAAA,CAAC;IAEF,MAAM,YAAY,GAAG,GAAS,EAAE;QAC9B,MAAM,0BAA0B,GAAG,MAAM,6BAA6B,EAAE,CAAC;QACzE,OAAO,0BAA0B,GAAG,CAAC,CAAC,CAAC;IACzC,CAAC,CAAA,CAAC;IAEF,MAAM,SAAS,GAAG,GAAS,EAAE;QAC3B,6EAA6E;QAC7E,gDAAgD;QAChD,wCAAwC;QACxC,mDAAmD;QACnD,wCAAwC;QACxC,wCAAwC;QACxC,wCAAwC;QACxC,iCAAiC;QACjC,wCAAwC;QACxC,EAAE;QACF,uEAAuE;QACvE,8DAA8D;QAC9D,sCAAsC;QACtC,wCAAwC;QACxC,EAAE;QACF,sEAAsE;QACtE,iBAAiB;QACjB,0CAA0C;QAC1C,0CAA0C;QAE1C,MAAM,UAAU,GAAG,MAAM,IAAA,sBAAc,EAAC,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QACtF,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,OAAO,GAAG,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAErE,MAAM,0BAA0B,GAAG,MAAM,6BAA6B,EAAE,CAAC;QACzE,IAAI,0BAA0B,KAAK,CAAC,CAAC;YACnC,MAAM,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAEvE,MAAM,iBAAiB,GAAG,YAAY,CAAC,0BAA0B,CAAC,CAAC;QACnE,MAAM,cAAc,GAAG,YAAY,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,CAAC,CAAC;QAE1E,GAAG,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC1C,MAAM,iBAAiB,CAAC,SAAS,EAAE,CAAC;QACpC,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;YACnC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC;YAC1B,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;SAC1B;IACH,CAAC,CAAA,CAAC;IAEF,+CAA+C;IAC/C,MAAM,SAAS,GAAG,GAAS,EAAE;QAC3B,MAAM,UAAU,EAAE,CAAC;QAEnB,8CAA8C;QAC9C,YAAY,GAAG,EAAE,CAAC;QAClB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;YAC5B,MAAM,eAAe,GAAG,iBAAiB;gBACvC,CAAC,CAAC,MAAM,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC;gBACnD,CAAC,CAAC,IAAI,CAAC;YACT,IAAI,CAAC,eAAe,EAAE;gBACpB,GAAG,CAAC,IAAI,CAAC,wBAAwB,MAAM,CAAC,IAAI,sBAAsB,CAAC,CAAC;gBACpE,MAAM;aACP;YAED,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;gBACpB,GAAG,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;gBAC7C,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;aAC1B;iBAAM;gBACL,GAAG,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,IAAI,qDAAqD,CAAC,CAAC;aACvF;YAED,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SAC3B;QAED,GAAG,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACjC,MAAM,CAAA,eAAe,aAAf,eAAe,uBAAf,eAAe,CAAG,OAAO,CAAC,CAAA,CAAC;IACnC,CAAC,CAAA,CAAC;IAEF,yBAAyB;IACzB,MAAM,UAAU,GAAG,GAAS,EAAE;QAC5B,GAAG,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACnC,MAAM,CAAA,cAAc,aAAd,cAAc,uBAAd,cAAc,CAAG,OAAO,CAAC,CAAA,CAAC;QAChC,YAAY,GAAG,OAAO,CAAC;QACvB,GAAG,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IAC1C,CAAC,CAAA,CAAC;IAEF,OAAO;QACL,SAAS;QACT,SAAS;QACT,YAAY;QACZ,QAAQ;KACT,CAAC;AACJ,CAAC,CAAC;AAEF,yFAAyF;AAClF,MAAM,oBAAoB,GAAG,CAClC,OAAgD,EAChD,EAAE;IACF,MAAM,EACJ,OAAO,EACP,SAAS,EACT,gBAAgB,GAAG,KAAK,EACxB,GAAG,EACH,MAAM,EACN,UAAU,EACV,iBAAiB,EAEjB,OAAO,GAAG,EAAE,EACZ,iBAAiB,EACjB,kBAAkB,GAAG,CAAC,EACtB,kBAAkB,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EACnD,eAAe,EACf,cAAc,EAEd,cAAc,EACd,qBAAqB,GAAG,CAAC,EACzB,qBAAqB,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EACtD,oBAAoB,EAEpB,cAAc,EACd,YAAY,GAAG,GAAG,GACnB,GAAG,OAAO,CAAC;IAEZ,4DAA4D;IAC5D,MAAM,KAAK,GAAc,EAAE,CAAC;IAE5B,MAAM,IAAA,sBAAc,EAAC,SAAS,EAAE,CAAO,QAAQ,EAAE,KAAK,EAAE,EAAE;QACxD,IAAI,gBAAgB,IAAI,KAAK,GAAG,CAAC;YAAE,OAAO;QAE1C,MAAM,KAAK,GAAG,GAAG,QAAQ,KAAK,KAAK,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;QAE/D,IAAI,eAAe,GAAG,KAAK,CAAC;QAC5B,MAAM,KAAK,GAAG,GAAG,EAAE,GAAG,eAAe,GAAG,IAAI,CAAA,CAAC,CAAC,CAAC,CAAC,kBAAkB;QAElE,0CAA0C;QAC1C,IAAI,SAAS,GAAmC,IAAI,CAAC;QACrD,MAAM,SAAS,GAAG,GAA4C,EAAE;;YAAC,OAAA,CAAC;gBAChE,OAAO;gBACP,GAAG;gBACH,QAAQ;gBACR,OAAO;gBACP,eAAe,EAAE,MAAA,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,SAAS,mCAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;gBACnD,KAAK;aACN,CAAC,CAAA;SAAA,CAAC;QAEH,GAAG,CAAC,KAAK,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC;QACrC,IAAA,iBAAW,EAAC,QAAkB,CAAC,CAAC;QAChC,GAAG,CAAC,IAAI,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC;QACpC,MAAM,CAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAG,SAAS,EAAE,EAAE,QAAQ,CAAC,CAAA,CAAC;QAC1C,GAAG,CAAC,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;QAE7C,SAAS,GAAG,mBAAmB,CAAC;YAC9B,OAAO,EAAE,SAAS,EAAE;YACpB,OAAO;YACP,iBAAiB;YACjB,eAAe;YACf,cAAc;YACd,GAAG;SACJ,CAAC,CAAC;QAEH,GAAG,CAAC,KAAK,CAAC,2CAA2C,KAAK,EAAE,CAAC,CAAC,CAAC,kBAAkB;QACjF,MAAM,CAAA,iBAAiB,aAAjB,iBAAiB,uBAAjB,iBAAiB,CAAG,SAAS,EAAE,CAAC,CAAA,CAAC;QACvC,GAAG,CAAC,KAAK,CAAC,gDAAgD,KAAK,GAAG,CAAC,CAAC,CAAC,kBAAkB;QAEvF,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAElE,IAAI,wBAAwB,GAAG,IAAI,CAAC;QACpC,OAAO,wBAAwB,IAAI,CAAC,eAAe,EAAE;YACnD,cAAc;YACd,0CAA0C;YAC1C,GAAG,CAAC,IAAI,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;YAChD,MAAM,IAAA,kBAAU,EACd,GAAS,EAAE;gBACT,IAAI,CAAC,SAAS;oBAAE,MAAM,KAAK,CAAC,+DAA+D,KAAK,EAAE,CAAC,CAAC,CAAC,kBAAkB;gBAEvH,MAAM,cAAc,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,CAAC;gBAClD,IAAI,CAAC,cAAc,IAAI,CAAC,cAAc,EAAE;oBACtC,GAAG,CAAC,IAAI,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAC;oBACjD,OAAO;iBACR;gBAED,GAAG,CAAC,KAAK,CAAC,2BAA2B,KAAK,EAAE,CAAC,CAAC;gBAC9C,MAAM,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC5B,MAAM,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC5B,GAAG,CAAC,KAAK,CAAC,gCAAgC,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC,CAAA,EACD;gBACE,UAAU,EAAE,kBAAkB;gBAC9B,OAAO,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,EAAE,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC;aAC/E,CACF,CAAC;YAEF,IAAI,iBAAiB,GAAG,IAAI,CAAC;YAC7B,OAAO,iBAAiB,IAAI,CAAC,eAAe,EAAE;gBAC5C,kBAAkB;gBAClB,IAAI,UAAU,GAAG,WAAW,CAAC;gBAC7B,IAAI,MAAM,EAAE;oBACV,GAAG,CAAC,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;oBAC7C,UAAU,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;oBACvC,GAAG,CAAC,KAAK,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAC;iBACnD;gBACD,MAAM,SAAS,GAAG,GAAG,KAAK,KAAK,UAAU,GAAG,CAAC;gBAE7C,qBAAqB;gBACrB,GAAG,CAAC,IAAI,CAAC,8BAA8B,SAAS,EAAE,CAAC,CAAC;gBACpD,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,kBAAU,EACjC,CAAO,UAAU,EAAE,EAAE,kDAAC,OAAA,cAAc,CAAC,SAAS,EAAE,EAAE,UAAU,CAAC,CAAA,GAAA,EAC7D;oBACE,UAAU,EAAE,qBAAqB;oBACjC,OAAO,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,EAAE,CAAC,qBAAqB,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC;iBAClF,CACF,CAAC;gBACF,GAAG,CAAC,KAAK,CAAC,mCAAmC,SAAS,EAAE,CAAC,CAAC;gBAE1D,MAAM,SAAS,GAAG,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,EAAE,CAAC;gBAC/B,KAAK,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBACzB,GAAG,CAAC,IAAI,CAAC,SAAS,SAAS,CAAC,MAAM,kBAAkB,SAAS,EAAE,CAAC,CAAC;gBAEjE,6CAA6C;gBAC7C,IAAI,gBAAgB,IAAI,eAAe,EAAE;oBACvC,iBAAiB,GAAG,KAAK,CAAC;oBAC1B,IAAI,gBAAgB;wBAAE,GAAG,CAAC,IAAI,CAAC,6DAA6D,SAAS,EAAE,CAAC,CAAC,CAAC,kBAAkB;yBACvH,IAAI,eAAe;wBAAE,GAAG,CAAC,IAAI,CAAC,iBAAiB,SAAS,EAAE,CAAC,CAAC;oBACjE,SAAS;iBACV;gBAED,GAAG,CAAC,KAAK,CAAC,8CAA8C,SAAS,EAAE,CAAC,CAAC,CAAC,kBAAkB;gBACxF,MAAM,CAAA,oBAAoB,aAApB,oBAAoB,uBAApB,oBAAoB,CAAG,SAAS,EAAE,EAAE,SAAS,CAAC,CAAA,CAAC;gBACrD,GAAG,CAAC,KAAK,CAAC,mDAAmD,SAAS,EAAE,CAAC,CAAC,CAAC,kBAAkB;gBAE7F,IAAI,cAAc,IAAI,CAAC,eAAe,EAAE;oBACtC,wFAAwF;oBACxF,IAAI;wBACF,GAAG,CAAC,IAAI,CAAC,oCAAoC,SAAS,EAAE,CAAC,CAAC;wBAC1D,MAAM,cAAc,CAAC,SAAS,EAAE,EAAE,SAAS,CAAC,CAAC;wBAC7C,GAAG,CAAC,KAAK,CAAC,yCAAyC,SAAS,EAAE,CAAC,CAAC,CAAC,kBAAkB;qBACpF;oBAAC,OAAO,CAAC,EAAE;wBACV,GAAG,CAAC,IAAI,CAAC,2CAA2C,SAAS,EAAE,CAAC,CAAC,CAAC,kBAAkB;wBACpF,GAAG,CAAC,KAAK,CAAE,CAAW,CAAC,QAAQ,EAAE,CAAC,CAAC;wBACnC,iBAAiB,GAAG,KAAK,CAAC;qBAC3B;iBACF;qBAAM;oBACL,IAAI,eAAe;wBAAE,GAAG,CAAC,IAAI,CAAC,iBAAiB,SAAS,EAAE,CAAC,CAAC;oBAC5D,iBAAiB,GAAG,KAAK,CAAC;iBAC3B;gBAED,8CAA8C;gBAC9C,MAAM,IAAI,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC,CAAC;aAC3D;YAED,sEAAsE;YACtE,GAAG,CAAC,KAAK,CAAC,8DAA8D,KAAK,EAAE,CAAC,CAAC;YACjF,wBAAwB,GAAG,cAAc,IAAI,CAAC,MAAM,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC;YAC9E,GAAG,CAAC,KAAK,CAAC,mEAAmE,KAAK,EAAE,CAAC,CAAC;YAEtF,IAAI,wBAAwB,EAAE;gBAC5B,IAAI,CAAC,eAAe;oBAAE,GAAG,CAAC,IAAI,CAAC,oEAAoE,KAAK,EAAE,CAAC,CAAC,CAAC,kBAAkB;;oBAC1H,GAAG,CAAC,IAAI,CAAC,+FAA+F,KAAK,EAAE,CAAC,CAAC,CAAC,kBAAkB;aAC1I;;gBAAM,GAAG,CAAC,IAAI,CAAC,uDAAuD,KAAK,EAAE,CAAC,CAAC,CAAC,kBAAkB;SACpG;QACD,GAAG,CAAC,IAAI,CAAC,gBAAgB,KAAK,EAAE,CAAC,CAAC;IACpC,CAAC,CAAA,CAAC,CAAC;IACH,OAAO,KAAK,CAAC;AACf,CAAC,CAAA,CAAC;AAxKW,QAAA,oBAAoB,wBAwK/B","sourcesContent":["import { findLastIndex } from 'lodash';\n\nimport { serialAsyncMap, retryAsync } from '../../utils/async';\nimport { validateUrl } from '../../utils/url';\nimport type { MaybePromise } from '../../utils/types';\n\nexport interface ListingLogger {\n  debug: (msg: string, data?: any) => void;\n  info: (msg: string, data?: any) => void;\n  warning: (msg: string, data?: any) => void;\n  error: (msg: string, data?: any) => void;\n}\n\nexport interface ListingPageFilter {\n  name: string;\n  disabled?: boolean;\n  initState: () => MaybePromise<boolean>;\n  resetState: () => MaybePromise<void>;\n  nextState: () => MaybePromise<void>;\n  hasNextState: () => MaybePromise<boolean>;\n  hasState: () => MaybePromise<boolean>;\n  loadState: () => MaybePromise<void>;\n}\n\nexport interface ListingFiltersSetupOptions<Ctx extends object, UrlType> {\n  context: ListingPageScraperContext<Ctx, UrlType>;\n  filters?: ListingPageFilter[];\n  shouldApplyFilter?: (\n    context: ListingPageScraperContext<Ctx, UrlType>,\n    filter: ListingPageFilter,\n    filters: ListingPageFilter[]\n  ) => MaybePromise<boolean>;\n  onResetFilters?: (context: ListingPageScraperContext<Ctx, UrlType>) => MaybePromise<void>;\n  onFiltersLoaded?: (context: ListingPageScraperContext<Ctx, UrlType>) => MaybePromise<void>;\n  log: ListingLogger;\n}\n\ntype ListingFilterController = Pick<ListingPageFilter, 'loadState' | 'nextState' | 'hasNextState' | 'hasState'>; // prettier-ignore\n\nexport interface ListingPageScraperContext<Ctx extends object, UrlType> {\n  context: Ctx;\n  log: ListingLogger;\n  startUrl: UrlType;\n  filters: ListingPageFilter[];\n  /** Use this if you need to load filters again (eg after reloading page manually) */\n  loadFilterState: () => MaybePromise<void>;\n  /** Call this function from any callback to stop scraping */\n  abort: () => void;\n}\n\n// prettier-ignore\nexport interface ListingPageScraperOptions<Ctx extends object, UrlType> extends Omit<ListingFiltersSetupOptions<Ctx, UrlType>, 'context'> {\n  context: Ctx;\n  startUrls: UrlType[];\n  listingCountOnly?: boolean;\n  /** Get ID of the current page in the pagination, so it can be logged */\n  pageId?: (context: ListingPageScraperContext<Ctx, UrlType>) => MaybePromise<string>;\n  log: ListingLogger;\n\n  onNavigate?: (context: ListingPageScraperContext<Ctx, UrlType>, url: UrlType) => MaybePromise<void>;\n  /**\n   * Hook triggered after navigating to the url using Page.goto().\n   *\n   * One use of this hook is to conditionally disable/enable filters based on the page content.\n   **/\n  onAfterNavigation?: (context: ListingPageScraperContext<Ctx, UrlType>) => MaybePromise<void>;\n\n  /** How many attempts are retried after filters failed to load. Defaults to 3 */\n  loadFiltersRetries?: number;\n  /**\n   * Hook triggered after a failed attempt at loading listings page filters.\n   *\n   * One use of this hook is to reload the page on failed attemp in case something didn't load correctly.\n   **/\n  onLoadFiltersError?: (\n    context: ListingPageScraperContext<Ctx, UrlType>,\n    error: any,\n    retryIndex: number\n  ) => MaybePromise<void>;\n\n  /** Main logic to extract entries from a page */\n  extractEntries: (context: ListingPageScraperContext<Ctx, UrlType>, retryIndex: number) => MaybePromise<UrlType[]>;\n  /** How many attempts are retried after failed to scrape entries from a listing. Defaults to 3 */\n  extractEntriesRetries?: number;\n  /**\n   * Hook triggered after a failed attempt at scraping entries from a listing.\n   *\n   * One use of this hook is to reload the page on failed attemp in case something didn't load correctly.\n   **/\n  onExtractEntriesError?: (\n    context: ListingPageScraperContext<Ctx, UrlType>,\n    error: any,\n    retryIndex: number\n  ) => MaybePromise<void>;\n  onExtractEntriesDone?: (\n    context: ListingPageScraperContext<Ctx, UrlType>,\n    entries: UrlType[] | null\n  ) => MaybePromise<void>;\n\n  /**\n   * If goToNextPage hook is defined, it will be called after each page. To indicate that there's no more\n   * pages left, throw an error.\n   **/\n  onGoToNextPage?: (\n    context: ListingPageScraperContext<Ctx, UrlType>,\n    entries: UrlType[] | null\n  ) => MaybePromise<void>;\n  /** How long to wait after we've navigated to the next page and before we start extracting? */\n  nextPageWait?: number;\n}\n\n/**\n * Given configuration for listing page filters, set up functions to\n * navigate through the different states of filters, to allow to paginate\n * through all states.\n */\nconst setupListingFilters = <Ctx extends object, UrlType>({\n  context,\n  filters = [],\n  shouldApplyFilter,\n  onResetFilters,\n  onFiltersLoaded,\n  log,\n}: ListingFiltersSetupOptions<Ctx, UrlType>): ListingFilterController => {\n  let filtersStack: ListingPageFilter[] = filters;\n\n  const getNextFilterStateChangeIndex = async () => {\n    const hasNextStates = await serialAsyncMap(filtersStack, (filter) => filter.hasNextState());\n    return findLastIndex(hasNextStates, (x) => x);\n  };\n\n  const hasState = async () => {\n    const hasStates = await serialAsyncMap(filtersStack, (filter) => filter.hasState());\n    return hasStates.some(Boolean);\n  };\n\n  const hasNextState = async () => {\n    const nextFilterStateChangeIndex = await getNextFilterStateChangeIndex();\n    return nextFilterStateChangeIndex > -1;\n  };\n\n  const nextState = async () => {\n    // Imagine we have 4 filters, each has 3 states (eg 3 options to select from)\n    // We start with all filters in the first state:\n    //   State 1: F1(1), F2(1), F3(1), F4(1)\n    // As we progress, we increment it akin to numbers:\n    //   State 2: F1(1), F2(1), F3(1), F4(2)\n    //   State 3: F1(1), F2(1), F3(1), F4(3)\n    //   State 4: F1(1), F2(1), F3(2), F4(1)\n    // All the way to the last state:\n    //   State n: F1(3), F2(3), F3(3), F4(3)\n    //\n    // When we want move to a next state, we identify the RIGHT-most filter\n    // whose state can be incremented (in this case we select F2):\n    //             YES    YES     NO    NO\n    //   State x: F1(1), F2(2), F3(3), F4(3)\n    //\n    // When we increment a filter state, all the other filter to the RIGHT\n    // will be reset:\n    //   State x:   F1(1), F2(2), F3(3), F4(3)\n    //   State x+1: F1(1), F2(3), F3(1), F4(1)\n\n    const initStates = await serialAsyncMap(filtersStack, (filter) => filter.initState());\n    if (initStates.some(Boolean)) return log.info('Initialised filters');\n\n    const nextFilterStateChangeIndex = await getNextFilterStateChangeIndex();\n    if (nextFilterStateChangeIndex === -1)\n      throw Error('Cannot select next filter state - reached end of list');\n\n    const filterToNextState = filtersStack[nextFilterStateChangeIndex];\n    const filtersToReset = filtersStack.slice(nextFilterStateChangeIndex + 1);\n\n    log.info('Setting filters to next state');\n    await filterToNextState.nextState();\n    for (const filter of filtersToReset) {\n      await filter.resetState();\n      await filter.nextState();\n    }\n  };\n\n  /** Load current filter state in the webpage */\n  const loadState = async () => {\n    await resetState();\n\n    // Load filters one by one, and only if needed\n    filtersStack = [];\n    for (const filter of filters) {\n      const shouldUseFilter = shouldApplyFilter\n        ? await shouldApplyFilter(context, filter, filters)\n        : true;\n      if (!shouldUseFilter) {\n        log.info(`Not applying filter \"${filter.name}\" or further filters`);\n        break;\n      }\n\n      if (!filter.disabled) {\n        log.info(`Applying filter \"${filter.name}\"`);\n        await filter.loadState();\n      } else {\n        log.info(`Filter \"${filter.name}\" recognised but not applied because it is disabled`);\n      }\n\n      filtersStack.push(filter);\n    }\n\n    log.info(`Done loading filters`);\n    await onFiltersLoaded?.(context);\n  };\n\n  /** Reset filter state */\n  const resetState = async () => {\n    log.info(`Resetting filter state`);\n    await onResetFilters?.(context);\n    filtersStack = filters;\n    log.info(`Resetting filter state done`);\n  };\n\n  return {\n    loadState,\n    nextState,\n    hasNextState,\n    hasState,\n  };\n};\n\n/** Get entries from a listing page (eg URLs to profiles that should be scraped later) */\nexport const scrapeListingEntries = async <Ctx extends object, UrlType>(\n  options: ListingPageScraperOptions<Ctx, UrlType>\n) => {\n  const {\n    context,\n    startUrls,\n    listingCountOnly = false,\n    log,\n    pageId,\n    onNavigate,\n    onAfterNavigation,\n\n    filters = [],\n    shouldApplyFilter,\n    loadFiltersRetries = 3,\n    onLoadFiltersError = (_, err) => console.error(err),\n    onFiltersLoaded,\n    onResetFilters,\n\n    extractEntries,\n    extractEntriesRetries = 3,\n    onExtractEntriesError = (_, err) => console.error(err),\n    onExtractEntriesDone,\n\n    onGoToNextPage,\n    nextPageWait = 500,\n  } = options;\n\n  /** Collection of ALL urls across all pages and startUrls */\n  const links: UrlType[] = [];\n\n  await serialAsyncMap(startUrls, async (startUrl, index) => {\n    if (listingCountOnly && index > 0) return;\n\n    const logId = `${startUrl} (${index + 1}/${startUrls.length})`;\n\n    let userAskedToStop = false;\n    const abort = () => { userAskedToStop = true }; // prettier-ignore\n\n    // Prepare context shared across all hooks\n    let filterObj: ListingFilterController | null = null;\n    const genCtxArg = (): ListingPageScraperContext<Ctx, UrlType> => ({\n      context,\n      log,\n      startUrl,\n      filters,\n      loadFilterState: filterObj?.loadState ?? (() => {}),\n      abort,\n    });\n\n    log.debug(`Validating URL ${logId}`);\n    validateUrl(startUrl as string);\n    log.info(`Navigating URL ${logId}`);\n    await onNavigate?.(genCtxArg(), startUrl);\n    log.debug(`Done navigating to URL ${logId}`);\n\n    filterObj = setupListingFilters({\n      context: genCtxArg(),\n      filters,\n      shouldApplyFilter,\n      onFiltersLoaded,\n      onResetFilters,\n      log,\n    });\n\n    log.debug(`Calling onAfterNavigation callback. URL ${logId}`); // prettier-ignore\n    await onAfterNavigation?.(genCtxArg());\n    log.debug(`Done calling onAfterNavigation callback. URL ${logId})`); // prettier-ignore\n\n    const isUsingFilters = filters.some((filter) => !filter.disabled);\n\n    let hasFilterStatesToProcess = true;\n    while (hasFilterStatesToProcess && !userAskedToStop) {\n      // Filter loop\n      // Load filters before we start paginating\n      log.info(`Setting up filters for URL ${logId}`);\n      await retryAsync(\n        async () => {\n          if (!filterObj) throw Error(`Filter controller is missing. This should never happen. URL ${logId}`); // prettier-ignore\n\n          const filterHasState = await filterObj.hasState();\n          if (!isUsingFilters || !filterHasState) {\n            log.info(`Not loading filters for URL ${logId}`);\n            return;\n          }\n\n          log.debug(`Loading filters for URL ${logId}`);\n          await filterObj.nextState();\n          await filterObj.loadState();\n          log.debug(`Done loading filters for URL ${logId}`);\n        },\n        {\n          maxRetries: loadFiltersRetries,\n          onError: (err, retryIndex) => onLoadFiltersError(genCtxArg(), err, retryIndex),\n        }\n      );\n\n      let nextPageAvailable = true;\n      while (nextPageAvailable && !userAskedToStop) {\n        // Pagination loop\n        let currPageId = 'next page';\n        if (pageId) {\n          log.debug(`Loading pageId for URL ${logId}`);\n          currPageId = await pageId(genCtxArg());\n          log.debug(`Done loading pageId for URL ${logId}`);\n        }\n        const pageLogId = `${logId} (${currPageId})`;\n\n        // Extract page links\n        log.info(`Extracting links from page ${pageLogId}`);\n        const { result } = await retryAsync(\n          async (retryIndex) => extractEntries(genCtxArg(), retryIndex),\n          {\n            maxRetries: extractEntriesRetries,\n            onError: (err, retryIndex) => onExtractEntriesError(genCtxArg(), err, retryIndex),\n          }\n        );\n        log.debug(`Done extracting links from page ${pageLogId}`);\n\n        const pageLinks = result ?? [];\n        links.push(...pageLinks);\n        log.info(`Found ${pageLinks.length} links on page ${pageLogId}`);\n\n        // Leave after printing the count or on abort\n        if (listingCountOnly || userAskedToStop) {\n          nextPageAvailable = false;\n          if (listingCountOnly) log.info(`Debugging mode. Entries are not scraped. Leaving now. URL ${pageLogId}`); // prettier-ignore\n          else if (userAskedToStop) log.info(`Aborting. URL ${pageLogId}`);\n          continue;\n        }\n\n        log.debug(`Calling onExtractEntriesDone callback. URL ${pageLogId}`); // prettier-ignore\n        await onExtractEntriesDone?.(genCtxArg(), pageLinks);\n        log.debug(`Done calling onExtractEntriesDone callback. URL ${pageLogId}`); // prettier-ignore\n\n        if (onGoToNextPage && !userAskedToStop) {\n          // If goToNextPage hook is defined, this will be called after each page, until it errors\n          try {\n            log.info(`Navigating to next page from URL ${pageLogId}`);\n            await onGoToNextPage(genCtxArg(), pageLinks);\n            log.debug(`Done navigating to next page from URL ${pageLogId}`); // prettier-ignore\n          } catch (e) {\n            log.info(`Failed navigating to next page from URL ${pageLogId}`); // prettier-ignore\n            log.error((e as Error).toString());\n            nextPageAvailable = false;\n          }\n        } else {\n          if (userAskedToStop) log.info(`Aborting. URL ${pageLogId}`);\n          nextPageAvailable = false;\n        }\n\n        // Wait before we start scraping the next page\n        await new Promise((res) => setTimeout(res, nextPageWait));\n      }\n\n      // Break out if we're not using filters or we've gone through them all\n      log.debug(`Checking if there are more filter states available for URL ${logId}`);\n      hasFilterStatesToProcess = isUsingFilters && (await filterObj.hasNextState());\n      log.debug(`Done checking if there are more filter states available for URL ${logId}`);\n\n      if (hasFilterStatesToProcess) {\n        if (!userAskedToStop) log.info(`Will repeat scraping this URL with different filter setting. URL ${logId}`); // prettier-ignore\n        else log.info(`There are unprocessed filter setting remaining for this URL, but stopping due to abort. URL ${logId}`); // prettier-ignore\n      } else log.info(`No filter setting remain for scraping this URL. URL ${logId}`); // prettier-ignore\n    }\n    log.info(`Finished URL ${logId}`);\n  });\n  return links;\n};\n"]}

package/dist/cjs/lib/test/actor.d.ts

@@ -0,0 +1,21 @@
+import type { vi } from 'vitest';
+import { Dictionary } from 'crawlee';
+import type { MaybeArray, MaybePromise } from '../../utils/types';
+import { OnBatchAddRequests } from './mockApifyClient';
+export declare const setupMockApifyActor: <TInput, TData extends MaybeArray<Dictionary> = MaybeArray<Dictionary>>({ vi: viInstance, actorInput, log, onPushData, onBatchAddRequests, onGetInfo, }: {
+    vi: typeof vi;
+    actorInput?: TInput | undefined;
+    log?: ((...args: any[]) => void) | undefined;
+    onPushData?: ((data: TData) => MaybePromise<void>) | undefined;
+    onBatchAddRequests?: OnBatchAddRequests | undefined;
+    onGetInfo?: ((...args: any[]) => MaybePromise<void>) | undefined;
+}) => Promise<void>;
+export declare const runCrawlerTest: <TData extends MaybeArray<Dictionary>, TInput>({ vi: viInstance, input, runCrawler, log, onPushData, onBatchAddRequests, onDone, }: {
+    vi: typeof vi;
+    input: TInput;
+    runCrawler: () => MaybePromise<void>;
+    log?: ((...args: any[]) => void) | undefined;
+    onPushData?: ((data: any, done: () => void) => MaybePromise<void>) | undefined;
+    onBatchAddRequests?: OnBatchAddRequests | undefined;
+    onDone?: ((done: () => void) => MaybePromise<void>) | undefined;
+}) => Promise<void>;

package/dist/cjs/lib/test/actor.js

@@ -0,0 +1,56 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runCrawlerTest = exports.setupMockApifyActor = void 0;
+const apify_1 = require("apify");
+const crawlee_1 = require("crawlee");
+const mockApifyClient_1 = require("./mockApifyClient");
+const setupMockApifyActor = ({ vi: viInstance, actorInput, log, onPushData, onBatchAddRequests, onGetInfo, }) => __awaiter(void 0, void 0, void 0, function* () {
+    const mockStorageClient = (0, mockApifyClient_1.createMockStorageClient)({ log, onBatchAddRequests });
+    viInstance.spyOn(apify_1.Actor, 'main').mockImplementation((fn) => __awaiter(void 0, void 0, void 0, function* () { return fn(); }));
+    viInstance.spyOn(apify_1.Actor, 'getInput').mockImplementation(() => Promise.resolve(actorInput));
+    viInstance.spyOn(apify_1.Actor, 'openDataset').mockImplementation((datasetId, options) => __awaiter(void 0, void 0, void 0, function* () {
+        console.log('Mock Actor.openDataset: ', datasetId);
+        return (0, mockApifyClient_1.createMockStorageDataset)(datasetId, options, { log, onPushData, onGetInfo });
+    }));
+    viInstance.spyOn(apify_1.Actor, 'pushData').mockImplementation((data) => __awaiter(void 0, void 0, void 0, function* () {
+        console.log('Mock Actor.pushData');
+        if (onPushData)
+            yield onPushData(data);
+    }));
+    viInstance.spyOn(apify_1.RequestQueue, 'open').mockImplementation(() => __awaiter(void 0, void 0, void 0, function* () {
+        const reqQueue = new apify_1.RequestQueue({
+            id: 'test',
+            client: mockStorageClient,
+        });
+        return reqQueue;
+    }));
+    viInstance
+        .spyOn(crawlee_1.KeyValueStore, 'open')
+        .mockImplementation(() => __awaiter(void 0, void 0, void 0, function* () { return new crawlee_1.KeyValueStore({ id: 'keyvalstore', client: mockStorageClient }); }));
+    yield apify_1.Actor.init();
+});
+exports.setupMockApifyActor = setupMockApifyActor;
+const runCrawlerTest = ({ vi: viInstance, input, runCrawler, log, onPushData, onBatchAddRequests, onDone = (done) => done(), }) => __awaiter(void 0, void 0, void 0, function* () {
+    yield new Promise((done) => __awaiter(void 0, void 0, void 0, function* () {
+        yield (0, exports.setupMockApifyActor)({
+            vi: viInstance,
+            actorInput: Object.assign({}, input),
+            log,
+            onPushData: (data) => onPushData === null || onPushData === void 0 ? void 0 : onPushData(data, done),
+            onBatchAddRequests,
+        });
+        yield runCrawler();
+        yield (onDone === null || onDone === void 0 ? void 0 : onDone(done));
+    }));
+});
+exports.runCrawlerTest = runCrawlerTest;
+//# sourceMappingURL=actor.js.map

package/dist/cjs/lib/test/actor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"actor.js","sourceRoot":"","sources":["../../../../src/lib/test/actor.ts"],"names":[],"mappings":";;;;;;;;;;;;AACA,iCAA4C;AAC5C,qCAAoD;AAGpD,uDAI2B;AAEpB,MAAM,mBAAmB,GAAG,CAGjC,EACA,EAAE,EAAE,UAAU,EACd,UAAU,EACV,GAAG,EACH,UAAU,EACV,kBAAkB,EAClB,SAAS,GAQV,EAAE,EAAE;IACH,MAAM,iBAAiB,GAAG,IAAA,yCAAuB,EAAC,EAAE,GAAG,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAE/E,UAAU,CAAC,KAAK,CAAC,aAAK,EAAE,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAO,EAAE,EAAE,EAAE,kDAAC,OAAA,EAAE,EAAE,CAAA,GAAA,CAAC,CAAC;IACvE,UAAU,CAAC,KAAK,CAAC,aAAK,EAAE,UAAU,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IAE1F,UAAU,CAAC,KAAK,CAAC,aAAK,EAAE,aAAa,CAAC,CAAC,kBAAkB,CAAC,CAAO,SAAS,EAAE,OAAO,EAAE,EAAE;QACrF,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE,SAAS,CAAC,CAAC;QACnD,OAAO,IAAA,0CAAwB,EAAC,SAAS,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC;IACtF,CAAC,CAAA,CAAC,CAAC;IACH,UAAU,CAAC,KAAK,CAAC,aAAK,EAAE,UAAU,CAAC,CAAC,kBAAkB,CAAC,CAAO,IAAI,EAAE,EAAE;QACpE,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACnC,IAAI,UAAU;YAAE,MAAM,UAAU,CAAC,IAAW,CAAC,CAAC;IAChD,CAAC,CAAA,CAAC,CAAC;IAEH,UAAU,CAAC,KAAK,CAAC,oBAAY,EAAE,MAAM,CAAC,CAAC,kBAAkB,CAAC,GAAS,EAAE;QACnE,MAAM,QAAQ,GAAG,IAAI,oBAAY,CAAC;YAChC,EAAE,EAAE,MAAM;YACV,MAAM,EAAE,iBAAiB;SAC1B,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAA,CAAC,CAAC;IAEH,UAAU;SACP,KAAK,CAAC,uBAAa,EAAE,MAAM,CAAC;SAC5B,kBAAkB,CACjB,GAAS,EAAE,kDAAC,OAAA,IAAI,uBAAa,CAAC,EAAE,EAAE,EAAE,aAAa,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAA,GAAA,CAChF,CAAC;IAEJ,MAAM,aAAK,CAAC,IAAI,EAAE,CAAC;AACrB,CAAC,CAAA,CAAC;AA/CW,QAAA,mBAAmB,uBA+C9B;AAEK,MAAM,cAAc,GAAG,CAAqD,EACjF,EAAE,EAAE,UAAU,EACd,KAAK,EACL,UAAU,EACV,GAAG,EACH,UAAU,EACV,kBAAkB,EAClB,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,GAS1B,EAAE,EAAE;IACH,MAAM,IAAI,OAAO,CAAO,CAAO,IAAI,EAAE,EAAE;QACrC,MAAM,IAAA,2BAAmB,EAAgB;YACvC,EAAE,EAAE,UAAU;YACd,UAAU,oBAAO,KAAK,CAAE;YACxB,GAAG;YACH,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAG,IAAI,EAAE,IAAI,CAAC;YAC9C,kBAAkB;SACnB,CAAC,CAAC;QAEH,MAAM,UAAU,EAAE,CAAC;QACnB,MAAM,CAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAG,IAAI,CAAC,CAAA,CAAC;IACvB,CAAC,CAAA,CAAC,CAAC;AACL,CAAC,CAAA,CAAC;AA7BW,QAAA,cAAc,kBA6BzB","sourcesContent":["import type { vi } from 'vitest';\nimport { Actor, RequestQueue } from 'apify';\nimport { Dictionary, KeyValueStore } from 'crawlee';\n\nimport type { MaybeArray, MaybePromise } from '../../utils/types';\nimport {\n  OnBatchAddRequests,\n  createMockStorageClient,\n  createMockStorageDataset,\n} from './mockApifyClient';\n\nexport const setupMockApifyActor = async <\n  TInput,\n  TData extends MaybeArray<Dictionary> = MaybeArray<Dictionary>\n>({\n  vi: viInstance,\n  actorInput,\n  log,\n  onPushData,\n  onBatchAddRequests,\n  onGetInfo,\n}: {\n  vi: typeof vi;\n  actorInput?: TInput;\n  log?: (...args: any[]) => void;\n  onPushData?: (data: TData) => MaybePromise<void>;\n  onBatchAddRequests?: OnBatchAddRequests;\n  onGetInfo?: (...args: any[]) => MaybePromise<void>;\n}) => {\n  const mockStorageClient = createMockStorageClient({ log, onBatchAddRequests });\n\n  viInstance.spyOn(Actor, 'main').mockImplementation(async (fn) => fn());\n  viInstance.spyOn(Actor, 'getInput').mockImplementation(() => Promise.resolve(actorInput));\n\n  viInstance.spyOn(Actor, 'openDataset').mockImplementation(async (datasetId, options) => {\n    console.log('Mock Actor.openDataset: ', datasetId);\n    return createMockStorageDataset(datasetId, options, { log, onPushData, onGetInfo });\n  });\n  viInstance.spyOn(Actor, 'pushData').mockImplementation(async (data) => {\n    console.log('Mock Actor.pushData');\n    if (onPushData) await onPushData(data as any);\n  });\n\n  viInstance.spyOn(RequestQueue, 'open').mockImplementation(async () => {\n    const reqQueue = new RequestQueue({\n      id: 'test',\n      client: mockStorageClient,\n    });\n    return reqQueue;\n  });\n\n  viInstance\n    .spyOn(KeyValueStore, 'open')\n    .mockImplementation(\n      async () => new KeyValueStore({ id: 'keyvalstore', client: mockStorageClient })\n    );\n\n  await Actor.init();\n};\n\nexport const runCrawlerTest = async <TData extends MaybeArray<Dictionary>, TInput>({\n  vi: viInstance,\n  input,\n  runCrawler,\n  log,\n  onPushData,\n  onBatchAddRequests,\n  onDone = (done) => done(),\n}: {\n  vi: typeof vi;\n  input: TInput;\n  runCrawler: () => MaybePromise<void>;\n  log?: (...args: any[]) => void;\n  onPushData?: (data: any, done: () => void) => MaybePromise<void>;\n  onBatchAddRequests?: OnBatchAddRequests;\n  onDone?: (done: () => void) => MaybePromise<void>;\n}) => {\n  await new Promise<void>(async (done) => {\n    await setupMockApifyActor<TInput, TData>({\n      vi: viInstance,\n      actorInput: { ...input },\n      log,\n      onPushData: (data) => onPushData?.(data, done),\n      onBatchAddRequests,\n    });\n\n    await runCrawler();\n    await onDone?.(done);\n  });\n};\n"]}

package/dist/cjs/lib/test/mockApifyClient.d.ts

@@ -0,0 +1,32 @@
+import type { Dataset, OpenStorageOptions } from 'apify';
+import type { RequestQueue as ClientRequestQueue, RequestQueueClientBatchAddRequestWithRetriesOptions, RequestQueueClientRequestSchema } from 'apify-client';
+import type { Dataset as ClientDataset, DatasetCollectionClient, KeyValueStoreClient, RequestQueueClient } from 'apify-client';
+import type { StorageClient } from 'crawlee';
+import type { MaybePromise } from '../../utils/types';
+type BatchAddRequestsArgs = [
+    requests: Omit<RequestQueueClientRequestSchema, 'id'>[],
+    options?: RequestQueueClientBatchAddRequestWithRetriesOptions
+];
+export type OnBatchAddRequests = (...args: BatchAddRequestsArgs) => MaybePromise<void>;
+export declare const createMockClientDataset: (overrides?: ClientDataset) => ClientDataset;
+export declare const createMockClientRequestQueue: (overrides?: ClientRequestQueue) => ClientRequestQueue;
+export declare const createMockKeyValueStoreClient: ({ log, }?: {
+    log?: ((args: any) => void) | undefined;
+}) => KeyValueStoreClient;
+export declare const createMockRequestQueueClient: ({ log, onBatchAddRequests, }?: {
+    log?: ((args: any) => void) | undefined;
+    onBatchAddRequests?: OnBatchAddRequests | undefined;
+}) => RequestQueueClient;
+export declare const createMockDatasetCollectionClient: ({ log, }?: {
+    log?: ((args: any) => void) | undefined;
+}) => DatasetCollectionClient;
+export declare const createMockStorageClient: ({ log, onBatchAddRequests, }?: {
+    log?: ((args: any) => void) | undefined;
+    onBatchAddRequests?: OnBatchAddRequests | undefined;
+}) => StorageClient;
+export declare const createMockStorageDataset: (datasetId?: string | null | undefined, options?: OpenStorageOptions | undefined, custom?: {
+    log?: ((...args: any[]) => void) | undefined;
+    onPushData?: ((...args: any[]) => MaybePromise<void>) | undefined;
+    onGetInfo?: ((...args: any[]) => MaybePromise<void>) | undefined;
+} | undefined) => Promise<Dataset<any>>;
+export {};