crawlee-one 1.0.0

package/dist/cjs/lib/readme/types.d.ts

@@ -0,0 +1,260 @@
+import { ArrVal } from '../../utils/types';
+import type { CrawleeOneScraperActorSpec } from '../actorSpec';
+export declare const README_FEATURE: readonly ["datasets", "modes", "filters", "noBrowser", "proxy", "crawlerConfig", "tests", "privacy", "integratedETL", "integratedCache", "errorMonitoring", "metamorph"];
+export declare const README_FEATURE_ENUM: {
+    proxy: "proxy";
+    datasets: "datasets";
+    metamorph: "metamorph";
+    filters: "filters";
+    modes: "modes";
+    noBrowser: "noBrowser";
+    crawlerConfig: "crawlerConfig";
+    tests: "tests";
+    privacy: "privacy";
+    integratedETL: "integratedETL";
+    integratedCache: "integratedCache";
+    errorMonitoring: "errorMonitoring";
+};
+export type ReadmeFeatureType = ArrVal<typeof README_FEATURE>;
+export declare const README_HOOK: readonly ["introAfterBegin", "introAfterDatasets", "introBeforeEnd", "featuresAfterBegin", "featuresBeforeEnd", "useCases", "usageAfterBegin", "usageBeforeEnd", "costAfterBegin", "costAfterPerfTables", "costBeforeEnd", "inputAfterBegin", "inputBeforeEnd", "filterAfterBegin", "filterBeforeEnd", "limitAfterBegin", "limitBeforeEnd", "inputExampleAfterBegin", "inputExampleBeforeEnd", "outputAfterBegin", "outputBeforeEnd", "outputExampleAfterBegin", "outputExampleBeforeEnd", "integrationAfterBegin", "integrationBeforeEnd", "apifyAfterBegin", "apifyBeforeEnd", "legalityAfterBegin", "legalityBeforeEnd", "contactAfterBegin", "contactBeforeEnd"];
+export declare const README_HOOK_ENUM: {
+    introAfterBegin: "introAfterBegin";
+    introAfterDatasets: "introAfterDatasets";
+    introBeforeEnd: "introBeforeEnd";
+    featuresAfterBegin: "featuresAfterBegin";
+    featuresBeforeEnd: "featuresBeforeEnd";
+    useCases: "useCases";
+    usageAfterBegin: "usageAfterBegin";
+    usageBeforeEnd: "usageBeforeEnd";
+    costAfterBegin: "costAfterBegin";
+    costAfterPerfTables: "costAfterPerfTables";
+    costBeforeEnd: "costBeforeEnd";
+    inputAfterBegin: "inputAfterBegin";
+    inputBeforeEnd: "inputBeforeEnd";
+    filterAfterBegin: "filterAfterBegin";
+    filterBeforeEnd: "filterBeforeEnd";
+    limitAfterBegin: "limitAfterBegin";
+    limitBeforeEnd: "limitBeforeEnd";
+    inputExampleAfterBegin: "inputExampleAfterBegin";
+    inputExampleBeforeEnd: "inputExampleBeforeEnd";
+    outputAfterBegin: "outputAfterBegin";
+    outputBeforeEnd: "outputBeforeEnd";
+    outputExampleAfterBegin: "outputExampleAfterBegin";
+    outputExampleBeforeEnd: "outputExampleBeforeEnd";
+    integrationAfterBegin: "integrationAfterBegin";
+    integrationBeforeEnd: "integrationBeforeEnd";
+    apifyAfterBegin: "apifyAfterBegin";
+    apifyBeforeEnd: "apifyBeforeEnd";
+    legalityAfterBegin: "legalityAfterBegin";
+    legalityBeforeEnd: "legalityBeforeEnd";
+    contactAfterBegin: "contactAfterBegin";
+    contactBeforeEnd: "contactBeforeEnd";
+};
+export type ReadmeHook = ArrVal<typeof README_HOOK>;
+/**
+ * Defines how to render a feature block for crawler README.
+ *
+ * Example rendered output:
+ * ```markdown
+ * - **3 kinds of datasets**
+ *   - test1
+ *   - Scrape details of organisations, researchers or projects.
+ *   - test2
+ * ```
+ */
+export interface ReadmeFeature<TData extends any = any> {
+    /** Feature is considered supported by this actor if truthy value is returned. */
+    supported: (it: RenderContext) => any;
+    /**
+     * Title template string for the feature. E.g.
+     * ```js
+     * '<%~ it.a.datasets.length %> kinds of datasets'
+     * ```
+     * */
+    title: string;
+    /**
+     * Template string that goes BEFORE the mainText. E.g.
+     * ```js
+     * '- Scrape details of <%~ it.fn.enumerate(it.a.datasets.map((d) => d.name)) %>.'
+     * ```
+     */
+    afterBegin?: string;
+    /**
+     * Main body template string for the feature. E.g.
+     * ```js
+     * '- Scrape details of <%~ it.fn.enumerate(it.a.datasets.map((d) => d.name)) %>.'
+     * ```
+     */
+    mainText: string;
+    /**
+     * Template string that goes AFTER the mainText. E.g.
+     * ```js
+     * '- Scrape details of <%~ it.fn.enumerate(it.a.datasets.map((d) => d.name)) %>.'
+     * ```
+     */
+    beforeEnd?: string;
+    /**
+     * If you need to store/access some variables during render, use this object.
+     *
+     * E.g.
+     * ```js
+     * filters: {
+     *   supported: (it) => it.a.datasets.some((d) => d.filters.length),
+     *   title: 'Filter support',
+     *   mainText:
+     *     `- Filter the results by <%~ it.fn.enumerate(it.fn.collectFilters(it)) %>.\n` +
+     *     `<% if (it.t.features.filters.data.maxEntriesSupported) { %>\n` +
+     *     `  - Limit the number of results.\n` +
+     *     `<% } %>`,
+     *   data: {
+     *     maxEntriesSupported: true,
+     *   },
+     * },
+     * ```
+     */
+    data?: TData;
+}
+/**
+ * Defines how to render an example input block for crawler README.
+ *
+ * Example rendered output:
+ * ```markdown
+ * #### Example 2: Same as above, but specified by providing a start URL
+ *
+ * ```json
+ * {
+ *   "startUrls": [
+ *     "https://www.skcris.sk/portal/web/guest/register-organizations"
+ *   ],
+ *   // Omit relationships to other entries
+ *   "entryIncludeLinkedResources": false,
+ *   "listingFilterMaxCount": 200,
+ *   "listingItemsPerPage": 200,
+ * }
+ * ```
+ */
+export interface ReadmeExampleInput<TData extends object = object> {
+    /**
+     * Title of the example, e.g.
+     *
+     * ```js
+     * `Get first 200 organisations (fast mode)`
+     * ```
+     */
+    title: string;
+    /** Example input data */
+    inputData: TData;
+    /**
+     * Comments related to individual fields of `inputData`
+     *
+     * These comments may be rendered as such:
+     *
+     * ```json
+     * {
+     *   "inputDataField1": 22,
+     *   // This is a comment from inputDataComments.inputDataField2
+     *   "inputDataField2": "Value from inputData.inputDataField2"
+     * }
+     * ```
+     */
+    inputDataComments?: Partial<Record<keyof TData, string>>;
+}
+/**
+ * Defines how to render a table block with data on performance
+ * for crawler README
+ *
+ * Example rendered output:
+ * ```markdown
+ * ### Organisations
+ *
+ * <table>
+ *   <thead>
+ *     <tr>
+ *       <td></td>
+ *       <td>
+ *         <strong>100 results</strong>
+ *       </td>
+ *       <td>
+ *         <strong>Full run (~ 2.6K results)</strong>
+ *       </td>
+ *     </tr>
+ *   </thead>
+ *   <tbody>
+ *     <tr>
+ *       <td>Fast run</td>
+ *       <td>$0.014 in 2m 0s</td>
+ *       <td>$0.289 in 42m 0s</td>
+ *     </tr>
+ *     <tr>
+ *       <td>Detailed run</td>
+ *       <td>$0.08 in 11m 37s</td>
+ *       <td>$2.008 in 4h 52m </td>
+ *     </tr>
+ *   </tbody>
+ * </table>
+ * ```
+ */
+export interface PerfTable {
+    /**
+     * Map rowIds to header text. E.g.
+     * ```ts
+     * [
+     *   { rowId: 'fast', template: 'Fast run' },
+     *   { rowId: 'detailed', template: 'Detailed run' },
+     * ]
+     * ```
+     */
+    rows: {
+        rowId: string;
+        template: string;
+    }[];
+    /**
+     * Map colIds to header text. E.g.
+     * ```ts
+     * [
+     *   { colId: '100items', template: '100 results' },
+     *   { colId: 'fullRun', template: 'Full run (~ <%~ it.fn.millify(it.dataset.size) %> results)' },
+     * ]
+     * ```
+     */
+    cols: {
+        colId: string;
+        template: string;
+    }[];
+}
+/** Defines how to render the crawler README. */
+export interface CrawleeOneReadmeTemplates {
+    input: {
+        /**
+         * ID (key on input JSON) of the actor input that sets the max number of entries.
+         *
+         * E.g. `'listingFilterMaxCount'`
+         */
+        maxCount: string;
+        /**
+         * Name (as mentioned in UI) of the actor input that sets to include personal data.
+         *
+         * E.g. `'Include personal data'`
+         */
+        privacyName: string;
+    };
+    /** Configure how to render the performance/cost tables */
+    perfTables: Record<string, PerfTable>;
+    /** Configure how to render readme actor features */
+    features: Record<ReadmeFeatureType, ReadmeFeature>;
+    /** Configure how to render readme example inputs */
+    exampleInputs: ReadmeExampleInput[];
+    /**
+     * Hooks are various places throughout the README template
+     * where you can insert own text
+     *
+     * Just as other templates, the hooks receive the whole context (`it`)
+     */
+    hooks?: Partial<Record<ReadmeHook, string>>;
+}
+/** Context available inside the rendering via `it` */
+export interface RenderContext {
+    fn: Record<string, any>;
+    t: CrawleeOneReadmeTemplates;
+    a: CrawleeOneScraperActorSpec;
+}

package/dist/cjs/lib/readme/types.js

@@ -0,0 +1,54 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.README_HOOK_ENUM = exports.README_HOOK = exports.README_FEATURE_ENUM = exports.README_FEATURE = void 0;
+const types_1 = require("../../utils/types");
+exports.README_FEATURE = [
+    'datasets',
+    'modes',
+    'filters',
+    'noBrowser',
+    'proxy',
+    'crawlerConfig',
+    'tests',
+    'privacy',
+    'integratedETL',
+    'integratedCache',
+    'errorMonitoring',
+    'metamorph',
+];
+exports.README_FEATURE_ENUM = (0, types_1.enumFromArray)(exports.README_FEATURE);
+exports.README_HOOK = [
+    'introAfterBegin',
+    'introAfterDatasets',
+    'introBeforeEnd',
+    'featuresAfterBegin',
+    'featuresBeforeEnd',
+    'useCases',
+    'usageAfterBegin',
+    'usageBeforeEnd',
+    'costAfterBegin',
+    'costAfterPerfTables',
+    'costBeforeEnd',
+    'inputAfterBegin',
+    'inputBeforeEnd',
+    'filterAfterBegin',
+    'filterBeforeEnd',
+    'limitAfterBegin',
+    'limitBeforeEnd',
+    'inputExampleAfterBegin',
+    'inputExampleBeforeEnd',
+    'outputAfterBegin',
+    'outputBeforeEnd',
+    'outputExampleAfterBegin',
+    'outputExampleBeforeEnd',
+    'integrationAfterBegin',
+    'integrationBeforeEnd',
+    'apifyAfterBegin',
+    'apifyBeforeEnd',
+    'legalityAfterBegin',
+    'legalityBeforeEnd',
+    'contactAfterBegin',
+    'contactBeforeEnd',
+];
+exports.README_HOOK_ENUM = (0, types_1.enumFromArray)(exports.README_HOOK);
+//# sourceMappingURL=types.js.map

package/dist/cjs/lib/readme/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/lib/readme/types.ts"],"names":[],"mappings":";;;AAAA,6CAA0D;AAG7C,QAAA,cAAc,GAAG;IAC5B,UAAU;IACV,OAAO;IACP,SAAS;IACT,WAAW;IACX,OAAO;IACP,eAAe;IACf,OAAO;IACP,SAAS;IACT,eAAe;IACf,iBAAiB;IACjB,iBAAiB;IACjB,WAAW;CACH,CAAC;AACE,QAAA,mBAAmB,GAAG,IAAA,qBAAa,EAAC,sBAAc,CAAC,CAAC;AAGpD,QAAA,WAAW,GAAG;IACzB,iBAAiB;IACjB,oBAAoB;IACpB,gBAAgB;IAChB,oBAAoB;IACpB,mBAAmB;IACnB,UAAU;IACV,iBAAiB;IACjB,gBAAgB;IAChB,gBAAgB;IAChB,qBAAqB;IACrB,eAAe;IACf,iBAAiB;IACjB,gBAAgB;IAChB,kBAAkB;IAClB,iBAAiB;IACjB,iBAAiB;IACjB,gBAAgB;IAChB,wBAAwB;IACxB,uBAAuB;IACvB,kBAAkB;IAClB,iBAAiB;IACjB,yBAAyB;IACzB,wBAAwB;IACxB,uBAAuB;IACvB,sBAAsB;IACtB,iBAAiB;IACjB,gBAAgB;IAChB,oBAAoB;IACpB,mBAAmB;IACnB,mBAAmB;IACnB,kBAAkB;CACV,CAAC;AACE,QAAA,gBAAgB,GAAG,IAAA,qBAAa,EAAC,mBAAW,CAAC,CAAC","sourcesContent":["import { ArrVal, enumFromArray } from '../../utils/types';\nimport type { CrawleeOneScraperActorSpec } from '../actorSpec';\n\nexport const README_FEATURE = [\n  'datasets',\n  'modes',\n  'filters',\n  'noBrowser',\n  'proxy',\n  'crawlerConfig',\n  'tests',\n  'privacy',\n  'integratedETL',\n  'integratedCache',\n  'errorMonitoring',\n  'metamorph',\n] as const;\nexport const README_FEATURE_ENUM = enumFromArray(README_FEATURE);\nexport type ReadmeFeatureType = ArrVal<typeof README_FEATURE>;\n\nexport const README_HOOK = [\n  'introAfterBegin',\n  'introAfterDatasets',\n  'introBeforeEnd',\n  'featuresAfterBegin',\n  'featuresBeforeEnd',\n  'useCases',\n  'usageAfterBegin',\n  'usageBeforeEnd',\n  'costAfterBegin',\n  'costAfterPerfTables',\n  'costBeforeEnd',\n  'inputAfterBegin',\n  'inputBeforeEnd',\n  'filterAfterBegin',\n  'filterBeforeEnd',\n  'limitAfterBegin',\n  'limitBeforeEnd',\n  'inputExampleAfterBegin',\n  'inputExampleBeforeEnd',\n  'outputAfterBegin',\n  'outputBeforeEnd',\n  'outputExampleAfterBegin',\n  'outputExampleBeforeEnd',\n  'integrationAfterBegin',\n  'integrationBeforeEnd',\n  'apifyAfterBegin',\n  'apifyBeforeEnd',\n  'legalityAfterBegin',\n  'legalityBeforeEnd',\n  'contactAfterBegin',\n  'contactBeforeEnd',\n] as const;\nexport const README_HOOK_ENUM = enumFromArray(README_HOOK);\nexport type ReadmeHook = ArrVal<typeof README_HOOK>;\n\n/**\n * Defines how to render a feature block for crawler README.\n *\n * Example rendered output:\n * ```markdown\n * - **3 kinds of datasets**\n *   - test1\n *   - Scrape details of organisations, researchers or projects.\n *   - test2\n * ```\n */\nexport interface ReadmeFeature<TData extends any = any> {\n  /** Feature is considered supported by this actor if truthy value is returned. */\n  supported: (it: RenderContext) => any;\n  /**\n   * Title template string for the feature. E.g.\n   * ```js\n   * '<%~ it.a.datasets.length %> kinds of datasets'\n   * ```\n   * */\n  title: string;\n  /**\n   * Template string that goes BEFORE the mainText. E.g.\n   * ```js\n   * '- Scrape details of <%~ it.fn.enumerate(it.a.datasets.map((d) => d.name)) %>.'\n   * ```\n   */\n  afterBegin?: string;\n  /**\n   * Main body template string for the feature. E.g.\n   * ```js\n   * '- Scrape details of <%~ it.fn.enumerate(it.a.datasets.map((d) => d.name)) %>.'\n   * ```\n   */\n  mainText: string;\n  /**\n   * Template string that goes AFTER the mainText. E.g.\n   * ```js\n   * '- Scrape details of <%~ it.fn.enumerate(it.a.datasets.map((d) => d.name)) %>.'\n   * ```\n   */\n  beforeEnd?: string;\n  /**\n   * If you need to store/access some variables during render, use this object.\n   *\n   * E.g.\n   * ```js\n   * filters: {\n   *   supported: (it) => it.a.datasets.some((d) => d.filters.length),\n   *   title: 'Filter support',\n   *   mainText:\n   *     `- Filter the results by <%~ it.fn.enumerate(it.fn.collectFilters(it)) %>.\\n` +\n   *     `<% if (it.t.features.filters.data.maxEntriesSupported) { %>\\n` +\n   *     `  - Limit the number of results.\\n` +\n   *     `<% } %>`,\n   *   data: {\n   *     maxEntriesSupported: true,\n   *   },\n   * },\n   * ```\n   */\n  data?: TData;\n}\n\n/**\n * Defines how to render an example input block for crawler README.\n *\n * Example rendered output:\n * ```markdown\n * #### Example 2: Same as above, but specified by providing a start URL\n *\n * ```json\n * {\n *   \"startUrls\": [\n *     \"https://www.skcris.sk/portal/web/guest/register-organizations\"\n *   ],\n *   // Omit relationships to other entries\n *   \"entryIncludeLinkedResources\": false,\n *   \"listingFilterMaxCount\": 200,\n *   \"listingItemsPerPage\": 200,\n * }\n * ```\n */\nexport interface ReadmeExampleInput<TData extends object = object> {\n  /**\n   * Title of the example, e.g.\n   *\n   * ```js\n   * `Get first 200 organisations (fast mode)`\n   * ```\n   */\n  title: string;\n  /** Example input data */\n  inputData: TData;\n  /**\n   * Comments related to individual fields of `inputData`\n   *\n   * These comments may be rendered as such:\n   *\n   * ```json\n   * {\n   *   \"inputDataField1\": 22,\n   *   // This is a comment from inputDataComments.inputDataField2\n   *   \"inputDataField2\": \"Value from inputData.inputDataField2\"\n   * }\n   * ```\n   */\n  inputDataComments?: Partial<Record<keyof TData, string>>;\n}\n\n/**\n * Defines how to render a table block with data on performance\n * for crawler README\n *\n * Example rendered output:\n * ```markdown\n * ### Organisations\n *\n * <table>\n *   <thead>\n *     <tr>\n *       <td></td>\n *       <td>\n *         <strong>100 results</strong>\n *       </td>\n *       <td>\n *         <strong>Full run (~ 2.6K results)</strong>\n *       </td>\n *     </tr>\n *   </thead>\n *   <tbody>\n *     <tr>\n *       <td>Fast run</td>\n *       <td>$0.014 in 2m 0s</td>\n *       <td>$0.289 in 42m 0s</td>\n *     </tr>\n *     <tr>\n *       <td>Detailed run</td>\n *       <td>$0.08 in 11m 37s</td>\n *       <td>$2.008 in 4h 52m </td>\n *     </tr>\n *   </tbody>\n * </table>\n * ```\n */\nexport interface PerfTable {\n  /**\n   * Map rowIds to header text. E.g.\n   * ```ts\n   * [\n   *   { rowId: 'fast', template: 'Fast run' },\n   *   { rowId: 'detailed', template: 'Detailed run' },\n   * ]\n   * ```\n   */\n  rows: {\n    rowId: string;\n    template: string;\n  }[];\n  /**\n   * Map colIds to header text. E.g.\n   * ```ts\n   * [\n   *   { colId: '100items', template: '100 results' },\n   *   { colId: 'fullRun', template: 'Full run (~ <%~ it.fn.millify(it.dataset.size) %> results)' },\n   * ]\n   * ```\n   */\n  cols: {\n    colId: string;\n    template: string;\n  }[];\n}\n\n/** Defines how to render the crawler README. */\nexport interface CrawleeOneReadmeTemplates {\n  input: {\n    /**\n     * ID (key on input JSON) of the actor input that sets the max number of entries.\n     *\n     * E.g. `'listingFilterMaxCount'`\n     */\n    maxCount: string;\n    /**\n     * Name (as mentioned in UI) of the actor input that sets to include personal data.\n     *\n     * E.g. `'Include personal data'`\n     */\n    privacyName: string;\n  };\n  /** Configure how to render the performance/cost tables */\n  perfTables: Record<string, PerfTable>;\n  /** Configure how to render readme actor features */\n  features: Record<ReadmeFeatureType, ReadmeFeature>;\n  /** Configure how to render readme example inputs */\n  exampleInputs: ReadmeExampleInput[];\n  /**\n   * Hooks are various places throughout the README template\n   * where you can insert own text\n   *\n   * Just as other templates, the hooks receive the whole context (`it`)\n   */\n  hooks?: Partial<Record<ReadmeHook, string>>;\n}\n\n/** Context available inside the rendering via `it` */\nexport interface RenderContext {\n  fn: Record<string, any>;\n  t: CrawleeOneReadmeTemplates;\n  a: CrawleeOneScraperActorSpec;\n}\n"]}

package/dist/cjs/lib/router.d.ts

@@ -0,0 +1,132 @@
+import type { BasicCrawler, BasicCrawlingContext, CheerioCrawlingContext, CrawlingContext, HttpCrawlingContext, JSDOMCrawlingContext, PlaywrightCrawlingContext, PuppeteerCrawlingContext, RouterHandler as CrawlerRouter, Request as CrawlerRequest } from 'crawlee';
+import type { MaybePromise } from '../utils/types';
+import type { CrawleeOneIO } from './integrations/types';
+/** Context object provided in CrawlerRouter */
+export type RouterHandlerCtx<CrawlerCtx extends CrawlingContext> = Parameters<Parameters<CrawlerRouter<CrawlerCtx>['addHandler']>[1]>[0];
+/** Function that's passed to `router.addHandler(label, handler)` */
+export type RouteHandler<CrawlerCtx extends CrawlingContext = CrawlingContext<BasicCrawler>, RouterCtx extends Record<string, any> = Record<string, any>> = Parameters<CrawlerRouter<RouterHandlerCtx<CrawlerCtx & RouterCtx>>['addHandler']>[1];
+/** Wrapper that modifies behavior of RouteHandler */
+export type CrawlerRouterWrapper<CrawlerCtx extends CrawlingContext = CrawlingContext<BasicCrawler>, RouterCtx extends Record<string, any> = Record<string, any>> = (handler: (ctx: RouterHandlerCtx<CrawlerCtx & RouterCtx>) => Promise<void>) => (ctx: RouterHandlerCtx<CrawlerCtx & RouterCtx>) => Promise<void>;
+/**
+ * Criteria that un-labelled requests are matched against.
+ *
+ * E.g. If `match` function returns truthy value,
+ * the request is passed to the `action` function for processing.
+ */
+export interface RouteMatcher<CrawlerCtx extends CrawlingContext = CrawlingContext<BasicCrawler>, RouterCtx extends Record<string, any> = Record<string, any>, Labels extends string = string> {
+    /** Human readable name */
+    name: string;
+    /**
+     * Label of the handler registered with `router.addHandler(label, handler)`
+     * that will process this request.
+     *
+     * NOTE: This value is used by the default `action` function. If you override
+     * the `action` function, `handlerLabel` is ignored and you have to process it yourself.
+     */
+    handlerLabel: Labels | null;
+    /**
+     * Function that decides whether the request will processed by this `action` function.
+     *
+     * @example
+     * [{
+     *   // If match returns true, the request is forwarded to handler
+     *   // with label JOB_DETAIL.
+     *   name: 'Job detail',
+     *   match: (url, ctx, route, handlers) => isUrlOfJobOffer(url),
+     *   handlerLabel: routeLabels.JOB_DETAIL,
+     * }]
+     */
+    match: (url: string, ctx: RouterHandlerCtx<CrawlerCtx & RouterCtx>, route: RouteMatcher<CrawlerCtx, RouterCtx, Labels>, handlers: Record<Labels, RouteHandler<CrawlerCtx, RouterCtx>>) => unknown;
+    /**
+     * Request is passed to this function if `match` returned truthy value.
+     *
+     * @example
+     * [{
+     *   // If match returns true, the request is forwarded to handler
+     *   // with label JOB_DETAIL.
+     *   name: 'Job detail',
+     *   match: (url, ctx, route, handlers) => isUrlOfJobOffer(url),
+     *   handlerLabel: routeLabels.JOB_DETAIL,
+     * }]
+     */
+    action?: (url: string, ctx: RouterHandlerCtx<CrawlerCtx>, route: RouteMatcher<CrawlerCtx, RouterCtx, Labels>, handlers: Record<Labels, RouteHandler<CrawlerCtx, RouterCtx>>) => MaybePromise<void>;
+}
+export declare const createRouteMatchers: <CrawlerCtx extends CrawlingContext<unknown, import("crawlee").Dictionary> = CrawlingContext<BasicCrawler<BasicCrawlingContext<import("crawlee").Dictionary>>, import("crawlee").Dictionary>, RouterCtx extends Record<string, any> = Record<string, any>, Labels extends string = string>(matchers: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[]) => RouteMatcher<CrawlerCtx, RouterCtx, Labels>[];
+export declare const createBasicRouteMatchers: <CrawlerCtx extends BasicCrawlingContext<import("crawlee").Dictionary> = BasicCrawlingContext<import("crawlee").Dictionary>, RouterCtx extends Record<string, any> = Record<string, any>, Labels extends string = string>(matchers: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[]) => RouteMatcher<CrawlerCtx, RouterCtx, Labels>[];
+export declare const createHttpRouteMatchers: <CrawlerCtx extends HttpCrawlingContext<any, any> = HttpCrawlingContext<any, any>, RouterCtx extends Record<string, any> = Record<string, any>, Labels extends string = string>(matchers: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[]) => RouteMatcher<CrawlerCtx, RouterCtx, Labels>[];
+export declare const createJsdomRouteMatchers: <CrawlerCtx extends JSDOMCrawlingContext<any, any> = JSDOMCrawlingContext<any, any>, RouterCtx extends Record<string, any> = Record<string, any>, Labels extends string = string>(matchers: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[]) => RouteMatcher<CrawlerCtx, RouterCtx, Labels>[];
+export declare const createCheerioRouteMatchers: <CrawlerCtx extends CheerioCrawlingContext<any, any> = CheerioCrawlingContext<any, any>, RouterCtx extends Record<string, any> = Record<string, any>, Labels extends string = string>(matchers: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[]) => RouteMatcher<CrawlerCtx, RouterCtx, Labels>[];
+export declare const createPlaywrightRouteMatchers: <CrawlerCtx extends PlaywrightCrawlingContext<import("crawlee").Dictionary> = PlaywrightCrawlingContext<import("crawlee").Dictionary>, RouterCtx extends Record<string, any> = Record<string, any>, Labels extends string = string>(matchers: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[]) => RouteMatcher<CrawlerCtx, RouterCtx, Labels>[];
+export declare const createPuppeteerRouteMatchers: <CrawlerCtx extends PuppeteerCrawlingContext<import("crawlee").Dictionary> = PuppeteerCrawlingContext<import("crawlee").Dictionary>, RouterCtx extends Record<string, any> = Record<string, any>, Labels extends string = string>(matchers: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[]) => RouteMatcher<CrawlerCtx, RouterCtx, Labels>[];
+export declare const registerHandlers: <CrawlerCtx extends CrawlingContext<unknown, import("crawlee").Dictionary>, RouterCtx extends Record<string, any> = Record<string, any>, Labels extends string = string>({ router, routerWrappers, routerContext, routeHandlers, }: {
+    router: CrawlerRouter<CrawlerCtx>;
+    routerWrappers?: CrawlerRouterWrapper<CrawlerCtx, RouterCtx>[] | undefined;
+    routerContext?: RouterCtx | undefined;
+    routeHandlers: Record<Labels, (ctx: Omit<Omit<CrawlerCtx & RouterCtx, "request"> & {
+        request: CrawlerRequest<import("crawlee").Dictionary>;
+    }, "request"> & {
+        request: CrawlerRequest<import("crawlee").Dictionary>;
+    }) => import("@crawlee/core/typedefs").Awaitable<void>>;
+}) => Promise<void>;
+/**
+ * Configures the default router handler to redirect URLs to labelled route handlers
+ * based on which route the URL matches first.
+ *
+ * NOTE: This does mean that the URLs passed to this default handler will be fetched
+ * twice (as the URL will be requeued to the correct handler). We recommend to use this
+ * function only in the scenarios where there is a small number of startUrls, yet these
+ * may need various ways of processing based on different paths or etc.
+ *
+ * @example
+ *
+ * const routeLabels = {
+ *   MAIN_PAGE: 'MAIN_PAGE',
+ *   JOB_LISTING: 'JOB_LISTING',
+ *   JOB_DETAIL: 'JOB_DETAIL',
+ *   JOB_RELATED_LIST: 'JOB_RELATED_LIST',
+ *   PARTNERS: 'PARTNERS',
+ * } as const;
+ *
+ * const router = createPlaywrightRouter();
+ *
+ * const routes = createPlaywrightRouteMatchers<typeof routeLabels>([
+ *  // URLs that match this route are redirected to router.addHandler(routeLabels.MAIN_PAGE)
+ *  {
+ *     route: routeLabels.MAIN_PAGE,
+ *     // Check for main page like https://www.profesia.sk/?#
+ *     match: (url) => url.match(/[\W]profesia\.sk\/?(?:[?#~]|$)/i),
+ *   },
+ *
+ *  // Optionally override the logic that assigns the URL to the route by specifying the `action` prop
+ *  {
+ *     route: routeLabels.MAIN_PAGE,
+ *     // Check for main page like https://www.profesia.sk/?#
+ *     match: (url) => url.match(/[\W]profesia\.sk\/?(?:[?#~]|$)/i),
+ *     action: async (ctx) => {
+ *       await ctx.crawler.addRequests([{
+ *         url: 'https://profesia.sk/praca',
+ *         label: routeLabels.JOB_LISTING,
+ *       }]);
+ *     },
+ *   },
+ * ]);
+ *
+ * // Set up default route to redirect to labelled routes
+ * setupDefaultRoute({ router, routes });
+ *
+ * // Now set up the labelled routes
+ * await router.addHandler(routeLabels.JOB_LISTING, async (ctx) => { ... }
+ */
+export declare const setupDefaultRoute: <CrawlerCtx extends CrawlingContext<unknown, import("crawlee").Dictionary>, RouterCtx extends Record<string, any> = Record<string, any>, Labels extends string = string, Input extends Record<string, any> = Record<string, any>>({ io, router, routerWrappers, routerContext, routes, routeHandlers, input, }: {
+    io: CrawleeOneIO;
+    router: CrawlerRouter<CrawlerCtx>;
+    routerWrappers?: CrawlerRouterWrapper<CrawlerCtx, RouterCtx>[] | undefined;
+    routerContext?: RouterCtx | undefined;
+    routes: RouteMatcher<CrawlerCtx, RouterCtx, Labels>[];
+    routeHandlers: Record<Labels, (ctx: Omit<Omit<CrawlerCtx & RouterCtx, "request"> & {
+        request: CrawlerRequest<import("crawlee").Dictionary>;
+    }, "request"> & {
+        request: CrawlerRequest<import("crawlee").Dictionary>;
+    }) => import("@crawlee/core/typedefs").Awaitable<void>>;
+    input?: Input | null | undefined;
+}) => Promise<void>;

package/dist/cjs/lib/router.js

@@ -0,0 +1,165 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setupDefaultRoute = exports.registerHandlers = exports.createPuppeteerRouteMatchers = exports.createPlaywrightRouteMatchers = exports.createCheerioRouteMatchers = exports.createJsdomRouteMatchers = exports.createHttpRouteMatchers = exports.createBasicRouteMatchers = exports.createRouteMatchers = void 0;
+const async_1 = require("../utils/async");
+const createRouteMatchers = (matchers) => matchers; // prettier-ignore
+exports.createRouteMatchers = createRouteMatchers;
+// Context-specific variants
+const createBasicRouteMatchers = (matchers) => matchers; // prettier-ignore
+exports.createBasicRouteMatchers = createBasicRouteMatchers;
+const createHttpRouteMatchers = (matchers) => matchers; // prettier-ignore
+exports.createHttpRouteMatchers = createHttpRouteMatchers;
+const createJsdomRouteMatchers = (matchers) => matchers; // prettier-ignore
+exports.createJsdomRouteMatchers = createJsdomRouteMatchers;
+const createCheerioRouteMatchers = (matchers) => matchers; // prettier-ignore
+exports.createCheerioRouteMatchers = createCheerioRouteMatchers;
+const createPlaywrightRouteMatchers = (matchers) => matchers; // prettier-ignore
+exports.createPlaywrightRouteMatchers = createPlaywrightRouteMatchers;
+const createPuppeteerRouteMatchers = (matchers) => matchers; // prettier-ignore
+exports.createPuppeteerRouteMatchers = createPuppeteerRouteMatchers;
+const registerHandlers = ({ router, routerWrappers, routerContext, routeHandlers, }) => __awaiter(void 0, void 0, void 0, function* () {
+    yield (0, async_1.serialAsyncMap)(Object.entries(routeHandlers), ([key, handler]) => __awaiter(void 0, void 0, void 0, function* () {
+        const wrappedHandler = (routerWrappers !== null && routerWrappers !== void 0 ? routerWrappers : []).reduceRight((fn, wrapper) => wrapper((ctx) => fn(ctx)), handler);
+        yield router.addHandler(key, (ctx) => __awaiter(void 0, void 0, void 0, function* () { return wrappedHandler(Object.assign(Object.assign({}, routerContext), ctx)); }));
+    }));
+});
+exports.registerHandlers = registerHandlers;
+/**
+ * Configures the default router handler to redirect URLs to labelled route handlers
+ * based on which route the URL matches first.
+ *
+ * NOTE: This does mean that the URLs passed to this default handler will be fetched
+ * twice (as the URL will be requeued to the correct handler). We recommend to use this
+ * function only in the scenarios where there is a small number of startUrls, yet these
+ * may need various ways of processing based on different paths or etc.
+ *
+ * @example
+ *
+ * const routeLabels = {
+ *   MAIN_PAGE: 'MAIN_PAGE',
+ *   JOB_LISTING: 'JOB_LISTING',
+ *   JOB_DETAIL: 'JOB_DETAIL',
+ *   JOB_RELATED_LIST: 'JOB_RELATED_LIST',
+ *   PARTNERS: 'PARTNERS',
+ * } as const;
+ *
+ * const router = createPlaywrightRouter();
+ *
+ * const routes = createPlaywrightRouteMatchers<typeof routeLabels>([
+ *  // URLs that match this route are redirected to router.addHandler(routeLabels.MAIN_PAGE)
+ *  {
+ *     route: routeLabels.MAIN_PAGE,
+ *     // Check for main page like https://www.profesia.sk/?#
+ *     match: (url) => url.match(/[\W]profesia\.sk\/?(?:[?#~]|$)/i),
+ *   },
+ *
+ *  // Optionally override the logic that assigns the URL to the route by specifying the `action` prop
+ *  {
+ *     route: routeLabels.MAIN_PAGE,
+ *     // Check for main page like https://www.profesia.sk/?#
+ *     match: (url) => url.match(/[\W]profesia\.sk\/?(?:[?#~]|$)/i),
+ *     action: async (ctx) => {
+ *       await ctx.crawler.addRequests([{
+ *         url: 'https://profesia.sk/praca',
+ *         label: routeLabels.JOB_LISTING,
+ *       }]);
+ *     },
+ *   },
+ * ]);
+ *
+ * // Set up default route to redirect to labelled routes
+ * setupDefaultRoute({ router, routes });
+ *
+ * // Now set up the labelled routes
+ * await router.addHandler(routeLabels.JOB_LISTING, async (ctx) => { ... }
+ */
+const setupDefaultRoute = ({ io, router, routerWrappers, routerContext, routes, routeHandlers, input, }) => __awaiter(void 0, void 0, void 0, function* () {
+    const { perfBatchSize, perfBatchWaitSecs, requestQueueId } = (input || {});
+    /** Redirect the URL to the labelled route identical to route's name */
+    // prettier-ignore
+    const defaultAction = (url, ctx, route) => __awaiter(void 0, void 0, void 0, function* () {
+        const handler = route.handlerLabel != null && routeHandlers[route.handlerLabel];
+        if (!handler) {
+            ctx.log.error(`No handler found for route ${route.name} (${route.handlerLabel}). URL will not be processed. URL: ${url}`); // prettier-ignore
+            return;
+        }
+        ctx.log.info(`Passing URL to handler ${route.handlerLabel}. URL: ${url}`);
+        yield handler(ctx);
+    });
+    const defaultHandler = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
+        var _a;
+        const { page, log: parentLog } = ctx;
+        const log = parentLog.child({ prefix: '[Router] ' });
+        const reqQueue = yield io.openRequestQueue(requestQueueId);
+        let handledRequestsCount = 0;
+        let req = ctx.request;
+        const closeRequest = () => __awaiter(void 0, void 0, void 0, function* () {
+            if (!req)
+                return;
+            yield reqQueue.markRequestHandled(req);
+            handledRequestsCount++;
+        });
+        const loadNextRequest = (suffix) => __awaiter(void 0, void 0, void 0, function* () {
+            log.debug(`Checking for new Request in the queue. ${suffix}`);
+            if (perfBatchWaitSecs)
+                yield (0, async_1.wait)(perfBatchWaitSecs);
+            const newReq = yield reqQueue.fetchNextRequest();
+            req = newReq !== null && newReq !== void 0 ? newReq : null;
+            if (req) {
+                log.debug(`Found new Request in the queue. ${suffix}`);
+                // WARNING - For each subsequent Request, it must be loaded manually
+                //           Hence, batching is suitable only for browser-based Crawlers
+                //           like Playwright or Puppeteer.
+                if (page && page.goto)
+                    yield page.goto(req.url);
+            }
+            else {
+                log.debug(`No more Requests in the queue. ${suffix}`);
+            }
+        });
+        const hasBatchReqs = () => perfBatchSize != null && req != null && handledRequestsCount < perfBatchSize;
+        try {
+            do {
+                const url = page ? yield page.url() : (req === null || req === void 0 ? void 0 : req.loadedUrl) || (req === null || req === void 0 ? void 0 : req.url);
+                const logSuffix = `Batch ${handledRequestsCount + 1} of ${perfBatchSize !== null && perfBatchSize !== void 0 ? perfBatchSize : 1}. URL: ${url}`;
+                // Find route handler for given URL
+                log.debug(`Searching for a handler for given Request. ${logSuffix}`);
+                const route = yield (0, async_1.serialAsyncFind)(routes, (currRoute) => __awaiter(void 0, void 0, void 0, function* () {
+                    const isMatch = yield currRoute.match(url, ctx, currRoute, routeHandlers);
+                    return isMatch;
+                }));
+                // Run the handler
+                if (route) {
+                    log.info(`URL matched route ${route.name} (handlerLabel: ${route.handlerLabel}). ${logSuffix}`); // prettier-ignore
+                    yield ((_a = route.action) !== null && _a !== void 0 ? _a : defaultAction)(url, ctx, route, routeHandlers);
+                }
+                else {
+                    log.error(`No route matched URL. URL will not be processed. ${logSuffix}`);
+                }
+                // Clean up and move onto another request
+                yield closeRequest();
+                yield loadNextRequest(logSuffix);
+            } while (hasBatchReqs());
+        }
+        catch (err) {
+            log.error(`Failed to process a request, returning it to the queue. URL: ${(req === null || req === void 0 ? void 0 : req.loadedUrl) || (req === null || req === void 0 ? void 0 : req.url)}.`); // prettier-ignore
+            log.error(err);
+            // Reinsert the request into the queue if we failed to process it due to an error
+            if (req)
+                yield reqQueue.reclaimRequest(req, { forefront: true });
+        }
+    });
+    const wrappedHandler = (routerWrappers !== null && routerWrappers !== void 0 ? routerWrappers : []).reduceRight((fn, wrapper) => wrapper(fn), defaultHandler);
+    yield router.addDefaultHandler((ctx) => wrappedHandler(Object.assign(Object.assign({}, routerContext), ctx)));
+});
+exports.setupDefaultRoute = setupDefaultRoute;
+//# sourceMappingURL=router.js.map