@crawlee/cheerio 3.0.0-beta.7 → 3.0.0-beta.72

package/internals/cheerio-crawler.d.ts

@@ -1,132 +1,68 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { BasicCrawler, BasicCrawlerOptions } from '@crawlee/basic';
-import { CrawlerExtension, CrawlerHandleFailedRequestInput, CrawlingContext, EnqueueLinksOptions, ProxyConfiguration, ProxyInfo, Request, RequestQueue, Session } from '@crawlee/core';
-import { BatchAddRequestsResult } from '@crawlee/types';
-import { Awaitable, CheerioRoot, Dictionary } from '@crawlee/utils';
-import { RequestLike, ResponseLike } from 'content-type';
-import { OptionsInit } from 'got-scraping';
-import { IncomingMessage } from 'http';
-export interface CheerioFailedRequestHandlerInput<JSONData = Dictionary> extends CrawlerHandleFailedRequestInput, CheerioRequestHandlerInputs<JSONData> {
-}
-export declare type CheerioFailedRequestHandler<JSONData = Dictionary> = (inputs: CheerioFailedRequestHandlerInput<JSONData>) => Awaitable<void>;
-export interface CheerioCrawlerOptions<JSONData = Dictionary> extends Omit<BasicCrawlerOptions<CheerioCrawlingContext<JSONData>>, 'requestHandler' | 'handleRequestFunction' | 'failedRequestHandler' | 'handleFailedRequestFunction' | 'handleRequestTimeoutSecs'> {
+import type { BasicCrawlerOptions } from '@crawlee/basic';
+import { BasicCrawler } from '@crawlee/basic';
+import type { CrawlingContext, EnqueueLinksOptions, ProxyConfiguration, ProxyInfo, Request, RequestQueue, Session } from '@crawlee/core';
+import { CrawlerExtension } from '@crawlee/core';
+import type { BatchAddRequestsResult, Awaitable, Dictionary } from '@crawlee/types';
+import type { CheerioRoot } from '@crawlee/utils';
+import type { RequestLike, ResponseLike } from 'content-type';
+import type { OptionsInit, Response as GotResponse, GotOptionsInit } from 'got-scraping';
+import type { IncomingMessage } from 'http';
+export declare type CheerioFailedRequestHandler<JSONData = Dictionary> = (inputs: CheerioCrawlingContext<JSONData>, error: Error) => Awaitable<void>;
+export interface CheerioCrawlerOptions<JSONData = Dictionary> extends Omit<BasicCrawlerOptions<CheerioCrawlingContext<JSONData>>, 'requestHandler' | 'handleRequestFunction' | 'failedRequestHandler' | 'handleFailedRequestFunction' | 'handleRequestTimeoutSecs' | 'errorHandler'> {
     /**
      * User-provided function that performs the logic of the crawler. It is called for each page
      * loaded and parsed by the crawler.
      *
-     * The function receives the following object as an argument:
-     * ```
-     * {
-     *   // The Cheerio object's function with the parsed HTML.
-     *   $: Cheerio,
-     *
-     *   // The request body of the web page, whose type depends on the content type.
-     *   body: String|Buffer,
-     *
-     *   // The parsed object from JSON for responses with the "application/json" content types.
-     *   // For other content types it's null.
-     *   json: Object,
-     *
-     *   // Request object with details of the requested web page
-     *   request: Request,
-     *
-     *   // Parsed Content-Type HTTP header: { type, encoding }
-     *   contentType: Object,
+     * The function receives the {@link CheerioCrawlingContext} as an argument,
+     * where the {@link CheerioCrawlingContext.request} instance represents the URL to crawl.
      *
-     *   // An instance of Node's http.IncomingMessage object,
-     *   response: Object,
-     *
-     *   // Session object, useful to work around anti-scraping protections
-     *   session: Session
-     *
-     *   // ProxyInfo object with information about currently used proxy
-     *   proxyInfo: ProxyInfo
-     *
-     *   // The running cheerio crawler instance.
-     *   crawler: CheerioCrawler
-     * }
-     * ```
-     *
-     * Type of `body` depends on the `Content-Type` header of the web page:
+     * Type of {@link CheerioCrawlingContext.body} depends on the `Content-Type` header of the web page:
      * - String for `text/html`, `application/xhtml+xml`, `application/xml` MIME content types
      * - Buffer for others MIME content types
      *
      * Parsed `Content-Type` header using
      * [content-type package](https://www.npmjs.com/package/content-type)
-     * is stored in `contentType`.
+     * is stored in {@link CheerioCrawlingContext.contentType}`.
      *
      * Cheerio is available only for HTML and XML content types.
      *
-     * With the {@link Request} object representing the URL to crawl.
-     *
      * If the function returns, the returned promise is awaited by the crawler.
      *
      * If the function throws an exception, the crawler will try to re-crawl the
      * request later, up to `option.maxRequestRetries` times.
      * If all the retries fail, the crawler calls the function
-     * provided to the `handleFailedRequestFunction` parameter.
+     * provided to the `failedRequestHandler` parameter.
      * To make this work, you should **always**
      * let your function throw exceptions rather than catch them.
      * The exceptions are logged to the request using the
      * {@link Request.pushErrorMessage} function.
      */
-    requestHandler: CheerioRequestHandler<JSONData>;
+    requestHandler?: CheerioRequestHandler<JSONData>;
     /**
      * User-provided function that performs the logic of the crawler. It is called for each page
      * loaded and parsed by the crawler.
      *
-     * The function receives the following object as an argument:
-     * ```
-     * {
-     *   // The Cheerio object's function with the parsed HTML.
-     *   $: Cheerio,
-     *
-     *   // The request body of the web page, whose type depends on the content type.
-     *   body: String|Buffer,
-     *
-     *   // The parsed object from JSON for responses with the "application/json" content types.
-     *   // For other content types it's null.
-     *   json: Object,
-     *
-     *   // Request object with details of the requested web page
-     *   request: Request,
-     *
-     *   // Parsed Content-Type HTTP header: { type, encoding }
-     *   contentType: Object,
-     *
-     *   // An instance of Node's http.IncomingMessage object,
-     *   response: Object,
+     * The function receives the {@link CheerioCrawlingContext} as an argument,
+     * where the {@link CheerioCrawlingContext.request} instance represents the URL to crawl.
      *
-     *   // Session object, useful to work around anti-scraping protections
-     *   session: Session
-     *
-     *   // ProxyInfo object with information about currently used proxy
-     *   proxyInfo: ProxyInfo
-     *
-     *   // The running cheerio crawler instance.
-     *   crawler: CheerioCrawler
-     * }
-     * ```
-     *
-     * Type of `body` depends on the `Content-Type` header of the web page:
+     * Type of {@link CheerioCrawlingContext.body} depends on the `Content-Type` header of the web page:
      * - String for `text/html`, `application/xhtml+xml`, `application/xml` MIME content types
      * - Buffer for others MIME content types
      *
      * Parsed `Content-Type` header using
      * [content-type package](https://www.npmjs.com/package/content-type)
-     * is stored in `contentType`.
+     * is stored in {@link CheerioCrawlingContext.contentType}`.
      *
      * Cheerio is available only for HTML and XML content types.
      *
-     * With the {@link Request} object representing the URL to crawl.
-     *
      * If the function returns, the returned promise is awaited by the crawler.
      *
      * If the function throws an exception, the crawler will try to re-crawl the
      * request later, up to `option.maxRequestRetries` times.
      * If all the retries fail, the crawler calls the function
-     * provided to the `handleFailedRequestFunction` parameter.
+     * provided to the `failedRequestHandler` parameter.
      * To make this work, you should **always**
      * let your function throw exceptions rather than catch them.
      * The exceptions are logged to the request using the
@@ -138,7 +74,7 @@ export interface CheerioCrawlerOptions<JSONData = Dictionary> extends Omit<Basic
     /**
      * Timeout in which the HTTP request to the resource needs to finish, given in seconds.
      */
-    requestTimeoutSecs?: number;
+    navigationTimeoutSecs?: number;
     /**
      * If set to true, SSL certificate errors will be ignored.
      */
@@ -149,51 +85,37 @@ export interface CheerioCrawlerOptions<JSONData = Dictionary> extends Omit<Basic
      * For more information, see the [documentation](https://docs.apify.com/proxy).
      */
     proxyConfiguration?: ProxyConfiguration;
+    /**
+     * User-provided function that allows modifying the request object before it gets retried by the crawler.
+     * It's executed before each retry for the requests that failed less than `option.maxRequestRetries` times.
+     *
+     * The function receives the {@link CheerioCrawlingContext} as the first argument,
+     * where the {@link CheerioCrawlingContext.request} corresponds to the request to be retried.
+     * Second argument is the `Error` instance that
+     * represents the last error thrown during processing of the request.
+     */
+    errorHandler?: CheerioFailedRequestHandler<JSONData>;
     /**
      * A function to handle requests that failed more than `option.maxRequestRetries` times.
-     * The function receives the following object as an argument:
-     * ```
-     * {
-     *     error: Error,
-     *     request: Request,
-     *     session: Session,
-     *     $: Cheerio,
-     *     body: String|Buffer,
-     *     json: Object,
-     *     contentType: Object,
-     *     response: Object,
-     *     proxyInfo: ProxyInfo,
-     *     crawler: CheerioCrawler,
-     * }
-     * ```
-     * where the {@link Request} instance corresponds to the failed request, and the `Error` instance
+     *
+     * The function receives the {@link CheerioCrawlingContext} as the first argument,
+     * where the {@link CheerioCrawlingContext.request} corresponds to the failed request.
+     * Second argument is the `Error` instance that
      * represents the last error thrown during processing of the request.
      *
-     * See [source code](https://github.com/apify/apify-js/blob/master/src/crawlers/cheerio_crawler.js#L13)
+     * See [source code](https://github.com/apify/apify-ts/blob/master/src/crawlers/cheerio_crawler.js#L13)
      * for the default implementation of this function.
      */
     failedRequestHandler?: CheerioFailedRequestHandler<JSONData>;
     /**
      * A function to handle requests that failed more than `option.maxRequestRetries` times.
-     * The function receives the following object as an argument:
-     * ```
-     * {
-     *     error: Error,
-     *     request: Request,
-     *     session: Session,
-     *     $: Cheerio,
-     *     body: String|Buffer,
-     *     json: Object,
-     *     contentType: Object,
-     *     response: Object,
-     *     proxyInfo: ProxyInfo,
-     *     crawler: CheerioCrawler,
-     * }
-     * ```
-     * where the {@link Request} instance corresponds to the failed request, and the `Error` instance
+     *
+     * The function receives the {@link CheerioCrawlingContext} as the first argument,
+     * where the {@link CheerioCrawlingContext.request} corresponds to the failed request.
+     * Second argument is the `Error` instance that
      * represents the last error thrown during processing of the request.
      *
-     * See [source code](https://github.com/apify/apify-js/blob/master/src/crawlers/cheerio_crawler.js#L13)
+     * See [source code](https://github.com/apify/apify-ts/blob/master/src/crawlers/cheerio_crawler.js#L13)
      * for the default implementation of this function.
      *
      * @deprecated `handleFailedRequestFunction` has been renamed to `failedRequestHandler` and will be removed in a future version.
@@ -300,7 +222,7 @@ export interface PostResponseInputs<JSONData = Dictionary> {
     crawler: CheerioCrawler<JSONData>;
 }
 export declare type PostResponse<JSONData = Dictionary> = (inputs: PostResponseInputs<JSONData>) => Awaitable<void>;
-export interface CheerioRequestHandlerInputs<JSONData = Dictionary> extends CrawlingContext {
+export interface CheerioCrawlingContext<JSONData extends Dictionary = Dictionary> extends CrawlingContext<JSONData> {
     /**
      * The [Cheerio](https://cheerio.js.org/) object with parsed HTML.
      */
@@ -323,9 +245,9 @@ export interface CheerioRequestHandlerInputs<JSONData = Dictionary> extends Craw
     crawler: CheerioCrawler<JSONData>;
     response: IncomingMessage;
     enqueueLinks: (options?: CheerioCrawlerEnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
+    sendRequest: (overrideOptions?: Partial<GotOptionsInit>) => Promise<GotResponse<string>>;
 }
-export declare type CheerioCrawlingContext<JSONData = Dictionary> = CheerioRequestHandlerInputs<JSONData>;
-export declare type CheerioRequestHandler<JSONData = Dictionary> = (inputs: CheerioRequestHandlerInputs<JSONData>) => Awaitable<void>;
+export declare type CheerioRequestHandler<JSONData = Dictionary> = (inputs: CheerioCrawlingContext<JSONData>) => Awaitable<void>;
 export declare type CheerioCrawlerEnqueueLinksOptions = Omit<EnqueueLinksOptions, 'urls' | 'requestQueue'>;
 /**
  * Provides a framework for the parallel crawling of web pages using plain HTTP requests and
@@ -380,18 +302,15 @@ export declare type CheerioCrawlerEnqueueLinksOptions = Omit<EnqueueLinksOptions
  *
  * ```javascript
  * // Prepare a list of URLs to crawl
- * const requestList = new RequestList({
- *   sources: [
- *       { url: 'http://www.example.com/page-1' },
- *       { url: 'http://www.example.com/page-2' },
- *   ],
- * });
- * await requestList.initialize();
+ * const requestList = await RequestList.open(null, [
+ *     { url: 'http://www.example.com/page-1' },
+ *     { url: 'http://www.example.com/page-2' },
+ * ]);
  *
  * // Crawl the URLs
  * const crawler = new CheerioCrawler({
  *     requestList,
- *     handlePageFunction: async ({ request, response, body, contentType, $ }) => {
+ *     async requestHandler({ request, response, body, contentType, $ }) {
  *         const data = [];
  *
  *         // Do some data extraction from the page with Cheerio.
@@ -400,7 +319,7 @@ export declare type CheerioCrawlerEnqueueLinksOptions = Omit<EnqueueLinksOptions
  *         });
  *
  *         // Save the data to dataset.
- *         await Actor.pushData({
+ *         await Dataset.pushData({
  *             url: request.url,
  *             html: body,
  *             data,
@@ -412,7 +331,7 @@ export declare type CheerioCrawlerEnqueueLinksOptions = Omit<EnqueueLinksOptions
  * ```
  * @category Crawlers
  */
-export declare class CheerioCrawler<JSONData = Dictionary> extends BasicCrawler<CheerioCrawlingContext<JSONData>, CheerioFailedRequestHandlerInput<JSONData>> {
+export declare class CheerioCrawler<JSONData = Dictionary> extends BasicCrawler<CheerioCrawlingContext<JSONData>> {
     /**
      * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
      * Only available if used by the crawler.
@@ -422,14 +341,14 @@ export declare class CheerioCrawler<JSONData = Dictionary> extends BasicCrawler<
     protected preNavigationHooks: CheerioHook<JSONData>[];
     protected postNavigationHooks: CheerioHook<JSONData>[];
     protected persistCookiesPerSession: boolean;
-    protected requestTimeoutMillis: number;
+    protected navigationTimeoutMillis: number;
     protected ignoreSslErrors: boolean;
     protected suggestResponseEncoding?: string;
     protected forceResponseEncoding?: string;
     protected readonly supportedMimeTypes: Set<string>;
     protected static optionsShape: {
         handlePageFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-        requestTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
         ignoreSslErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
         additionalMimeTypes: import("ow").ArrayPredicate<string>;
         suggestResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
@@ -442,11 +361,9 @@ export declare class CheerioCrawler<JSONData = Dictionary> extends BasicCrawler<
         requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
         requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
         handleRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-        requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>; /**
-         * An object with information about currently used proxy by the crawler
-         * and configured by the {@link ProxyConfiguration} class.
-         */
+        requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
         handleRequestTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
         failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
         handleFailedRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
         maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
@@ -456,12 +373,13 @@ export declare class CheerioCrawler<JSONData = Dictionary> extends BasicCrawler<
         useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
         minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
         maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
         log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
     };
     /**
      * All `CheerioCrawler` parameters are passed via an options object.
      */
-    constructor(options: CheerioCrawlerOptions<JSONData>);
+    constructor(options?: CheerioCrawlerOptions<JSONData>);
     /**
      * **EXPERIMENTAL**
      * Function for attaching CrawlerExtensions such as the Unblockers.
@@ -469,7 +387,7 @@ export declare class CheerioCrawler<JSONData = Dictionary> extends BasicCrawler<
      */
     use(extension: CrawlerExtension): void;
     /**
-     * Wrapper around handlePageFunction that opens and closes pages etc.
+     * Wrapper around requestHandler that opens and closes pages etc.
      */
     protected _runRequestHandler(crawlingContext: CheerioCrawlingContext<JSONData>): Promise<void>;
     protected _handleNavigation(crawlingContext: CheerioCrawlingContext<JSONData>): Promise<void>;
@@ -545,5 +463,30 @@ interface RequestFunctionOptions {
     proxyUrl?: string;
     gotOptions: OptionsInit;
 }
+/**
+ * Creates new {@link Router} instance that works based on request labels.
+ * This instance can then serve as a `requestHandler` of your {@link CheerioCrawler}.
+ * Defaults to the {@link CheerioCrawlingContext}.
+ *
+ * > Serves as a shortcut for using `Router.create<CheerioCrawlingContext>()`.
+ *
+ * ```ts
+ * import { CheerioCrawler, createCheerioRouter } from 'crawlee';
+ *
+ * const router = createCheerioRouter();
+ * router.addHandler('label-a', async (ctx) => {
+ *    ctx.log.info('...');
+ * });
+ * router.addDefaultHandler(async (ctx) => {
+ *    ctx.log.info('...');
+ * });
+ *
+ * const crawler = new CheerioCrawler({
+ *     requestHandler: router,
+ * });
+ * await crawler.run();
+ * ```
+ */
+export declare function createCheerioRouter<Context extends CheerioCrawlingContext = CheerioCrawlingContext>(): import("@crawlee/basic").RouterHandler<Context>;
 export {};
 //# sourceMappingURL=cheerio-crawler.d.ts.map

package/internals/cheerio-crawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cheerio-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/cheerio-crawler.ts"],"names":[],"mappings":";;AAEA,OAAO,EACH,YAAY,EACZ,mBAAmB,EAEtB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,gBAAgB,EAChB,+BAA+B,EAC/B,eAAe,EAEf,mBAAmB,EAEnB,kBAAkB,EAClB,SAAS,EACT,OAAO,EACP,YAAY,EAEZ,OAAO,EAEV,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAyC,MAAM,gBAAgB,CAAC;AAE3G,OAA0B,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5E,OAAO,EAAe,WAAW,EAA+C,MAAM,cAAc,CAAC;AAGrG,OAAO,EAAuB,eAAe,EAAE,MAAM,MAAM,CAAC;AAoB5D,MAAM,WAAW,gCAAgC,CAAC,QAAQ,GAAG,UAAU,CAAE,SAAQ,+BAA+B,EAAE,2BAA2B,CAAC,QAAQ,CAAC;CAAG;AAE1J,oBAAY,2BAA2B,CAAC,QAAQ,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,gCAAgC,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEzI,MAAM,WAAW,qBAAqB,CAAC,QAAQ,GAAG,UAAU,CAAE,SAAQ,IAAI,CACtE,mBAAmB,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC,EAEnD,gBAAgB,GAChB,uBAAuB,GAEvB,sBAAsB,GACtB,6BAA6B,GAC7B,0BAA0B,CAC/B;IACG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2DG;IACH,cAAc,EAAE,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAEhD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6DG;IACH,kBAAkB,CAAC,EAAE,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAErD;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,oBAAoB,CAAC,EAAE,2BAA2B,CAAC,QAAQ,CAAC,CAAC;IAE7D;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,2BAA2B,CAAC,EAAE,2BAA2B,CAAC,QAAQ,CAAC,CAAC;IAEpE;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;IAE7C;;;;;;;;;;;OAWG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;IAE9C;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;;;;;;;;;OAUG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;CACtC;AAED,MAAM,WAAW,oBAAoB,CAAC,QAAQ,GAAG,UAAU;IACvD;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,OAAO,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;CACtC;AAED,oBAAY,cAAc,CAAC,QAAQ,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,oBAAoB,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAChH,oBAAY,WAAW,CAAC,QAAQ,GAAG,UAAU,IAAI,CAC7C,eAAe,EAAE,sBAAsB,CAAC,QAAQ,CAAC,EACjD,UAAU,EAAE,WAAW,KACtB,SAAS,CAAC,IAAI,CAAC,CAAC;AAErB,MAAM,WAAW,kBAAkB,CAAC,QAAQ,GAAG,UAAU;IACrD;;OAEG;IACH,QAAQ,CAAC,EAAE,eAAe,CAAC;IAE3B;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;CACrC;AAED,oBAAY,YAAY,CAAC,QAAQ,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE5G,MAAM,WAAW,2BAA2B,CAAC,QAAQ,GAAG,UAAU,CAAE,SAAQ,eAAe;IACvF;;OAEG;IACH,CAAC,EAAE,WAAW,CAAC;IAEf;;OAEG;IACH,IAAI,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;IAExB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;IAClC,QAAQ,EAAE,eAAe,CAAC;IAC1B,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,iCAAiC,KAAK,OAAO,CAAC,sBAAsB,CAAC,CAAC;CAClG;AAED,oBAAY,sBAAsB,CAAC,QAAQ,GAAG,UAAU,IAAI,2BAA2B,CAAC,QAAQ,CAAC,CAAC;AAClG,oBAAY,qBAAqB,CAAC,QAAQ,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,2BAA2B,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAC9H,oBAAY,iCAAiC,GAAG,IAAI,CAAC,mBAAmB,EAAE,MAAM,GAAG,cAAc,CAAC,CAAC;AAEnG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoFG;AACH,qBAAa,cAAc,CAAC,QAAQ,GAAG,UAAU,CAAE,SAAQ,YAAY,CACnE,sBAAsB,CAAC,QAAQ,CAAC,EAChC,gCAAgC,CAAC,QAAQ,CAAC,CAC7C;IACG;;;OAGG;IACI,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAE/C,SAAS,CAAC,+BAA+B,EAAE,MAAM,CAAC;IAClD,SAAS,CAAC,kBAAkB,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;IACtD,SAAS,CAAC,mBAAmB,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;IACvD,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAC5C,SAAS,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACvC,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAC3C,SAAS,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnD,iBAA0B,YAAY;;;;;;;;;;;;;;;kHAhJtC;;;WAGG;;;;;;;;;;;;MA2JD;IAEF;;OAEG;gBACS,OAAO,EAAE,qBAAqB,CAAC,QAAQ,CAAC;IA2EpD;;;;OAIG;IACH,GAAG,CAAC,SAAS,EAAE,gBAAgB;IA4B/B;;OAEG;cACsB,kBAAkB,CAAC,eAAe,EAAE,sBAAsB,CAAC,QAAQ,CAAC;cA2E7E,iBAAiB,CAAC,eAAe,EAAE,sBAAsB,CAAC,QAAQ,CAAC;IA2BnF;;OAEG;IACH,OAAO,CAAC,aAAa;IAUrB;;;;OAIG;cACa,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,sBAAsB,GAAG,OAAO,CAAC,eAAe,CAAC;IAe9H;;OAEG;cACa,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,eAAe;;;;;;;;;;;;;;;;;;;IA6BhF;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,WAAW;kBACzD,IAAI;;IA+BxD,SAAS,CAAC,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,GAAG;QAC9F,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,eAAe,CAAC;KAC7B;cAqCe,eAAe,CAAC,QAAQ,EAAE,eAAe;IAczD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC,EAAE;IAWhG;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,OAAO,CAAC,EAAE,OAAO;IAKjD,OAAO,CAAC,oBAAoB;IAgB5B;;OAEG;IACH,OAAO,CAAC,iBAAiB,CASvB;CACL;AAED,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,iCAAiC,CAAC;IAC5C,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACtB,YAAY,EAAE,YAAY,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,wBAAsB,0BAA0B,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,kBAAkB,EAAE,eAAe,EAAE,EAAE,2BAA2B,mCAoB9I;AAED,UAAU,sBAAsB;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,WAAW,CAAC;CAC3B"}
+{"version":3,"file":"cheerio-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/cheerio-crawler.ts"],"names":[],"mappings":";;AAEA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EACH,YAAY,EAEf,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EACR,eAAe,EACf,mBAAmB,EACnB,kBAAkB,EAClB,SAAS,EACT,OAAO,EACP,YAAY,EACZ,OAAO,EACV,MAAM,eAAe,CAAC;AACvB,OAAO,EACH,gBAAgB,EAMnB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,sBAAsB,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACpF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAIlD,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE9D,OAAO,KAAK,EAAE,WAAW,EAAiC,QAAQ,IAAI,WAAW,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAIxH,OAAO,KAAK,EAAuB,eAAe,EAAE,MAAM,MAAM,CAAC;AAoBjE,oBAAY,2BAA2B,CAAC,QAAQ,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,sBAAsB,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,KAAK,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE7I,MAAM,WAAW,qBAAqB,CAAC,QAAQ,GAAG,UAAU,CAAE,SAAQ,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC,EAE1H,gBAAgB,GAChB,uBAAuB,GAEvB,sBAAsB,GACtB,6BAA6B,GAC7B,0BAA0B,GAE1B,cAAc,CACnB;IACG;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,cAAc,CAAC,EAAE,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAEjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,kBAAkB,CAAC,EAAE,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAErD;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;OAQG;IACH,YAAY,CAAC,EAAE,2BAA2B,CAAC,QAAQ,CAAC,CAAC;IAErD;;;;;;;;;;OAUG;IACH,oBAAoB,CAAC,EAAE,2BAA2B,CAAC,QAAQ,CAAC,CAAC;IAE7D;;;;;;;;;;;;OAYG;IACH,2BAA2B,CAAC,EAAE,2BAA2B,CAAC,QAAQ,CAAC,CAAC;IAEpE;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;IAE7C;;;;;;;;;;;OAWG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;IAE9C;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;;;;;;;;;OAUG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;CACtC;AAED,MAAM,WAAW,oBAAoB,CAAC,QAAQ,GAAG,UAAU;IACvD;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,OAAO,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;CACtC;AAED,oBAAY,cAAc,CAAC,QAAQ,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,oBAAoB,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAChH,oBAAY,WAAW,CAAC,QAAQ,GAAG,UAAU,IAAI,CAC7C,eAAe,EAAE,sBAAsB,CAAC,QAAQ,CAAC,EACjD,UAAU,EAAE,WAAW,KACtB,SAAS,CAAC,IAAI,CAAC,CAAC;AAErB,MAAM,WAAW,kBAAkB,CAAC,QAAQ,GAAG,UAAU;IACrD;;OAEG;IACH,QAAQ,CAAC,EAAE,eAAe,CAAC;IAE3B;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;CACrC;AAED,oBAAY,YAAY,CAAC,QAAQ,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE5G,MAAM,WAAW,sBAAsB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/G;;OAEG;IACH,CAAC,EAAE,WAAW,CAAC;IAEf;;OAEG;IACH,IAAI,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;IAExB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;IAClC,QAAQ,EAAE,eAAe,CAAC;IAC1B,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,iCAAiC,KAAK,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC/F,WAAW,EAAE,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;CAC5F;AAED,oBAAY,qBAAqB,CAAC,QAAQ,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,sBAAsB,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AACzH,oBAAY,iCAAiC,GAAG,IAAI,CAAC,mBAAmB,EAAE,MAAM,GAAG,cAAc,CAAC,CAAC;AAEnG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiFG;AACH,qBAAa,cAAc,CAAC,QAAQ,GAAG,UAAU,CAAE,SAAQ,YAAY,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IACrG;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,SAAS,CAAC,+BAA+B,EAAE,MAAM,CAAC;IAClD,SAAS,CAAC,kBAAkB,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;IACtD,SAAS,CAAC,mBAAmB,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;IACvD,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAC5C,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAC3C,SAAS,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnD,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAcpC;IAEF;;OAEG;gBACS,OAAO,GAAE,qBAAqB,CAAC,QAAQ,CAAM;IAgFzD;;;;OAIG;IACH,GAAG,CAAC,SAAS,EAAE,gBAAgB;IA4B/B;;OAEG;cACsB,kBAAkB,CAAC,eAAe,EAAE,sBAAsB,CAAC,QAAQ,CAAC;cA2E7E,iBAAiB,CAAC,eAAe,EAAE,sBAAsB,CAAC,QAAQ,CAAC;IA2BnF;;OAEG;IACH,OAAO,CAAC,aAAa;IAUrB;;;;OAIG;cACa,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,sBAAsB,GAAG,OAAO,CAAC,eAAe,CAAC;IAe9H;;OAEG;cACa,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,eAAe;;;;;;;;;;;;;;;;;;;IA6BhF;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,WAAW;kBACzD,IAAI;;IA+BxD,SAAS,CAAC,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,GAAG;QAC9F,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,eAAe,CAAC;KAC7B;cAqCe,eAAe,CAAC,QAAQ,EAAE,eAAe;IAczD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC,EAAE;IAWhG;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,OAAO,CAAC,EAAE,OAAO;IAKjD,OAAO,CAAC,oBAAoB;IAgB5B;;OAEG;IACH,OAAO,CAAC,iBAAiB,CASvB;CACL;AAED,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,iCAAiC,CAAC;IAC5C,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACtB,YAAY,EAAE,YAAY,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,wBAAsB,0BAA0B,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,kBAAkB,EAAE,eAAe,EAAE,EAAE,2BAA2B,mCAoB9I;AAED,UAAU,sBAAsB;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,WAAW,CAAC;CAC3B;AAsED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,qDAElG"}

package/internals/cheerio-crawler.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.cheerioCrawlerEnqueueLinks = exports.CheerioCrawler = void 0;
+exports.createCheerioRouter = exports.cheerioCrawlerEnqueueLinks = exports.CheerioCrawler = void 0;
 const tslib_1 = require("tslib");
 const timeout_1 = require("@apify/timeout");
 const utilities_1 = require("@apify/utilities");
@@ -82,18 +82,15 @@ const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  *
  * ```javascript
  * // Prepare a list of URLs to crawl
- * const requestList = new RequestList({
- *   sources: [
- *       { url: 'http://www.example.com/page-1' },
- *       { url: 'http://www.example.com/page-2' },
- *   ],
- * });
- * await requestList.initialize();
+ * const requestList = await RequestList.open(null, [
+ *     { url: 'http://www.example.com/page-1' },
+ *     { url: 'http://www.example.com/page-2' },
+ * ]);
  *
  * // Crawl the URLs
  * const crawler = new CheerioCrawler({
  *     requestList,
- *     handlePageFunction: async ({ request, response, body, contentType, $ }) => {
+ *     async requestHandler({ request, response, body, contentType, $ }) {
  *         const data = [];
  *
  *         // Do some data extraction from the page with Cheerio.
@@ -102,7 +99,7 @@ const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  *         });
  *
  *         // Save the data to dataset.
- *         await Actor.pushData({
+ *         await Dataset.pushData({
  *             url: request.url,
  *             html: body,
  *             data,
@@ -118,9 +115,9 @@ class CheerioCrawler extends basic_1.BasicCrawler {
     /**
      * All `CheerioCrawler` parameters are passed via an options object.
      */
-    constructor(options) {
+    constructor(options = {}) {
         (0, ow_1.default)(options, 'CheerioCrawlerOptions', ow_1.default.object.exactShape(CheerioCrawler.optionsShape));
-        const { requestHandler, handlePageFunction, requestHandlerTimeoutSecs = 60, requestTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, proxyConfiguration, persistCookiesPerSession, preNavigationHooks = [], postNavigationHooks = [], 
+        const { requestHandler, handlePageFunction, requestHandlerTimeoutSecs = 60, navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, proxyConfiguration, persistCookiesPerSession, preNavigationHooks = [], postNavigationHooks = [], 
         // Ignored
         handleRequestFunction, 
         // BasicCrawler
@@ -132,7 +129,7 @@ class CheerioCrawler extends basic_1.BasicCrawler {
             autoscaledPoolOptions,
             // We need to add some time for internal functions to finish,
             // but not too much so that we would stall the crawler.
-            requestHandlerTimeoutSecs: requestTimeoutSecs + requestHandlerTimeoutSecs + basic_1.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
+            requestHandlerTimeoutSecs: navigationTimeoutSecs + requestHandlerTimeoutSecs + basic_1.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
         });
         /**
          * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
@@ -168,7 +165,7 @@ class CheerioCrawler extends basic_1.BasicCrawler {
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, "requestTimeoutMillis", {
+        Object.defineProperty(this, "navigationTimeoutMillis", {
             enumerable: true,
             configurable: true,
             writable: true,
@@ -221,7 +218,11 @@ class CheerioCrawler extends basic_1.BasicCrawler {
             propertyKey: 'requestHandler',
             newProperty: requestHandler,
             oldProperty: handlePageFunction,
+            allowUndefined: true,
         });
+        if (!this.requestHandler) {
+            this.requestHandler = this.router;
+        }
         // Cookies should be persisted per session only if session pool is used
         if (!this.useSessionPool && persistCookiesPerSession) {
             throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
@@ -233,7 +234,7 @@ class CheerioCrawler extends basic_1.BasicCrawler {
             this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
         }
         this.userRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
-        this.requestTimeoutMillis = requestTimeoutSecs * 1000;
+        this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
         this.ignoreSslErrors = ignoreSslErrors;
         this.suggestResponseEncoding = suggestResponseEncoding;
         this.forceResponseEncoding = forceResponseEncoding;
@@ -275,7 +276,7 @@ class CheerioCrawler extends basic_1.BasicCrawler {
         }
     }
     /**
-     * Wrapper around handlePageFunction that opens and closes pages etc.
+     * Wrapper around requestHandler that opens and closes pages etc.
      */
     async _runRequestHandler(crawlingContext) {
         const { request, session } = crawlingContext;
@@ -349,7 +350,7 @@ class CheerioCrawler extends basic_1.BasicCrawler {
         const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
         this._applyCookies(crawlingContext, gotOptions, preNavigationHooksCookies, postNavigationHooksCookies);
         const proxyUrl = crawlingContext.proxyInfo?.url;
-        crawlingContext.response = await (0, timeout_1.addTimeoutToPromise)(() => this._requestFunction({ request, session, proxyUrl, gotOptions }), this.requestTimeoutMillis, `request timed out after ${this.requestTimeoutMillis / 1000} seconds.`);
+        crawlingContext.response = await (0, timeout_1.addTimeoutToPromise)(() => this._requestFunction({ request, session, proxyUrl, gotOptions }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
         (0, timeout_1.tryCancel)();
         await this._executeHooks(this.postNavigationHooks, crawlingContext, gotOptions);
         (0, timeout_1.tryCancel)();
@@ -421,7 +422,7 @@ class CheerioCrawler extends basic_1.BasicCrawler {
             url: request.url,
             method: request.method,
             proxyUrl,
-            timeout: { request: this.requestTimeoutMillis },
+            timeout: { request: this.navigationTimeoutMillis },
             sessionToken: session,
             ...gotOptions,
             headers: { ...request.headers, ...gotOptions?.headers },
@@ -434,7 +435,7 @@ class CheerioCrawler extends basic_1.BasicCrawler {
         // TODO this is incorrect, the check for man in the middle needs to be done
         //   on individual proxy level, not on the `proxyConfiguration` level,
         //   because users can use normal + MITM proxies in a single configuration.
-        //   Disable SSL verification for MITM proxies
+        // Disable SSL verification for MITM proxies
         if (this.proxyConfiguration && this.proxyConfiguration.isManInTheMiddle) {
             requestOptions.https = {
                 ...requestOptions.https,
@@ -534,7 +535,7 @@ Object.defineProperty(CheerioCrawler, "optionsShape", {
     value: {
         ...basic_1.BasicCrawler.optionsShape,
         handlePageFunction: ow_1.default.optional.function,
-        requestTimeoutSecs: ow_1.default.optional.number,
+        navigationTimeoutSecs: ow_1.default.optional.number,
         ignoreSslErrors: ow_1.default.optional.boolean,
         additionalMimeTypes: ow_1.default.optional.array.ofType(ow_1.default.string),
         suggestResponseEncoding: ow_1.default.optional.string,
@@ -581,10 +582,19 @@ function extractUrlsFromCheerio($, selector, baseUrl) {
             throw new Error(`An extracted URL: ${href} is relative and options.baseUrl is not set. `
                 + 'Use options.baseUrl in enqueueLinks() to automatically resolve relative URLs.');
         }
+        const tryAbsolute = () => {
+            try {
+                return (new URL(href, baseUrl)).href;
+            }
+            catch {
+                return undefined;
+            }
+        };
         return baseUrl
-            ? (new URL(href, baseUrl)).href
+            ? tryAbsolute()
             : href;
-    });
+    })
+        .filter((href) => !!href);
 }
 /**
  * The stream object returned from got does not have the below properties.
@@ -618,4 +628,32 @@ function addResponsePropertiesToStream(stream) {
     }
     return stream;
 }
+/**
+ * Creates new {@link Router} instance that works based on request labels.
+ * This instance can then serve as a `requestHandler` of your {@link CheerioCrawler}.
+ * Defaults to the {@link CheerioCrawlingContext}.
+ *
+ * > Serves as a shortcut for using `Router.create<CheerioCrawlingContext>()`.
+ *
+ * ```ts
+ * import { CheerioCrawler, createCheerioRouter } from 'crawlee';
+ *
+ * const router = createCheerioRouter();
+ * router.addHandler('label-a', async (ctx) => {
+ *    ctx.log.info('...');
+ * });
+ * router.addDefaultHandler(async (ctx) => {
+ *    ctx.log.info('...');
+ * });
+ *
+ * const crawler = new CheerioCrawler({
+ *     requestHandler: router,
+ * });
+ * await crawler.run();
+ * ```
+ */
+function createCheerioRouter() {
+    return core_1.Router.create();
+}
+exports.createCheerioRouter = createCheerioRouter;
 //# sourceMappingURL=cheerio-crawler.js.map