@crawlee/cheerio 3.0.3-beta.8 → 3.0.4-beta.0

package/README.md

@@ -63,7 +63,7 @@ Additionally, the package provides various helper functions to simplify running
 
 ## Quick Start
 
-This short tutorial will set you up to start using Crawlee in a minute or two. If you want to learn more, proceed to the [Getting Started](https://crawlee.dev/docs/guides/getting-started) tutorial that will take you step by step through creating your first scraper.
+This short tutorial will set you up to start using Crawlee in a minute or two. If you want to learn more, proceed to the [Getting Started](https://crawlee.dev/docs/introduction) tutorial that will take you step by step through creating your first scraper.
 
 ### Local stand-alone usage
 

package/index.d.ts

@@ -1,3 +1,3 @@
-export * from '@crawlee/basic';
+export * from '@crawlee/http';
 export * from './internals/cheerio-crawler';
 //# sourceMappingURL=index.d.ts.map

package/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,6BAA6B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,6BAA6B,CAAC"}

package/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const tslib_1 = require("tslib");
-tslib_1.__exportStar(require("@crawlee/basic"), exports);
+tslib_1.__exportStar(require("@crawlee/http"), exports);
 tslib_1.__exportStar(require("./internals/cheerio-crawler"), exports);
 //# sourceMappingURL=index.js.map

package/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,yDAA+B;AAC/B,sEAA4C"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,wDAA8B;AAC9B,sEAA4C"}

package/index.mjs

@@ -16,6 +16,7 @@ export const EVENT_SESSION_RETIRED = mod.EVENT_SESSION_RETIRED;
 export const EnqueueStrategy = mod.EnqueueStrategy;
 export const EventManager = mod.EventManager;
 export const EventType = mod.EventType;
+export const HttpCrawler = mod.HttpCrawler;
 export const KeyValueStore = mod.KeyValueStore;
 export const LocalEventManager = mod.LocalEventManager;
 export const Log = mod.Log;
@@ -34,6 +35,7 @@ export const REQUESTS_PERSISTENCE_KEY = mod.REQUESTS_PERSISTENCE_KEY;
 export const Request = mod.Request;
 export const RequestList = mod.RequestList;
 export const RequestQueue = mod.RequestQueue;
+export const RetryRequestError = mod.RetryRequestError;
 export const Router = mod.Router;
 export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
 export const STORAGE_CONSISTENCY_DELAY_MILLIS = mod.STORAGE_CONSISTENCY_DELAY_MILLIS;
@@ -54,6 +56,7 @@ export const cookieStringToToughCookie = mod.cookieStringToToughCookie;
 export const createBasicRouter = mod.createBasicRouter;
 export const createCheerioRouter = mod.createCheerioRouter;
 export const createDeserialize = mod.createDeserialize;
+export const createHttpRouter = mod.createHttpRouter;
 export const createRequestOptions = mod.createRequestOptions;
 export const createRequests = mod.createRequests;
 export const deserializeArray = mod.deserializeArray;

package/internals/cheerio-crawler.d.ts

@@ -1,222 +1,27 @@
 /// <reference types="node" />
-/// <reference types="node" />
-import type { BasicCrawlerOptions, ErrorHandler, RequestHandler } from '@crawlee/basic';
-import { BasicCrawler } from '@crawlee/basic';
-import type { CrawlingContext, EnqueueLinksOptions, ProxyConfiguration, Request, RequestQueue, Session } from '@crawlee/core';
-import { CrawlerExtension, Configuration } from '@crawlee/core';
-import type { BatchAddRequestsResult, Awaitable, Dictionary } from '@crawlee/types';
-import type { CheerioRoot } from '@crawlee/utils';
-import type { RequestLike, ResponseLike } from 'content-type';
-import type { OptionsInit, Response as GotResponse, GotOptionsInit } from 'got-scraping';
+import type { HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, ErrorHandler, RequestHandler, EnqueueLinksOptions, RequestQueue, Configuration } from '@crawlee/http';
+import { HttpCrawler } from '@crawlee/http';
+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
+import * as cheerio from 'cheerio';
 import type { IncomingMessage } from 'http';
 export declare type CheerioErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = Dictionary> = ErrorHandler<CheerioCrawlingContext<UserData, JSONData>>;
+JSONData extends Dictionary = any> = ErrorHandler<CheerioCrawlingContext<UserData, JSONData>>;
 export interface CheerioCrawlerOptions<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = Dictionary> extends Omit<BasicCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>>, 'requestHandler' | 'handleRequestFunction' | 'failedRequestHandler' | 'handleFailedRequestFunction' | 'handleRequestTimeoutSecs' | 'errorHandler'> {
-    /**
-     * User-provided function that performs the logic of the crawler. It is called for each page
-     * loaded and parsed by the crawler.
-     *
-     * The function receives the {@link CheerioCrawlingContext} as an argument,
-     * where the {@link CheerioCrawlingContext.request} instance represents the URL to crawl.
-     *
-     * Type of {@link CheerioCrawlingContext.body} depends on the `Content-Type` header of the web page:
-     * - String for `text/html`, `application/xhtml+xml`, `application/xml` MIME content types
-     * - Buffer for others MIME content types
-     *
-     * Parsed `Content-Type` header using
-     * [content-type package](https://www.npmjs.com/package/content-type)
-     * is stored in {@link CheerioCrawlingContext.contentType}`.
-     *
-     * Cheerio is available only for HTML and XML content types.
-     *
-     * If the function returns, the returned promise is awaited by the crawler.
-     *
-     * If the function throws an exception, the crawler will try to re-crawl the
-     * request later, up to `option.maxRequestRetries` times.
-     * If all the retries fail, the crawler calls the function
-     * provided to the `failedRequestHandler` parameter.
-     * To make this work, you should **always**
-     * let your function throw exceptions rather than catch them.
-     * The exceptions are logged to the request using the
-     * {@link Request.pushErrorMessage} function.
-     */
-    requestHandler?: CheerioRequestHandler<UserData, JSONData>;
-    /**
-     * User-provided function that performs the logic of the crawler. It is called for each page
-     * loaded and parsed by the crawler.
-     *
-     * The function receives the {@link CheerioCrawlingContext} as an argument,
-     * where the {@link CheerioCrawlingContext.request} instance represents the URL to crawl.
-     *
-     * Type of {@link CheerioCrawlingContext.body} depends on the `Content-Type` header of the web page:
-     * - String for `text/html`, `application/xhtml+xml`, `application/xml` MIME content types
-     * - Buffer for others MIME content types
-     *
-     * Parsed `Content-Type` header using
-     * [content-type package](https://www.npmjs.com/package/content-type)
-     * is stored in {@link CheerioCrawlingContext.contentType}`.
-     *
-     * Cheerio is available only for HTML and XML content types.
-     *
-     * If the function returns, the returned promise is awaited by the crawler.
-     *
-     * If the function throws an exception, the crawler will try to re-crawl the
-     * request later, up to `option.maxRequestRetries` times.
-     * If all the retries fail, the crawler calls the function
-     * provided to the `failedRequestHandler` parameter.
-     * To make this work, you should **always**
-     * let your function throw exceptions rather than catch them.
-     * The exceptions are logged to the request using the
-     * {@link Request.pushErrorMessage} function.
-     *
-     * @deprecated `handlePageFunction` has been renamed to `requestHandler` and will be removed in a future version.
-     * @ignore
-     */
-    handlePageFunction?: CheerioRequestHandler<UserData, JSONData>;
-    /**
-     * Timeout in which the HTTP request to the resource needs to finish, given in seconds.
-     */
-    navigationTimeoutSecs?: number;
-    /**
-     * If set to true, SSL certificate errors will be ignored.
-     */
-    ignoreSslErrors?: boolean;
-    /**
-     * If set, `CheerioCrawler` will be configured for all connections to use
-     * [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
-     * For more information, see the [documentation](https://docs.apify.com/proxy).
-     */
-    proxyConfiguration?: ProxyConfiguration;
-    /**
-     * User-provided function that allows modifying the request object before it gets retried by the crawler.
-     * It's executed before each retry for the requests that failed less than `option.maxRequestRetries` times.
-     *
-     * The function receives the {@link CheerioCrawlingContext} as the first argument,
-     * where the {@link CheerioCrawlingContext.request} corresponds to the request to be retried.
-     * Second argument is the `Error` instance that
-     * represents the last error thrown during processing of the request.
-     */
-    errorHandler?: CheerioErrorHandler<UserData, JSONData>;
-    /**
-     * A function to handle requests that failed more than `option.maxRequestRetries` times.
-     *
-     * The function receives the {@link CheerioCrawlingContext} as the first argument,
-     * where the {@link CheerioCrawlingContext.request} corresponds to the failed request.
-     * Second argument is the `Error` instance that
-     * represents the last error thrown during processing of the request.
-     *
-     * See [source code](https://github.com/apify/crawlee/blob/master/src/crawlers/cheerio_crawler.js#L13)
-     * for the default implementation of this function.
-     */
-    failedRequestHandler?: CheerioErrorHandler<UserData, JSONData>;
-    /**
-     * A function to handle requests that failed more than `option.maxRequestRetries` times.
-     *
-     * The function receives the {@link CheerioCrawlingContext} as the first argument,
-     * where the {@link CheerioCrawlingContext.request} corresponds to the failed request.
-     * Second argument is the `Error` instance that
-     * represents the last error thrown during processing of the request.
-     *
-     * See [source code](https://github.com/apify/crawlee/blob/master/src/crawlers/cheerio_crawler.js#L13)
-     * for the default implementation of this function.
-     *
-     * @deprecated `handleFailedRequestFunction` has been renamed to `failedRequestHandler` and will be removed in a future version.
-     * @ignore
-     */
-    handleFailedRequestFunction?: CheerioErrorHandler<UserData, JSONData>;
-    /**
-     * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
-     * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotOptions`,
-     * which are passed to the `requestAsBrowser()` function the crawler calls to navigate.
-     * Example:
-     * ```
-     * preNavigationHooks: [
-     *     async (crawlingContext, gotOptions) => {
-     *         // ...
-     *     },
-     * ]
-     * ```
-     */
-    preNavigationHooks?: CheerioHook<UserData, JSONData>[];
-    /**
-     * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
-     * The function accepts `crawlingContext` as the only parameter.
-     * Example:
-     * ```
-     * postNavigationHooks: [
-     *     async (crawlingContext) => {
-     *         // ...
-     *     },
-     * ]
-     * ```
-     */
-    postNavigationHooks?: CheerioHook<UserData, JSONData>[];
-    /**
-     * An array of [MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types)
-     * you want the crawler to load and process. By default, only `text/html` and `application/xhtml+xml` MIME types are supported.
-     */
-    additionalMimeTypes?: string[];
-    /**
-     * By default `CheerioCrawler` will extract correct encoding from the HTTP response headers.
-     * Sadly, there are some websites which use invalid headers. Those are encoded using the UTF-8 encoding.
-     * If those sites actually use a different encoding, the response will be corrupted. You can use
-     * `suggestResponseEncoding` to fall back to a certain encoding, if you know that your target website uses it.
-     * To force a certain encoding, disregarding the response headers, use {@link CheerioCrawlerOptions.forceResponseEncoding}
-     * ```
-     * // Will fall back to windows-1250 encoding if none found
-     * suggestResponseEncoding: 'windows-1250'
-     * ```
-     */
-    suggestResponseEncoding?: string;
-    /**
-     * By default `CheerioCrawler` will extract correct encoding from the HTTP response headers. Use `forceResponseEncoding`
-     * to force a certain encoding, disregarding the response headers.
-     * To only provide a default for missing encodings, use {@link CheerioCrawlerOptions.suggestResponseEncoding}
-     * ```
-     * // Will force windows-1250 encoding even if headers say otherwise
-     * forceResponseEncoding: 'windows-1250'
-     * ```
-     */
-    forceResponseEncoding?: string;
-    /**
-     * Automatically saves cookies to Session. Works only if Session Pool is used.
-     *
-     * It parses cookie from response "set-cookie" header saves or updates cookies for session and once the session is used for next request.
-     * It passes the "Cookie" header to the request with the session cookies.
-     */
-    persistCookiesPerSession?: boolean;
+JSONData extends Dictionary = any> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>> {
 }
 export declare type CheerioHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = Dictionary> = (crawlingContext: CheerioCrawlingContext<UserData, JSONData>, gotOptions: OptionsInit) => Awaitable<void>;
+JSONData extends Dictionary = any> = InternalHttpHook<CheerioCrawlingContext<UserData, JSONData>>;
 export interface CheerioCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
+JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData, CheerioCrawler> {
     /**
      * The [Cheerio](https://cheerio.js.org/) object with parsed HTML.
+     * Cheerio is available only for HTML and XML content types.
      */
-    $: CheerioRoot;
-    /**
-     * The request body of the web page.
-     */
-    body: (string | Buffer);
-    /**
-     * The parsed object from JSON string if the response contains the content type application/json.
-     */
-    json: JSONData;
-    /**
-     * Parsed `Content-Type header: { type, encoding }`.
-     */
-    contentType: {
-        type: string;
-        encoding: string;
-    };
-    crawler: CheerioCrawler;
-    response: IncomingMessage;
+    $: cheerio.CheerioAPI;
     enqueueLinks: (options?: CheerioCrawlerEnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
-    sendRequest: (overrideOptions?: Partial<GotOptionsInit>) => Promise<GotResponse<string>>;
 }
 export declare type CheerioRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = Dictionary> = RequestHandler<CheerioCrawlingContext<UserData, JSONData>>;
+JSONData extends Dictionary = any> = RequestHandler<CheerioCrawlingContext<UserData, JSONData>>;
 export interface CheerioCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestQueue'> {
 }
 /**
@@ -227,23 +32,23 @@ export interface CheerioCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOpti
  *
  * Since `CheerioCrawler` uses raw HTTP requests to download web pages,
  * it is very fast and efficient on data bandwidth. However, if the target website requires JavaScript
- * to display the content, you might need to use {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead,
+ * to display the content, you might need to use {@apilink PuppeteerCrawler} or {@apilink PlaywrightCrawler} instead,
  * because it loads the pages using full-featured headless Chrome browser.
  *
  * `CheerioCrawler` downloads each URL using a plain HTTP request,
  * parses the HTML content using [Cheerio](https://www.npmjs.com/package/cheerio)
- * and then invokes the user-provided {@link CheerioCrawlerOptions.requestHandler} to extract page data
+ * and then invokes the user-provided {@apilink CheerioCrawlerOptions.requestHandler} to extract page data
  * using a [jQuery](https://jquery.com/)-like interface to the parsed HTML DOM.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link CheerioCrawlerOptions.requestList}
- * or {@link CheerioCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@apilink Request} objects that are fed from
+ * {@apilink RequestList} or {@apilink RequestQueue} instances provided by the {@apilink CheerioCrawlerOptions.requestList}
+ * or {@apilink CheerioCrawlerOptions.requestQueue} constructor options, respectively.
  *
- * If both {@link CheerioCrawlerOptions.requestList} and {@link CheerioCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * If both {@apilink CheerioCrawlerOptions.requestList} and {@apilink CheerioCrawlerOptions.requestQueue} are used,
+ * the instance first processes URLs from the {@apilink RequestList} and automatically enqueues all of them
+ * to {@apilink RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
  *
- * The crawler finishes when there are no more {@link Request} objects to crawl.
+ * The crawler finishes when there are no more {@apilink Request} objects to crawl.
  *
  * We can use the `preNavigationHooks` to adjust `gotOptions`:
  *
@@ -258,28 +63,20 @@ export interface CheerioCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOpti
  * By default, `CheerioCrawler` only processes web pages with the `text/html`
  * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),
  * and skips pages with other content types. If you want the crawler to process other content types,
- * use the {@link CheerioCrawlerOptions.additionalMimeTypes} constructor option.
+ * use the {@apilink CheerioCrawlerOptions.additionalMimeTypes} constructor option.
  * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
- * For details, see {@link CheerioCrawlerOptions.requestHandler}.
+ * For more details, see {@apilink CheerioCrawlerOptions.requestHandler}.
  *
  * New requests are only dispatched when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
+ * using the functionality provided by the {@apilink AutoscaledPool} class.
+ * All {@apilink AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
  * parameter of the `CheerioCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPool} options are available directly in the `CheerioCrawler` constructor.
+ * {@apilink AutoscaledPool} options are available directly in the `CheerioCrawler` constructor.
  *
  * **Example usage:**
  *
  * ```javascript
- * // Prepare a list of URLs to crawl
- * const requestList = await RequestList.open(null, [
- *     { url: 'http://www.example.com/page-1' },
- *     { url: 'http://www.example.com/page-2' },
- * ]);
- *
- * // Crawl the URLs
  * const crawler = new CheerioCrawler({
- *     requestList,
  *     async requestHandler({ request, response, body, contentType, $ }) {
  *         const data = [];
  *
@@ -297,147 +94,39 @@ export interface CheerioCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOpti
  *     },
  * });
  *
- * await crawler.run();
+ * await crawler.run([
+ *     'http://www.example.com/page-1',
+ *     'http://www.example.com/page-2',
+ * ]);
  * ```
  * @category Crawlers
  */
-export declare class CheerioCrawler extends BasicCrawler<CheerioCrawlingContext> {
-    readonly config: Configuration;
-    /**
-     * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
-     * Only available if used by the crawler.
-     */
-    proxyConfiguration?: ProxyConfiguration;
-    protected userRequestHandlerTimeoutMillis: number;
-    protected preNavigationHooks: CheerioHook[];
-    protected postNavigationHooks: CheerioHook[];
-    protected persistCookiesPerSession: boolean;
-    protected navigationTimeoutMillis: number;
-    protected ignoreSslErrors: boolean;
-    protected suggestResponseEncoding?: string;
-    protected forceResponseEncoding?: string;
-    protected readonly supportedMimeTypes: Set<string>;
-    protected static optionsShape: {
-        handlePageFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-        navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-        ignoreSslErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-        additionalMimeTypes: import("ow").ArrayPredicate<string>;
-        suggestResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-        forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-        proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-        persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-        preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
-        postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
-        requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-        requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-        requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-        handleRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-        requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-        handleRequestTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-        errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-        failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-        handleFailedRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-        maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-        maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-        autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-        sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-        useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-        minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-        maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-        maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-        log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-    };
+export declare class CheerioCrawler extends HttpCrawler<CheerioCrawlingContext> {
     /**
      * All `CheerioCrawler` parameters are passed via an options object.
      */
     constructor(options?: CheerioCrawlerOptions, config?: Configuration);
-    /**
-     * **EXPERIMENTAL**
-     * Function for attaching CrawlerExtensions such as the Unblockers.
-     * @param extension Crawler extension that overrides the crawler configuration.
-     */
-    use(extension: CrawlerExtension): void;
-    /**
-     * Wrapper around requestHandler that opens and closes pages etc.
-     */
-    protected _runRequestHandler(crawlingContext: CheerioCrawlingContext): Promise<void>;
-    protected _handleNavigation(crawlingContext: CheerioCrawlingContext): Promise<void>;
-    /**
-     * Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks.
-     */
-    private _applyCookies;
-    /**
-     * Function to make the HTTP request. It performs optimizations
-     * on the request such as only downloading the request body if the
-     * received content type matches text/html, application/xml, application/xhtml+xml.
-     */
-    protected _requestFunction({ request, session, proxyUrl, gotOptions }: RequestFunctionOptions): Promise<IncomingMessage>;
-    /**
-     * Encodes and parses response according to the provided content type
-     */
-    protected _parseResponse(request: Request, responseStream: IncomingMessage): Promise<{
+    protected _parseHTML(response: IncomingMessage, isXml: boolean, crawlingContext: CheerioCrawlingContext): Promise<{
         dom: unknown;
-        isXml: boolean;
-        response: IncomingMessage;
-        contentType: {
-            type: string;
-            encoding: BufferEncoding;
-        };
-        body?: undefined;
-    } | {
-        body: Buffer;
-        response: IncomingMessage;
-        contentType: {
-            type: string;
-            encoding: BufferEncoding;
-        };
-        dom?: undefined;
-        isXml?: undefined;
+        $: cheerio.CheerioAPI;
+        readonly body: string;
+        enqueueLinks: (enqueueOptions?: CheerioCrawlerEnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
     }>;
-    /**
-     * Combines the provided `requestOptions` with mandatory (non-overridable) values.
-     */
-    protected _getRequestOptions(request: Request, session?: Session, proxyUrl?: string, gotOptions?: OptionsInit): OptionsInit & {
-        isStream: true;
-    };
-    protected _encodeResponse(request: Request, response: IncomingMessage, encoding: BufferEncoding): {
-        encoding: BufferEncoding;
-        response: IncomingMessage;
-    };
     protected _parseHtmlToDom(response: IncomingMessage): Promise<unknown>;
-    /**
-     * Checks and extends supported mime types
-     */
-    protected _extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]): void;
-    /**
-     * Handles timeout request
-     */
-    protected _handleRequestTimeout(session?: Session): void;
-    private _abortDownloadOfBody;
-    /**
-     * @internal wraps public utility for mocking purposes
-     */
-    private _requestAsBrowser;
 }
 interface EnqueueLinksInternalOptions {
     options?: CheerioCrawlerEnqueueLinksOptions;
-    $: CheerioRoot | null;
+    $: cheerio.CheerioAPI | null;
     requestQueue: RequestQueue;
     originalRequestUrl: string;
     finalRequestUrl?: string;
 }
 /** @internal */
 export declare function cheerioCrawlerEnqueueLinks({ options, $, requestQueue, originalRequestUrl, finalRequestUrl }: EnqueueLinksInternalOptions): Promise<BatchAddRequestsResult>;
-interface RequestFunctionOptions {
-    request: Request;
-    session?: Session;
-    proxyUrl?: string;
-    gotOptions: OptionsInit;
-}
 /**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link CheerioCrawler}.
- * Defaults to the {@link CheerioCrawlingContext}.
+ * Creates new {@apilink Router} instance that works based on request labels.
+ * This instance can then serve as a `requestHandler` of your {@apilink CheerioCrawler}.
+ * Defaults to the {@apilink CheerioCrawlingContext}.
  *
  * > Serves as a shortcut for using `Router.create<CheerioCrawlingContext>()`.
  *
@@ -458,6 +147,6 @@ interface RequestFunctionOptions {
  * await crawler.run();
  * ```
  */
-export declare function createCheerioRouter<Context extends CheerioCrawlingContext = CheerioCrawlingContext>(): import("@crawlee/basic").RouterHandler<Context>;
+export declare function createCheerioRouter<Context extends CheerioCrawlingContext = CheerioCrawlingContext>(): import("@crawlee/http").RouterHandler<Context>;
 export {};
 //# sourceMappingURL=cheerio-crawler.d.ts.map

package/internals/cheerio-crawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cheerio-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/cheerio-crawler.ts"],"names":[],"mappings":";;AAEA,OAAO,KAAK,EAAyB,mBAAmB,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAC/G,OAAO,EAAE,YAAY,EAAqC,MAAM,gBAAgB,CAAC;AACjF,OAAO,KAAK,EAAE,eAAe,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAC9H,OAAO,EAAE,gBAAgB,EAA0F,aAAa,EAAE,MAAM,eAAe,CAAC;AACxJ,OAAO,KAAK,EAAE,sBAAsB,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACpF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAIlD,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE9D,OAAO,KAAK,EAAE,WAAW,EAAiC,QAAQ,IAAI,WAAW,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAIxH,OAAO,KAAK,EAAuB,eAAe,EAAE,MAAM,MAAM,CAAC;AAqBjE,oBAAY,mBAAmB,CAC3B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,UAAU,IACpC,YAAY,CAAC,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEjE,MAAM,WAAW,qBAAqB,CAClC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,UAAU,CACtC,SAAQ,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,EAE5E,gBAAgB,GAChB,uBAAuB,GAEvB,sBAAsB,GACtB,6BAA6B,GAC7B,0BAA0B,GAE1B,cAAc,CACf;IACD;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,cAAc,CAAC,EAAE,qBAAqB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAE3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,kBAAkB,CAAC,EAAE,qBAAqB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAE/D;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;OAQG;IACH,YAAY,CAAC,EAAE,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAEvD;;;;;;;;;;OAUG;IACH,oBAAoB,CAAC,EAAE,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAE/D;;;;;;;;;;;;;OAaG;IACH,2BAA2B,CAAC,EAAE,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAEtE;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,WAAW,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;IAEvD;;;;;;;;;;;OAWG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;IAExD;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;;;;;;;;;OAUG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;CACtC;AAED,oBAAY,WAAW,CACnB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,UAAU,IACpC,CACJ,eAAe,EAAE,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAC3D,UAAU,EAAE,WAAW,KACtB,SAAS,CAAC,IAAI,CAAC,CAAC;AAErB,MAAM,WAAW,sBAAsB,CACnC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,UAAU,CACtC,SAAQ,eAAe,CAAC,QAAQ,CAAC;IACnC;;OAEG;IACH,CAAC,EAAE,WAAW,CAAC;IAEf;;OAEG;IACH,IAAI,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;IAExB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,OAAO,EAAE,cAAc,CAAC;IACxB,QAAQ,EAAE,eAAe,CAAC;IAC1B,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,iCAAiC,KAAK,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC/F,WAAW,EAAE,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;CAC5F;AAED,oBAAY,qBAAqB,CAC7B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,UAAU,IACpC,cAAc,CAAC,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACnE,MAAM,WAAW,iCAAkC,SAAQ,IAAI,CAAC,mBAAmB,EAAE,MAAM,GAAG,cAAc,CAAC;CAAG;AAEhH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiFG;AACH,qBAAa,cAAe,SAAQ,YAAY,CAAC,sBAAsB,CAAC;aAoCD,MAAM;IAnCzE;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,SAAS,CAAC,+BAA+B,EAAE,MAAM,CAAC;IAClD,SAAS,CAAC,kBAAkB,EAAE,WAAW,EAAE,CAAC;IAC5C,SAAS,CAAC,mBAAmB,EAAE,WAAW,EAAE,CAAC;IAC7C,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAC5C,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAC3C,SAAS,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnD,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAcpC;IAEF;;OAEG;gBACS,OAAO,GAAE,qBAA0B,EAAoB,MAAM,gBAAkC;IA+E3G;;;;OAIG;IACH,GAAG,CAAC,SAAS,EAAE,gBAAgB;IA4B/B;;OAEG;cACsB,kBAAkB,CAAC,eAAe,EAAE,sBAAsB;cA2EnE,iBAAiB,CAAC,eAAe,EAAE,sBAAsB;IA2BzE;;OAEG;IACH,OAAO,CAAC,aAAa;IAmDrB;;;;OAIG;cACa,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,sBAAsB,GAAG,OAAO,CAAC,eAAe,CAAC;IAe9H;;OAEG;cACa,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,eAAe;;;;;;;;;;;;;;;;;;;IA6BhF;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,WAAW;kBACzD,IAAI;;IAkCxD,SAAS,CAAC,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,GAAG;QAC9F,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,eAAe,CAAC;KAC7B;cAqCe,eAAe,CAAC,QAAQ,EAAE,eAAe;IAczD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC,EAAE;IAWhG;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,OAAO,CAAC,EAAE,OAAO;IAKjD,OAAO,CAAC,oBAAoB;IAgB5B;;OAEG;IACH,OAAO,CAAC,iBAAiB,CASvB;CACL;AAED,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,iCAAiC,CAAC;IAC5C,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACtB,YAAY,EAAE,YAAY,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,wBAAsB,0BAA0B,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,kBAAkB,EAAE,eAAe,EAAE,EAAE,2BAA2B,mCAoB9I;AAED,UAAU,sBAAsB;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,WAAW,CAAC;CAC3B;AAsED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,qDAElG"}
+{"version":3,"file":"cheerio-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/cheerio-crawler.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EACR,kBAAkB,EAClB,2BAA2B,EAC3B,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,YAAY,EACZ,aAAa,EAChB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,WAAW,EAAgE,MAAM,eAAe,CAAC;AAC1G,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,MAAM,CAAC;AAE5C,oBAAY,mBAAmB,CAC3B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IAC7B,YAAY,CAAC,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEjE,MAAM,WAAW,qBAAqB,CAClC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CAC/B,SAAQ,kBAAkB,CAAC,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;CAAG;AAE/E,oBAAY,WAAW,CACnB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IAC7B,gBAAgB,CAAC,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAErE,MAAM,WAAW,sBAAsB,CACnC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CAC/B,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,EAAE,cAAc,CAAC;IACzE;;;OAGG;IACH,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC;IAEtB,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,iCAAiC,KAAK,OAAO,CAAC,sBAAsB,CAAC,CAAC;CAClG;AAED,oBAAY,qBAAqB,CAC7B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IAC7B,cAAc,CAAC,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACnE,MAAM,WAAW,iCAAkC,SAAQ,IAAI,CAAC,mBAAmB,EAAE,MAAM,GAAG,cAAc,CAAC;CAAG;AAEhH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4EG;AACH,qBAAa,cAAe,SAAQ,WAAW,CAAC,sBAAsB,CAAC;IACnE;;OAEG;gBAES,OAAO,CAAC,EAAE,qBAAqB,EAAE,MAAM,CAAC,EAAE,aAAa;cAI1C,UAAU,CAAC,QAAQ,EAAE,eAAe,EAAE,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,sBAAsB;;;;wCAiBxE,iCAAiC;;cAY/D,eAAe,CAAC,QAAQ,EAAE,eAAe;CAa5D;AAED,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,iCAAiC,CAAC;IAC5C,CAAC,EAAE,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAC7B,YAAY,EAAE,YAAY,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,wBAAsB,0BAA0B,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,kBAAkB,EAAE,eAAe,EAAE,EAAE,2BAA2B,mCAoB9I;AAgCD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,oDAElG"}