@crawlee/cheerio 3.0.0-alpha.2 → 3.0.0-alpha.5

package/internals/cheerio-crawler.d.ts

@@ -0,0 +1,620 @@
+/// <reference types="node" />
+import { BasicCrawler, BasicCrawlerOptions } from '@crawlee/basic';
+import { CrawlerExtension, CrawlerHandleFailedRequestInput, CrawlingContext, EnqueueLinksOptions, ProxyConfiguration, ProxyInfo, Request, RequestQueue, Session, storage } from '@crawlee/core';
+import { Awaitable, CheerioRoot } from '@crawlee/utils';
+import { RequestLike, ResponseLike } from 'content-type';
+import { OptionsInit } from 'got-scraping';
+import { IncomingMessage } from 'http';
+export interface CheerioFailedRequestHandlerInput<JSONData = unknown> extends CrawlerHandleFailedRequestInput, CheerioRequestHandlerInputs<JSONData> {
+}
+export declare type CheerioFailedRequestHandler<JSONData = unknown> = (inputs: CheerioFailedRequestHandlerInput<JSONData>) => Awaitable<void>;
+export interface CheerioCrawlerOptions<JSONData = unknown> extends Omit<BasicCrawlerOptions<CheerioCrawlingContext<JSONData>>, 'requestHandler' | 'handleRequestFunction' | 'failedRequestHandler' | 'handleFailedRequestFunction' | 'handleRequestTimeoutSecs'> {
+    /**
+     * User-provided function that performs the logic of the crawler. It is called for each page
+     * loaded and parsed by the crawler.
+     *
+     * The function receives the following object as an argument:
+     * ```
+     * {
+     *   // The Cheerio object's function with the parsed HTML.
+     *   $: Cheerio,
+     *
+     *   // The request body of the web page, whose type depends on the content type.
+     *   body: String|Buffer,
+     *
+     *   // The parsed object from JSON for responses with the "application/json" content types.
+     *   // For other content types it's null.
+     *   json: Object,
+     *
+     *   // Request object with details of the requested web page
+     *   request: Request,
+     *
+     *   // Parsed Content-Type HTTP header: { type, encoding }
+     *   contentType: Object,
+     *
+     *   // An instance of Node's http.IncomingMessage object,
+     *   response: Object,
+     *
+     *   // Session object, useful to work around anti-scraping protections
+     *   session: Session
+     *
+     *   // ProxyInfo object with information about currently used proxy
+     *   proxyInfo: ProxyInfo
+     *
+     *   // The running cheerio crawler instance.
+     *   crawler: CheerioCrawler
+     * }
+     * ```
+     *
+     * Type of `body` depends on the `Content-Type` header of the web page:
+     * - String for `text/html`, `application/xhtml+xml`, `application/xml` MIME content types
+     * - Buffer for others MIME content types
+     *
+     * Parsed `Content-Type` header using
+     * [content-type package](https://www.npmjs.com/package/content-type)
+     * is stored in `contentType`.
+     *
+     * Cheerio is available only for HTML and XML content types.
+     *
+     * With the {@link Request} object representing the URL to crawl.
+     *
+     * If the function returns, the returned promise is awaited by the crawler.
+     *
+     * If the function throws an exception, the crawler will try to re-crawl the
+     * request later, up to `option.maxRequestRetries` times.
+     * If all the retries fail, the crawler calls the function
+     * provided to the `handleFailedRequestFunction` parameter.
+     * To make this work, you should **always**
+     * let your function throw exceptions rather than catch them.
+     * The exceptions are logged to the request using the
+     * {@link Request.pushErrorMessage} function.
+     */
+    requestHandler: CheerioRequestHandler<JSONData>;
+    /**
+     * User-provided function that performs the logic of the crawler. It is called for each page
+     * loaded and parsed by the crawler.
+     *
+     * The function receives the following object as an argument:
+     * ```
+     * {
+     *   // The Cheerio object's function with the parsed HTML.
+     *   $: Cheerio,
+     *
+     *   // The request body of the web page, whose type depends on the content type.
+     *   body: String|Buffer,
+     *
+     *   // The parsed object from JSON for responses with the "application/json" content types.
+     *   // For other content types it's null.
+     *   json: Object,
+     *
+     *   // Request object with details of the requested web page
+     *   request: Request,
+     *
+     *   // Parsed Content-Type HTTP header: { type, encoding }
+     *   contentType: Object,
+     *
+     *   // An instance of Node's http.IncomingMessage object,
+     *   response: Object,
+     *
+     *   // Session object, useful to work around anti-scraping protections
+     *   session: Session
+     *
+     *   // ProxyInfo object with information about currently used proxy
+     *   proxyInfo: ProxyInfo
+     *
+     *   // The running cheerio crawler instance.
+     *   crawler: CheerioCrawler
+     * }
+     * ```
+     *
+     * Type of `body` depends on the `Content-Type` header of the web page:
+     * - String for `text/html`, `application/xhtml+xml`, `application/xml` MIME content types
+     * - Buffer for others MIME content types
+     *
+     * Parsed `Content-Type` header using
+     * [content-type package](https://www.npmjs.com/package/content-type)
+     * is stored in `contentType`.
+     *
+     * Cheerio is available only for HTML and XML content types.
+     *
+     * With the {@link Request} object representing the URL to crawl.
+     *
+     * If the function returns, the returned promise is awaited by the crawler.
+     *
+     * If the function throws an exception, the crawler will try to re-crawl the
+     * request later, up to `option.maxRequestRetries` times.
+     * If all the retries fail, the crawler calls the function
+     * provided to the `handleFailedRequestFunction` parameter.
+     * To make this work, you should **always**
+     * let your function throw exceptions rather than catch them.
+     * The exceptions are logged to the request using the
+     * {@link Request.pushErrorMessage} function.
+     *
+     * @deprecated `handlePageFunction` has been renamed to `requestHandler` and will be removed in a future version.
+     */
+    handlePageFunction?: CheerioRequestHandler<JSONData>;
+    /**
+     * > This option is deprecated, use `preNavigationHooks` instead.
+     *
+     * A function that executes before the HTTP request is made to the target resource.
+     * This function is suitable for setting dynamic properties such as cookies to the {@link Request}.
+     *
+     * The function receives the following object as an argument:
+     * ```
+     * {
+     *     request: Request,
+     *     session: Session,
+     *     proxyInfo: ProxyInfo,
+     *     crawler: CheerioCrawler,
+     * }
+     * ```
+     * where the {@link Request} instance corresponds to the initialized request
+     * and the {@link Session} instance corresponds to used session.
+     *
+     * The function should modify the properties of the passed {@link Request} instance
+     * in place because there are already earlier references to it. Making a copy and returning it from
+     * this function is therefore not supported, because it would create inconsistencies where
+     * different parts of SDK would have access to a different {@link Request} instance.
+     */
+    prepareRequestFunction?: PrepareRequest<JSONData>;
+    /**
+     * > This option is deprecated, use `postNavigationHooks` instead.
+     *
+     * A function that executes right after the HTTP request is made to the target resource and response is returned.
+     * This function is suitable for overriding custom properties of response e.g. setting headers because of response parsing.
+     *
+     * **Example usage:**
+     *
+     * ```javascript
+     * const cheerioCrawlerOptions = {
+     *     // ...
+     *     postResponseFunction: ({ request, response }) => {
+     *         if (request.userData.parseAsJSON) {
+     *             response.headers['content-type'] = 'application/json; charset=utf-8';
+     *         }
+     *     }
+     * }
+     * ```
+     * The function receives the following object as an argument:
+     * ```
+     * {
+     *     response: Object,
+     *     request: Request,
+     *     session: Session,
+     *     proxyInfo: ProxyInfo,
+     *     crawler: CheerioCrawler,
+     * }
+     * ```
+     * The response is an instance of Node's http.IncomingMessage object.
+     */
+    postResponseFunction?: PostResponse<JSONData>;
+    /**
+     * Timeout in which the HTTP request to the resource needs to finish, given in seconds.
+     */
+    requestTimeoutSecs?: number;
+    /**
+     * If set to true, SSL certificate errors will be ignored.
+     */
+    ignoreSslErrors?: boolean;
+    /**
+     * If set, `CheerioCrawler` will be configured for all connections to use
+     * [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
+     * For more information, see the [documentation](https://docs.apify.com/proxy).
+     */
+    proxyConfiguration?: ProxyConfiguration;
+    /**
+     * A function to handle requests that failed more than `option.maxRequestRetries` times.
+     * The function receives the following object as an argument:
+     * ```
+     * {
+     *     error: Error,
+     *     request: Request,
+     *     session: Session,
+     *     $: Cheerio,
+     *     body: String|Buffer,
+     *     json: Object,
+     *     contentType: Object,
+     *     response: Object,
+     *     proxyInfo: ProxyInfo,
+     *     crawler: CheerioCrawler,
+     * }
+     * ```
+     * where the {@link Request} instance corresponds to the failed request, and the `Error` instance
+     * represents the last error thrown during processing of the request.
+     *
+     * See [source code](https://github.com/apify/apify-js/blob/master/src/crawlers/cheerio_crawler.js#L13)
+     * for the default implementation of this function.
+     */
+    failedRequestHandler?: CheerioFailedRequestHandler<JSONData>;
+    /**
+     * A function to handle requests that failed more than `option.maxRequestRetries` times.
+     * The function receives the following object as an argument:
+     * ```
+     * {
+     *     error: Error,
+     *     request: Request,
+     *     session: Session,
+     *     $: Cheerio,
+     *     body: String|Buffer,
+     *     json: Object,
+     *     contentType: Object,
+     *     response: Object,
+     *     proxyInfo: ProxyInfo,
+     *     crawler: CheerioCrawler,
+     * }
+     * ```
+     * where the {@link Request} instance corresponds to the failed request, and the `Error` instance
+     * represents the last error thrown during processing of the request.
+     *
+     * See [source code](https://github.com/apify/apify-js/blob/master/src/crawlers/cheerio_crawler.js#L13)
+     * for the default implementation of this function.
+     *
+     * @deprecated `handleFailedRequestFunction` has been renamed to `failedRequestHandler` and will be removed in a future version.
+     */
+    handleFailedRequestFunction?: CheerioFailedRequestHandler<JSONData>;
+    /**
+     * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
+     * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotOptions`,
+     * which are passed to the `requestAsBrowser()` function the crawler calls to navigate.
+     * Example:
+     * ```
+     * preNavigationHooks: [
+     *     async (crawlingContext, gotOptions) => {
+     *         // ...
+     *     },
+     * ]
+     * ```
+     */
+    preNavigationHooks?: CheerioHook<JSONData>[];
+    /**
+     * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
+     * The function accepts `crawlingContext` as the only parameter.
+     * Example:
+     * ```
+     * postNavigationHooks: [
+     *     async (crawlingContext) => {
+     *         // ...
+     *     },
+     * ]
+     * ```
+     */
+    postNavigationHooks?: CheerioHook<JSONData>[];
+    /**
+     * An array of [MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types)
+     * you want the crawler to load and process. By default, only `text/html` and `application/xhtml+xml` MIME types are supported.
+     */
+    additionalMimeTypes?: string[];
+    /**
+     * By default `CheerioCrawler` will extract correct encoding from the HTTP response headers.
+     * Sadly, there are some websites which use invalid headers. Those are encoded using the UTF-8 encoding.
+     * If those sites actually use a different encoding, the response will be corrupted. You can use
+     * `suggestResponseEncoding` to fall back to a certain encoding, if you know that your target website uses it.
+     * To force a certain encoding, disregarding the response headers, use {@link CheerioCrawlerOptions.forceResponseEncoding}
+     * ```
+     * // Will fall back to windows-1250 encoding if none found
+     * suggestResponseEncoding: 'windows-1250'
+     * ```
+     */
+    suggestResponseEncoding?: string;
+    /**
+     * By default `CheerioCrawler` will extract correct encoding from the HTTP response headers. Use `forceResponseEncoding`
+     * to force a certain encoding, disregarding the response headers.
+     * To only provide a default for missing encodings, use {@link CheerioCrawlerOptions.suggestResponseEncoding}
+     * ```
+     * // Will force windows-1250 encoding even if headers say otherwise
+     * forceResponseEncoding: 'windows-1250'
+     * ```
+     */
+    forceResponseEncoding?: string;
+    /**
+     * Automatically saves cookies to Session. Works only if Session Pool is used.
+     *
+     * It parses cookie from response "set-cookie" header saves or updates cookies for session and once the session is used for next request.
+     * It passes the "Cookie" header to the request with the session cookies.
+     */
+    persistCookiesPerSession?: boolean;
+}
+export interface PrepareRequestInputs<JSONData = unknown> {
+    /**
+     *  Original instance of the {@link Request} object. Must be modified in-place.
+     */
+    request: Request;
+    /**
+     * The current session
+     */
+    session?: Session;
+    /**
+     * An object with information about currently used proxy by the crawler
+     * and configured by the {@link ProxyConfiguration} class.
+     */
+    proxyInfo?: ProxyInfo;
+    crawler?: CheerioCrawler<JSONData>;
+}
+export declare type PrepareRequest<JSONData = unknown> = (inputs: PrepareRequestInputs<JSONData>) => Awaitable<void>;
+export declare type CheerioHook<JSONData = unknown> = (crawlingContext: CheerioCrawlingContext<JSONData>, gotOptions: OptionsInit) => Awaitable<void>;
+export interface PostResponseInputs<JSONData = unknown> {
+    /**
+     * stream
+     */
+    response?: IncomingMessage;
+    /**
+     * Original instance fo the {Request} object. Must be modified in-place.
+     */
+    request: Request;
+    /**
+     * The current session
+     */
+    session?: Session;
+    /**
+     * An object with information about currently used proxy by the crawler
+     * and configured by the {@link ProxyConfiguration} class.
+     */
+    proxyInfo?: ProxyInfo;
+    crawler: CheerioCrawler<JSONData>;
+}
+export declare type PostResponse<JSONData = unknown> = (inputs: PostResponseInputs<JSONData>) => Awaitable<void>;
+export interface CheerioRequestHandlerInputs<JSONData = unknown> extends CrawlingContext {
+    /**
+     * The [Cheerio](https://cheerio.js.org/) object with parsed HTML.
+     */
+    $: CheerioRoot;
+    /**
+     * The request body of the web page.
+     */
+    body: (string | Buffer);
+    /**
+     * The parsed object from JSON string if the response contains the content type application/json.
+     */
+    json: JSONData;
+    /**
+     * Parsed `Content-Type header: { type, encoding }`.
+     */
+    contentType: {
+        type: string;
+        encoding: string;
+    };
+    crawler: CheerioCrawler<JSONData>;
+    response: IncomingMessage;
+    enqueueLinks: (options?: CheerioCrawlerEnqueueLinksOptions) => Promise<storage.BatchAddRequestsResult>;
+}
+export declare type CheerioCrawlingContext<JSONData = unknown> = CheerioRequestHandlerInputs<JSONData>;
+export declare type CheerioRequestHandler<JSONData = unknown> = (inputs: CheerioRequestHandlerInputs<JSONData>) => Awaitable<void>;
+export declare type CheerioCrawlerEnqueueLinksOptions = Omit<EnqueueLinksOptions, 'urls' | 'requestQueue'>;
+/**
+ * Provides a framework for the parallel crawling of web pages using plain HTTP requests and
+ * [cheerio](https://www.npmjs.com/package/cheerio) HTML parser.
+ * The URLs to crawl are fed either from a static list of URLs
+ * or from a dynamic queue of URLs enabling recursive crawling of websites.
+ *
+ * Since `CheerioCrawler` uses raw HTTP requests to download web pages,
+ * it is very fast and efficient on data bandwidth. However, if the target website requires JavaScript
+ * to display the content, you might need to use {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead,
+ * because it loads the pages using full-featured headless Chrome browser.
+ *
+ * `CheerioCrawler` downloads each URL using a plain HTTP request,
+ * parses the HTML content using [Cheerio](https://www.npmjs.com/package/cheerio)
+ * and then invokes the user-provided {@link CheerioCrawlerOptions.requestHandler} to extract page data
+ * using a [jQuery](https://jquery.com/)-like interface to the parsed HTML DOM.
+ *
+ * The source URLs are represented using {@link Request} objects that are fed from
+ * {@link RequestList} or {@link RequestQueue} instances provided by the {@link CheerioCrawlerOptions.requestList}
+ * or {@link CheerioCrawlerOptions.requestQueue} constructor options, respectively.
+ *
+ * If both {@link CheerioCrawlerOptions.requestList} and {@link CheerioCrawlerOptions.requestQueue} are used,
+ * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
+ * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ *
+ * The crawler finishes when there are no more {@link Request} objects to crawl.
+ *
+ * We can use the `preNavigationHooks` to adjust `gotOptions`:
+ *
+ * ```
+ * preNavigationHooks: [
+ *     (crawlingContext, gotOptions) => {
+ *         // ...
+ *     },
+ * ]
+ * ```
+ *
+ * By default, `CheerioCrawler` only processes web pages with the `text/html`
+ * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),
+ * and skips pages with other content types. If you want the crawler to process other content types,
+ * use the {@link CheerioCrawlerOptions.additionalMimeTypes} constructor option.
+ * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
+ * For details, see {@link CheerioCrawlerOptions.requestHandler}.
+ *
+ * New requests are only dispatched when there is enough free CPU and memory available,
+ * using the functionality provided by the {@link AutoscaledPool} class.
+ * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
+ * parameter of the `CheerioCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+ * {@link AutoscaledPool} options are available directly in the `CheerioCrawler` constructor.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * // Prepare a list of URLs to crawl
+ * const requestList = new RequestList({
+ *   sources: [
+ *       { url: 'http://www.example.com/page-1' },
+ *       { url: 'http://www.example.com/page-2' },
+ *   ],
+ * });
+ * await requestList.initialize();
+ *
+ * // Crawl the URLs
+ * const crawler = new CheerioCrawler({
+ *     requestList,
+ *     handlePageFunction: async ({ request, response, body, contentType, $ }) => {
+ *         const data = [];
+ *
+ *         // Do some data extraction from the page with Cheerio.
+ *         $('.some-collection').each((index, el) => {
+ *             data.push({ title: $(el).find('.some-title').text() });
+ *         });
+ *
+ *         // Save the data to dataset.
+ *         await Actor.pushData({
+ *             url: request.url,
+ *             html: body,
+ *             data,
+ *         })
+ *     },
+ * });
+ *
+ * await crawler.run();
+ * ```
+ * @category Crawlers
+ */
+export declare class CheerioCrawler<JSONData = unknown> extends BasicCrawler<CheerioCrawlingContext<JSONData>, CheerioFailedRequestHandlerInput<JSONData>> {
+    /**
+     * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
+     * Only available if used by the crawler.
+     */
+    proxyConfiguration?: ProxyConfiguration;
+    protected userRequestHandlerTimeoutMillis: number;
+    protected defaultGotoOptions: {
+        timeout: number;
+    };
+    protected preNavigationHooks: CheerioHook<JSONData>[];
+    protected postNavigationHooks: CheerioHook<JSONData>[];
+    protected persistCookiesPerSession: boolean;
+    protected requestTimeoutMillis: number;
+    protected ignoreSslErrors: boolean;
+    protected suggestResponseEncoding?: string;
+    protected forceResponseEncoding?: string;
+    protected prepareRequestFunction?: PrepareRequest<JSONData>;
+    protected postResponseFunction?: PostResponse<JSONData>;
+    protected readonly supportedMimeTypes: Set<string>;
+    protected static optionsShape: {
+        handlePageFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        requestTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        ignoreSslErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        additionalMimeTypes: import("ow").ArrayPredicate<string>;
+        suggestResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        prepareRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        postResponseFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
+        postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
+        requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        handleRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        handleRequestTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        handleFailedRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+    };
+    /**
+     * All `CheerioCrawler` parameters are passed via an options object.
+     */
+    constructor(options: CheerioCrawlerOptions<JSONData>);
+    /**
+     * **EXPERIMENTAL**
+     * Function for attaching CrawlerExtensions such as the Unblockers.
+     * @param extension Crawler extension that overrides the crawler configuration.
+     */
+    use(extension: CrawlerExtension): void;
+    /**
+     * Wrapper around handlePageFunction that opens and closes pages etc.
+     */
+    protected _runRequestHandler(crawlingContext: CheerioCrawlingContext<JSONData>): Promise<void>;
+    protected _handleNavigation(crawlingContext: CheerioCrawlingContext<JSONData>): Promise<void>;
+    /**
+     * When users change `request.headers.cookie` inside preNavigationHook, the change would be ignored,
+     * as `request.headers` are already merged into the `gotOptions`. This method is using
+     * old `request.headers` snapshot (before hooks are executed), makes a diff with the cookie value
+     * after hooks are executed, and merges any new cookies back to `gotOptions`.
+     *
+     * This way we can still use both `gotOptions` and `context.request` in the hooks (not both).
+     */
+    private _mergeRequestCookieDiff;
+    /**
+     * Function to make the HTTP request. It performs optimizations
+     * on the request such as only downloading the request body if the
+     * received content type matches text/html, application/xml, application/xhtml+xml.
+     */
+    protected _requestFunction({ request, session, proxyUrl, gotOptions }: RequestFunctionOptions): Promise<IncomingMessage>;
+    /**
+     * Sets the cookie header to `gotOptions` based on provided session and request. If some cookies were already set,
+     * the session cookie will be merged with them. User provided cookies on `request` object have precedence.
+     */
+    private _applySessionCookie;
+    /**
+     * Encodes and parses response according to the provided content type
+     */
+    protected _parseResponse(request: Request, responseStream: IncomingMessage): Promise<{
+        dom: unknown;
+        isXml: boolean;
+        response: IncomingMessage;
+        contentType: {
+            type: string;
+            encoding: BufferEncoding;
+        };
+        body?: undefined;
+    } | {
+        body: Buffer;
+        response: IncomingMessage;
+        contentType: {
+            type: string;
+            encoding: BufferEncoding;
+        };
+        dom?: undefined;
+        isXml?: undefined;
+    }>;
+    /**
+     * Combines the provided `requestOptions` with mandatory (non-overridable) values.
+     */
+    protected _getRequestOptions(request: Request, session?: Session, proxyUrl?: string, gotOptions?: OptionsInit): OptionsInit & {
+        isStream: true;
+    };
+    protected _encodeResponse(request: Request, response: IncomingMessage, encoding: BufferEncoding): {
+        encoding: BufferEncoding;
+        response: IncomingMessage;
+    };
+    protected _parseHtmlToDom(response: IncomingMessage): Promise<unknown>;
+    /**
+     * Checks and extends supported mime types
+     */
+    protected _extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]): void;
+    /**
+     * Handles blocked request
+     */
+    protected _throwOnBlockedRequest(session: Session, statusCode: number): void;
+    /**
+     * Handles timeout request
+     */
+    protected _handleRequestTimeout(session?: Session): void;
+    private _abortDownloadOfBody;
+    /**
+     * @internal wraps public utility for mocking purposes
+     */
+    private _requestAsBrowser;
+}
+interface EnqueueLinksInternalOptions {
+    options?: CheerioCrawlerEnqueueLinksOptions;
+    $: CheerioRoot | null;
+    requestQueue: RequestQueue;
+    originalRequestUrl: string;
+    finalRequestUrl?: string;
+}
+/** @internal */
+export declare function cheerioCrawlerEnqueueLinks({ options, $, requestQueue, originalRequestUrl, finalRequestUrl }: EnqueueLinksInternalOptions): Promise<storage.BatchAddRequestsResult>;
+interface RequestFunctionOptions {
+    request: Request;
+    session?: Session;
+    proxyUrl?: string;
+    gotOptions: OptionsInit;
+}
+export {};
+//# sourceMappingURL=cheerio-crawler.d.ts.map

package/internals/cheerio-crawler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cheerio-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/cheerio-crawler.ts"],"names":[],"mappings":";AAEA,OAAO,EACH,YAAY,EACZ,mBAAmB,EAEtB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,gBAAgB,EAChB,+BAA+B,EAC/B,eAAe,EAGf,mBAAmB,EAEnB,kBAAkB,EAClB,SAAS,EACT,OAAO,EACP,YAAY,EAEZ,OAAO,EACP,OAAO,EAEV,MAAM,eAAe,CAAC;AACvB,OAAO,EACH,SAAS,EACT,WAAW,EAGd,MAAM,gBAAgB,CAAC;AAExB,OAA0B,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5E,OAAO,EAAe,WAAW,EAA+C,MAAM,cAAc,CAAC;AAGrG,OAAO,EAAuB,eAAe,EAAE,MAAM,MAAM,CAAC;AAoB5D,MAAM,WAAW,gCAAgC,CAAC,QAAQ,GAAG,OAAO,CAAE,SAAQ,+BAA+B,EAAE,2BAA2B,CAAC,QAAQ,CAAC;CAAG;AAEvJ,oBAAY,2BAA2B,CAAC,QAAQ,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,gCAAgC,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEtI,MAAM,WAAW,qBAAqB,CAAC,QAAQ,GAAG,OAAO,CAAE,SAAQ,IAAI,CACnE,mBAAmB,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC,EAEnD,gBAAgB,GAChB,uBAAuB,GAEvB,sBAAsB,GACtB,6BAA6B,GAC7B,0BAA0B,CAC/B;IACG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2DG;IACH,cAAc,EAAE,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAEhD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6DG;IACH,kBAAkB,CAAC,EAAE,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAErD;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,sBAAsB,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;IAElD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,oBAAoB,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAE9C;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,oBAAoB,CAAC,EAAE,2BAA2B,CAAC,QAAQ,CAAC,CAAC;IAE7D;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,2BAA2B,CAAC,EAAE,2BAA2B,CAAC,QAAQ,CAAC,CAAC;IAEpE;;;;;;;;;;;;OAYG;IACH,kBAAkB,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;IAE7C;;;;;;;;;;;OAWG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;IAE9C;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;;;;;;;;;OAUG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;CACtC;AAED,MAAM,WAAW,oBAAoB,CAAC,QAAQ,GAAG,OAAO;IACpD;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,OAAO,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;CACtC;AAED,oBAAY,cAAc,CAAC,QAAQ,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,oBAAoB,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAC7G,oBAAY,WAAW,CAAC,QAAQ,GAAG,OAAO,IAAI,CAC1C,eAAe,EAAE,sBAAsB,CAAC,QAAQ,CAAC,EACjD,UAAU,EAAE,WAAW,KACtB,SAAS,CAAC,IAAI,CAAC,CAAC;AAErB,MAAM,WAAW,kBAAkB,CAAC,QAAQ,GAAG,OAAO;IAClD;;OAEG;IACH,QAAQ,CAAC,EAAE,eAAe,CAAC;IAE3B;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;CACrC;AAED,oBAAY,YAAY,CAAC,QAAQ,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEzG,MAAM,WAAW,2BAA2B,CAAC,QAAQ,GAAG,OAAO,CAAE,SAAQ,eAAe;IACpF;;OAEG;IACH,CAAC,EAAE,WAAW,CAAC;IAEf;;OAEG;IACH,IAAI,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;IAExB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;IAClC,QAAQ,EAAE,eAAe,CAAC;IAC1B,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,iCAAiC,KAAK,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;CAC1G;AAED,oBAAY,sBAAsB,CAAC,QAAQ,GAAG,OAAO,IAAI,2BAA2B,CAAC,QAAQ,CAAC,CAAC;AAC/F,oBAAY,qBAAqB,CAAC,QAAQ,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,2BAA2B,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAC3H,oBAAY,iCAAiC,GAAG,IAAI,CAAC,mBAAmB,EAAE,MAAM,GAAG,cAAc,CAAC,CAAC;AAEnG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoFG;AACH,qBAAa,cAAc,CAAC,QAAQ,GAAG,OAAO,CAAE,SAAQ,YAAY,CAChE,sBAAsB,CAAC,QAAQ,CAAC,EAChC,gCAAgC,CAAC,QAAQ,CAAC,CAC7C;IACG;;;OAGG;IACI,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAE/C,SAAS,CAAC,+BAA+B,EAAE,MAAM,CAAC;IAClD,SAAS,CAAC,kBAAkB,EAAG;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IACnD,SAAS,CAAC,kBAAkB,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;IACtD,SAAS,CAAC,mBAAmB,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;IACvD,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAC5C,SAAS,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACvC,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAC3C,SAAS,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,sBAAsB,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC5D,SAAS,CAAC,oBAAoB,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IACxD,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnD,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAgBpC;IAEF;;OAEG;gBACS,OAAO,EAAE,qBAAqB,CAAC,QAAQ,CAAC;IA+EpD;;;;OAIG;IACH,GAAG,CAAC,SAAS,EAAE,gBAAgB;IA4B/B;;OAEG;cACsB,kBAAkB,CAAC,eAAe,EAAE,sBAAsB,CAAC,QAAQ,CAAC;cA0E7E,iBAAiB,CAAC,eAAe,EAAE,sBAAsB,CAAC,QAAQ,CAAC;IAqCnF;;;;;;;OAOG;IACH,OAAO,CAAC,uBAAuB;IAY/B;;;;OAIG;cACa,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,sBAAsB,GAAG,OAAO,CAAC,eAAe,CAAC;IAe9H;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAY3B;;OAEG;cACa,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,eAAe;;;;;;;;;;;;;;;;;;;IA6BhF;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,WAAW;kBACzD,IAAI;;IA+BxD,SAAS,CAAC,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,GAAG;QAC9F,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,eAAe,CAAC;KAC7B;cAqCe,eAAe,CAAC,QAAQ,EAAE,eAAe;IAczD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC,EAAE;IAWhG;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM;IAQrE;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,OAAO,CAAC,EAAE,OAAO;IAKjD,OAAO,CAAC,oBAAoB;IAgB5B;;OAEG;IACH,OAAO,CAAC,iBAAiB,CASvB;CACL;AAED,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,iCAAiC,CAAC;IAC5C,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACtB,YAAY,EAAE,YAAY,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,wBAAsB,0BAA0B,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,kBAAkB,EAAE,eAAe,EAAE,EAAE,2BAA2B,2CAoB9I;AAED,UAAU,sBAAsB;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,WAAW,CAAC;CAC3B"}