apify 2.3.1-beta.4 → 3.0.0-alpha.10

package/build/crawlers/cheerio_crawler.d.ts

@@ -1,931 +0,0 @@
-/// <reference types="node" />
-export default CheerioCrawler;
-export type CheerioCrawlerOptions = {
-    /**
-     *   User-provided function that performs the logic of the crawler. It is called for each page
-     *   loaded and parsed by the crawler.
-     *
-     *   The function receives the following object as an argument:
-     * ```
-     * {
-     *   // The Cheerio object's function with the parsed HTML.
-     *   $: Cheerio,
-     *
-     *   // The request body of the web page, whose type depends on the content type.
-     *   body: String|Buffer,
-     *
-     *   // The parsed object from JSON for responses with the "application/json" content types.
-     *   // For other content types it's null.
-     *   json: Object,
-     *
-     *   // Apify.Request object with details of the requested web page
-     *   request: Request,
-     *
-     *   // Parsed Content-Type HTTP header: { type, encoding }
-     *   contentType: Object,
-     *
-     *   // An instance of Node's http.IncomingMessage object,
-     *   response: Object,
-     *
-     *   // Session object, useful to work around anti-scraping protections
-     *   session: Session
-     *
-     *   // ProxyInfo object with information about currently used proxy
-     *   proxyInfo: ProxyInfo
-     *
-     *   // The running cheerio crawler instance.
-     *   crawler: CheerioCrawler
-     * }
-     * ```
-     *
-     *   Type of `body` depends on the `Content-Type` header of the web page:
-     *   - String for `text/html`, `application/xhtml+xml`, `application/xml` MIME content types
-     *   - Buffer for others MIME content types
-     *
-     *   Parsed `Content-Type` header using
-     *   [content-type package](https://www.npmjs.com/package/content-type)
-     *   is stored in `contentType`.
-     *
-     *   Cheerio is available only for HTML and XML content types.
-     *
-     *   With the {@link Request } object representing the URL to crawl.
-     *
-     *   If the function returns, the returned promise is awaited by the crawler.
-     *
-     *   If the function throws an exception, the crawler will try to re-crawl the
-     *   request later, up to `option.maxRequestRetries` times.
-     *   If all the retries fail, the crawler calls the function
-     *   provided to the `handleFailedRequestFunction` parameter.
-     *   To make this work, you should **always**
-     *   let your function throw exceptions rather than catch them.
-     *   The exceptions are logged to the request using the
-     *   {@link RequestpushErrorMessage } function.
-     */
-    handlePageFunction: CheerioHandlePage;
-    /**
-     * Static list of URLs to be processed.
-     * Either `requestList` or `requestQueue` option must be provided (or both).
-     */
-    requestList?: RequestList | undefined;
-    /**
-     * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
-     * Either `requestList` or `requestQueue` option must be provided (or both).
-     */
-    requestQueue?: RequestQueue | undefined;
-    /**
-     * > This option is deprecated, use `preNavigationHooks` instead.
-     *
-     * A function that executes before the HTTP request is made to the target resource.
-     * This function is suitable for setting dynamic properties such as cookies to the {@link Request }.
-     *
-     * The function receives the following object as an argument:
-     * ```
-     * {
-     * request: Request,
-     * session: Session,
-     * proxyInfo: ProxyInfo,
-     * crawler: CheerioCrawler,
-     * }
-     * ```
-     * where the {@link Request } instance corresponds to the initialized request
-     * and the {@link Session } instance corresponds to used session.
-     *
-     * The function should modify the properties of the passed {@link Request } instance
-     * in place because there are already earlier references to it. Making a copy and returning it from
-     * this function is therefore not supported, because it would create inconsistencies where
-     * different parts of SDK would have access to a different {@link Request } instance.
-     */
-    prepareRequestFunction?: PrepareRequest | undefined;
-    /**
-     * > This option is deprecated, use `postNavigationHooks` instead.
-     *
-     * A function that executes right after the HTTP request is made to the target resource and response is returned.
-     * This function is suitable for overriding custom properties of response e.g. setting headers because of response parsing.
-     *
-     * **Example usage:**
-     *
-     * ```javascript
-     * const cheerioCrawlerOptions = {
-     * // ...
-     * postResponseFunction: ({ request, response }) => {
-     * if (request.userData.parseAsJSON) {
-     * response.headers['content-type'] = 'application/json; charset=utf-8';
-     * }
-     * }
-     * }
-     * ```
-     * The function receives the following object as an argument:
-     * ```
-     * {
-     * response: Object,
-     * request: Request,
-     * session: Session,
-     * proxyInfo: ProxyInfo,
-     * crawler: CheerioCrawler,
-     * }
-     * ```
-     * The response is an instance of Node's http.IncomingMessage object.
-     */
-    postResponseFunction?: PostResponse | undefined;
-    /**
-     * Timeout in which the function passed as `handlePageFunction` needs to finish, given in seconds.
-     */
-    handlePageTimeoutSecs?: number | undefined;
-    /**
-     * Timeout in which the HTTP request to the resource needs to finish, given in seconds.
-     */
-    requestTimeoutSecs?: number | undefined;
-    /**
-     * If set to true, SSL certificate errors will be ignored.
-     */
-    ignoreSslErrors?: boolean | undefined;
-    /**
-     * If set, `CheerioCrawler` will be configured for all connections to use
-     * [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
-     * For more information, see the [documentation](https://docs.apify.com/proxy).
-     */
-    proxyConfiguration?: ProxyConfiguration | undefined;
-    /**
-     * A function to handle requests that failed more than `option.maxRequestRetries` times.
-     * The function receives the following object as an argument:
-     * ```
-     * {
-     * error: Error,
-     * request: Request,
-     * session: Session,
-     * $: Cheerio,
-     * body: String|Buffer,
-     * json: Object,
-     * contentType: Object,
-     * response: Object,
-     * proxyInfo: ProxyInfo,
-     * crawler: CheerioCrawler,
-     * }
-     * ```
-     * where the {@link Request } instance corresponds to the failed request, and the `Error` instance
-     * represents the last error thrown during processing of the request.
-     *
-     * See [source code](https://github.com/apify/apify-js/blob/master/src/crawlers/cheerio_crawler.js#L13)
-     * for the default implementation of this function.
-     */
-    handleFailedRequestFunction?: HandleFailedRequest | undefined;
-    /**
-     * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
-     * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `requestAsBrowserOptions`,
-     * which are passed to the `requestAsBrowser()` function the crawler calls to navigate.
-     * Example:
-     * ```
-     * preNavigationHooks: [
-     * async (crawlingContext, requestAsBrowserOptions) => {
-     * requestAsBrowserOptions.forceUrlEncoding = true;
-     * },
-     * ]
-     * ```
-     */
-    preNavigationHooks?: Hook[] | undefined;
-    /**
-     * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
-     * The function accepts `crawlingContext` as the only parameter.
-     * Example:
-     * ```
-     * postNavigationHooks: [
-     * async (crawlingContext) => {
-     * // ...
-     * },
-     * ]
-     * ```
-     */
-    postNavigationHooks?: Hook[] | undefined;
-    /**
-     * An array of <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types"
-     * target="_blank">MIME types</a> you want the crawler to load and process.
-     * By default, only `text/html` and `application/xhtml+xml` MIME types are supported.
-     */
-    additionalMimeTypes?: string[] | undefined;
-    /**
-     * By default `CheerioCrawler` will extract correct encoding from the HTTP response headers.
-     * Sadly, there are some websites which use invalid headers. Those are encoded using the UTF-8 encoding.
-     * If those sites actually use a different encoding, the response will be corrupted. You can use
-     * `suggestResponseEncoding` to fall back to a certain encoding, if you know that your target website uses it.
-     * To force a certain encoding, disregarding the response headers, use {@link CheerioCrawlerOptions.forceResponseEncoding }```
-     * // Will fall back to windows-1250 encoding if none found
-     * suggestResponseEncoding: 'windows-1250'
-     * ```
-     */
-    suggestResponseEncoding?: string | undefined;
-    /**
-     * By default `CheerioCrawler` will extract correct encoding from the HTTP response headers. Use `forceResponseEncoding`
-     * to force a certain encoding, disregarding the response headers.
-     * To only provide a default for missing encodings, use {@link CheerioCrawlerOptions.suggestResponseEncoding }```
-     * // Will force windows-1250 encoding even if headers say otherwise
-     * forceResponseEncoding: 'windows-1250'
-     * ```
-     */
-    forceResponseEncoding?: string | undefined;
-    /**
-     * Indicates how many times the request is retried if either `requestFunction` or `handlePageFunction` fails.
-     */
-    maxRequestRetries?: number | undefined;
-    /**
-     * Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
-     * Always set this value in order to prevent infinite loops in misconfigured crawlers.
-     * Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
-     */
-    maxRequestsPerCrawl?: number | undefined;
-    /**
-     * Custom options passed to the underlying {@link AutoscaledPool } constructor.
-     * Note that the `runTaskFunction`, `isTaskReadyFunction` and `isFinishedFunction` options
-     * are provided by `CheerioCrawler` and cannot be overridden. Reasonable {@link Snapshotter }
-     * and {@link SystemStatus } defaults are provided to account for the fact that `cheerio`
-     * parses HTML synchronously and therefore blocks the event loop.
-     */
-    autoscaledPoolOptions?: AutoscaledPoolOptions | undefined;
-    /**
-     * Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool } option.
-     *
-     * *WARNING:* If you set this value too high with respect to the available system memory and CPU, your crawler will run extremely slow or crash.
-     * If you're not sure, just keep the default value and the concurrency will scale up automatically.
-     */
-    minConcurrency?: number | undefined;
-    /**
-     * Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool } option.
-     */
-    maxConcurrency?: number | undefined;
-    /**
-     * If set to true Crawler will automatically use Session Pool. It will automatically retire sessions on 403, 401 and 429 status codes.
-     * It also marks Session as bad after a request timeout.
-     */
-    useSessionPool?: boolean | undefined;
-    /**
-     * Custom options passed to the underlying {@link SessionPool } constructor.
-     */
-    sessionPoolOptions?: SessionPoolOptions | undefined;
-    /**
-     * Automatically saves cookies to Session. Works only if Session Pool is used.
-     *
-     * It parses cookie from response "set-cookie" header saves or updates cookies for session and once the session is used for next request.
-     * It passes the "Cookie" header to the request with the session cookies.
-     */
-    persistCookiesPerSession?: boolean | undefined;
-};
-export type PrepareRequestInputs = {
-    /**
-     *  Original instance fo the {Request} object. Must be modified in-place.
-     */
-    request: Request;
-    /**
-     * The current session
-     */
-    session?: Session | undefined;
-    /**
-     * An object with information about currently used proxy by the crawler
-     * and configured by the {@link ProxyConfiguration } class.
-     */
-    proxyInfo?: ProxyInfo | undefined;
-    crawler?: CheerioCrawler | undefined;
-};
-export type PrepareRequest = (inputs: PrepareRequestInputs) => (void | Promise<void>);
-export type PostResponseInputs = {
-    /**
-     * stream
-     */
-    response: (IncomingMessage | Readable);
-    /**
-     *  Original instance fo the {Request} object. Must be modified in-place.
-     */
-    request: Request;
-    /**
-     * The current session
-     */
-    session?: Session | undefined;
-    /**
-     * An object with information about currently used proxy by the crawler
-     * and configured by the {@link ProxyConfiguration } class.
-     */
-    proxyInfo?: ProxyInfo | undefined;
-    crawler: CheerioCrawler;
-};
-export type PostResponse = (inputs: PostResponseInputs) => (void | Promise<void>);
-export type CheerioHandlePageInputs = {
-    /**
-     *  The [Cheerio](https://cheerio.js.org/) object with parsed HTML.
-     */
-    $: CheerioAPI;
-    /**
-     *  The request body of the web page.
-     */
-    body: (string | Buffer);
-    /**
-     *  The parsed object from JSON string if the response contains the content type application/json.
-     */
-    json: any;
-    /**
-     *   The original {@link Request } object.
-     */
-    request: Request;
-    /**
-     *  Parsed `Content-Type header: { type, encoding }`.
-     */
-    contentType: {
-        type: string;
-        encoding: string;
-    };
-    /**
-     *   An instance of Node's [http.IncomingMessage](https://nodejs.org/api/http.html#http_class_http_incomingmessage) object,
-     */
-    response: IncomingMessage;
-    session: Session;
-    /**
-     *   An object with information about currently used proxy by the crawler
-     *   and configured by the {@link ProxyConfiguration } class.
-     */
-    proxyInfo: ProxyInfo;
-    crawler: CheerioCrawler;
-};
-export type CheerioHandlePage = (inputs: CheerioHandlePageInputs) => Promise<void>;
-/**
- * @typedef CheerioCrawlerOptions
- * @property {CheerioHandlePage} handlePageFunction
- *   User-provided function that performs the logic of the crawler. It is called for each page
- *   loaded and parsed by the crawler.
- *
- *   The function receives the following object as an argument:
- * ```
- * {
- *   // The Cheerio object's function with the parsed HTML.
- *   $: Cheerio,
- *
- *   // The request body of the web page, whose type depends on the content type.
- *   body: String|Buffer,
- *
- *   // The parsed object from JSON for responses with the "application/json" content types.
- *   // For other content types it's null.
- *   json: Object,
- *
- *   // Apify.Request object with details of the requested web page
- *   request: Request,
- *
- *   // Parsed Content-Type HTTP header: { type, encoding }
- *   contentType: Object,
- *
- *   // An instance of Node's http.IncomingMessage object,
- *   response: Object,
- *
- *   // Session object, useful to work around anti-scraping protections
- *   session: Session
- *
- *   // ProxyInfo object with information about currently used proxy
- *   proxyInfo: ProxyInfo
- *
- *   // The running cheerio crawler instance.
- *   crawler: CheerioCrawler
- * }
- * ```
- *
- *   Type of `body` depends on the `Content-Type` header of the web page:
- *   - String for `text/html`, `application/xhtml+xml`, `application/xml` MIME content types
- *   - Buffer for others MIME content types
- *
- *   Parsed `Content-Type` header using
- *   [content-type package](https://www.npmjs.com/package/content-type)
- *   is stored in `contentType`.
- *
- *   Cheerio is available only for HTML and XML content types.
- *
- *   With the {@link Request} object representing the URL to crawl.
- *
- *   If the function returns, the returned promise is awaited by the crawler.
- *
- *   If the function throws an exception, the crawler will try to re-crawl the
- *   request later, up to `option.maxRequestRetries` times.
- *   If all the retries fail, the crawler calls the function
- *   provided to the `handleFailedRequestFunction` parameter.
- *   To make this work, you should **always**
- *   let your function throw exceptions rather than catch them.
- *   The exceptions are logged to the request using the
- *   {@link Request#pushErrorMessage} function.
- * @property {RequestList} [requestList]
- *   Static list of URLs to be processed.
- *   Either `requestList` or `requestQueue` option must be provided (or both).
- * @property {RequestQueue} [requestQueue]
- *   Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
- *   Either `requestList` or `requestQueue` option must be provided (or both).
- * @property {PrepareRequest} [prepareRequestFunction]
- * > This option is deprecated, use `preNavigationHooks` instead.
- *
- *   A function that executes before the HTTP request is made to the target resource.
- *   This function is suitable for setting dynamic properties such as cookies to the {@link Request}.
- *
- *   The function receives the following object as an argument:
- * ```
- * {
- *   request: Request,
- *   session: Session,
- *   proxyInfo: ProxyInfo,
- *   crawler: CheerioCrawler,
- * }
- * ```
- *   where the {@link Request} instance corresponds to the initialized request
- *   and the {@link Session} instance corresponds to used session.
- *
- *   The function should modify the properties of the passed {@link Request} instance
- *   in place because there are already earlier references to it. Making a copy and returning it from
- *   this function is therefore not supported, because it would create inconsistencies where
- *   different parts of SDK would have access to a different {@link Request} instance.
- *
- * @property {PostResponse} [postResponseFunction]
- * > This option is deprecated, use `postNavigationHooks` instead.
- *
- * A function that executes right after the HTTP request is made to the target resource and response is returned.
- * This function is suitable for overriding custom properties of response e.g. setting headers because of response parsing.
- *
- * **Example usage:**
- *
- * ```javascript
- *  const cheerioCrawlerOptions = {
- *      // ...
- *      postResponseFunction: ({ request, response }) => {
- *          if (request.userData.parseAsJSON) {
- *              response.headers['content-type'] = 'application/json; charset=utf-8';
- *          }
- *      }
- *  }
- * ```
- * The function receives the following object as an argument:
- * ```
- * {
- *   response: Object,
- *   request: Request,
- *   session: Session,
- *   proxyInfo: ProxyInfo,
- *   crawler: CheerioCrawler,
- * }
- * ```
- * The response is an instance of Node's http.IncomingMessage object.
- *
- * @property {number} [handlePageTimeoutSecs=60]
- *   Timeout in which the function passed as `handlePageFunction` needs to finish, given in seconds.
- * @property {number} [requestTimeoutSecs=30]
- *   Timeout in which the HTTP request to the resource needs to finish, given in seconds.
- * @property {boolean} [ignoreSslErrors=true]
- *   If set to true, SSL certificate errors will be ignored.
- * @property {ProxyConfiguration} [proxyConfiguration]
- *   If set, `CheerioCrawler` will be configured for all connections to use
- *   [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
- *   For more information, see the [documentation](https://docs.apify.com/proxy).
- * @property {HandleFailedRequest} [handleFailedRequestFunction]
- *   A function to handle requests that failed more than `option.maxRequestRetries` times.
- *   The function receives the following object as an argument:
- * ```
- * {
- *   error: Error,
- *   request: Request,
- *   session: Session,
- *   $: Cheerio,
- *   body: String|Buffer,
- *   json: Object,
- *   contentType: Object,
- *   response: Object,
- *   proxyInfo: ProxyInfo,
- *   crawler: CheerioCrawler,
- * }
- * ```
- *   where the {@link Request} instance corresponds to the failed request, and the `Error` instance
- *   represents the last error thrown during processing of the request.
- *
- *   See [source code](https://github.com/apify/apify-js/blob/master/src/crawlers/cheerio_crawler.js#L13)
- *   for the default implementation of this function.
- * @property {Array<Hook>} [preNavigationHooks]
- *   Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
- *   or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `requestAsBrowserOptions`,
- *   which are passed to the `requestAsBrowser()` function the crawler calls to navigate.
- *   Example:
- * ```
- * preNavigationHooks: [
- *     async (crawlingContext, requestAsBrowserOptions) => {
- *         requestAsBrowserOptions.forceUrlEncoding = true;
- *     },
- * ]
- * ```
- * @property {Array<Hook>} [postNavigationHooks]
- *   Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
- *   The function accepts `crawlingContext` as the only parameter.
- *   Example:
- * ```
- * postNavigationHooks: [
- *     async (crawlingContext) => {
- *         // ...
- *     },
- * ]
- * ```
- * @property {string[]} [additionalMimeTypes]
- *   An array of <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types"
- *   target="_blank">MIME types</a> you want the crawler to load and process.
- *   By default, only `text/html` and `application/xhtml+xml` MIME types are supported.
- * @property {string} [suggestResponseEncoding]
- *   By default `CheerioCrawler` will extract correct encoding from the HTTP response headers.
- *   Sadly, there are some websites which use invalid headers. Those are encoded using the UTF-8 encoding.
- *   If those sites actually use a different encoding, the response will be corrupted. You can use
- *   `suggestResponseEncoding` to fall back to a certain encoding, if you know that your target website uses it.
- *   To force a certain encoding, disregarding the response headers, use {@link CheerioCrawlerOptions.forceResponseEncoding}
- *   ```
- *   // Will fall back to windows-1250 encoding if none found
- *   suggestResponseEncoding: 'windows-1250'
- *   ```
- * @property {string} [forceResponseEncoding]
- *   By default `CheerioCrawler` will extract correct encoding from the HTTP response headers. Use `forceResponseEncoding`
- *   to force a certain encoding, disregarding the response headers.
- *   To only provide a default for missing encodings, use {@link CheerioCrawlerOptions.suggestResponseEncoding}
- *   ```
- *   // Will force windows-1250 encoding even if headers say otherwise
- *   forceResponseEncoding: 'windows-1250'
- *   ```
- * @property {number} [maxRequestRetries=3]
- *   Indicates how many times the request is retried if either `requestFunction` or `handlePageFunction` fails.
- * @property {number} [maxRequestsPerCrawl]
- *   Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
- *   Always set this value in order to prevent infinite loops in misconfigured crawlers.
- *   Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
- * @property {AutoscaledPoolOptions} [autoscaledPoolOptions]
- *   Custom options passed to the underlying {@link AutoscaledPool} constructor.
- *   Note that the `runTaskFunction`, `isTaskReadyFunction` and `isFinishedFunction` options
- *   are provided by `CheerioCrawler` and cannot be overridden. Reasonable {@link Snapshotter}
- *   and {@link SystemStatus} defaults are provided to account for the fact that `cheerio`
- *   parses HTML synchronously and therefore blocks the event loop.
- * @property {number} [minConcurrency=1]
- *   Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
- *
- *   *WARNING:* If you set this value too high with respect to the available system memory and CPU, your crawler will run extremely slow or crash.
- *   If you're not sure, just keep the default value and the concurrency will scale up automatically.
- * @property {number} [maxConcurrency=1000]
- *   Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
- * @property {boolean} [useSessionPool=true]
- *   If set to true Crawler will automatically use Session Pool. It will automatically retire sessions on 403, 401 and 429 status codes.
- *   It also marks Session as bad after a request timeout.
- * @property {SessionPoolOptions} [sessionPoolOptions]
- *   Custom options passed to the underlying {@link SessionPool} constructor.
- * @property {boolean} [persistCookiesPerSession]
- *   Automatically saves cookies to Session. Works only if Session Pool is used.
- *
- *   It parses cookie from response "set-cookie" header saves or updates cookies for session and once the session is used for next request.
- *   It passes the "Cookie" header to the request with the session cookies.
- */
-/**
- * Provides a framework for the parallel crawling of web pages using plain HTTP requests and
- * [cheerio](https://www.npmjs.com/package/cheerio) HTML parser.
- * The URLs to crawl are fed either from a static list of URLs
- * or from a dynamic queue of URLs enabling recursive crawling of websites.
- *
- * Since `CheerioCrawler` uses raw HTTP requests to download web pages,
- * it is very fast and efficient on data bandwidth. However, if the target website requires JavaScript
- * to display the content, you might need to use {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead,
- * because it loads the pages using full-featured headless Chrome browser.
- *
- * `CheerioCrawler` downloads each URL using a plain HTTP request,
- * parses the HTML content using [Cheerio](https://www.npmjs.com/package/cheerio)
- * and then invokes the user-provided {@link CheerioCrawlerOptions.handlePageFunction} to extract page data
- * using a [jQuery](https://jquery.com/)-like interface to the parsed HTML DOM.
- *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link CheerioCrawlerOptions.requestList}
- * or {@link CheerioCrawlerOptions.requestQueue} constructor options, respectively.
- *
- * If both {@link CheerioCrawlerOptions.requestList} and {@link CheerioCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
- *
- * The crawler finishes when there are no more {@link Request} objects to crawl.
- *
- * `CheerioCrawler` downloads the web pages using the `{@link utils#requestAsBrowser}` utility function.
- * As opposed to the browser based crawlers that are automatically encoding the URLs, the
- * `{@link utils#requestAsBrowser}` function will not do so. We either need to manually encode the URLs
- * via `encodeURI()` function, or set `forceUrlEncoding: true` in the `requestAsBrowserOptions`,
- * which will automatically encode all the URLs before accessing them.
- *
- * > We can either use `forceUrlEncoding` or encode manually, but not both - it would
- * > result in double encoding and therefore lead to invalid URLs.
- *
- * We can use the `preNavigationHooks` to adjust `requestAsBrowserOptions`:
- *
- * ```
- * preNavigationHooks: [
- *     (crawlingContext, requestAsBrowserOptions) => {
- *         requestAsBrowserOptions.forceUrlEncoding = true;
- *     },
- * ]
- * ```
- *
- * By default, `CheerioCrawler` only processes web pages with the `text/html`
- * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),
- * and skips pages with other content types. If you want the crawler to process other content types,
- * use the {@link CheerioCrawlerOptions.additionalMimeTypes} constructor option.
- * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
- * For details, see {@link CheerioCrawlerOptions.handlePageFunction}.
- *
- * New requests are only dispatched when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
- * parameter of the `CheerioCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPool} options are available directly in the `CheerioCrawler` constructor.
- *
- * **Example usage:**
- *
- * ```javascript
- * // Prepare a list of URLs to crawl
- * const requestList = new Apify.RequestList({
- *   sources: [
- *       { url: 'http://www.example.com/page-1' },
- *       { url: 'http://www.example.com/page-2' },
- *   ],
- * });
- * await requestList.initialize();
- *
- * // Crawl the URLs
- * const crawler = new Apify.CheerioCrawler({
- *     requestList,
- *     handlePageFunction: async ({ request, response, body, contentType, $ }) => {
- *         const data = [];
- *
- *         // Do some data extraction from the page with Cheerio.
- *         $('.some-collection').each((index, el) => {
- *             data.push({ title: $(el).find('.some-title').text() });
- *         });
- *
- *         // Save the data to dataset.
- *         await Apify.pushData({
- *             url: request.url,
- *             html: body,
- *             data,
- *         })
- *     },
- * });
- *
- * await crawler.run();
- * ```
- * @property {Statistics} stats
- *  Contains statistics about the current run.
- * @property {?RequestList} requestList
- *  A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request}s.
- *  Only available if used by the crawler.
- * @property {?RequestQueue} requestQueue
- *  A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request}s.
- *  Only available if used by the crawler.
- * @property {?SessionPool} sessionPool
- *  A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session}s.
- *  Only available if used by the crawler.
- * @property {?ProxyConfiguration} proxyConfiguration
- *  A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
- *  Only available if used by the crawler.
- * @property {AutoscaledPool} autoscaledPool
- *  A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
- *  Note that this property is only initialized after calling the {@link CheerioCrawler#run} function.
- *  You can use it to change the concurrency settings on the fly,
- *  to pause the crawler by calling {@link AutoscaledPool#pause}
- *  or to abort it by calling {@link AutoscaledPool#abort}.
- */
-declare class CheerioCrawler extends BasicCrawler {
-    /**
-     * @param {CheerioCrawlerOptions} options
-     * All `CheerioCrawler` parameters are passed via an options object.
-     */
-    constructor(options: CheerioCrawlerOptions);
-    supportedMimeTypes: Set<string>;
-    handlePageTimeoutMillis: number;
-    requestTimeoutMillis: number;
-    ignoreSslErrors: boolean;
-    suggestResponseEncoding: string | undefined;
-    forceResponseEncoding: string | undefined;
-    prepareRequestFunction: PrepareRequest | undefined;
-    postResponseFunction: PostResponse | undefined;
-    proxyConfiguration: ProxyConfiguration | undefined;
-    /**
-     * @type {Array<any>}
-     * @ignore
-     * */
-    preNavigationHooks: Array<any>;
-    /**
-     * @type {Array<any>}
-     * @ignore
-     * */
-    postNavigationHooks: Array<any>;
-    persistCookiesPerSession: boolean;
-    /**
-     * **EXPERIMENTAL**
-     * Function for attaching CrawlerExtensions such as the Unblockers.
-     * @param {CrawlerExtension} extension - Crawler extension that overrides the crawler configuration.
-     */
-    use(extension: CrawlerExtension): void;
-    /**
-     * @param {CrawlingContext} crawlingContext
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _handleNavigation(crawlingContext: CrawlingContext): Promise<void>;
-    /**
-     * When users change `request.headers.cookie` inside preNavigationHook, the change would be ignored,
-     * as `request.headers` are already merged into the `requestAsBrowserOptions`. This method is using
-     * old `request.headers` snapshot (before hooks are executed), makes a diff with the cookie value
-     * after hooks are executed, and merges any new cookies back to `requestAsBrowserOptions`.
-     *
-     * This way we can still use both `requestAsBrowserOptions` and `context.request` in the hooks (not both).
-     *
-     * @param {Request} request
-     * @param {string} cookieSnapshot
-     * @param {RequestAsBrowserOptions} requestAsBrowserOptions
-     * @private
-     * @ignore
-     * @internal
-     */
-    private _mergeRequestCookieDiff;
-    /**
-     * Function to make the HTTP request. It performs optimizations
-     * on the request such as only downloading the request body if the
-     * received content type matches text/html, application/xml, application/xhtml+xml.
-     *
-     * @param {object} options
-     * @param {Request} options.request
-     * @param {Session} options.session
-     * @param {string} options.proxyUrl
-     * @param {RequestAsBrowserOptions} options.requestAsBrowserOptions
-     * @returns {Promise<IncomingMessage|Readable>}
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _requestFunction({ request, session, proxyUrl, requestAsBrowserOptions }: {
-        request: Request;
-        session: Session;
-        proxyUrl: string;
-        requestAsBrowserOptions: RequestAsBrowserOptions;
-    }): Promise<IncomingMessage | Readable>;
-    /**
-     * Sets the cookie header to `requestAsBrowserOptions` based on provided session and request. If some cookies were already set,
-     * the session cookie will be merged with them. User provided cookies on `request` object have precedence.
-     *
-     * @param {CrawlingContext} crawlingContext
-     * @param {RequestAsBrowserOptions} requestAsBrowserOptions
-     * @return {void}
-     * @ignore
-     * @private
-     * @internal
-     */
-    private _applySessionCookie;
-    /**
-     * Encodes and parses response according to the provided content type
-     * @param {Request} request
-     * @param {IncomingMessage|Readable} responseStream
-     * @returns {Promise<object>}
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _parseResponse(request: Request, responseStream: IncomingMessage | Readable): Promise<object>;
-    /**
-     * Combines the provided `requestOptions` with mandatory (non-overridable) values.
-     * @param {Request} request
-     * @param {Session} [session]
-     * @param {string} [proxyUrl]
-     * @param {RequestAsBrowserOptions} [requestAsBrowserOptions]
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _getRequestOptions(request: Request, session?: Session | undefined, proxyUrl?: string | undefined, requestAsBrowserOptions?: RequestAsBrowserOptions | undefined): {
-        headers: {
-            [x: string]: string;
-        };
-        https: any;
-        isStream: boolean;
-        /**
-         *  URL of the target endpoint. Supports both HTTP and HTTPS schemes.
-         */
-        url: string;
-        /**
-         * HTTP method.
-         */
-        method: string;
-        /**
-         * An HTTP proxy to be passed down to the HTTP request. Supports proxy authentication with Basic Auth.
-         */
-        proxyUrl: string | undefined;
-        /**
-         * Configuration to be used for generating correct browser headers.
-         * See the [`header-generator`](https://github.com/apify/header-generator) library.
-         */
-        headerGeneratorOptions?: object | undefined;
-        /**
-         * Two-letter ISO 639 language code.
-         */
-        languageCode?: string | undefined;
-        /**
-         * Two-letter ISO 3166 country code.
-         */
-        countryCode?: string | undefined;
-        /**
-         * If `true`, the function uses User-Agent of a mobile browser.
-         */
-        useMobileVersion?: boolean | undefined;
-        /**
-         * If set to true, SSL/TLS certificate errors will be ignored.
-         */
-        ignoreSslErrors?: boolean | undefined;
-        /**
-         * Node.js' HTTP parser is stricter than parsers used by web browsers, which prevents scraping of websites
-         * whose servers do not comply with HTTP specs, either by accident or due to some anti-scraping protections,
-         * causing e.g. the `invalid header value char` error. The `useInsecureHttpParser` option forces
-         * the HTTP parser to ignore certain errors which lets you scrape such websites.
-         * However, it will also open your application to some security vulnerabilities,
-         * although the risk should be negligible as these vulnerabilities mainly relate to server applications, not clients.
-         * Learn more in this [blog post](https://snyk.io/blog/node-js-release-fixes-a-critical-http-security-vulnerability/).
-         */
-        useInsecureHttpParser?: boolean | undefined;
-        /**
-         * Function accepts `response` object as a single parameter and should return `true` or `false`.
-         * If function returns true, request gets aborted.
-         */
-        abortFunction?: import("../utils_request").AbortFunction | undefined;
-        /**
-         * If set to false, it will prevent use of HTTP2 requests. This is strongly discouraged. Websites
-         * expect HTTP2 connections, because browsers use HTTP2 by default. It will automatically downgrade
-         * to HTTP/1.1 for websites that do not support HTTP2.
-         */
-        useHttp2?: boolean | undefined;
-        /**
-         * A unique object used to generate browser headers. By default, new headers are generated on every call.
-         * Set this option to make these headers persistent.
-         */
-        sessionToken: object | Session | undefined;
-        timeout: {
-            request: number;
-        };
-    };
-    /**
-     * @param {*} request
-     * @param {*} response
-     * @param {*} encoding
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _encodeResponse(request: any, response: any, encoding: any): {
-        response: any;
-        encoding: string;
-    };
-    /**
-     * @param {*} response
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _parseHtmlToDom(response: any): Promise<any>;
-    /**
-     * Checks and extends supported mime types
-     * @param {Array<(string|Object)>} additionalMimeTypes
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _extendSupportedMimeTypes(additionalMimeTypes: Array<(string | Object)>): void;
-    /**
-     * Handles blocked request
-     * @param {Session} session
-     * @param {number} statusCode
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _throwOnBlockedRequest(session: Session, statusCode: number): void;
-    /**
-     * Handles timeout request
-     * @param {Session} session
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _handleRequestTimeout(session: Session): void;
-    /**
-     * @param {Request} request
-     * @param {IncomingMessage|Readable} response
-     * @private
-     */
-    private _abortDownloadOfBody;
-}
-import { RequestList } from "../request_list";
-import { RequestQueue } from "../storages/request_queue";
-import { ProxyConfiguration } from "../proxy_configuration";
-import { HandleFailedRequest } from "./basic_crawler";
-import { Hook } from "./browser_crawler";
-import { AutoscaledPoolOptions } from "../autoscaling/autoscaled_pool";
-import { SessionPoolOptions } from "../session_pool/session_pool";
-import Request from "../request";
-import { Session } from "../session_pool/session";
-import { ProxyInfo } from "../proxy_configuration";
-import { IncomingMessage } from "http";
-import { Readable } from "stream";
-import { CheerioAPI } from "cheerio/lib/load";
-import { BasicCrawler } from "./basic_crawler";
-import CrawlerExtension from "./crawler_extension";
-import { CrawlingContext } from "./basic_crawler";
-import { RequestAsBrowserOptions } from "../utils_request";
-//# sourceMappingURL=cheerio_crawler.d.ts.map