@crawlee/cheerio 3.0.0-alpha.2 → 3.0.0-alpha.5

package/internals/cheerio-crawler.js

@@ -0,0 +1,682 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cheerioCrawlerEnqueueLinks = exports.CheerioCrawler = void 0;
+const tslib_1 = require("tslib");
+const timeout_1 = require("@apify/timeout");
+const utilities_1 = require("@apify/utilities");
+const basic_1 = require("@crawlee/basic");
+const core_1 = require("@crawlee/core");
+const utils_1 = require("@crawlee/utils");
+const cheerio_1 = tslib_1.__importDefault(require("cheerio"));
+const content_type_1 = tslib_1.__importDefault(require("content-type"));
+const got_scraping_1 = require("got-scraping");
+const htmlparser2_1 = require("htmlparser2");
+const WritableStream_1 = require("htmlparser2/lib/WritableStream");
+const iconv_lite_1 = tslib_1.__importDefault(require("iconv-lite"));
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const util_1 = tslib_1.__importDefault(require("util"));
+/**
+ * Default mime types, which CheerioScraper supports.
+ */
+const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml', 'application/xml'];
+const APPLICATION_JSON_MIME_TYPE = 'application/json';
+const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
+    snapshotterOptions: {
+        eventLoopSnapshotIntervalSecs: 2,
+        maxBlockedMillis: 100,
+    },
+    systemStatusOptions: {
+        maxEventLoopOverloadedRatio: 0.7,
+    },
+};
+/**
+ * Provides a framework for the parallel crawling of web pages using plain HTTP requests and
+ * [cheerio](https://www.npmjs.com/package/cheerio) HTML parser.
+ * The URLs to crawl are fed either from a static list of URLs
+ * or from a dynamic queue of URLs enabling recursive crawling of websites.
+ *
+ * Since `CheerioCrawler` uses raw HTTP requests to download web pages,
+ * it is very fast and efficient on data bandwidth. However, if the target website requires JavaScript
+ * to display the content, you might need to use {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead,
+ * because it loads the pages using full-featured headless Chrome browser.
+ *
+ * `CheerioCrawler` downloads each URL using a plain HTTP request,
+ * parses the HTML content using [Cheerio](https://www.npmjs.com/package/cheerio)
+ * and then invokes the user-provided {@link CheerioCrawlerOptions.requestHandler} to extract page data
+ * using a [jQuery](https://jquery.com/)-like interface to the parsed HTML DOM.
+ *
+ * The source URLs are represented using {@link Request} objects that are fed from
+ * {@link RequestList} or {@link RequestQueue} instances provided by the {@link CheerioCrawlerOptions.requestList}
+ * or {@link CheerioCrawlerOptions.requestQueue} constructor options, respectively.
+ *
+ * If both {@link CheerioCrawlerOptions.requestList} and {@link CheerioCrawlerOptions.requestQueue} are used,
+ * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
+ * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ *
+ * The crawler finishes when there are no more {@link Request} objects to crawl.
+ *
+ * We can use the `preNavigationHooks` to adjust `gotOptions`:
+ *
+ * ```
+ * preNavigationHooks: [
+ *     (crawlingContext, gotOptions) => {
+ *         // ...
+ *     },
+ * ]
+ * ```
+ *
+ * By default, `CheerioCrawler` only processes web pages with the `text/html`
+ * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),
+ * and skips pages with other content types. If you want the crawler to process other content types,
+ * use the {@link CheerioCrawlerOptions.additionalMimeTypes} constructor option.
+ * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
+ * For details, see {@link CheerioCrawlerOptions.requestHandler}.
+ *
+ * New requests are only dispatched when there is enough free CPU and memory available,
+ * using the functionality provided by the {@link AutoscaledPool} class.
+ * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
+ * parameter of the `CheerioCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+ * {@link AutoscaledPool} options are available directly in the `CheerioCrawler` constructor.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * // Prepare a list of URLs to crawl
+ * const requestList = new RequestList({
+ *   sources: [
+ *       { url: 'http://www.example.com/page-1' },
+ *       { url: 'http://www.example.com/page-2' },
+ *   ],
+ * });
+ * await requestList.initialize();
+ *
+ * // Crawl the URLs
+ * const crawler = new CheerioCrawler({
+ *     requestList,
+ *     handlePageFunction: async ({ request, response, body, contentType, $ }) => {
+ *         const data = [];
+ *
+ *         // Do some data extraction from the page with Cheerio.
+ *         $('.some-collection').each((index, el) => {
+ *             data.push({ title: $(el).find('.some-title').text() });
+ *         });
+ *
+ *         // Save the data to dataset.
+ *         await Actor.pushData({
+ *             url: request.url,
+ *             html: body,
+ *             data,
+ *         })
+ *     },
+ * });
+ *
+ * await crawler.run();
+ * ```
+ * @category Crawlers
+ */
+class CheerioCrawler extends basic_1.BasicCrawler {
+    /**
+     * All `CheerioCrawler` parameters are passed via an options object.
+     */
+    constructor(options) {
+        (0, ow_1.default)(options, 'CheerioCrawlerOptions', ow_1.default.object.exactShape(CheerioCrawler.optionsShape));
+        const { requestHandler, handlePageFunction, requestHandlerTimeoutSecs = 60, requestTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, proxyConfiguration, prepareRequestFunction, postResponseFunction, persistCookiesPerSession, preNavigationHooks = [], postNavigationHooks = [], 
+        // Ignored
+        handleRequestFunction, 
+        // BasicCrawler
+        autoscaledPoolOptions = CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS, ...basicCrawlerOptions } = options;
+        super({
+            ...basicCrawlerOptions,
+            // Will be overridden below
+            requestHandler: () => { },
+            autoscaledPoolOptions,
+            // We need to add some time for internal functions to finish,
+            // but not too much so that we would stall the crawler.
+            requestHandlerTimeoutSecs: requestTimeoutSecs + requestHandlerTimeoutSecs + basic_1.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
+        });
+        /**
+         * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
+         * Only available if used by the crawler.
+         */
+        Object.defineProperty(this, "proxyConfiguration", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "userRequestHandlerTimeoutMillis", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "defaultGotoOptions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "preNavigationHooks", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "postNavigationHooks", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "persistCookiesPerSession", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "requestTimeoutMillis", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "ignoreSslErrors", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "suggestResponseEncoding", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "forceResponseEncoding", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "prepareRequestFunction", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "postResponseFunction", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "supportedMimeTypes", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * @internal wraps public utility for mocking purposes
+         */
+        Object.defineProperty(this, "_requestAsBrowser", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: (options) => {
+                return new Promise((resolve, reject) => {
+                    const stream = (0, got_scraping_1.gotScraping)(options);
+                    stream.on('error', reject);
+                    stream.on('response', () => {
+                        resolve(addResponsePropertiesToStream(stream));
+                    });
+                });
+            }
+        });
+        this._handlePropertyNameChange({
+            newName: 'requestHandler',
+            oldName: 'handlePageFunction',
+            propertyKey: 'requestHandler',
+            newProperty: requestHandler,
+            oldProperty: handlePageFunction,
+        });
+        // Cookies should be persisted per session only if session pool is used
+        if (!this.useSessionPool && persistCookiesPerSession) {
+            throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
+        }
+        this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
+        if (additionalMimeTypes.length)
+            this._extendSupportedMimeTypes(additionalMimeTypes);
+        if (suggestResponseEncoding && forceResponseEncoding) {
+            this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
+        }
+        this.userRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
+        this.requestTimeoutMillis = requestTimeoutSecs * 1000;
+        this.ignoreSslErrors = ignoreSslErrors;
+        this.suggestResponseEncoding = suggestResponseEncoding;
+        this.forceResponseEncoding = forceResponseEncoding;
+        this.prepareRequestFunction = prepareRequestFunction;
+        this.postResponseFunction = postResponseFunction;
+        this.proxyConfiguration = proxyConfiguration;
+        this.preNavigationHooks = preNavigationHooks;
+        this.postNavigationHooks = [
+            ({ request, response }) => this._abortDownloadOfBody(request, response),
+            ...postNavigationHooks,
+        ];
+        if (this.useSessionPool) {
+            this.persistCookiesPerSession = persistCookiesPerSession ?? true;
+        }
+        else {
+            this.persistCookiesPerSession = false;
+        }
+    }
+    /**
+     * **EXPERIMENTAL**
+     * Function for attaching CrawlerExtensions such as the Unblockers.
+     * @param extension Crawler extension that overrides the crawler configuration.
+     */
+    use(extension) {
+        (0, ow_1.default)(extension, ow_1.default.object.instanceOf(core_1.CrawlerExtension));
+        const extensionOptions = extension.getCrawlerOptions();
+        for (const [key, value] of (0, utils_1.entries)(extensionOptions)) {
+            const isConfigurable = this.hasOwnProperty(key); // eslint-disable-line
+            const originalType = typeof this[key];
+            const extensionType = typeof value; // What if we want to null something? It is really needed?
+            const isSameType = originalType === extensionType || value == null; // fast track for deleting keys
+            const exists = this[key] != null;
+            if (!isConfigurable) { // Test if the property can be configured on the crawler
+                throw new Error(`${extension.name} tries to set property "${key}" that is not configurable on CheerioCrawler instance.`);
+            }
+            if (!isSameType && exists) { // Assuming that extensions will only add up configuration
+                throw new Error(`${extension.name} tries to set property of different type "${extensionType}". "CheerioCrawler.${key}: ${originalType}".`);
+            }
+            this.log.warning(`${extension.name} is overriding "CheerioCrawler.${key}: ${originalType}" with ${value}.`);
+            this[key] = value;
+        }
+    }
+    /**
+     * Wrapper around handlePageFunction that opens and closes pages etc.
+     */
+    async _runRequestHandler(crawlingContext) {
+        const { request, session } = crawlingContext;
+        if (this.proxyConfiguration) {
+            const sessionId = session ? session.id : undefined;
+            crawlingContext.proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId);
+        }
+        await this._handleNavigation(crawlingContext);
+        (0, timeout_1.tryCancel)();
+        const { dom, isXml, body, contentType, response } = await this._parseResponse(request, crawlingContext.response);
+        (0, timeout_1.tryCancel)();
+        if (this.useSessionPool) {
+            this._throwOnBlockedRequest(session, response.statusCode);
+        }
+        if (this.persistCookiesPerSession) {
+            session.setCookiesFromResponse(response);
+        }
+        request.loadedUrl = response.url;
+        const $ = dom
+            ? cheerio_1.default.load(dom, {
+                xmlMode: isXml,
+                // Recent versions of cheerio use parse5 as the HTML parser/serializer. It's more strict than htmlparser2
+                // and not good for scraping. It also does not have a great streaming interface.
+                // Here we tell cheerio to use htmlparser2 for serialization, otherwise the conflict produces weird errors.
+                _useHtmlParser2: true,
+            })
+            : null;
+        crawlingContext.$ = $;
+        crawlingContext.contentType = contentType;
+        crawlingContext.response = response;
+        crawlingContext.enqueueLinks = async (enqueueOptions) => {
+            return cheerioCrawlerEnqueueLinks({
+                options: enqueueOptions,
+                $,
+                requestQueue: await this.getRequestQueue(),
+                originalRequestUrl: crawlingContext.request.url,
+                finalRequestUrl: crawlingContext.request.loadedUrl,
+            });
+        };
+        Object.defineProperty(crawlingContext, 'json', {
+            get() {
+                if (contentType.type !== APPLICATION_JSON_MIME_TYPE)
+                    return null;
+                const jsonString = body.toString(contentType.encoding);
+                return JSON.parse(jsonString);
+            },
+        });
+        Object.defineProperty(crawlingContext, 'body', {
+            get() {
+                // NOTE: For XML/HTML documents, we don't store the original body and only reconstruct it from Cheerio's DOM.
+                // This is to save memory for high-concurrency crawls. The downside is that changes
+                // made to DOM are reflected in the HTML, but we can live with that...
+                if (dom) {
+                    return isXml ? $.xml() : $.html({ decodeEntities: false });
+                }
+                return body;
+            },
+        });
+        return (0, timeout_1.addTimeoutToPromise)(() => Promise.resolve(this.requestHandler(crawlingContext)), this.userRequestHandlerTimeoutMillis, `requestHandler timed out after ${this.userRequestHandlerTimeoutMillis / 1000} seconds.`);
+    }
+    async _handleNavigation(crawlingContext) {
+        if (this.prepareRequestFunction) {
+            this.log.deprecated('Option "prepareRequestFunction" is deprecated. Use "preNavigationHooks" instead.');
+            await this.prepareRequestFunction(crawlingContext);
+            (0, timeout_1.tryCancel)();
+        }
+        const gotOptions = {};
+        if (this.useSessionPool) {
+            this._applySessionCookie(crawlingContext, gotOptions);
+        }
+        const { request, session } = crawlingContext;
+        const cookieSnapshot = request.headers?.Cookie ?? request.headers?.cookie;
+        await this._executeHooks(this.preNavigationHooks, crawlingContext, gotOptions);
+        (0, timeout_1.tryCancel)();
+        const proxyUrl = crawlingContext.proxyInfo?.url;
+        this._mergeRequestCookieDiff(request, cookieSnapshot, gotOptions);
+        crawlingContext.response = await (0, timeout_1.addTimeoutToPromise)(() => this._requestFunction({ request, session, proxyUrl, gotOptions }), this.requestTimeoutMillis, `request timed out after ${this.requestTimeoutMillis / 1000} seconds.`);
+        (0, timeout_1.tryCancel)();
+        await this._executeHooks(this.postNavigationHooks, crawlingContext, gotOptions);
+        (0, timeout_1.tryCancel)();
+        if (this.postResponseFunction) {
+            this.log.deprecated('Option "postResponseFunction" is deprecated. Use "postNavigationHooks" instead.');
+            await this.postResponseFunction(crawlingContext);
+            (0, timeout_1.tryCancel)();
+        }
+    }
+    /**
+     * When users change `request.headers.cookie` inside preNavigationHook, the change would be ignored,
+     * as `request.headers` are already merged into the `gotOptions`. This method is using
+     * old `request.headers` snapshot (before hooks are executed), makes a diff with the cookie value
+     * after hooks are executed, and merges any new cookies back to `gotOptions`.
+     *
+     * This way we can still use both `gotOptions` and `context.request` in the hooks (not both).
+     */
+    _mergeRequestCookieDiff(request, cookieSnapshot, gotOptions) {
+        const cookieDiff = (0, core_1.diffCookies)(request.url, cookieSnapshot, request.headers?.Cookie ?? request.headers?.cookie);
+        if (cookieDiff.length > 0) {
+            gotOptions.headers ?? (gotOptions.headers = {});
+            gotOptions.headers.Cookie = (0, core_1.mergeCookies)(request.url, [
+                gotOptions.headers.Cookie,
+                cookieDiff,
+            ]);
+        }
+    }
+    /**
+     * Function to make the HTTP request. It performs optimizations
+     * on the request such as only downloading the request body if the
+     * received content type matches text/html, application/xml, application/xhtml+xml.
+     */
+    async _requestFunction({ request, session, proxyUrl, gotOptions }) {
+        const opts = this._getRequestOptions(request, session, proxyUrl, gotOptions);
+        try {
+            return await this._requestAsBrowser(opts);
+        }
+        catch (e) {
+            if (e instanceof got_scraping_1.TimeoutError) {
+                this._handleRequestTimeout(session);
+                return undefined;
+            }
+            throw e;
+        }
+    }
+    /**
+     * Sets the cookie header to `gotOptions` based on provided session and request. If some cookies were already set,
+     * the session cookie will be merged with them. User provided cookies on `request` object have precedence.
+     */
+    _applySessionCookie({ request, session }, gotOptions) {
+        const userCookie = request.headers?.Cookie ?? request.headers?.cookie;
+        const sessionCookie = session.getCookieString(request.url);
+        const mergedCookies = (0, core_1.mergeCookies)(request.url, [sessionCookie, userCookie]);
+        // merge cookies from all possible sources
+        if (mergedCookies) {
+            gotOptions.headers ?? (gotOptions.headers = {});
+            gotOptions.headers.Cookie = mergedCookies;
+        }
+    }
+    /**
+     * Encodes and parses response according to the provided content type
+     */
+    async _parseResponse(request, responseStream) {
+        const { statusCode } = responseStream;
+        const { type, charset } = (0, utils_1.parseContentTypeFromResponse)(responseStream);
+        const { response, encoding } = this._encodeResponse(request, responseStream, charset);
+        const contentType = { type, encoding };
+        if (statusCode >= 500) {
+            const body = await (0, utilities_1.readStreamToString)(response, encoding);
+            // Errors are often sent as JSON, so attempt to parse them,
+            // despite Accept header being set to text/html.
+            if (type === APPLICATION_JSON_MIME_TYPE) {
+                const errorResponse = JSON.parse(body);
+                let { message } = errorResponse;
+                if (!message)
+                    message = util_1.default.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
+                throw new Error(`${statusCode} - ${message}`);
+            }
+            // It's not a JSON so it's probably some text. Get the first 100 chars of it.
+            throw new Error(`${statusCode} - Internal Server Error: ${body.substr(0, 100)}`);
+        }
+        else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
+            const dom = await this._parseHtmlToDom(response);
+            return ({ dom, isXml: type.includes('xml'), response, contentType });
+        }
+        else {
+            const body = await (0, utilities_1.concatStreamToBuffer)(response);
+            return { body, response, contentType };
+        }
+    }
+    /**
+     * Combines the provided `requestOptions` with mandatory (non-overridable) values.
+     */
+    _getRequestOptions(request, session, proxyUrl, gotOptions) {
+        const requestOptions = {
+            url: request.url,
+            method: request.method,
+            proxyUrl,
+            timeout: { request: this.requestTimeoutMillis },
+            sessionToken: session,
+            ...gotOptions,
+            headers: { ...request.headers, ...gotOptions?.headers },
+            https: {
+                ...gotOptions?.https,
+                rejectUnauthorized: !this.ignoreSslErrors,
+            },
+            isStream: true,
+        };
+        // TODO this is incorrect, the check for man in the middle needs to be done
+        //   on individual proxy level, not on the `proxyConfiguration` level,
+        //   because users can use normal + MITM proxies in a single configuration.
+        //   Disable SSL verification for MITM proxies
+        if (this.proxyConfiguration && this.proxyConfiguration.isManInTheMiddle) {
+            requestOptions.https = {
+                ...requestOptions.https,
+                rejectUnauthorized: false,
+            };
+        }
+        if (/PATCH|POST|PUT/.test(request.method))
+            requestOptions.body = request.payload;
+        return requestOptions;
+    }
+    _encodeResponse(request, response, encoding) {
+        if (this.forceResponseEncoding) {
+            encoding = this.forceResponseEncoding;
+        }
+        else if (!encoding && this.suggestResponseEncoding) {
+            encoding = this.suggestResponseEncoding;
+        }
+        // Fall back to utf-8 if we still don't have encoding.
+        const utf8 = 'utf8';
+        if (!encoding)
+            return { response, encoding: utf8 };
+        // This means that the encoding is one of Node.js supported
+        // encodings and we don't need to re-encode it.
+        if (Buffer.isEncoding(encoding))
+            return { response, encoding };
+        // Try to re-encode a variety of unsupported encodings to utf-8
+        if (iconv_lite_1.default.encodingExists(encoding)) {
+            const encodeStream = iconv_lite_1.default.encodeStream(utf8);
+            const decodeStream = iconv_lite_1.default.decodeStream(encoding).on('error', (err) => encodeStream.emit('error', err));
+            response.on('error', (err) => decodeStream.emit('error', err));
+            const encodedResponse = response.pipe(decodeStream).pipe(encodeStream);
+            encodedResponse.statusCode = response.statusCode;
+            encodedResponse.headers = response.headers;
+            encodedResponse.url = response.url;
+            return {
+                response: encodedResponse,
+                encoding: utf8,
+            };
+        }
+        throw new Error(`Resource ${request.url} served with unsupported charset/encoding: ${encoding}`);
+    }
+    async _parseHtmlToDom(response) {
+        return new Promise((resolve, reject) => {
+            const domHandler = new htmlparser2_1.DomHandler((err, dom) => {
+                if (err)
+                    reject(err);
+                else
+                    resolve(dom);
+            });
+            const parser = new WritableStream_1.WritableStream(domHandler, { decodeEntities: true });
+            parser.on('error', reject);
+            response
+                .on('error', reject)
+                .pipe(parser);
+        });
+    }
+    /**
+     * Checks and extends supported mime types
+     */
+    _extendSupportedMimeTypes(additionalMimeTypes) {
+        additionalMimeTypes.forEach((mimeType) => {
+            try {
+                const parsedType = content_type_1.default.parse(mimeType);
+                this.supportedMimeTypes.add(parsedType.type);
+            }
+            catch (err) {
+                throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
+            }
+        });
+    }
+    /**
+     * Handles blocked request
+     */
+    _throwOnBlockedRequest(session, statusCode) {
+        const isBlocked = session.retireOnBlockedStatusCodes(statusCode);
+        if (isBlocked) {
+            throw new Error(`Request blocked - received ${statusCode} status code`);
+        }
+    }
+    /**
+     * Handles timeout request
+     */
+    _handleRequestTimeout(session) {
+        session?.markBad();
+        throw new Error(`request timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds.`);
+    }
+    _abortDownloadOfBody(request, response) {
+        const { statusCode } = response;
+        const { type } = (0, utils_1.parseContentTypeFromResponse)(response);
+        if (statusCode === 406) {
+            request.noRetry = true;
+            throw new Error(`Resource ${request.url} is not available in the format requested by the Accept header. Skipping resource.`);
+        }
+        if (!this.supportedMimeTypes.has(type) && statusCode < 500) {
+            request.noRetry = true;
+            throw new Error(`Resource ${request.url} served Content-Type ${type}, `
+                + `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
+        }
+    }
+}
+exports.CheerioCrawler = CheerioCrawler;
+Object.defineProperty(CheerioCrawler, "optionsShape", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: {
+        ...basic_1.BasicCrawler.optionsShape,
+        handlePageFunction: ow_1.default.optional.function,
+        requestTimeoutSecs: ow_1.default.optional.number,
+        ignoreSslErrors: ow_1.default.optional.boolean,
+        additionalMimeTypes: ow_1.default.optional.array.ofType(ow_1.default.string),
+        suggestResponseEncoding: ow_1.default.optional.string,
+        forceResponseEncoding: ow_1.default.optional.string,
+        proxyConfiguration: ow_1.default.optional.object.validate(core_1.validators.proxyConfiguration),
+        prepareRequestFunction: ow_1.default.optional.function,
+        postResponseFunction: ow_1.default.optional.function,
+        persistCookiesPerSession: ow_1.default.optional.boolean,
+        preNavigationHooks: ow_1.default.optional.array,
+        postNavigationHooks: ow_1.default.optional.array,
+    }
+});
+/** @internal */
+async function cheerioCrawlerEnqueueLinks({ options, $, requestQueue, originalRequestUrl, finalRequestUrl }) {
+    if (!$) {
+        throw new Error('Cannot enqueue links because the DOM is not available.');
+    }
+    const baseUrl = (0, core_1.resolveBaseUrl)({
+        enqueueStrategy: options?.strategy,
+        finalRequestUrl,
+        originalRequestUrl,
+        userProvidedBaseUrl: options?.baseUrl,
+    });
+    const urls = extractUrlsFromCheerio($, options?.selector ?? 'a', baseUrl);
+    return (0, core_1.enqueueLinks)({
+        requestQueue,
+        urls,
+        baseUrl,
+        ...options,
+    });
+}
+exports.cheerioCrawlerEnqueueLinks = cheerioCrawlerEnqueueLinks;
+/**
+ * Extracts URLs from a given Cheerio object.
+ * @ignore
+ */
+function extractUrlsFromCheerio($, selector, baseUrl) {
+    return $(selector)
+        .map((_i, el) => $(el).attr('href'))
+        .get()
+        .filter((href) => !!href)
+        .map((href) => {
+        // Throw a meaningful error when only a relative URL would be extracted instead of waiting for the Request to fail later.
+        const isHrefAbsolute = /^[a-z][a-z0-9+.-]*:/.test(href); // Grabbed this in 'is-absolute-url' package.
+        if (!isHrefAbsolute && !baseUrl) {
+            throw new Error(`An extracted URL: ${href} is relative and options.baseUrl is not set. `
+                + 'Use options.baseUrl in enqueueLinks() to automatically resolve relative URLs.');
+        }
+        return baseUrl
+            ? (new URL(href, baseUrl)).href
+            : href;
+    });
+}
+/**
+ * The stream object returned from got does not have the below properties.
+ * At the same time, you can't read data directly from the response stream,
+ * because they won't get emitted unless you also read from the primary
+ * got stream. To be able to work with only one stream, we move the expected props
+ * from the response stream to the got stream.
+ * @internal
+ */
+function addResponsePropertiesToStream(stream) {
+    const properties = [
+        'statusCode', 'statusMessage', 'headers',
+        'complete', 'httpVersion', 'rawHeaders',
+        'rawTrailers', 'trailers', 'url',
+        'request',
+    ];
+    const response = stream.response;
+    response.on('end', () => {
+        // @ts-expect-error
+        Object.assign(stream.rawTrailers, response.rawTrailers);
+        // @ts-expect-error
+        Object.assign(stream.trailers, response.trailers);
+        // @ts-expect-error
+        stream.complete = response.complete;
+    });
+    for (const prop of properties) {
+        if (!(prop in stream)) {
+            // @ts-expect-error
+            stream[prop] = response[prop];
+        }
+    }
+    return stream;
+}
+//# sourceMappingURL=cheerio-crawler.js.map