@crawlee/cheerio 3.0.3-beta.11 → 3.0.3-beta.14

package/internals/cheerio-crawler.js

@@ -2,34 +2,10 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createCheerioRouter = exports.cheerioCrawlerEnqueueLinks = exports.CheerioCrawler = void 0;
 const tslib_1 = require("tslib");
-const timeout_1 = require("@apify/timeout");
-const utilities_1 = require("@apify/utilities");
-const basic_1 = require("@crawlee/basic");
-const core_1 = require("@crawlee/core");
-const utils_1 = require("@crawlee/utils");
+const http_1 = require("@crawlee/http");
 const cheerio = tslib_1.__importStar(require("cheerio"));
-const content_type_1 = tslib_1.__importDefault(require("content-type"));
-const got_scraping_1 = require("got-scraping");
 const htmlparser2_1 = require("htmlparser2");
 const WritableStream_1 = require("htmlparser2/lib/WritableStream");
-const iconv_lite_1 = tslib_1.__importDefault(require("iconv-lite"));
-const ow_1 = tslib_1.__importDefault(require("ow"));
-const util_1 = tslib_1.__importDefault(require("util"));
-/**
- * Default mime types, which CheerioScraper supports.
- */
-const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml', 'application/xml'];
-const APPLICATION_JSON_MIME_TYPE = 'application/json';
-const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
-    desiredConcurrency: 10,
-    snapshotterOptions: {
-        eventLoopSnapshotIntervalSecs: 2,
-        maxBlockedMillis: 100,
-    },
-    systemStatusOptions: {
-        maxEventLoopOverloadedRatio: 0.7,
-    },
-};
 /**
  * Provides a framework for the parallel crawling of web pages using plain HTTP requests and
  * [cheerio](https://www.npmjs.com/package/cheerio) HTML parser.
@@ -71,7 +47,7 @@ const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  * and skips pages with other content types. If you want the crawler to process other content types,
  * use the {@apilink CheerioCrawlerOptions.additionalMimeTypes} constructor option.
  * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
- * For details, see {@apilink CheerioCrawlerOptions.requestHandler}.
+ * For more details, see {@apilink CheerioCrawlerOptions.requestHandler}.
  *
  * New requests are only dispatched when there is enough free CPU and memory available,
  * using the functionality provided by the {@apilink AutoscaledPool} class.
@@ -107,209 +83,30 @@ const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  * ```
  * @category Crawlers
  */
-class CheerioCrawler extends basic_1.BasicCrawler {
+class CheerioCrawler extends http_1.HttpCrawler {
     /**
      * All `CheerioCrawler` parameters are passed via an options object.
      */
-    constructor(options = {}, config = core_1.Configuration.getGlobalConfig()) {
-        (0, ow_1.default)(options, 'CheerioCrawlerOptions', ow_1.default.object.exactShape(CheerioCrawler.optionsShape));
-        const { requestHandler, handlePageFunction, requestHandlerTimeoutSecs = 60, navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, proxyConfiguration, persistCookiesPerSession, preNavigationHooks = [], postNavigationHooks = [], 
-        // Ignored
-        handleRequestFunction, 
-        // BasicCrawler
-        autoscaledPoolOptions = CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS, ...basicCrawlerOptions } = options;
-        super({
-            ...basicCrawlerOptions,
-            requestHandler,
-            autoscaledPoolOptions,
-            // We need to add some time for internal functions to finish,
-            // but not too much so that we would stall the crawler.
-            requestHandlerTimeoutSecs: navigationTimeoutSecs + requestHandlerTimeoutSecs + basic_1.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
-        }, config);
-        Object.defineProperty(this, "config", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: config
-        });
-        /**
-         * A reference to the underlying {@apilink ProxyConfiguration} class that manages the crawler's proxies.
-         * Only available if used by the crawler.
-         */
-        Object.defineProperty(this, "proxyConfiguration", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "userRequestHandlerTimeoutMillis", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "preNavigationHooks", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "postNavigationHooks", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "persistCookiesPerSession", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "navigationTimeoutMillis", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "ignoreSslErrors", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "suggestResponseEncoding", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "forceResponseEncoding", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "supportedMimeTypes", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
+    // eslint-disable-next-line @typescript-eslint/no-useless-constructor
+    constructor(options, config) {
+        super(options, config);
+    }
+    async _parseHTML(response, isXml, crawlingContext) {
+        const dom = await this._parseHtmlToDom(response);
+        const $ = cheerio.load(dom, {
+            xmlMode: isXml,
+            // Recent versions of cheerio use parse5 as the HTML parser/serializer. It's more strict than htmlparser2
+            // and not good for scraping. It also does not have a great streaming interface.
+            // Here we tell cheerio to use htmlparser2 for serialization, otherwise the conflict produces weird errors.
+            _useHtmlParser2: true,
         });
-        /**
-         * @internal wraps public utility for mocking purposes
-         */
-        Object.defineProperty(this, "_requestAsBrowser", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: (options) => {
-                return new Promise((resolve, reject) => {
-                    const stream = (0, got_scraping_1.gotScraping)(options);
-                    stream.on('error', reject);
-                    stream.on('response', () => {
-                        resolve(addResponsePropertiesToStream(stream));
-                    });
-                });
-            }
-        });
-        this._handlePropertyNameChange({
-            newName: 'requestHandler',
-            oldName: 'handlePageFunction',
-            propertyKey: 'requestHandler',
-            newProperty: requestHandler,
-            oldProperty: handlePageFunction,
-            allowUndefined: true,
-        });
-        if (!this.requestHandler) {
-            this.requestHandler = this.router;
-        }
-        // Cookies should be persisted per session only if session pool is used
-        if (!this.useSessionPool && persistCookiesPerSession) {
-            throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
-        }
-        this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
-        if (additionalMimeTypes.length)
-            this._extendSupportedMimeTypes(additionalMimeTypes);
-        if (suggestResponseEncoding && forceResponseEncoding) {
-            this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
-        }
-        this.userRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
-        this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
-        this.ignoreSslErrors = ignoreSslErrors;
-        this.suggestResponseEncoding = suggestResponseEncoding;
-        this.forceResponseEncoding = forceResponseEncoding;
-        this.proxyConfiguration = proxyConfiguration;
-        this.preNavigationHooks = preNavigationHooks;
-        this.postNavigationHooks = [
-            ({ request, response }) => this._abortDownloadOfBody(request, response),
-            ...postNavigationHooks,
-        ];
-        if (this.useSessionPool) {
-            this.persistCookiesPerSession = persistCookiesPerSession ?? true;
-        }
-        else {
-            this.persistCookiesPerSession = false;
-        }
-    }
-    /**
-     * **EXPERIMENTAL**
-     * Function for attaching CrawlerExtensions such as the Unblockers.
-     * @param extension Crawler extension that overrides the crawler configuration.
-     */
-    use(extension) {
-        (0, ow_1.default)(extension, ow_1.default.object.instanceOf(core_1.CrawlerExtension));
-        const extensionOptions = extension.getCrawlerOptions();
-        for (const [key, value] of (0, utils_1.entries)(extensionOptions)) {
-            const isConfigurable = this.hasOwnProperty(key);
-            const originalType = typeof this[key];
-            const extensionType = typeof value; // What if we want to null something? It is really needed?
-            const isSameType = originalType === extensionType || value == null; // fast track for deleting keys
-            const exists = this[key] != null;
-            if (!isConfigurable) { // Test if the property can be configured on the crawler
-                throw new Error(`${extension.name} tries to set property "${key}" that is not configurable on CheerioCrawler instance.`);
-            }
-            if (!isSameType && exists) { // Assuming that extensions will only add up configuration
-                throw new Error(`${extension.name} tries to set property of different type "${extensionType}". "CheerioCrawler.${key}: ${originalType}".`);
-            }
-            this.log.warning(`${extension.name} is overriding "CheerioCrawler.${key}: ${originalType}" with ${value}.`);
-            this[key] = value;
-        }
-    }
-    /**
-     * Wrapper around requestHandler that opens and closes pages etc.
-     */
-    async _runRequestHandler(crawlingContext) {
-        const { request, session } = crawlingContext;
-        if (this.proxyConfiguration) {
-            const sessionId = session ? session.id : undefined;
-            crawlingContext.proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId);
-        }
-        if (!request.skipNavigation) {
-            await this._handleNavigation(crawlingContext);
-            (0, timeout_1.tryCancel)();
-            const { dom, isXml, body, contentType, response } = await this._parseResponse(request, crawlingContext.response);
-            (0, timeout_1.tryCancel)();
-            if (this.useSessionPool) {
-                this._throwOnBlockedRequest(session, response.statusCode);
-            }
-            if (this.persistCookiesPerSession) {
-                session.setCookiesFromResponse(response);
-            }
-            request.loadedUrl = response.url;
-            const $ = dom
-                ? cheerio.load(dom, {
-                    xmlMode: isXml,
-                    // Recent versions of cheerio use parse5 as the HTML parser/serializer. It's more strict than htmlparser2
-                    // and not good for scraping. It also does not have a great streaming interface.
-                    // Here we tell cheerio to use htmlparser2 for serialization, otherwise the conflict produces weird errors.
-                    _useHtmlParser2: true,
-                })
-                : null;
-            crawlingContext.$ = $;
-            crawlingContext.contentType = contentType;
-            crawlingContext.response = response;
-            crawlingContext.enqueueLinks = async (enqueueOptions) => {
+        return {
+            dom,
+            $,
+            get body() {
+                return isXml ? $.xml() : $.html({ decodeEntities: false });
+            },
+            enqueueLinks: async (enqueueOptions) => {
                 return cheerioCrawlerEnqueueLinks({
                     options: enqueueOptions,
                     $,
@@ -317,200 +114,8 @@ class CheerioCrawler extends basic_1.BasicCrawler {
                     originalRequestUrl: crawlingContext.request.url,
                     finalRequestUrl: crawlingContext.request.loadedUrl,
                 });
-            };
-            Object.defineProperty(crawlingContext, 'json', {
-                get() {
-                    if (contentType.type !== APPLICATION_JSON_MIME_TYPE)
-                        return null;
-                    const jsonString = body.toString(contentType.encoding);
-                    return JSON.parse(jsonString);
-                },
-            });
-            Object.defineProperty(crawlingContext, 'body', {
-                get() {
-                    // NOTE: For XML/HTML documents, we don't store the original body and only reconstruct it from Cheerio's DOM.
-                    // This is to save memory for high-concurrency crawls. The downside is that changes
-                    // made to DOM are reflected in the HTML, but we can live with that...
-                    if (dom) {
-                        return isXml ? $.xml() : $.html({ decodeEntities: false });
-                    }
-                    return body;
-                },
-            });
-        }
-        return (0, timeout_1.addTimeoutToPromise)(() => Promise.resolve(this.requestHandler(crawlingContext)), this.userRequestHandlerTimeoutMillis, `requestHandler timed out after ${this.userRequestHandlerTimeoutMillis / 1000} seconds.`);
-    }
-    async _handleNavigation(crawlingContext) {
-        const gotOptions = {};
-        const { request, session } = crawlingContext;
-        const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
-        // Execute pre navigation hooks before applying session pool cookies,
-        // as they may also set cookies in the session
-        await this._executeHooks(this.preNavigationHooks, crawlingContext, gotOptions);
-        (0, timeout_1.tryCancel)();
-        const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
-        this._applyCookies(crawlingContext, gotOptions, preNavigationHooksCookies, postNavigationHooksCookies);
-        const proxyUrl = crawlingContext.proxyInfo?.url;
-        crawlingContext.response = await (0, timeout_1.addTimeoutToPromise)(() => this._requestFunction({ request, session, proxyUrl, gotOptions }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
-        (0, timeout_1.tryCancel)();
-        await this._executeHooks(this.postNavigationHooks, crawlingContext, gotOptions);
-        (0, timeout_1.tryCancel)();
-    }
-    /**
-     * Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks.
-     */
-    _applyCookies({ session, request }, gotOptions, preHookCookies, postHookCookies) {
-        const sessionCookie = session?.getCookieString(request.url) ?? '';
-        let alteredGotOptionsCookies = (gotOptions.headers?.Cookie || gotOptions.headers?.cookie || '');
-        if (gotOptions.headers?.Cookie && gotOptions.headers?.cookie) {
-            const { Cookie: upperCaseHeader, cookie: lowerCaseHeader, } = gotOptions.headers;
-            // eslint-disable-next-line max-len
-            this.log.warning(`Encountered mixed casing for the cookie headers in the got options for request ${request.url} (${request.id}). Their values will be merged`);
-            const sourceCookies = [];
-            if (Array.isArray(lowerCaseHeader)) {
-                sourceCookies.push(...lowerCaseHeader);
-            }
-            else {
-                sourceCookies.push(lowerCaseHeader);
-            }
-            if (Array.isArray(upperCaseHeader)) {
-                sourceCookies.push(...upperCaseHeader);
-            }
-            else {
-                sourceCookies.push(upperCaseHeader);
-            }
-            alteredGotOptionsCookies = (0, core_1.mergeCookies)(request.url, sourceCookies);
-        }
-        const sourceCookies = [
-            sessionCookie,
-            preHookCookies,
-        ];
-        if (Array.isArray(alteredGotOptionsCookies)) {
-            sourceCookies.push(...alteredGotOptionsCookies);
-        }
-        else {
-            sourceCookies.push(alteredGotOptionsCookies);
-        }
-        sourceCookies.push(postHookCookies);
-        const mergedCookie = (0, core_1.mergeCookies)(request.url, sourceCookies);
-        gotOptions.headers ?? (gotOptions.headers = {});
-        Reflect.deleteProperty(gotOptions.headers, 'Cookie');
-        Reflect.deleteProperty(gotOptions.headers, 'cookie');
-        gotOptions.headers.Cookie = mergedCookie;
-    }
-    /**
-     * Function to make the HTTP request. It performs optimizations
-     * on the request such as only downloading the request body if the
-     * received content type matches text/html, application/xml, application/xhtml+xml.
-     */
-    async _requestFunction({ request, session, proxyUrl, gotOptions }) {
-        const opts = this._getRequestOptions(request, session, proxyUrl, gotOptions);
-        try {
-            return await this._requestAsBrowser(opts);
-        }
-        catch (e) {
-            if (e instanceof got_scraping_1.TimeoutError) {
-                this._handleRequestTimeout(session);
-                return undefined;
-            }
-            throw e;
-        }
-    }
-    /**
-     * Encodes and parses response according to the provided content type
-     */
-    async _parseResponse(request, responseStream) {
-        const { statusCode } = responseStream;
-        const { type, charset } = (0, utils_1.parseContentTypeFromResponse)(responseStream);
-        const { response, encoding } = this._encodeResponse(request, responseStream, charset);
-        const contentType = { type, encoding };
-        if (statusCode >= 500) {
-            const body = await (0, utilities_1.readStreamToString)(response, encoding);
-            // Errors are often sent as JSON, so attempt to parse them,
-            // despite Accept header being set to text/html.
-            if (type === APPLICATION_JSON_MIME_TYPE) {
-                const errorResponse = JSON.parse(body);
-                let { message } = errorResponse;
-                if (!message)
-                    message = util_1.default.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
-                throw new Error(`${statusCode} - ${message}`);
-            }
-            // It's not a JSON, so it's probably some text. Get the first 100 chars of it.
-            throw new Error(`${statusCode} - Internal Server Error: ${body.substr(0, 100)}`);
-        }
-        else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
-            const dom = await this._parseHtmlToDom(response);
-            return ({ dom, isXml: type.includes('xml'), response, contentType });
-        }
-        else {
-            const body = await (0, utilities_1.concatStreamToBuffer)(response);
-            return { body, response, contentType };
-        }
-    }
-    /**
-     * Combines the provided `requestOptions` with mandatory (non-overridable) values.
-     */
-    _getRequestOptions(request, session, proxyUrl, gotOptions) {
-        const requestOptions = {
-            url: request.url,
-            method: request.method,
-            proxyUrl,
-            timeout: { request: this.navigationTimeoutMillis },
-            sessionToken: session,
-            ...gotOptions,
-            headers: { ...request.headers, ...gotOptions?.headers },
-            https: {
-                ...gotOptions?.https,
-                rejectUnauthorized: !this.ignoreSslErrors,
             },
-            isStream: true,
         };
-        // Delete any possible lowercased header for cookie as they are merged in _applyCookies under the uppercase Cookie header
-        Reflect.deleteProperty(requestOptions.headers, 'cookie');
-        // TODO this is incorrect, the check for man in the middle needs to be done
-        //   on individual proxy level, not on the `proxyConfiguration` level,
-        //   because users can use normal + MITM proxies in a single configuration.
-        // Disable SSL verification for MITM proxies
-        if (this.proxyConfiguration && this.proxyConfiguration.isManInTheMiddle) {
-            requestOptions.https = {
-                ...requestOptions.https,
-                rejectUnauthorized: false,
-            };
-        }
-        if (/PATCH|POST|PUT/.test(request.method))
-            requestOptions.body = request.payload;
-        return requestOptions;
-    }
-    _encodeResponse(request, response, encoding) {
-        if (this.forceResponseEncoding) {
-            encoding = this.forceResponseEncoding;
-        }
-        else if (!encoding && this.suggestResponseEncoding) {
-            encoding = this.suggestResponseEncoding;
-        }
-        // Fall back to utf-8 if we still don't have encoding.
-        const utf8 = 'utf8';
-        if (!encoding)
-            return { response, encoding: utf8 };
-        // This means that the encoding is one of Node.js supported
-        // encodings and we don't need to re-encode it.
-        if (Buffer.isEncoding(encoding))
-            return { response, encoding };
-        // Try to re-encode a variety of unsupported encodings to utf-8
-        if (iconv_lite_1.default.encodingExists(encoding)) {
-            const encodeStream = iconv_lite_1.default.encodeStream(utf8);
-            const decodeStream = iconv_lite_1.default.decodeStream(encoding).on('error', (err) => encodeStream.emit('error', err));
-            response.on('error', (err) => decodeStream.emit('error', err));
-            const encodedResponse = response.pipe(decodeStream).pipe(encodeStream);
-            encodedResponse.statusCode = response.statusCode;
-            encodedResponse.headers = response.headers;
-            encodedResponse.url = response.url;
-            return {
-                response: encodedResponse,
-                encoding: utf8,
-            };
-        }
-        throw new Error(`Resource ${request.url} served with unsupported charset/encoding: ${encoding}`);
     }
     async _parseHtmlToDom(response) {
         return new Promise((resolve, reject) => {
@@ -527,73 +132,21 @@ class CheerioCrawler extends basic_1.BasicCrawler {
                 .pipe(parser);
         });
     }
-    /**
-     * Checks and extends supported mime types
-     */
-    _extendSupportedMimeTypes(additionalMimeTypes) {
-        additionalMimeTypes.forEach((mimeType) => {
-            try {
-                const parsedType = content_type_1.default.parse(mimeType);
-                this.supportedMimeTypes.add(parsedType.type);
-            }
-            catch (err) {
-                throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
-            }
-        });
-    }
-    /**
-     * Handles timeout request
-     */
-    _handleRequestTimeout(session) {
-        session?.markBad();
-        throw new Error(`request timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds.`);
-    }
-    _abortDownloadOfBody(request, response) {
-        const { statusCode } = response;
-        const { type } = (0, utils_1.parseContentTypeFromResponse)(response);
-        if (statusCode === 406) {
-            request.noRetry = true;
-            throw new Error(`Resource ${request.url} is not available in the format requested by the Accept header. Skipping resource.`);
-        }
-        if (!this.supportedMimeTypes.has(type) && statusCode < 500) {
-            request.noRetry = true;
-            throw new Error(`Resource ${request.url} served Content-Type ${type}, `
-                + `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
-        }
-    }
 }
 exports.CheerioCrawler = CheerioCrawler;
-Object.defineProperty(CheerioCrawler, "optionsShape", {
-    enumerable: true,
-    configurable: true,
-    writable: true,
-    value: {
-        ...basic_1.BasicCrawler.optionsShape,
-        handlePageFunction: ow_1.default.optional.function,
-        navigationTimeoutSecs: ow_1.default.optional.number,
-        ignoreSslErrors: ow_1.default.optional.boolean,
-        additionalMimeTypes: ow_1.default.optional.array.ofType(ow_1.default.string),
-        suggestResponseEncoding: ow_1.default.optional.string,
-        forceResponseEncoding: ow_1.default.optional.string,
-        proxyConfiguration: ow_1.default.optional.object.validate(core_1.validators.proxyConfiguration),
-        persistCookiesPerSession: ow_1.default.optional.boolean,
-        preNavigationHooks: ow_1.default.optional.array,
-        postNavigationHooks: ow_1.default.optional.array,
-    }
-});
 /** @internal */
 async function cheerioCrawlerEnqueueLinks({ options, $, requestQueue, originalRequestUrl, finalRequestUrl }) {
     if (!$) {
         throw new Error('Cannot enqueue links because the DOM is not available.');
     }
-    const baseUrl = (0, core_1.resolveBaseUrlForEnqueueLinksFiltering)({
+    const baseUrl = (0, http_1.resolveBaseUrlForEnqueueLinksFiltering)({
         enqueueStrategy: options?.strategy,
         finalRequestUrl,
         originalRequestUrl,
         userProvidedBaseUrl: options?.baseUrl,
     });
     const urls = extractUrlsFromCheerio($, options?.selector ?? 'a', options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
-    return (0, core_1.enqueueLinks)({
+    return (0, http_1.enqueueLinks)({
         requestQueue,
         urls,
         baseUrl,
@@ -631,38 +184,6 @@ function extractUrlsFromCheerio($, selector, baseUrl) {
     })
         .filter((href) => !!href);
 }
-/**
- * The stream object returned from got does not have the below properties.
- * At the same time, you can't read data directly from the response stream,
- * because they won't get emitted unless you also read from the primary
- * got stream. To be able to work with only one stream, we move the expected props
- * from the response stream to the got stream.
- * @internal
- */
-function addResponsePropertiesToStream(stream) {
-    const properties = [
-        'statusCode', 'statusMessage', 'headers',
-        'complete', 'httpVersion', 'rawHeaders',
-        'rawTrailers', 'trailers', 'url',
-        'request',
-    ];
-    const response = stream.response;
-    response.on('end', () => {
-        // @ts-expect-error
-        Object.assign(stream.rawTrailers, response.rawTrailers);
-        // @ts-expect-error
-        Object.assign(stream.trailers, response.trailers);
-        // @ts-expect-error
-        stream.complete = response.complete;
-    });
-    for (const prop of properties) {
-        if (!(prop in stream)) {
-            // @ts-expect-error
-            stream[prop] = response[prop];
-        }
-    }
-    return stream;
-}
 /**
  * Creates new {@apilink Router} instance that works based on request labels.
  * This instance can then serve as a `requestHandler` of your {@apilink CheerioCrawler}.
@@ -688,7 +209,7 @@ function addResponsePropertiesToStream(stream) {
  * ```
  */
 function createCheerioRouter() {
-    return core_1.Router.create();
+    return http_1.Router.create();
 }
 exports.createCheerioRouter = createCheerioRouter;
 //# sourceMappingURL=cheerio-crawler.js.map

package/internals/cheerio-crawler.js.map

@@ -1 +1 @@
-{"version":3,"file":"cheerio-crawler.js","sourceRoot":"","sources":["../../src/internals/cheerio-crawler.ts"],"names":[],"mappings":";;;;AAAA,4CAAgE;AAChE,gDAA4E;AAE5E,0CAAiF;AAEjF,wCAAwJ;AAGxJ,0CAAuE;AAEvE,yDAAmC;AAEnC,wEAA6C;AAE7C,+CAAyD;AACzD,6CAAyC;AACzC,mEAAgE;AAEhE,oEAA+B;AAC/B,oDAAoB;AACpB,wDAAwB;AAExB;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,yCAAyC,GAA0B;IACrE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AA4PF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4EG;AACH,MAAa,cAAe,SAAQ,oBAAoC;IAiCpE;;OAEG;IACH,YAAY,UAAiC,EAAE,EAAoB,SAAS,oBAAa,CAAC,eAAe,EAAE;QACvG,IAAA,YAAE,EAAC,OAAO,EAAE,uBAAuB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QAExF,MAAM,EACF,cAAc,EACd,kBAAkB,EAElB,yBAAyB,GAAG,EAAE,EAC9B,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,kBAAkB,EAClB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE;QAExB,UAAU;QACV,qBAAqB;QAErB,eAAe;QACf,qBAAqB,GAAG,yCAAyC,EACjE,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,cAAc;YACd,qBAAqB;YACrB,6DAA6D;YAC7D,uDAAuD;YACvD,yBAAyB,EAAE,qBAAqB,GAAG,yBAAyB,GAAG,yCAAiC;SACnH,EAAE,MAAM,CAAC,CAAC;;;;;mBAjCoD;;QAnCnE;;;WAGG;QACH;;;;;WAAwC;QAExC;;;;;WAAkD;QAClD;;;;;WAA4C;QAC5C;;;;;WAA6C;QAC7C;;;;;WAA4C;QAC5C;;;;;WAA0C;QAC1C;;;;;WAAmC;QACnC;;;;;WAA2C;QAC3C;;;;;WAAyC;QACzC;;;;;WAAmD;QA0dnD;;WAEG;QACH;;;;mBAA4B,CAAC,OAAyC,EAAE,EAAE;gBACtE,OAAO,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBACpD,MAAM,MAAM,GAAG,IAAA,0BAAW,EAAC,OAAO,CAAC,CAAC;oBAEpC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;oBAC3B,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE;wBACvB,OAAO,CAAC,6BAA6B,CAAC,MAAM,CAAC,CAAC,CAAC;oBACnD,CAAC,CAAC,CAAC;gBACP,CAAC,CAAC,CAAC;YACP,CAAC;WAAC;QA9aE,IAAI,CAAC,yBAAyB,CAAC;YAC3B,OAAO,EAAE,gBAAgB;YACzB,OAAO,EAAE,oBAAoB;YAC7B,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,cAAc;YAC3B,WAAW,EAAE,kBAAkB;YAC/B,cAAc,EAAE,IAAI;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;YACtB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;SACrC;QAED,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE;YAClD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;SACtG;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE;YAClD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,sGAAsG,CAAC,CAAC;SAC5H;QAED,IAAI,CAAC,+BAA+B,GAAG,yBAAyB,GAAG,IAAI,CAAC;QACxE,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,IAAI,IAAI,CAAC;SACpE;aAAM;YACH,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;SACzC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAAC,SAA2B;QAC3B,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,uBAAgB,CAAC,CAAC,CAAC;QAEtD,MAAM,gBAAgB,GAAG,SAAS,CAAC,iBAAiB,EAAE,CAAC;QAEvD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAA,eAAO,EAAC,gBAAgB,CAAC,EAAE;YAClD,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAChD,MAAM,YAAY,GAAG,OAAO,IAAI,CAAC,GAAiB,CAAC,CAAC;YACpD,MAAM,aAAa,GAAG,OAAO,KAAK,CAAC,CAAC,0DAA0D;YAC9F,MAAM,UAAU,GAAG,YAAY,KAAK,aAAa,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,+BAA+B;YACnG,MAAM,MAAM,GAAG,IAAI,CAAC,GAAiB,CAAC,IAAI,IAAI,CAAC;YAE/C,IAAI,CAAC,cAAc,EAAE,EAAE,wDAAwD;gBAC3E,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,CAAC,IAAI,2BAA2B,GAAG,wDAAwD,CAAC,CAAC;aAC5H;YAED,IAAI,CAAC,UAAU,IAAI,MAAM,EAAE,EAAE,0DAA0D;gBACnF,MAAM,IAAI,KAAK,CACX,GAAG,SAAS,CAAC,IAAI,6CAA6C,aAAa,sBAAsB,GAAG,KAAK,YAAY,IAAI,CAC5H,CAAC;aACL;YAED,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,IAAI,kCAAkC,GAAG,KAAK,YAAY,UAAU,KAAK,GAAG,CAAC,CAAC;YAE5G,IAAI,CAAC,GAAiB,CAAC,GAAG,KAAyB,CAAC;SACvD;IACL,CAAC;IAED;;OAEG;IACgB,KAAK,CAAC,kBAAkB,CAAC,eAAuC;QAC/E,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACnD,eAAe,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;SACrF;QACD,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE;YACzB,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAA,mBAAS,GAAE,CAAC;YAEZ,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,eAAe,CAAC,QAAS,CAAC,CAAC;YAClH,IAAA,mBAAS,GAAE,CAAC;YAEZ,IAAI,IAAI,CAAC,cAAc,EAAE;gBACrB,IAAI,CAAC,sBAAsB,CAAC,OAAQ,EAAE,QAAQ,CAAC,UAAW,CAAC,CAAC;aAC/D;YAED,IAAI,IAAI,CAAC,wBAAwB,EAAE;gBAC/B,OAAQ,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;aAC7C;YAED,OAAO,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC;YAEjC,MAAM,CAAC,GAAG,GAAG;gBACT,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,GAAa,EAAE;oBAC1B,OAAO,EAAE,KAAK;oBACd,yGAAyG;oBACzG,gFAAgF;oBAChF,2GAA2G;oBAC3G,eAAe,EAAE,IAAI;iBACN,CAAC;gBACpB,CAAC,CAAC,IAAI,CAAC;YAEX,eAAe,CAAC,CAAC,GAAG,CAAE,CAAC;YACvB,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;YAC1C,eAAe,CAAC,QAAQ,GAAG,QAAQ,CAAC;YACpC,eAAe,CAAC,YAAY,GAAG,KAAK,EAAE,cAAc,EAAE,EAAE;gBACpD,OAAO,0BAA0B,CAAC;oBAC9B,OAAO,EAAE,cAAc;oBACvB,CAAC;oBACD,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;oBAC1C,kBAAkB,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG;oBAC/C,eAAe,EAAE,eAAe,CAAC,OAAO,CAAC,SAAS;iBACrD,CAAC,CAAC;YACP,CAAC,CAAC;YAEF,MAAM,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE;gBAC3C,GAAG;oBACC,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;wBAAE,OAAO,IAAI,CAAC;oBACjE,MAAM,UAAU,GAAG,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;oBACxD,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBAClC,CAAC;aACJ,CAAC,CAAC;YAEH,MAAM,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE;gBAC3C,GAAG;oBACC,6GAA6G;oBAC7G,mFAAmF;oBACnF,sEAAsE;oBACtE,IAAI,GAAG,EAAE;wBACL,OAAO,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;qBAChE;oBACD,OAAO,IAAI,CAAC;gBAChB,CAAC;aACJ,CAAC,CAAC;SACN;QAED,OAAO,IAAA,6BAAmB,EACtB,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC,EAC3D,IAAI,CAAC,+BAA+B,EACpC,kCAAkC,IAAI,CAAC,+BAA+B,GAAG,IAAI,WAAW,CAC3F,CAAC;IACN,CAAC;IAES,KAAK,CAAC,iBAAiB,CAAC,eAAuC;QACrE,MAAM,UAAU,GAAG,EAAiB,CAAC;QACrC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAC7C,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE5E,qEAAqE;QACrE,8CAA8C;QAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAC/E,IAAA,mBAAS,GAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE7E,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,UAAU,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEvG,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,eAAe,CAAC,QAAQ,GAAG,MAAM,IAAA,6BAAmB,EAChD,GAAG,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,EACvE,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,IAAA,mBAAS,GAAE,CAAC;QAEZ,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAChF,IAAA,mBAAS,GAAE,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,EAAE,OAAO,EAAE,OAAO,EAAmB,EAAE,UAAuB,EAAE,cAAsB,EAAE,eAAuB;QACjI,MAAM,aAAa,GAAG,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAClE,IAAI,wBAAwB,GAAG,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC;QAEhG,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE;YAC1D,MAAM,EACF,MAAM,EAAE,eAAe,EACvB,MAAM,EAAE,eAAe,GAC1B,GAAG,UAAU,CAAC,OAAO,CAAC;YAEvB,mCAAmC;YACnC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,kFAAkF,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,gCAAgC,CAAC,CAAC;YAE/J,MAAM,aAAa,GAAG,EAAE,CAAC;YAEzB,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE;gBAChC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;aAC1C;iBAAM;gBACH,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;aACvC;YAED,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE;gBAChC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;aAC1C;iBAAM;gBACH,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;aACvC;YAED,wBAAwB,GAAG,IAAA,mBAAY,EAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;SACvE;QAED,MAAM,aAAa,GAAG;YAClB,aAAa;YACb,cAAc;SACjB,CAAC;QAEF,IAAI,KAAK,CAAC,OAAO,CAAC,wBAAwB,CAAC,EAAE;YACzC,aAAa,CAAC,IAAI,CAAC,GAAG,wBAAwB,CAAC,CAAC;SACnD;aAAM;YACH,aAAa,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;SAChD;QAED,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpC,MAAM,YAAY,GAAG,IAAA,mBAAY,EAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAE9D,UAAU,CAAC,OAAO,KAAlB,UAAU,CAAC,OAAO,GAAK,EAAE,EAAC;QAC1B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACrD,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACrD,UAAU,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAA0B;QAC/F,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE7E,IAAI;YACA,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;SAC7C;QAAC,OAAO,CAAC,EAAE;YACR,IAAI,CAAC,YAAY,2BAAY,EAAE;gBAC3B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,SAAuC,CAAC;aAClD;YAED,MAAM,CAAC,CAAC;SACX;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAgB,EAAE,cAA+B;QAC5E,MAAM,EAAE,UAAU,EAAE,GAAG,cAAc,CAAC;QACtC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAA,oCAA4B,EAAC,cAAc,CAAC,CAAC;QACvE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QACtF,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,UAAW,IAAI,GAAG,EAAE;YACpB,MAAM,IAAI,GAAG,MAAM,IAAA,8BAAkB,EAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAE1D,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE;gBACrC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,cAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,MAAM,OAAO,EAAE,CAAC,CAAC;aACjD;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,6BAA6B,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;SACpF;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YAC/C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YACjD,OAAO,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;SACxE;aAAM;YACH,MAAM,IAAI,GAAG,MAAM,IAAA,gCAAoB,EAAC,QAAQ,CAAC,CAAC;YAClD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;SAC1C;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAgB,EAAE,OAAiB,EAAE,QAAiB,EAAE,UAAwB;QACzG,MAAM,cAAc,GAAqC;YACrD,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAgB;YAChC,QAAQ;YACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE;YAClD,YAAY,EAAE,OAAO;YACrB,GAAG,UAAU;YACb,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE;YACvD,KAAK,EAAE;gBACH,GAAG,UAAU,EAAE,KAAK;gBACpB,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,QAAQ,EAAE,IAAI;SACjB,CAAC;QAEF,yHAAyH;QACzH,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC,OAAQ,EAAE,QAAQ,CAAC,CAAC;QAE1D,2EAA2E;QAC3E,sEAAsE;QACtE,2EAA2E;QAC3E,4CAA4C;QAC5C,IAAI,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;YACrE,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;SACL;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC;QAEjF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CAAC,OAAgB,EAAE,QAAyB,EAAE,QAAwB;QAI3F,IAAI,IAAI,CAAC,qBAAqB,EAAE;YAC5B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;SAC3D;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE;YAClD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;SAC7D;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,oBAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE;YAChC,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACtE,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,YAAY,CAIpE,CAAC;YACF,eAAe,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;YACjD,eAAe,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;YAC3C,eAAe,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;YACnC,OAAO;gBACH,QAAQ,EAAE,eAAsB;gBAChC,QAAQ,EAAE,IAAI;aACjB,CAAC;SACL;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAES,KAAK,CAAC,eAAe,CAAC,QAAyB;QACrD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,UAAU,GAAG,IAAI,wBAAU,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;gBAC3C,IAAI,GAAG;oBAAE,MAAM,CAAC,GAAG,CAAC,CAAC;;oBAChB,OAAO,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,+BAAc,CAAC,UAAU,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;YACxE,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,QAAQ;iBACH,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;iBACnB,IAAI,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,mBAAmB,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;YACrC,IAAI;gBACA,MAAM,UAAU,GAAG,sBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;aAChD;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;aAC9F;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,EAAE,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,2BAA2B,GAAG,IAAI,WAAW,CAAC,CAAC;IACnG,CAAC;IAEO,oBAAoB,CAAC,OAAgB,EAAE,QAAyB;QACpE,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAC;QAChC,MAAM,EAAE,IAAI,EAAE,GAAG,IAAA,oCAA4B,EAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,UAAU,KAAK,GAAG,EAAE;YACpB,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,oFAAoF,CAAC,CAAC;SAChI;QAED,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,UAAW,GAAG,GAAG,EAAE;YACzD,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;kBACjE,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;SACvG;IACL,CAAC;;AAveL,wCAsfC;AAreG;;;;WAAyC;QACrC,GAAG,oBAAY,CAAC,YAAY;QAC5B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAExC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAU,CAAC,kBAAkB,CAAC;QAC9E,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;KACzC;GAAC;AAieN,gBAAgB;AACT,KAAK,UAAU,0BAA0B,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,kBAAkB,EAAE,eAAe,EAA+B;IAC3I,IAAI,CAAC,CAAC,EAAE;QACJ,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;KAC7E;IAED,MAAM,OAAO,GAAG,IAAA,6CAAsC,EAAC;QACnD,eAAe,EAAE,OAAO,EAAE,QAAQ;QAClC,eAAe;QACf,kBAAkB;QAClB,mBAAmB,EAAE,OAAO,EAAE,OAAO;KACxC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,sBAAsB,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,IAAI,GAAG,EAAE,OAAO,EAAE,OAAO,IAAI,eAAe,IAAI,kBAAkB,CAAC,CAAC;IAE5H,OAAO,IAAA,mBAAY,EAAC;QAChB,YAAY;QACZ,IAAI;QACJ,OAAO;QACP,GAAG,OAAO;KACb,CAAC,CAAC;AACP,CAAC;AApBD,gEAoBC;AASD;;;GAGG;AACH,SAAS,sBAAsB,CAAC,CAAc,EAAE,QAAgB,EAAE,OAAgB;IAC9E,OAAO,CAAC,CAAC,QAAQ,CAAC;SACb,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnC,GAAG,EAAE;SACL,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SACxB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE;YAC7B,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,+CAA+C;kBAClF,+EAA+E,CAAC,CAAC;SAC1F;QACD,MAAM,WAAW,GAAG,GAAG,EAAE;YACrB,IAAI;gBACA,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;aACxC;YAAC,MAAM;gBACJ,OAAO,SAAS,CAAC;aACpB;QACL,CAAC,CAAC;QACF,OAAO,OAAO;YACV,CAAC,CAAC,WAAW,EAAE;YACf,CAAC,CAAC,IAAI,CAAC;IACf,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAa,CAAC;AAC9C,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,6BAA6B,CAAC,MAAkB;IACrD,MAAM,UAAU,GAAG;QACf,YAAY,EAAE,eAAe,EAAE,SAAS;QACxC,UAAU,EAAE,aAAa,EAAE,YAAY;QACvC,aAAa,EAAE,UAAU,EAAE,KAAK;QAChC,SAAS;KACZ,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;IAElC,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACpB,mBAAmB;QACnB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;QACxD,mBAAmB;QACnB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAElD,mBAAmB;QACnB,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE;QAC3B,IAAI,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE;YACnB,mBAAmB;YACnB,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,IAA6B,CAAC,CAAC;SAC1D;KACJ;IAED,OAAO,MAAoC,CAAC;AAChD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAgB,mBAAmB;IAC/B,OAAO,aAAM,CAAC,MAAM,EAAW,CAAC;AACpC,CAAC;AAFD,kDAEC"}
+{"version":3,"file":"cheerio-crawler.js","sourceRoot":"","sources":["../../src/internals/cheerio-crawler.ts"],"names":[],"mappings":";;;;AAUA,wCAA0G;AAG1G,yDAAmC;AACnC,6CAAyC;AACzC,mEAAgE;AAqChE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4EG;AACH,MAAa,cAAe,SAAQ,kBAAmC;IACnE;;OAEG;IACH,qEAAqE;IACrE,YAAY,OAA+B,EAAE,MAAsB;QAC/D,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAEkB,KAAK,CAAC,UAAU,CAAC,QAAyB,EAAE,KAAc,EAAE,eAAuC;QAClH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QAEjD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAa,EAAE;YAClC,OAAO,EAAE,KAAK;YACd,yGAAyG;YACzG,gFAAgF;YAChF,2GAA2G;YAC3G,eAAe,EAAE,IAAI;SACN,CAAC,CAAC;QAErB,OAAO;YACH,GAAG;YACH,CAAC;YACD,IAAI,IAAI;gBACJ,OAAO,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,YAAY,EAAE,KAAK,EAAE,cAAkD,EAAE,EAAE;gBACvE,OAAO,0BAA0B,CAAC;oBAC9B,OAAO,EAAE,cAAc;oBACvB,CAAC;oBACD,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;oBAC1C,kBAAkB,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG;oBAC/C,eAAe,EAAE,eAAe,CAAC,OAAO,CAAC,SAAS;iBACrD,CAAC,CAAC;YACP,CAAC;SACJ,CAAC;IACN,CAAC;IAES,KAAK,CAAC,eAAe,CAAC,QAAyB;QACrD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,UAAU,GAAG,IAAI,wBAAU,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;gBAC3C,IAAI,GAAG;oBAAE,MAAM,CAAC,GAAG,CAAC,CAAC;;oBAChB,OAAO,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,+BAAc,CAAC,UAAU,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;YACxE,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,QAAQ;iBACH,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;iBACnB,IAAI,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;IACP,CAAC;CACJ;AAnDD,wCAmDC;AAUD,gBAAgB;AACT,KAAK,UAAU,0BAA0B,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,kBAAkB,EAAE,eAAe,EAA+B;IAC3I,IAAI,CAAC,CAAC,EAAE;QACJ,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;KAC7E;IAED,MAAM,OAAO,GAAG,IAAA,6CAAsC,EAAC;QACnD,eAAe,EAAE,OAAO,EAAE,QAAQ;QAClC,eAAe;QACf,kBAAkB;QAClB,mBAAmB,EAAE,OAAO,EAAE,OAAO;KACxC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,sBAAsB,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,IAAI,GAAG,EAAE,OAAO,EAAE,OAAO,IAAI,eAAe,IAAI,kBAAkB,CAAC,CAAC;IAE5H,OAAO,IAAA,mBAAY,EAAC;QAChB,YAAY;QACZ,IAAI;QACJ,OAAO;QACP,GAAG,OAAO;KACb,CAAC,CAAC;AACP,CAAC;AApBD,gEAoBC;AAED;;;GAGG;AACH,SAAS,sBAAsB,CAAC,CAAqB,EAAE,QAAgB,EAAE,OAAgB;IACrF,OAAO,CAAC,CAAC,QAAQ,CAAC;SACb,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnC,GAAG,EAAE;SACL,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SACxB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE;YAC7B,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,+CAA+C;kBAClF,+EAA+E,CAAC,CAAC;SAC1F;QACD,MAAM,WAAW,GAAG,GAAG,EAAE;YACrB,IAAI;gBACA,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;aACxC;YAAC,MAAM;gBACJ,OAAO,SAAS,CAAC;aACpB;QACL,CAAC,CAAC;QACF,OAAO,OAAO;YACV,CAAC,CAAC,WAAW,EAAE;YACf,CAAC,CAAC,IAAI,CAAC;IACf,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAa,CAAC;AAC9C,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAgB,mBAAmB;IAC/B,OAAO,aAAM,CAAC,MAAM,EAAW,CAAC;AACpC,CAAC;AAFD,kDAEC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/cheerio",
-    "version": "3.0.3-beta.11",
+    "version": "3.0.3-beta.14",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -53,14 +53,9 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/timeout": "^0.3.0",
-        "@apify/utilities": "^2.0.0",
-        "@crawlee/basic": "^3.0.3-beta.11",
-        "@types/content-type": "^1.1.5",
+        "@crawlee/http": "^3.0.3-beta.14",
+        "@crawlee/types": "^3.0.3-beta.14",
         "cheerio": "1.0.0-rc.12",
-        "content-type": "^1.0.4",
-        "htmlparser2": "^8.0.1",
-        "iconv-lite": "^0.6.3",
-        "ow": "^0.28.1"
+        "htmlparser2": "^8.0.1"
     }
 }