@crawlee/cheerio 3.0.3-beta.8 → 3.0.4-beta.0

package/internals/cheerio-crawler.js

@@ -2,34 +2,10 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createCheerioRouter = exports.cheerioCrawlerEnqueueLinks = exports.CheerioCrawler = void 0;
 const tslib_1 = require("tslib");
-const timeout_1 = require("@apify/timeout");
-const utilities_1 = require("@apify/utilities");
-const basic_1 = require("@crawlee/basic");
-const core_1 = require("@crawlee/core");
-const utils_1 = require("@crawlee/utils");
+const http_1 = require("@crawlee/http");
 const cheerio = tslib_1.__importStar(require("cheerio"));
-const content_type_1 = tslib_1.__importDefault(require("content-type"));
-const got_scraping_1 = require("got-scraping");
 const htmlparser2_1 = require("htmlparser2");
 const WritableStream_1 = require("htmlparser2/lib/WritableStream");
-const iconv_lite_1 = tslib_1.__importDefault(require("iconv-lite"));
-const ow_1 = tslib_1.__importDefault(require("ow"));
-const util_1 = tslib_1.__importDefault(require("util"));
-/**
- * Default mime types, which CheerioScraper supports.
- */
-const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml', 'application/xml'];
-const APPLICATION_JSON_MIME_TYPE = 'application/json';
-const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
-    desiredConcurrency: 10,
-    snapshotterOptions: {
-        eventLoopSnapshotIntervalSecs: 2,
-        maxBlockedMillis: 100,
-    },
-    systemStatusOptions: {
-        maxEventLoopOverloadedRatio: 0.7,
-    },
-};
 /**
  * Provides a framework for the parallel crawling of web pages using plain HTTP requests and
  * [cheerio](https://www.npmjs.com/package/cheerio) HTML parser.
@@ -38,23 +14,23 @@ const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  *
  * Since `CheerioCrawler` uses raw HTTP requests to download web pages,
  * it is very fast and efficient on data bandwidth. However, if the target website requires JavaScript
- * to display the content, you might need to use {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead,
+ * to display the content, you might need to use {@apilink PuppeteerCrawler} or {@apilink PlaywrightCrawler} instead,
  * because it loads the pages using full-featured headless Chrome browser.
  *
  * `CheerioCrawler` downloads each URL using a plain HTTP request,
  * parses the HTML content using [Cheerio](https://www.npmjs.com/package/cheerio)
- * and then invokes the user-provided {@link CheerioCrawlerOptions.requestHandler} to extract page data
+ * and then invokes the user-provided {@apilink CheerioCrawlerOptions.requestHandler} to extract page data
  * using a [jQuery](https://jquery.com/)-like interface to the parsed HTML DOM.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link CheerioCrawlerOptions.requestList}
- * or {@link CheerioCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@apilink Request} objects that are fed from
+ * {@apilink RequestList} or {@apilink RequestQueue} instances provided by the {@apilink CheerioCrawlerOptions.requestList}
+ * or {@apilink CheerioCrawlerOptions.requestQueue} constructor options, respectively.
  *
- * If both {@link CheerioCrawlerOptions.requestList} and {@link CheerioCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * If both {@apilink CheerioCrawlerOptions.requestList} and {@apilink CheerioCrawlerOptions.requestQueue} are used,
+ * the instance first processes URLs from the {@apilink RequestList} and automatically enqueues all of them
+ * to {@apilink RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
  *
- * The crawler finishes when there are no more {@link Request} objects to crawl.
+ * The crawler finishes when there are no more {@apilink Request} objects to crawl.
  *
  * We can use the `preNavigationHooks` to adjust `gotOptions`:
  *
@@ -69,28 +45,20 @@ const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  * By default, `CheerioCrawler` only processes web pages with the `text/html`
  * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),
  * and skips pages with other content types. If you want the crawler to process other content types,
- * use the {@link CheerioCrawlerOptions.additionalMimeTypes} constructor option.
+ * use the {@apilink CheerioCrawlerOptions.additionalMimeTypes} constructor option.
  * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
- * For details, see {@link CheerioCrawlerOptions.requestHandler}.
+ * For more details, see {@apilink CheerioCrawlerOptions.requestHandler}.
  *
  * New requests are only dispatched when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
+ * using the functionality provided by the {@apilink AutoscaledPool} class.
+ * All {@apilink AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
  * parameter of the `CheerioCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPool} options are available directly in the `CheerioCrawler` constructor.
+ * {@apilink AutoscaledPool} options are available directly in the `CheerioCrawler` constructor.
  *
  * **Example usage:**
  *
  * ```javascript
- * // Prepare a list of URLs to crawl
- * const requestList = await RequestList.open(null, [
- *     { url: 'http://www.example.com/page-1' },
- *     { url: 'http://www.example.com/page-2' },
- * ]);
- *
- * // Crawl the URLs
  * const crawler = new CheerioCrawler({
- *     requestList,
  *     async requestHandler({ request, response, body, contentType, $ }) {
  *         const data = [];
  *
@@ -108,213 +76,37 @@ const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  *     },
  * });
  *
- * await crawler.run();
+ * await crawler.run([
+ *     'http://www.example.com/page-1',
+ *     'http://www.example.com/page-2',
+ * ]);
  * ```
  * @category Crawlers
  */
-class CheerioCrawler extends basic_1.BasicCrawler {
+class CheerioCrawler extends http_1.HttpCrawler {
     /**
      * All `CheerioCrawler` parameters are passed via an options object.
      */
-    constructor(options = {}, config = core_1.Configuration.getGlobalConfig()) {
-        (0, ow_1.default)(options, 'CheerioCrawlerOptions', ow_1.default.object.exactShape(CheerioCrawler.optionsShape));
-        const { requestHandler, handlePageFunction, requestHandlerTimeoutSecs = 60, navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, proxyConfiguration, persistCookiesPerSession, preNavigationHooks = [], postNavigationHooks = [], 
-        // Ignored
-        handleRequestFunction, 
-        // BasicCrawler
-        autoscaledPoolOptions = CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS, ...basicCrawlerOptions } = options;
-        super({
-            ...basicCrawlerOptions,
-            requestHandler,
-            autoscaledPoolOptions,
-            // We need to add some time for internal functions to finish,
-            // but not too much so that we would stall the crawler.
-            requestHandlerTimeoutSecs: navigationTimeoutSecs + requestHandlerTimeoutSecs + basic_1.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
-        }, config);
-        Object.defineProperty(this, "config", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: config
-        });
-        /**
-         * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
-         * Only available if used by the crawler.
-         */
-        Object.defineProperty(this, "proxyConfiguration", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "userRequestHandlerTimeoutMillis", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "preNavigationHooks", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "postNavigationHooks", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "persistCookiesPerSession", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "navigationTimeoutMillis", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
+    // eslint-disable-next-line @typescript-eslint/no-useless-constructor
+    constructor(options, config) {
+        super(options, config);
+    }
+    async _parseHTML(response, isXml, crawlingContext) {
+        const dom = await this._parseHtmlToDom(response);
+        const $ = cheerio.load(dom, {
+            xmlMode: isXml,
+            // Recent versions of cheerio use parse5 as the HTML parser/serializer. It's more strict than htmlparser2
+            // and not good for scraping. It also does not have a great streaming interface.
+            // Here we tell cheerio to use htmlparser2 for serialization, otherwise the conflict produces weird errors.
+            _useHtmlParser2: true,
         });
-        Object.defineProperty(this, "ignoreSslErrors", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "suggestResponseEncoding", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "forceResponseEncoding", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "supportedMimeTypes", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        /**
-         * @internal wraps public utility for mocking purposes
-         */
-        Object.defineProperty(this, "_requestAsBrowser", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: (options) => {
-                return new Promise((resolve, reject) => {
-                    const stream = (0, got_scraping_1.gotScraping)(options);
-                    stream.on('error', reject);
-                    stream.on('response', () => {
-                        resolve(addResponsePropertiesToStream(stream));
-                    });
-                });
-            }
-        });
-        this._handlePropertyNameChange({
-            newName: 'requestHandler',
-            oldName: 'handlePageFunction',
-            propertyKey: 'requestHandler',
-            newProperty: requestHandler,
-            oldProperty: handlePageFunction,
-            allowUndefined: true,
-        });
-        if (!this.requestHandler) {
-            this.requestHandler = this.router;
-        }
-        // Cookies should be persisted per session only if session pool is used
-        if (!this.useSessionPool && persistCookiesPerSession) {
-            throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
-        }
-        this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
-        if (additionalMimeTypes.length)
-            this._extendSupportedMimeTypes(additionalMimeTypes);
-        if (suggestResponseEncoding && forceResponseEncoding) {
-            this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
-        }
-        this.userRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
-        this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
-        this.ignoreSslErrors = ignoreSslErrors;
-        this.suggestResponseEncoding = suggestResponseEncoding;
-        this.forceResponseEncoding = forceResponseEncoding;
-        this.proxyConfiguration = proxyConfiguration;
-        this.preNavigationHooks = preNavigationHooks;
-        this.postNavigationHooks = [
-            ({ request, response }) => this._abortDownloadOfBody(request, response),
-            ...postNavigationHooks,
-        ];
-        if (this.useSessionPool) {
-            this.persistCookiesPerSession = persistCookiesPerSession ?? true;
-        }
-        else {
-            this.persistCookiesPerSession = false;
-        }
-    }
-    /**
-     * **EXPERIMENTAL**
-     * Function for attaching CrawlerExtensions such as the Unblockers.
-     * @param extension Crawler extension that overrides the crawler configuration.
-     */
-    use(extension) {
-        (0, ow_1.default)(extension, ow_1.default.object.instanceOf(core_1.CrawlerExtension));
-        const extensionOptions = extension.getCrawlerOptions();
-        for (const [key, value] of (0, utils_1.entries)(extensionOptions)) {
-            const isConfigurable = this.hasOwnProperty(key);
-            const originalType = typeof this[key];
-            const extensionType = typeof value; // What if we want to null something? It is really needed?
-            const isSameType = originalType === extensionType || value == null; // fast track for deleting keys
-            const exists = this[key] != null;
-            if (!isConfigurable) { // Test if the property can be configured on the crawler
-                throw new Error(`${extension.name} tries to set property "${key}" that is not configurable on CheerioCrawler instance.`);
-            }
-            if (!isSameType && exists) { // Assuming that extensions will only add up configuration
-                throw new Error(`${extension.name} tries to set property of different type "${extensionType}". "CheerioCrawler.${key}: ${originalType}".`);
-            }
-            this.log.warning(`${extension.name} is overriding "CheerioCrawler.${key}: ${originalType}" with ${value}.`);
-            this[key] = value;
-        }
-    }
-    /**
-     * Wrapper around requestHandler that opens and closes pages etc.
-     */
-    async _runRequestHandler(crawlingContext) {
-        const { request, session } = crawlingContext;
-        if (this.proxyConfiguration) {
-            const sessionId = session ? session.id : undefined;
-            crawlingContext.proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId);
-        }
-        if (!request.skipNavigation) {
-            await this._handleNavigation(crawlingContext);
-            (0, timeout_1.tryCancel)();
-            const { dom, isXml, body, contentType, response } = await this._parseResponse(request, crawlingContext.response);
-            (0, timeout_1.tryCancel)();
-            if (this.useSessionPool) {
-                this._throwOnBlockedRequest(session, response.statusCode);
-            }
-            if (this.persistCookiesPerSession) {
-                session.setCookiesFromResponse(response);
-            }
-            request.loadedUrl = response.url;
-            const $ = dom
-                ? cheerio.load(dom, {
-                    xmlMode: isXml,
-                    // Recent versions of cheerio use parse5 as the HTML parser/serializer. It's more strict than htmlparser2
-                    // and not good for scraping. It also does not have a great streaming interface.
-                    // Here we tell cheerio to use htmlparser2 for serialization, otherwise the conflict produces weird errors.
-                    _useHtmlParser2: true,
-                })
-                : null;
-            crawlingContext.$ = $;
-            crawlingContext.contentType = contentType;
-            crawlingContext.response = response;
-            crawlingContext.enqueueLinks = async (enqueueOptions) => {
+        return {
+            dom,
+            $,
+            get body() {
+                return isXml ? $.xml() : $.html({ decodeEntities: false });
+            },
+            enqueueLinks: async (enqueueOptions) => {
                 return cheerioCrawlerEnqueueLinks({
                     options: enqueueOptions,
                     $,
@@ -322,200 +114,8 @@ class CheerioCrawler extends basic_1.BasicCrawler {
                     originalRequestUrl: crawlingContext.request.url,
                     finalRequestUrl: crawlingContext.request.loadedUrl,
                 });
-            };
-            Object.defineProperty(crawlingContext, 'json', {
-                get() {
-                    if (contentType.type !== APPLICATION_JSON_MIME_TYPE)
-                        return null;
-                    const jsonString = body.toString(contentType.encoding);
-                    return JSON.parse(jsonString);
-                },
-            });
-            Object.defineProperty(crawlingContext, 'body', {
-                get() {
-                    // NOTE: For XML/HTML documents, we don't store the original body and only reconstruct it from Cheerio's DOM.
-                    // This is to save memory for high-concurrency crawls. The downside is that changes
-                    // made to DOM are reflected in the HTML, but we can live with that...
-                    if (dom) {
-                        return isXml ? $.xml() : $.html({ decodeEntities: false });
-                    }
-                    return body;
-                },
-            });
-        }
-        return (0, timeout_1.addTimeoutToPromise)(() => Promise.resolve(this.requestHandler(crawlingContext)), this.userRequestHandlerTimeoutMillis, `requestHandler timed out after ${this.userRequestHandlerTimeoutMillis / 1000} seconds.`);
-    }
-    async _handleNavigation(crawlingContext) {
-        const gotOptions = {};
-        const { request, session } = crawlingContext;
-        const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
-        // Execute pre navigation hooks before applying session pool cookies,
-        // as they may also set cookies in the session
-        await this._executeHooks(this.preNavigationHooks, crawlingContext, gotOptions);
-        (0, timeout_1.tryCancel)();
-        const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
-        this._applyCookies(crawlingContext, gotOptions, preNavigationHooksCookies, postNavigationHooksCookies);
-        const proxyUrl = crawlingContext.proxyInfo?.url;
-        crawlingContext.response = await (0, timeout_1.addTimeoutToPromise)(() => this._requestFunction({ request, session, proxyUrl, gotOptions }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
-        (0, timeout_1.tryCancel)();
-        await this._executeHooks(this.postNavigationHooks, crawlingContext, gotOptions);
-        (0, timeout_1.tryCancel)();
-    }
-    /**
-     * Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks.
-     */
-    _applyCookies({ session, request }, gotOptions, preHookCookies, postHookCookies) {
-        const sessionCookie = session?.getCookieString(request.url) ?? '';
-        let alteredGotOptionsCookies = (gotOptions.headers?.Cookie || gotOptions.headers?.cookie || '');
-        if (gotOptions.headers?.Cookie && gotOptions.headers?.cookie) {
-            const { Cookie: upperCaseHeader, cookie: lowerCaseHeader, } = gotOptions.headers;
-            // eslint-disable-next-line max-len
-            this.log.warning(`Encountered mixed casing for the cookie headers in the got options for request ${request.url} (${request.id}). Their values will be merged`);
-            const sourceCookies = [];
-            if (Array.isArray(lowerCaseHeader)) {
-                sourceCookies.push(...lowerCaseHeader);
-            }
-            else {
-                sourceCookies.push(lowerCaseHeader);
-            }
-            if (Array.isArray(upperCaseHeader)) {
-                sourceCookies.push(...upperCaseHeader);
-            }
-            else {
-                sourceCookies.push(upperCaseHeader);
-            }
-            alteredGotOptionsCookies = (0, core_1.mergeCookies)(request.url, sourceCookies);
-        }
-        const sourceCookies = [
-            sessionCookie,
-            preHookCookies,
-        ];
-        if (Array.isArray(alteredGotOptionsCookies)) {
-            sourceCookies.push(...alteredGotOptionsCookies);
-        }
-        else {
-            sourceCookies.push(alteredGotOptionsCookies);
-        }
-        sourceCookies.push(postHookCookies);
-        const mergedCookie = (0, core_1.mergeCookies)(request.url, sourceCookies);
-        gotOptions.headers ?? (gotOptions.headers = {});
-        Reflect.deleteProperty(gotOptions.headers, 'Cookie');
-        Reflect.deleteProperty(gotOptions.headers, 'cookie');
-        gotOptions.headers.Cookie = mergedCookie;
-    }
-    /**
-     * Function to make the HTTP request. It performs optimizations
-     * on the request such as only downloading the request body if the
-     * received content type matches text/html, application/xml, application/xhtml+xml.
-     */
-    async _requestFunction({ request, session, proxyUrl, gotOptions }) {
-        const opts = this._getRequestOptions(request, session, proxyUrl, gotOptions);
-        try {
-            return await this._requestAsBrowser(opts);
-        }
-        catch (e) {
-            if (e instanceof got_scraping_1.TimeoutError) {
-                this._handleRequestTimeout(session);
-                return undefined;
-            }
-            throw e;
-        }
-    }
-    /**
-     * Encodes and parses response according to the provided content type
-     */
-    async _parseResponse(request, responseStream) {
-        const { statusCode } = responseStream;
-        const { type, charset } = (0, utils_1.parseContentTypeFromResponse)(responseStream);
-        const { response, encoding } = this._encodeResponse(request, responseStream, charset);
-        const contentType = { type, encoding };
-        if (statusCode >= 500) {
-            const body = await (0, utilities_1.readStreamToString)(response, encoding);
-            // Errors are often sent as JSON, so attempt to parse them,
-            // despite Accept header being set to text/html.
-            if (type === APPLICATION_JSON_MIME_TYPE) {
-                const errorResponse = JSON.parse(body);
-                let { message } = errorResponse;
-                if (!message)
-                    message = util_1.default.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
-                throw new Error(`${statusCode} - ${message}`);
-            }
-            // It's not a JSON, so it's probably some text. Get the first 100 chars of it.
-            throw new Error(`${statusCode} - Internal Server Error: ${body.substr(0, 100)}`);
-        }
-        else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
-            const dom = await this._parseHtmlToDom(response);
-            return ({ dom, isXml: type.includes('xml'), response, contentType });
-        }
-        else {
-            const body = await (0, utilities_1.concatStreamToBuffer)(response);
-            return { body, response, contentType };
-        }
-    }
-    /**
-     * Combines the provided `requestOptions` with mandatory (non-overridable) values.
-     */
-    _getRequestOptions(request, session, proxyUrl, gotOptions) {
-        const requestOptions = {
-            url: request.url,
-            method: request.method,
-            proxyUrl,
-            timeout: { request: this.navigationTimeoutMillis },
-            sessionToken: session,
-            ...gotOptions,
-            headers: { ...request.headers, ...gotOptions?.headers },
-            https: {
-                ...gotOptions?.https,
-                rejectUnauthorized: !this.ignoreSslErrors,
             },
-            isStream: true,
         };
-        // Delete any possible lowercased header for cookie as they are merged in _applyCookies under the uppercase Cookie header
-        Reflect.deleteProperty(requestOptions.headers, 'cookie');
-        // TODO this is incorrect, the check for man in the middle needs to be done
-        //   on individual proxy level, not on the `proxyConfiguration` level,
-        //   because users can use normal + MITM proxies in a single configuration.
-        // Disable SSL verification for MITM proxies
-        if (this.proxyConfiguration && this.proxyConfiguration.isManInTheMiddle) {
-            requestOptions.https = {
-                ...requestOptions.https,
-                rejectUnauthorized: false,
-            };
-        }
-        if (/PATCH|POST|PUT/.test(request.method))
-            requestOptions.body = request.payload;
-        return requestOptions;
-    }
-    _encodeResponse(request, response, encoding) {
-        if (this.forceResponseEncoding) {
-            encoding = this.forceResponseEncoding;
-        }
-        else if (!encoding && this.suggestResponseEncoding) {
-            encoding = this.suggestResponseEncoding;
-        }
-        // Fall back to utf-8 if we still don't have encoding.
-        const utf8 = 'utf8';
-        if (!encoding)
-            return { response, encoding: utf8 };
-        // This means that the encoding is one of Node.js supported
-        // encodings and we don't need to re-encode it.
-        if (Buffer.isEncoding(encoding))
-            return { response, encoding };
-        // Try to re-encode a variety of unsupported encodings to utf-8
-        if (iconv_lite_1.default.encodingExists(encoding)) {
-            const encodeStream = iconv_lite_1.default.encodeStream(utf8);
-            const decodeStream = iconv_lite_1.default.decodeStream(encoding).on('error', (err) => encodeStream.emit('error', err));
-            response.on('error', (err) => decodeStream.emit('error', err));
-            const encodedResponse = response.pipe(decodeStream).pipe(encodeStream);
-            encodedResponse.statusCode = response.statusCode;
-            encodedResponse.headers = response.headers;
-            encodedResponse.url = response.url;
-            return {
-                response: encodedResponse,
-                encoding: utf8,
-            };
-        }
-        throw new Error(`Resource ${request.url} served with unsupported charset/encoding: ${encoding}`);
     }
     async _parseHtmlToDom(response) {
         return new Promise((resolve, reject) => {
@@ -532,73 +132,21 @@ class CheerioCrawler extends basic_1.BasicCrawler {
                 .pipe(parser);
         });
     }
-    /**
-     * Checks and extends supported mime types
-     */
-    _extendSupportedMimeTypes(additionalMimeTypes) {
-        additionalMimeTypes.forEach((mimeType) => {
-            try {
-                const parsedType = content_type_1.default.parse(mimeType);
-                this.supportedMimeTypes.add(parsedType.type);
-            }
-            catch (err) {
-                throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
-            }
-        });
-    }
-    /**
-     * Handles timeout request
-     */
-    _handleRequestTimeout(session) {
-        session?.markBad();
-        throw new Error(`request timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds.`);
-    }
-    _abortDownloadOfBody(request, response) {
-        const { statusCode } = response;
-        const { type } = (0, utils_1.parseContentTypeFromResponse)(response);
-        if (statusCode === 406) {
-            request.noRetry = true;
-            throw new Error(`Resource ${request.url} is not available in the format requested by the Accept header. Skipping resource.`);
-        }
-        if (!this.supportedMimeTypes.has(type) && statusCode < 500) {
-            request.noRetry = true;
-            throw new Error(`Resource ${request.url} served Content-Type ${type}, `
-                + `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
-        }
-    }
 }
 exports.CheerioCrawler = CheerioCrawler;
-Object.defineProperty(CheerioCrawler, "optionsShape", {
-    enumerable: true,
-    configurable: true,
-    writable: true,
-    value: {
-        ...basic_1.BasicCrawler.optionsShape,
-        handlePageFunction: ow_1.default.optional.function,
-        navigationTimeoutSecs: ow_1.default.optional.number,
-        ignoreSslErrors: ow_1.default.optional.boolean,
-        additionalMimeTypes: ow_1.default.optional.array.ofType(ow_1.default.string),
-        suggestResponseEncoding: ow_1.default.optional.string,
-        forceResponseEncoding: ow_1.default.optional.string,
-        proxyConfiguration: ow_1.default.optional.object.validate(core_1.validators.proxyConfiguration),
-        persistCookiesPerSession: ow_1.default.optional.boolean,
-        preNavigationHooks: ow_1.default.optional.array,
-        postNavigationHooks: ow_1.default.optional.array,
-    }
-});
 /** @internal */
 async function cheerioCrawlerEnqueueLinks({ options, $, requestQueue, originalRequestUrl, finalRequestUrl }) {
     if (!$) {
         throw new Error('Cannot enqueue links because the DOM is not available.');
     }
-    const baseUrl = (0, core_1.resolveBaseUrlForEnqueueLinksFiltering)({
+    const baseUrl = (0, http_1.resolveBaseUrlForEnqueueLinksFiltering)({
         enqueueStrategy: options?.strategy,
         finalRequestUrl,
         originalRequestUrl,
         userProvidedBaseUrl: options?.baseUrl,
     });
     const urls = extractUrlsFromCheerio($, options?.selector ?? 'a', options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
-    return (0, core_1.enqueueLinks)({
+    return (0, http_1.enqueueLinks)({
         requestQueue,
         urls,
         baseUrl,
@@ -637,41 +185,9 @@ function extractUrlsFromCheerio($, selector, baseUrl) {
         .filter((href) => !!href);
 }
 /**
- * The stream object returned from got does not have the below properties.
- * At the same time, you can't read data directly from the response stream,
- * because they won't get emitted unless you also read from the primary
- * got stream. To be able to work with only one stream, we move the expected props
- * from the response stream to the got stream.
- * @internal
- */
-function addResponsePropertiesToStream(stream) {
-    const properties = [
-        'statusCode', 'statusMessage', 'headers',
-        'complete', 'httpVersion', 'rawHeaders',
-        'rawTrailers', 'trailers', 'url',
-        'request',
-    ];
-    const response = stream.response;
-    response.on('end', () => {
-        // @ts-expect-error
-        Object.assign(stream.rawTrailers, response.rawTrailers);
-        // @ts-expect-error
-        Object.assign(stream.trailers, response.trailers);
-        // @ts-expect-error
-        stream.complete = response.complete;
-    });
-    for (const prop of properties) {
-        if (!(prop in stream)) {
-            // @ts-expect-error
-            stream[prop] = response[prop];
-        }
-    }
-    return stream;
-}
-/**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link CheerioCrawler}.
- * Defaults to the {@link CheerioCrawlingContext}.
+ * Creates new {@apilink Router} instance that works based on request labels.
+ * This instance can then serve as a `requestHandler` of your {@apilink CheerioCrawler}.
+ * Defaults to the {@apilink CheerioCrawlingContext}.
  *
  * > Serves as a shortcut for using `Router.create<CheerioCrawlingContext>()`.
  *
@@ -693,7 +209,7 @@ function addResponsePropertiesToStream(stream) {
  * ```
  */
 function createCheerioRouter() {
-    return core_1.Router.create();
+    return http_1.Router.create();
 }
 exports.createCheerioRouter = createCheerioRouter;
 //# sourceMappingURL=cheerio-crawler.js.map