@crawlee/http 4.0.0-beta.9 → 4.0.0-beta.91

package/internals/http-crawler.js

@@ -1,28 +1,38 @@
-import { extname } from 'node:path';
+import { Readable } from 'node:stream';
 import util from 'node:util';
-import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS, BasicCrawler, Configuration, CrawlerExtension, mergeCookies, processHttpRequestOptions, RequestState, Router, SessionError, validators, } from '@crawlee/basic';
+import { BasicCrawler, ContextPipeline, NavigationSkippedError, RequestState, Router, SessionError, } from '@crawlee/basic';
+import { getCookiesFromResponse } from '@crawlee/core';
+import { ResponseWithUrl } from '@crawlee/http-client';
 import { RETRY_CSS_SELECTORS } from '@crawlee/utils';
-import * as cheerio from 'cheerio';
 import contentTypeParser from 'content-type';
 import iconv from 'iconv-lite';
-import mime from 'mime-types';
-import ow, { ObjectPredicate } from 'ow';
+import ow from 'ow';
 import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
-import { concatStreamToBuffer, readStreamToString } from '@apify/utilities';
-let TimeoutError;
+import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
 /**
  * Default mime types, which HttpScraper supports.
  */
 const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml', 'application/xml'];
 const APPLICATION_JSON_MIME_TYPE = 'application/json';
-const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
+/**
+ * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
+ * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default.
+ *
+ * A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
+ * wholesale, tuning included, so spread these options in if you want to keep it:
+ *
+ * ```typescript
+ * new ConcurrencySystem({ ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS, maxConcurrency: 50 });
+ * ```
+ */
+export const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS = {
     desiredConcurrency: 10,
-    snapshotterOptions: {
-        eventLoopSnapshotIntervalSecs: 2,
-        maxBlockedMillis: 100,
-    },
-    systemStatusOptions: {
-        maxEventLoopOverloadedRatio: 0.7,
+    loadSignals: {
+        eventLoop: {
+            snapshotIntervalSecs: 2,
+            maxBlockedMillis: 100,
+            overloadedRatio: 0.7,
+        },
     },
 };
 /**
@@ -36,28 +46,30 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  *
  * This crawler downloads each URL using a plain HTTP request and doesn't do any HTML parsing.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link HttpCrawlerOptions.requestList}
- * or {@link HttpCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link HttpCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
+ * result as `requestManager`.
  *
- * If both {@link HttpCrawlerOptions.requestList} and {@link HttpCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link HttpCrawlerOptions.requestList|`requestList`} and {@link HttpCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
  *
  * ```javascript
  * preNavigationHooks: [
- *     (crawlingContext, gotOptions) => {
+ *     (crawlingContext) => {
  *         // ...
  *     },
  * ]
  * ```
  *
- * By default, this crawler only processes web pages with the `text/html`
- * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),
+ * By default, this crawler only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
+ * and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),
  * and skips pages with other content types. If you want the crawler to process other content types,
  * use the {@link HttpCrawlerOptions.additionalMimeTypes} constructor option.
  * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
@@ -65,9 +77,9 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  *
  * New requests are only dispatched when there is enough free CPU and memory available,
  * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
- * parameter of the constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPool} options are available directly in the constructor.
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
+ * constructor, or, for finer control, by injecting a pre-configured
+ * {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * **Example usage:**
  *
@@ -93,22 +105,17 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  * @category Crawlers
  */
 export class HttpCrawler extends BasicCrawler {
-    config;
-    /**
-     * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
-     * Only available if used by the crawler.
-     */
-    proxyConfiguration;
-    userRequestHandlerTimeoutMillis;
+    // Internal storage uses the base (non-extended) context types. The public option types are
+    // extension-aware for consumer DX, but internally the pipeline composes hooks against the
+    // concrete crawling context, which does not statically carry `ContextExtension`. The members
+    // added by `extendContext` are present at runtime regardless.
     preNavigationHooks;
     postNavigationHooks;
-    persistCookiesPerSession;
+    saveResponseCookies;
     navigationTimeoutMillis;
     ignoreSslErrors;
     suggestResponseEncoding;
     forceResponseEncoding;
-    additionalHttpErrorStatusCodes;
-    ignoreHttpErrorStatusCodes;
     supportedMimeTypes;
     static optionsShape = {
         ...BasicCrawler.optionsShape,
@@ -117,157 +124,183 @@ export class HttpCrawler extends BasicCrawler {
         additionalMimeTypes: ow.optional.array.ofType(ow.string),
         suggestResponseEncoding: ow.optional.string,
         forceResponseEncoding: ow.optional.string,
-        proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
-        persistCookiesPerSession: ow.optional.boolean,
-        additionalHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
-        ignoreHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
+        saveResponseCookies: ow.optional.boolean,
         preNavigationHooks: ow.optional.array,
         postNavigationHooks: ow.optional.array,
     };
     /**
      * All `HttpCrawlerOptions` parameters are passed via an options object.
      */
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
+    constructor(options = {}) {
         ow(options, 'HttpCrawlerOptions', ow.object.exactShape(HttpCrawler.optionsShape));
-        const { requestHandler, requestHandlerTimeoutSecs = 60, navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, proxyConfiguration, persistCookiesPerSession, preNavigationHooks = [], postNavigationHooks = [], additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [], 
+        const { navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, saveResponseCookies = true, preNavigationHooks = [], postNavigationHooks = [], 
         // BasicCrawler
-        autoscaledPoolOptions = HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS, ...basicCrawlerOptions } = options;
+        contextPipelineBuilder, ...basicCrawlerOptions } = options;
         super({
             ...basicCrawlerOptions,
-            requestHandler,
-            autoscaledPoolOptions,
-            // We need to add some time for internal functions to finish,
-            // but not too much so that we would stall the crawler.
-            requestHandlerTimeoutSecs: navigationTimeoutSecs + requestHandlerTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
-        }, config);
-        this.config = config;
-        this.requestHandler = requestHandler ?? this.router;
-        // Cookies should be persisted per session only if session pool is used
-        if (!this.useSessionPool && persistCookiesPerSession) {
-            throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
-        }
+            contextPipelineBuilder: contextPipelineBuilder ??
+                (() => this.buildContextPipeline()),
+        });
         this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
         if (additionalMimeTypes.length)
-            this._extendSupportedMimeTypes(additionalMimeTypes);
+            this.extendSupportedMimeTypes(additionalMimeTypes);
         if (suggestResponseEncoding && forceResponseEncoding) {
             this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
         }
-        this.userRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
         this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
         this.ignoreSslErrors = ignoreSslErrors;
         this.suggestResponseEncoding = suggestResponseEncoding;
         this.forceResponseEncoding = forceResponseEncoding;
-        this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
-        this.ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
-        this.proxyConfiguration = proxyConfiguration;
+        // Cast away the extension-aware option types to the base internal storage types (see the field
+        // declarations above). This is sound - the hooks only ever receive the base context plus the
+        // members `extendContext` added at runtime.
         this.preNavigationHooks = preNavigationHooks;
         this.postNavigationHooks = [
             ({ request, response }) => this._abortDownloadOfBody(request, response),
             ...postNavigationHooks,
         ];
-        if (this.useSessionPool) {
-            this.persistCookiesPerSession = persistCookiesPerSession ?? true;
-        }
-        else {
-            this.persistCookiesPerSession = false;
-        }
+        this.saveResponseCookies = saveResponseCookies;
     }
     /**
-     * **EXPERIMENTAL**
-     * Function for attaching CrawlerExtensions such as the Unblockers.
-     * @param extension Crawler extension that overrides the crawler configuration.
+     * Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
+     * concurrency shortcuts on top. Not called for a supplied
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} — spread the constant into it yourself to
+     * keep the tuning.
      */
-    use(extension) {
-        ow(extension, ow.object.instanceOf(CrawlerExtension));
-        const className = this.constructor.name;
-        const extensionOptions = extension.getCrawlerOptions();
-        for (const [key, value] of Object.entries(extensionOptions)) {
-            const isConfigurable = Object.hasOwn(this, key);
-            const originalType = typeof this[key];
-            const extensionType = typeof value; // What if we want to null something? It is really needed?
-            const isSameType = originalType === extensionType || value == null; // fast track for deleting keys
-            const exists = this[key] != null;
-            if (!isConfigurable) {
-                // Test if the property can be configured on the crawler
-                throw new Error(`${extension.name} tries to set property "${key}" that is not configurable on ${className} instance.`);
-            }
-            if (!isSameType && exists) {
-                // Assuming that extensions will only add up configuration
-                throw new Error(`${extension.name} tries to set property of different type "${extensionType}". "${className}.${key}: ${originalType}".`);
-            }
-            this.log.warning(`${extension.name} is overriding "${className}.${key}: ${originalType}" with ${value}.`);
-            this[key] = value;
-        }
+    createDefaultConcurrencySystem(options) {
+        return super.createDefaultConcurrencySystem({
+            ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS,
+            ...options,
+        });
     }
-    /**
-     * Wrapper around requestHandler that opens and closes pages etc.
-     */
-    async _runRequestHandler(crawlingContext) {
-        const { request, session } = crawlingContext;
-        if (this.proxyConfiguration) {
-            const sessionId = session ? session.id : undefined;
-            crawlingContext.proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId, { request });
+    buildContextPipeline() {
+        // When navigation is skipped, `prepareHttpRequest` has already installed throwing getters for
+        // the response-derived members, so the guarded action is bypassed and the context left untouched.
+        const skipGuard = (action) => ({
+            action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
+        });
+        let pipeline = ContextPipeline.create().compose({
+            action: this.prepareHttpRequest.bind(this),
+        });
+        for (const hook of this.preNavigationHooks) {
+            pipeline = pipeline.compose(skipGuard(hook));
         }
-        if (!request.skipNavigation) {
-            await this._handleNavigation(crawlingContext);
-            tryCancel();
-            const parsed = await this._parseResponse(request, crawlingContext.response, crawlingContext);
-            const response = parsed.response;
-            const contentType = parsed.contentType;
-            tryCancel();
-            // `??=` because descendant classes may already set optimized version
-            crawlingContext.waitForSelector ??= async (selector, _timeoutMs) => {
-                const $ = cheerio.load(parsed.body.toString());
-                if ($(selector).get().length === 0) {
-                    throw new Error(`Selector '${selector}' not found.`);
-                }
+        let pipelineWithNavigation = pipeline.compose(skipGuard(this.makeHttpRequest.bind(this)));
+        for (const hook of this.postNavigationHooks) {
+            pipelineWithNavigation = pipelineWithNavigation.compose(skipGuard(hook));
+        }
+        return pipelineWithNavigation
+            .compose({ action: this.processHttpResponse.bind(this) })
+            .compose({ action: this.handleBlockedRequestByContent.bind(this) });
+    }
+    async prepareHttpRequest(crawlingContext) {
+        const { request } = crawlingContext;
+        if (request.skipNavigation) {
+            return {
+                request: new Proxy(request, {
+                    get(target, propertyName, receiver) {
+                        if (propertyName === 'loadedUrl') {
+                            throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
+                        }
+                        return Reflect.get(target, propertyName, receiver);
+                    },
+                }),
+                get response() {
+                    throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
+                },
             };
-            crawlingContext.parseWithCheerio ??= async (selector, timeoutMs) => {
-                const $ = cheerio.load(parsed.body.toString());
-                if (selector) {
-                    await crawlingContext.waitForSelector(selector, timeoutMs);
-                }
-                return $;
+        }
+        request.state = RequestState.BEFORE_NAV;
+        return {};
+    }
+    async makeHttpRequest(crawlingContext) {
+        tryCancel();
+        const { request, session } = crawlingContext;
+        const proxyUrl = crawlingContext.proxyInfo?.url;
+        const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        tryCancel();
+        request.loadedUrl = httpResponse?.url;
+        request.state = RequestState.AFTER_NAV;
+        return { request: request, response: httpResponse };
+    }
+    async processHttpResponse(crawlingContext) {
+        if (crawlingContext.request.skipNavigation) {
+            return {
+                get contentType() {
+                    throw new NavigationSkippedError('The `contentType` property is not available - `skipNavigation` was used');
+                },
+                get body() {
+                    throw new NavigationSkippedError('The `body` property is not available - `skipNavigation` was used');
+                },
+                get json() {
+                    throw new NavigationSkippedError('The `json` property is not available - `skipNavigation` was used');
+                },
+                get waitForSelector() {
+                    throw new NavigationSkippedError('The `waitForSelector` method is not available - `skipNavigation` was used');
+                },
+                get parseWithCheerio() {
+                    throw new NavigationSkippedError('The `parseWithCheerio` method is not available - `skipNavigation` was used');
+                },
             };
-            if (this.useSessionPool) {
-                this._throwOnBlockedRequest(crawlingContext.session, response.statusCode);
+        }
+        tryCancel();
+        const parsed = await this.parseResponse(crawlingContext.request, crawlingContext.response);
+        tryCancel();
+        const response = parsed.response;
+        const contentType = parsed.contentType;
+        const waitForSelector = async (selector, _timeoutMs) => {
+            const cheerio = await import('cheerio');
+            const $ = cheerio.load(parsed.body.toString());
+            if ($(selector).get().length === 0) {
+                throw new Error(`Selector '${selector}' not found.`);
             }
-            if (this.persistCookiesPerSession) {
-                crawlingContext.session.setCookiesFromResponse(response);
+        };
+        const parseWithCheerio = async (selector, timeoutMs) => {
+            const cheerio = await import('cheerio');
+            const $ = cheerio.load(parsed.body.toString());
+            if (selector) {
+                await crawlingContext.waitForSelector(selector, timeoutMs);
             }
-            request.loadedUrl = response.url;
-            if (!this.requestMatchesEnqueueStrategy(request)) {
-                this.log.debug(
-                // eslint-disable-next-line dot-notation
-                `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`);
-                request.noRetry = true;
-                request.state = RequestState.SKIPPED;
-                return;
+            return $;
+        };
+        this._throwOnBlockedRequest(response.status);
+        if (this.saveResponseCookies) {
+            try {
+                for (const cookie of getCookiesFromResponse(response)) {
+                    if (!cookie)
+                        continue;
+                    try {
+                        crawlingContext.session.cookieJar.setCookieSync(cookie, response.url, { ignoreError: false });
+                    }
+                    catch (e) {
+                        this.log.debug(`Could not set cookie: ${e.message}`);
+                    }
+                }
+            }
+            catch (e) {
+                this.log.exception(e, 'Could not get cookies from response');
             }
-            Object.assign(crawlingContext, parsed);
-            Object.defineProperty(crawlingContext, 'json', {
-                get() {
-                    if (contentType.type !== APPLICATION_JSON_MIME_TYPE)
-                        return null;
-                    const jsonString = parsed.body.toString(contentType.encoding);
-                    return JSON.parse(jsonString);
-                },
-            });
         }
+        return {
+            get json() {
+                if (contentType.type !== APPLICATION_JSON_MIME_TYPE)
+                    return null;
+                const jsonString = parsed.body.toString(contentType.encoding);
+                return JSON.parse(jsonString);
+            },
+            waitForSelector,
+            parseWithCheerio,
+            contentType,
+            body: parsed.body,
+        };
+    }
+    async handleBlockedRequestByContent(crawlingContext) {
         if (this.retryOnBlocked) {
             const error = await this.isRequestBlocked(crawlingContext);
             if (error)
                 throw new SessionError(error);
         }
-        request.state = RequestState.REQUEST_HANDLER;
-        try {
-            await addTimeoutToPromise(async () => Promise.resolve(this.requestHandler(crawlingContext)), this.userRequestHandlerTimeoutMillis, `requestHandler timed out after ${this.userRequestHandlerTimeoutMillis / 1000} seconds.`);
-            request.state = RequestState.DONE;
-        }
-        catch (e) {
-            request.state = RequestState.ERROR;
-            throw e;
-        }
+        return {};
     }
     async isRequestBlocked(crawlingContext) {
         if (HTML_AND_XML_MIME_TYPES.includes(crawlingContext.contentType.type)) {
@@ -277,84 +310,25 @@ export class HttpCrawler extends BasicCrawler {
                 return `Found selectors: ${foundSelectors.join(', ')}`;
             }
         }
-        return false;
-    }
-    async _handleNavigation(crawlingContext) {
-        const gotOptions = {};
-        const { request, session } = crawlingContext;
-        const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
-        request.state = RequestState.BEFORE_NAV;
-        // Execute pre navigation hooks before applying session pool cookies,
-        // as they may also set cookies in the session
-        await this._executeHooks(this.preNavigationHooks, crawlingContext, gotOptions);
-        tryCancel();
-        const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
-        this._applyCookies(crawlingContext, gotOptions, preNavigationHooksCookies, postNavigationHooksCookies);
-        const proxyUrl = crawlingContext.proxyInfo?.url;
-        crawlingContext.response = await addTimeoutToPromise(async () => this._requestFunction({ request, session, proxyUrl, gotOptions }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
-        tryCancel();
-        request.state = RequestState.AFTER_NAV;
-        await this._executeHooks(this.postNavigationHooks, crawlingContext, gotOptions);
-        tryCancel();
-    }
-    /**
-     * Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks.
-     */
-    _applyCookies({ session, request }, gotOptions, preHookCookies, postHookCookies) {
-        const sessionCookie = session?.getCookieString(request.url) ?? '';
-        let alteredGotOptionsCookies = gotOptions.headers?.Cookie || gotOptions.headers?.cookie || '';
-        if (gotOptions.headers?.Cookie && gotOptions.headers?.cookie) {
-            const { Cookie: upperCaseHeader, cookie: lowerCaseHeader } = gotOptions.headers;
-            this.log.warning(`Encountered mixed casing for the cookie headers in the got options for request ${request.url} (${request.id}). Their values will be merged`);
-            const sourceCookies = [];
-            if (Array.isArray(lowerCaseHeader)) {
-                sourceCookies.push(...lowerCaseHeader);
-            }
-            else {
-                sourceCookies.push(lowerCaseHeader);
-            }
-            if (Array.isArray(upperCaseHeader)) {
-                sourceCookies.push(...upperCaseHeader);
-            }
-            else {
-                sourceCookies.push(upperCaseHeader);
-            }
-            alteredGotOptionsCookies = mergeCookies(request.url, sourceCookies);
-        }
-        const sourceCookies = [sessionCookie, preHookCookies];
-        if (Array.isArray(alteredGotOptionsCookies)) {
-            sourceCookies.push(...alteredGotOptionsCookies);
-        }
-        else {
-            sourceCookies.push(alteredGotOptionsCookies);
-        }
-        sourceCookies.push(postHookCookies);
-        const mergedCookie = mergeCookies(request.url, sourceCookies);
-        gotOptions.headers ??= {};
-        Reflect.deleteProperty(gotOptions.headers, 'Cookie');
-        Reflect.deleteProperty(gotOptions.headers, 'cookie');
-        if (mergedCookie !== '') {
-            gotOptions.headers.Cookie = mergedCookie;
+        if (this.blockedStatusCodes.has(crawlingContext.response.status)) {
+            return `Blocked by status code ${crawlingContext.response.status}`;
         }
+        return false;
     }
     /**
      * Function to make the HTTP request. It performs optimizations
      * on the request such as only downloading the request body if the
      * received content type matches text/html, application/xml, application/xhtml+xml.
      */
-    async _requestFunction({ request, session, proxyUrl, gotOptions, }) {
-        if (!TimeoutError) {
-            // @ts-ignore
-            ({ TimeoutError } = await import('got-scraping'));
-        }
-        const opts = this._getRequestOptions(request, session, proxyUrl, gotOptions);
+    async requestFunction({ request, session, proxyUrl }) {
+        const opts = this.getRequestOptions(request, session, proxyUrl);
         try {
             return await this._requestAsBrowser(opts, session);
         }
         catch (e) {
-            if (e instanceof TimeoutError) {
-                this._handleRequestTimeout(session);
-                return undefined;
+            if (e instanceof Error && e.constructor.name === 'TimeoutError') {
+                this.handleRequestTimeout(session);
+                return new Response(); // this will never happen, as handleRequestTimeout always throws
             }
             if (this.isProxyError(e)) {
                 throw new SessionError(this._getMessageFromError(e));
@@ -367,18 +341,16 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Encodes and parses response according to the provided content type
      */
-    async _parseResponse(request, responseStream, crawlingContext) {
-        const { statusCode } = responseStream;
-        const { type, charset } = parseContentTypeFromResponse(responseStream);
-        const { response, encoding } = this._encodeResponse(request, responseStream, charset);
+    async parseResponse(request, response) {
+        const { status } = response;
+        const { type, charset } = parseContentTypeFromResponse(response);
+        const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);
         const contentType = { type, encoding };
-        if (statusCode >= 400 && statusCode <= 599) {
-            this.stats.registerStatusCode(statusCode);
+        if (status >= 400 && status <= 599) {
+            this.stats.registerStatusCode(status);
         }
-        const excludeError = this.ignoreHttpErrorStatusCodes.has(statusCode);
-        const includeError = this.additionalHttpErrorStatusCodes.has(statusCode);
-        if ((statusCode >= 500 && !excludeError) || includeError) {
-            const body = await readStreamToString(response, encoding);
+        if (this.isErrorStatusCode(status)) {
+            const body = await reencodedResponse.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
             // Errors are often sent as JSON, so attempt to parse them,
             // despite Accept header being set to text/html.
             if (type === APPLICATION_JSON_MIME_TYPE) {
@@ -386,59 +358,57 @@ export class HttpCrawler extends BasicCrawler {
                 let { message } = errorResponse;
                 if (!message)
                     message = util.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
-                throw new Error(`${statusCode} - ${message}`);
+                throw new Error(`${status} - ${message}`);
             }
-            if (includeError) {
-                throw new Error(`${statusCode} - Error status code was set by user.`);
+            if (this.additionalHttpErrorStatusCodes.has(status)) {
+                throw new Error(`${status} - Error status code was set by user.`);
             }
             // It's not a JSON, so it's probably some text. Get the first 100 chars of it.
-            throw new Error(`${statusCode} - Internal Server Error: ${body.slice(0, 100)}`);
+            throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
         }
         else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
-            const isXml = type.includes('xml');
-            const parsed = await this._parseHTML(response, isXml, crawlingContext);
-            return { ...parsed, isXml, response, contentType };
+            if (!charset && !this.forceResponseEncoding) {
+                const rawBytes = Buffer.from(await response.arrayBuffer());
+                const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
+                const charsetToUse = metaCharset ?? this.suggestResponseEncoding ?? 'utf-8';
+                const body = iconv.encodingExists(charsetToUse)
+                    ? iconv.decode(rawBytes, charsetToUse)
+                    : rawBytes.toString('utf8');
+                return { response, contentType: { type, encoding: 'utf-8' }, body };
+            }
+            return { response, contentType, body: await reencodedResponse.text() };
         }
         else {
-            const body = await concatStreamToBuffer(response);
+            const body = Buffer.from(await reencodedResponse.bytes());
             return {
                 body,
                 response,
                 contentType,
-                enqueueLinks: async () => Promise.resolve({ processedRequests: [], unprocessedRequests: [] }),
             };
         }
     }
-    async _parseHTML(response, _isXml, _crawlingContext) {
-        return {
-            body: await concatStreamToBuffer(response),
-        };
-    }
     /**
      * Combines the provided `requestOptions` with mandatory (non-overridable) values.
      */
-    _getRequestOptions(request, session, proxyUrl, gotOptions) {
+    getRequestOptions(request, session, proxyUrl) {
         const requestOptions = {
             url: request.url,
             method: request.method,
             proxyUrl,
-            timeout: { request: this.navigationTimeoutMillis },
+            timeout: this.navigationTimeoutMillis,
             sessionToken: session,
-            ...gotOptions,
-            headers: { ...request.headers, ...gotOptions?.headers },
+            headers: request.headers,
             https: {
-                ...gotOptions?.https,
                 rejectUnauthorized: !this.ignoreSslErrors,
             },
-            isStream: true,
+            body: undefined,
         };
-        // Delete any possible lowercased header for cookie as they are merged in _applyCookies under the uppercase Cookie header
-        Reflect.deleteProperty(requestOptions.headers, 'cookie');
-        // TODO this is incorrect, the check for man in the middle needs to be done
-        //   on individual proxy level, not on the `proxyConfiguration` level,
-        //   because users can use normal + MITM proxies in a single configuration.
+        if (requestOptions.headers?.cookie || requestOptions.headers?.Cookie) {
+            requestOptions.headers.Cookie = this._getCookieHeaderFromRequest(request);
+            delete requestOptions.headers.cookie;
+        }
         // Disable SSL verification for MITM proxies
-        if (this.proxyConfiguration && this.proxyConfiguration.isManInTheMiddle) {
+        if (session.proxyInfo?.ignoreTlsErrors) {
             requestOptions.https = {
                 ...requestOptions.https,
                 rejectUnauthorized: false,
@@ -448,7 +418,7 @@ export class HttpCrawler extends BasicCrawler {
             requestOptions.body = request.payload ?? '';
         return requestOptions;
     }
-    _encodeResponse(request, response, encoding) {
+    encodeResponse(request, response, encoding) {
         if (this.forceResponseEncoding) {
             encoding = this.forceResponseEncoding;
         }
@@ -466,14 +436,16 @@ export class HttpCrawler extends BasicCrawler {
         // Try to re-encode a variety of unsupported encodings to utf-8
         if (iconv.encodingExists(encoding)) {
             const encodeStream = iconv.encodeStream(utf8);
-            const decodeStream = iconv.decodeStream(encoding).on('error', (err) => encodeStream.emit('error', err));
-            response.on('error', (err) => decodeStream.emit('error', err));
-            const encodedResponse = response.pipe(decodeStream).pipe(encodeStream);
-            encodedResponse.statusCode = response.statusCode;
-            encodedResponse.headers = response.headers;
-            encodedResponse.url = response.url;
+            const decodeStream = iconv
+                .decodeStream(encoding)
+                .on('error', (err) => encodeStream.emit('error', err));
+            const reencodedBody = response.body
+                ? Readable.toWeb(Readable.from(Readable.fromWeb(response.body)
+                    .pipe(decodeStream)
+                    .pipe(encodeStream)))
+                : null;
             return {
-                response: encodedResponse,
+                response: new ResponseWithUrl(reencodedBody, response),
                 encoding: utf8,
             };
         }
@@ -482,7 +454,7 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Checks and extends supported mime types
      */
-    _extendSupportedMimeTypes(additionalMimeTypes) {
+    extendSupportedMimeTypes(additionalMimeTypes) {
         for (const mimeType of additionalMimeTypes) {
             if (mimeType === '*/*') {
                 this.supportedMimeTypes.add(mimeType);
@@ -500,17 +472,14 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Handles timeout request
      */
-    _handleRequestTimeout(session) {
-        session?.markBad();
-        throw new Error(`request timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds.`);
+    handleRequestTimeout(session) {
+        session.markBad();
+        throw new Error(`request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
     }
     _abortDownloadOfBody(request, response) {
-        const { statusCode } = response;
+        const { status } = response;
         const { type } = parseContentTypeFromResponse(response);
-        // eslint-disable-next-line dot-notation -- accessing private property
-        const blockedStatusCodes = this.sessionPool ? this.sessionPool['blockedStatusCodes'] : [];
-        // if we retry the request, can the Content-Type change?
-        const isTransientContentType = statusCode >= 500 || blockedStatusCodes.includes(statusCode);
+        const isTransientContentType = status >= 500 || this.blockedStatusCodes.has(status);
         if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && !isTransientContentType) {
             request.noRetry = true;
             throw new Error(`Resource ${request.url} served Content-Type ${type}, ` +
@@ -521,115 +490,28 @@ export class HttpCrawler extends BasicCrawler {
      * @internal wraps public utility for mocking purposes
      */
     _requestAsBrowser = async (options, session) => {
-        const response = await this.httpClient.stream(processHttpRequestOptions({
+        const opts = processHttpRequestOptions({
             ...options,
-            cookieJar: options.cookieJar, // HACK - the type of ToughCookieJar in got is wrong
             responseType: 'text',
-        }), (redirectResponse, updatedRequest) => {
-            if (this.persistCookiesPerSession) {
-                session.setCookiesFromResponse(redirectResponse);
-                const cookieString = session.getCookieString(updatedRequest.url.toString());
-                if (cookieString !== '') {
-                    updatedRequest.headers.Cookie = cookieString;
-                }
-            }
         });
-        return addResponsePropertiesToStream(response.stream, response);
-    };
-}
-/**
- * The stream object returned from got does not have the below properties.
- * At the same time, you can't read data directly from the response stream,
- * because they won't get emitted unless you also read from the primary
- * got stream. To be able to work with only one stream, we move the expected props
- * from the response stream to the got stream.
- * @internal
- */
-function addResponsePropertiesToStream(stream, response) {
-    const properties = [
-        'statusCode',
-        'statusMessage',
-        'headers',
-        'complete',
-        'httpVersion',
-        'rawHeaders',
-        'rawTrailers',
-        'trailers',
-        'url',
-        'request',
-    ];
-    stream.on('end', () => {
-        // @ts-expect-error
-        if (stream.rawTrailers)
-            stream.rawTrailers = response.rawTrailers; // TODO BC with got - remove in 4.0
-        // @ts-expect-error
-        if (stream.trailers)
-            stream.trailers = response.trailers;
-        // @ts-expect-error
-        stream.complete = response.complete;
-    });
-    for (const prop of properties) {
-        if (!(prop in stream)) {
-            stream[prop] = response[prop];
-        }
-    }
-    return stream;
-}
-/**
- * Gets parsed content type from response object
- * @param response HTTP response object
- */
-function parseContentTypeFromResponse(response) {
-    ow(response, ow.object.partialShape({
-        url: ow.string.url,
-        headers: new ObjectPredicate(),
-    }));
-    const { url, headers } = response;
-    let parsedContentType;
-    if (headers['content-type']) {
-        try {
-            parsedContentType = contentTypeParser.parse(headers['content-type']);
-        }
-        catch {
-            // Can not parse content type from Content-Type header. Try to parse it from file extension.
-        }
-    }
-    // Parse content type from file extension as fallback
-    if (!parsedContentType) {
-        const parsedUrl = new URL(url);
-        const contentTypeFromExtname = mime.contentType(extname(parsedUrl.pathname)) || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5
-        parsedContentType = contentTypeParser.parse(contentTypeFromExtname);
-    }
-    return {
-        type: parsedContentType.type,
-        charset: parsedContentType.parameters.charset,
+        // When saveResponseCookies is false, the response cookies must not mutate the
+        // session jar. Reads still go through the session (so session.setCookie() in pre-nav
+        // hooks keeps working) but a per-request clone is passed in so writes are discarded.
+        const cookieJar = this.saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
+        const response = await this.httpClient.sendRequest(new Request(opts.url, {
+            body: opts.body ? Readable.toWeb(opts.body) : undefined,
+            headers: new Headers(opts.headers),
+            method: opts.method,
+            // Node-specific option to make the request body work with streams
+            duplex: 'half',
+        }), {
+            session,
+            cookieJar,
+            timeoutMillis: opts.timeout,
+        });
+        return response;
     };
 }
-/**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link HttpCrawler}.
- * Defaults to the {@link HttpCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<HttpCrawlingContext>()`.
- *
- * ```ts
- * import { HttpCrawler, createHttpRouter } from 'crawlee';
- *
- * const router = createHttpRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
- *
- * const crawler = new HttpCrawler({
- *     requestHandler: router,
- * });
- * await crawler.run();
- * ```
- */
-export function createHttpRouter(routes) {
-    return Router.create(routes);
+export function createHttpRouter(routesOrSchemas) {
+    return Router.create(routesOrSchemas);
 }
-//# sourceMappingURL=http-crawler.js.map