@crawlee/http 4.0.0-beta.11 → 4.0.0-beta.110

package/internals/http-crawler.js

@@ -1,27 +1,38 @@
+import { Readable } from 'node:stream';
 import util from 'node:util';
-import { BasicCrawler, Configuration, ContextPipeline, mergeCookies, processHttpRequestOptions, RequestState, Router, SessionError, validators, } from '@crawlee/basic';
+import { BasicCrawler, ContextPipeline, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, Router, SessionError, } from '@crawlee/basic';
+import { getCookiesFromResponse } from '@crawlee/core';
+import { ResponseWithUrl } from '@crawlee/http-client';
 import { RETRY_CSS_SELECTORS } from '@crawlee/utils';
-import * as cheerio from 'cheerio';
 import contentTypeParser from 'content-type';
 import iconv from 'iconv-lite';
 import ow from 'ow';
-import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
-import { concatStreamToBuffer, readStreamToString } from '@apify/utilities';
-import { parseContentTypeFromResponse } from './utils.js';
-let TimeoutError;
+import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
+import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
 /**
  * Default mime types, which HttpScraper supports.
  */
 const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml', 'application/xml'];
 const APPLICATION_JSON_MIME_TYPE = 'application/json';
-const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
+/**
+ * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
+ * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default.
+ *
+ * A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
+ * wholesale, tuning included, so spread these options in if you want to keep it:
+ *
+ * ```typescript
+ * new ConcurrencySystem({ ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS, maxConcurrency: 50 });
+ * ```
+ */
+export const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS = {
     desiredConcurrency: 10,
-    snapshotterOptions: {
-        eventLoopSnapshotIntervalSecs: 2,
-        maxBlockedMillis: 100,
-    },
-    systemStatusOptions: {
-        maxEventLoopOverloadedRatio: 0.7,
+    loadSignals: {
+        eventLoop: {
+            snapshotIntervalSecs: 2,
+            maxBlockedMillis: 100,
+            overloadedRatio: 0.7,
+        },
     },
 };
 /**
@@ -35,38 +46,40 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  *
  * This crawler downloads each URL using a plain HTTP request and doesn't do any HTML parsing.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link HttpCrawlerOptions.requestList}
- * or {@link HttpCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link HttpCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
+ * result as `requestManager`.
  *
- * If both {@link HttpCrawlerOptions.requestList} and {@link HttpCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link HttpCrawlerOptions.requestList|`requestList`} and {@link HttpCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
  *
  * ```javascript
  * preNavigationHooks: [
- *     (crawlingContext, gotOptions) => {
+ *     (crawlingContext) => {
  *         // ...
  *     },
  * ]
  * ```
  *
- * By default, this crawler only processes web pages with the `text/html`
- * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),
+ * By default, this crawler only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
+ * and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),
  * and skips pages with other content types. If you want the crawler to process other content types,
  * use the {@link HttpCrawlerOptions.additionalMimeTypes} constructor option.
  * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
  * For details, see {@link HttpCrawlerOptions.requestHandler}.
  *
- * New requests are only dispatched when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
- * parameter of the constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPool} options are available directly in the constructor.
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
+ * {@link ConcurrencySystem}.
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
+ * constructor, or, for finer control, by injecting a pre-configured
+ * {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * **Example usage:**
  *
@@ -92,22 +105,19 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  * @category Crawlers
  */
 export class HttpCrawler extends BasicCrawler {
-    config;
-    /**
-     * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
-     * Only available if used by the crawler.
-     */
-    proxyConfiguration;
-    preNavigationHooks;
-    postNavigationHooks;
-    persistCookiesPerSession;
-    navigationTimeoutMillis;
+    // Internal storage uses the base (non-extended) context types. The public option types are
+    // extension-aware for consumer DX, but internally the pipeline composes hooks against the
+    // concrete crawling context, which does not statically carry `ContextExtension`. The members
+    // added by `extendContext` are present at runtime regardless.
+    #preNavigationHooks;
+    #postNavigationHooks;
+    #saveResponseCookies;
+    #navigationTimeoutMillis;
+    // kept as TS-private: tests read it at runtime
     ignoreSslErrors;
-    suggestResponseEncoding;
-    forceResponseEncoding;
-    additionalHttpErrorStatusCodes;
-    ignoreHttpErrorStatusCodes;
-    supportedMimeTypes;
+    #suggestResponseEncoding;
+    #forceResponseEncoding;
+    #supportedMimeTypes;
     static optionsShape = {
         ...BasicCrawler.optionsShape,
         navigationTimeoutSecs: ow.optional.number,
@@ -115,105 +125,119 @@ export class HttpCrawler extends BasicCrawler {
         additionalMimeTypes: ow.optional.array.ofType(ow.string),
         suggestResponseEncoding: ow.optional.string,
         forceResponseEncoding: ow.optional.string,
-        proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
-        persistCookiesPerSession: ow.optional.boolean,
-        additionalHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
-        ignoreHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
+        saveResponseCookies: ow.optional.boolean,
         preNavigationHooks: ow.optional.array,
         postNavigationHooks: ow.optional.array,
     };
     /**
      * All `HttpCrawlerOptions` parameters are passed via an options object.
      */
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
+    constructor(options = {}) {
         ow(options, 'HttpCrawlerOptions', ow.object.exactShape(HttpCrawler.optionsShape));
-        const { navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, proxyConfiguration, persistCookiesPerSession, preNavigationHooks = [], postNavigationHooks = [], additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [], 
+        const { navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, saveResponseCookies = true, preNavigationHooks = [], postNavigationHooks = [], 
         // BasicCrawler
-        autoscaledPoolOptions = HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS, contextPipelineBuilder, ...basicCrawlerOptions } = options;
+        contextPipelineBuilder, ...basicCrawlerOptions } = options;
         super({
             ...basicCrawlerOptions,
-            autoscaledPoolOptions,
             contextPipelineBuilder: contextPipelineBuilder ??
                 (() => this.buildContextPipeline()),
-        }, config);
-        this.config = config;
-        // Cookies should be persisted per session only if session pool is used
-        if (!this.useSessionPool && persistCookiesPerSession) {
-            throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
-        }
-        this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
+        });
+        this.#supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
         if (additionalMimeTypes.length)
-            this._extendSupportedMimeTypes(additionalMimeTypes);
+            this.extendSupportedMimeTypes(additionalMimeTypes);
         if (suggestResponseEncoding && forceResponseEncoding) {
             this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
         }
-        this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
+        this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
         this.ignoreSslErrors = ignoreSslErrors;
-        this.suggestResponseEncoding = suggestResponseEncoding;
-        this.forceResponseEncoding = forceResponseEncoding;
-        this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
-        this.ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
-        this.proxyConfiguration = proxyConfiguration;
-        this.preNavigationHooks = preNavigationHooks;
-        this.postNavigationHooks = [
-            ({ request, response }) => this._abortDownloadOfBody(request, response),
+        this.#suggestResponseEncoding = suggestResponseEncoding;
+        this.#forceResponseEncoding = forceResponseEncoding;
+        // Cast away the extension-aware option types to the base internal storage types (see the field
+        // declarations above). This is sound - the hooks only ever receive the base context plus the
+        // members `extendContext` added at runtime.
+        this.#preNavigationHooks = preNavigationHooks;
+        this.#postNavigationHooks = [
+            ({ request, response }) => this.abortDownloadOfBody(request, response),
             ...postNavigationHooks,
         ];
-        if (this.useSessionPool) {
-            this.persistCookiesPerSession = persistCookiesPerSession ?? true;
-        }
-        else {
-            this.persistCookiesPerSession = false;
-        }
+        this.#saveResponseCookies = saveResponseCookies;
+    }
+    getNavigationTimeoutMillis() {
+        return this.#navigationTimeoutMillis;
+    }
+    /**
+     * Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
+     * concurrency shortcuts on top. Not called for a supplied
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} — spread the constant into it yourself to
+     * keep the tuning.
+     */
+    createDefaultConcurrencySystem(options) {
+        return super.createDefaultConcurrencySystem({
+            ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS,
+            ...options,
+        });
     }
     buildContextPipeline() {
-        return ContextPipeline.create()
-            .compose({ action: this.prepareProxyInfo.bind(this) })
-            .compose({
-            action: this.makeHttpRequest.bind(this),
-        })
+        // When navigation is skipped, `prepareHttpRequest` has already installed throwing getters for
+        // the response-derived members, so the guarded action is bypassed and the context left untouched.
+        const skipGuard = (action) => ({
+            action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
+        });
+        // A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation
+        // hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same
+        // window the navigation uses instead of each step being timed on its own.
+        const navigationTimedOut = `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`;
+        const windowGuard = (step) => skipGuard(async (ctx) => {
+            const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);
+            if (remaining <= 0) {
+                throw new TimeoutError(navigationTimedOut);
+            }
+            return addTimeoutToPromise(async () => step(ctx), remaining, navigationTimedOut);
+        });
+        let pipeline = ContextPipeline.create().compose({
+            action: this.prepareHttpRequest.bind(this),
+        });
+        for (const hook of this.#preNavigationHooks) {
+            pipeline = pipeline.compose(windowGuard(hook));
+        }
+        let pipelineWithNavigation = pipeline.compose(skipGuard(this.makeHttpRequest.bind(this)));
+        for (const hook of this.#postNavigationHooks) {
+            pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));
+        }
+        return pipelineWithNavigation
             .compose({ action: this.processHttpResponse.bind(this) })
             .compose({ action: this.handleBlockedRequestByContent.bind(this) });
     }
-    async prepareProxyInfo(crawlingContext) {
-        const { request, session } = crawlingContext;
-        let proxyInfo;
-        if (this.proxyConfiguration) {
-            const sessionId = session ? session.id : undefined;
-            proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId, { request });
-        }
-        return { proxyInfo };
-    }
-    async makeHttpRequest(crawlingContext) {
-        const { request, session } = crawlingContext;
+    async prepareHttpRequest(crawlingContext) {
+        const { request } = crawlingContext;
         if (request.skipNavigation) {
             return {
                 request: new Proxy(request, {
                     get(target, propertyName, receiver) {
                         if (propertyName === 'loadedUrl') {
-                            throw new Error('The `request.loadedUrl` property is not available - `skipNavigation` was used');
+                            throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
                         }
                         return Reflect.get(target, propertyName, receiver);
                     },
                 }),
                 get response() {
-                    throw new Error('The `response` property is not available - `skipNavigation` was used');
+                    throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
                 },
             };
         }
-        const gotOptions = {};
-        const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
         request.state = RequestState.BEFORE_NAV;
-        // Execute pre navigation hooks before applying session pool cookies,
-        // as they may also set cookies in the session
-        await this._executeHooks(this.preNavigationHooks, crawlingContext, gotOptions);
+        return {};
+    }
+    async makeHttpRequest(crawlingContext) {
         tryCancel();
-        const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
-        this._applyCookies(crawlingContext, gotOptions, preNavigationHooksCookies, postNavigationHooksCookies);
+        const { request, session } = crawlingContext;
         const proxyUrl = crawlingContext.proxyInfo?.url;
-        const httpResponse = await addTimeoutToPromise(async () => this._requestFunction({ request, session, proxyUrl, gotOptions }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        // Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
+        // have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
+        // client abort.
+        const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
-        request.loadedUrl = httpResponse.url;
+        request.loadedUrl = httpResponse?.url;
         request.state = RequestState.AFTER_NAV;
         return { request: request, response: httpResponse };
     }
@@ -221,46 +245,68 @@ export class HttpCrawler extends BasicCrawler {
         if (crawlingContext.request.skipNavigation) {
             return {
                 get contentType() {
-                    throw new Error('The `contentType` property is not available - `skipNavigation` was used');
+                    throw new NavigationSkippedError('The `contentType` property is not available - `skipNavigation` was used');
                 },
                 get body() {
-                    throw new Error('The `body` property is not available - `skipNavigation` was used');
+                    throw new NavigationSkippedError('The `body` property is not available - `skipNavigation` was used');
                 },
                 get json() {
-                    throw new Error('The `json` property is not available - `skipNavigation` was used');
+                    throw new NavigationSkippedError('The `json` property is not available - `skipNavigation` was used');
                 },
                 get waitForSelector() {
-                    throw new Error('The `waitForSelector` method is not available - `skipNavigation` was used');
+                    throw new NavigationSkippedError('The `waitForSelector` method is not available - `skipNavigation` was used');
                 },
                 get parseWithCheerio() {
-                    throw new Error('The `parseWithCheerio` method is not available - `skipNavigation` was used');
+                    throw new NavigationSkippedError('The `parseWithCheerio` method is not available - `skipNavigation` was used');
                 },
             };
         }
-        await this._executeHooks(this.postNavigationHooks, crawlingContext);
         tryCancel();
-        const parsed = await this._parseResponse(crawlingContext.request, crawlingContext.response);
+        // Reading the body is still part of the navigation, so it draws from the same shared window: on a server
+        // that streams the body slowly the request completes (headers arrive) but the body read would otherwise
+        // run unbounded. `extendTimeout` from a post-navigation hook has already pushed this deadline out if asked.
+        const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);
+        if (remaining <= 0) {
+            throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
+        }
+        const parsed = await addTimeoutToPromise(async () => this.parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
         const response = parsed.response;
         const contentType = parsed.contentType;
         const waitForSelector = async (selector, _timeoutMs) => {
+            const cheerio = await import('cheerio');
             const $ = cheerio.load(parsed.body.toString());
             if ($(selector).get().length === 0) {
                 throw new Error(`Selector '${selector}' not found.`);
             }
         };
         const parseWithCheerio = async (selector, timeoutMs) => {
+            const cheerio = await import('cheerio');
             const $ = cheerio.load(parsed.body.toString());
             if (selector) {
                 await crawlingContext.waitForSelector(selector, timeoutMs);
             }
             return $;
         };
-        if (this.useSessionPool) {
-            this._throwOnBlockedRequest(crawlingContext.session, response.statusCode);
-        }
-        if (this.persistCookiesPerSession) {
-            crawlingContext.session.setCookiesFromResponse(response);
+        this.throwOnBlockedRequest(response.status);
+        if (this.#saveResponseCookies) {
+            try {
+                for (const cookie of getCookiesFromResponse(response)) {
+                    if (!cookie)
+                        continue;
+                    try {
+                        await crawlingContext.session.cookieJar.setCookie(cookie, response.url, {
+                            ignoreError: false,
+                        });
+                    }
+                    catch (e) {
+                        this.log.debug(`Could not set cookie: ${e.message}`);
+                    }
+                }
+            }
+            catch (e) {
+                this.log.exception(e, 'Could not get cookies from response');
+            }
         }
         return {
             get json() {
@@ -291,69 +337,28 @@ export class HttpCrawler extends BasicCrawler {
                 return `Found selectors: ${foundSelectors.join(', ')}`;
             }
         }
-        return false;
-    }
-    /**
-     * Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks.
-     */
-    _applyCookies({ session, request }, gotOptions, preHookCookies, postHookCookies) {
-        const sessionCookie = session?.getCookieString(request.url) ?? '';
-        let alteredGotOptionsCookies = gotOptions.headers?.Cookie || gotOptions.headers?.cookie || '';
-        if (gotOptions.headers?.Cookie && gotOptions.headers?.cookie) {
-            const { Cookie: upperCaseHeader, cookie: lowerCaseHeader } = gotOptions.headers;
-            this.log.warning(`Encountered mixed casing for the cookie headers in the got options for request ${request.url} (${request.id}). Their values will be merged`);
-            const sourceCookies = [];
-            if (Array.isArray(lowerCaseHeader)) {
-                sourceCookies.push(...lowerCaseHeader);
-            }
-            else {
-                sourceCookies.push(lowerCaseHeader);
-            }
-            if (Array.isArray(upperCaseHeader)) {
-                sourceCookies.push(...upperCaseHeader);
-            }
-            else {
-                sourceCookies.push(upperCaseHeader);
-            }
-            alteredGotOptionsCookies = mergeCookies(request.url, sourceCookies);
-        }
-        const sourceCookies = [sessionCookie, preHookCookies];
-        if (Array.isArray(alteredGotOptionsCookies)) {
-            sourceCookies.push(...alteredGotOptionsCookies);
-        }
-        else {
-            sourceCookies.push(alteredGotOptionsCookies);
-        }
-        sourceCookies.push(postHookCookies);
-        const mergedCookie = mergeCookies(request.url, sourceCookies);
-        gotOptions.headers ??= {};
-        Reflect.deleteProperty(gotOptions.headers, 'Cookie');
-        Reflect.deleteProperty(gotOptions.headers, 'cookie');
-        if (mergedCookie !== '') {
-            gotOptions.headers.Cookie = mergedCookie;
+        if (this.blockedStatusCodes.has(crawlingContext.response.status)) {
+            return `Blocked by status code ${crawlingContext.response.status}`;
         }
+        return false;
     }
     /**
      * Function to make the HTTP request. It performs optimizations
      * on the request such as only downloading the request body if the
      * received content type matches text/html, application/xml, application/xhtml+xml.
      */
-    async _requestFunction({ request, session, proxyUrl, gotOptions, }) {
-        if (!TimeoutError) {
-            // @ts-ignore
-            ({ TimeoutError } = await import('got-scraping'));
-        }
-        const opts = this._getRequestOptions(request, session, proxyUrl, gotOptions);
+    async requestFunction({ request, session, proxyUrl }) {
+        const opts = this.getRequestOptions(request, session, proxyUrl);
         try {
-            return await this._requestAsBrowser(opts, session);
+            return await this.requestAsBrowser(opts, session);
         }
         catch (e) {
-            if (e instanceof TimeoutError) {
-                this._handleRequestTimeout(session);
-                return undefined;
+            if (e instanceof Error && e.constructor.name === 'TimeoutError') {
+                this.handleRequestTimeout(session);
+                return new Response(); // this will never happen, as handleRequestTimeout always throws
             }
             if (this.isProxyError(e)) {
-                throw new SessionError(this._getMessageFromError(e));
+                throw new SessionError(this.getMessageFromError(e));
             }
             else {
                 throw e;
@@ -363,18 +368,16 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Encodes and parses response according to the provided content type
      */
-    async _parseResponse(request, responseStream) {
-        const { statusCode } = responseStream;
-        const { type, charset } = parseContentTypeFromResponse(responseStream);
-        const { response, encoding } = this._encodeResponse(request, responseStream, charset);
+    async parseResponse(request, response) {
+        const { status } = response;
+        const { type, charset } = parseContentTypeFromResponse(response);
+        const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);
         const contentType = { type, encoding };
-        if (statusCode >= 400 && statusCode <= 599) {
-            this.stats.registerStatusCode(statusCode);
+        if (status >= 400 && status <= 599) {
+            this.stats.registerStatusCode(status);
         }
-        const excludeError = this.ignoreHttpErrorStatusCodes.has(statusCode);
-        const includeError = this.additionalHttpErrorStatusCodes.has(statusCode);
-        if ((statusCode >= 500 && !excludeError) || includeError) {
-            const body = await readStreamToString(response, encoding);
+        if (this.isErrorStatusCode(status)) {
+            const body = await reencodedResponse.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
             // Errors are often sent as JSON, so attempt to parse them,
             // despite Accept header being set to text/html.
             if (type === APPLICATION_JSON_MIME_TYPE) {
@@ -382,19 +385,28 @@ export class HttpCrawler extends BasicCrawler {
                 let { message } = errorResponse;
                 if (!message)
                     message = util.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
-                throw new Error(`${statusCode} - ${message}`);
+                throw new Error(`${status} - ${message}`);
             }
-            if (includeError) {
-                throw new Error(`${statusCode} - Error status code was set by user.`);
+            if (this.additionalHttpErrorStatusCodes.has(status)) {
+                throw new Error(`${status} - Error status code was set by user.`);
             }
             // It's not a JSON, so it's probably some text. Get the first 100 chars of it.
-            throw new Error(`${statusCode} - Internal Server Error: ${body.slice(0, 100)}`);
+            throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
         }
         else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
-            return { response, contentType, body: await readStreamToString(response) };
+            if (!charset && !this.#forceResponseEncoding) {
+                const rawBytes = Buffer.from(await response.arrayBuffer());
+                const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
+                const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';
+                const body = iconv.encodingExists(charsetToUse)
+                    ? iconv.decode(rawBytes, charsetToUse)
+                    : rawBytes.toString('utf8');
+                return { response, contentType: { type, encoding: 'utf-8' }, body };
+            }
+            return { response, contentType, body: await reencodedResponse.text() };
         }
         else {
-            const body = await concatStreamToBuffer(response);
+            const body = Buffer.from(await reencodedResponse.bytes());
             return {
                 body,
                 response,
@@ -405,28 +417,25 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Combines the provided `requestOptions` with mandatory (non-overridable) values.
      */
-    _getRequestOptions(request, session, proxyUrl, gotOptions) {
+    getRequestOptions(request, session, proxyUrl) {
         const requestOptions = {
             url: request.url,
             method: request.method,
             proxyUrl,
-            timeout: { request: this.navigationTimeoutMillis },
+            timeout: this.#navigationTimeoutMillis,
             sessionToken: session,
-            ...gotOptions,
-            headers: { ...request.headers, ...gotOptions?.headers },
+            headers: request.headers,
             https: {
-                ...gotOptions?.https,
                 rejectUnauthorized: !this.ignoreSslErrors,
             },
-            isStream: true,
+            body: undefined,
         };
-        // Delete any possible lowercased header for cookie as they are merged in _applyCookies under the uppercase Cookie header
-        Reflect.deleteProperty(requestOptions.headers, 'cookie');
-        // TODO this is incorrect, the check for man in the middle needs to be done
-        //   on individual proxy level, not on the `proxyConfiguration` level,
-        //   because users can use normal + MITM proxies in a single configuration.
+        if (requestOptions.headers?.cookie || requestOptions.headers?.Cookie) {
+            requestOptions.headers.Cookie = this.getCookieHeaderFromRequest(request);
+            delete requestOptions.headers.cookie;
+        }
         // Disable SSL verification for MITM proxies
-        if (this.proxyConfiguration && this.proxyConfiguration.isManInTheMiddle) {
+        if (session.proxyInfo?.ignoreTlsErrors) {
             requestOptions.https = {
                 ...requestOptions.https,
                 rejectUnauthorized: false,
@@ -436,12 +445,12 @@ export class HttpCrawler extends BasicCrawler {
             requestOptions.body = request.payload ?? '';
         return requestOptions;
     }
-    _encodeResponse(request, response, encoding) {
-        if (this.forceResponseEncoding) {
-            encoding = this.forceResponseEncoding;
+    encodeResponse(request, response, encoding) {
+        if (this.#forceResponseEncoding) {
+            encoding = this.#forceResponseEncoding;
         }
-        else if (!encoding && this.suggestResponseEncoding) {
-            encoding = this.suggestResponseEncoding;
+        else if (!encoding && this.#suggestResponseEncoding) {
+            encoding = this.#suggestResponseEncoding;
         }
         // Fall back to utf-8 if we still don't have encoding.
         const utf8 = 'utf8';
@@ -454,14 +463,16 @@ export class HttpCrawler extends BasicCrawler {
         // Try to re-encode a variety of unsupported encodings to utf-8
         if (iconv.encodingExists(encoding)) {
             const encodeStream = iconv.encodeStream(utf8);
-            const decodeStream = iconv.decodeStream(encoding).on('error', (err) => encodeStream.emit('error', err));
-            response.on('error', (err) => decodeStream.emit('error', err));
-            const encodedResponse = response.pipe(decodeStream).pipe(encodeStream);
-            encodedResponse.statusCode = response.statusCode;
-            encodedResponse.headers = response.headers;
-            encodedResponse.url = response.url;
+            const decodeStream = iconv
+                .decodeStream(encoding)
+                .on('error', (err) => encodeStream.emit('error', err));
+            const reencodedBody = response.body
+                ? Readable.toWeb(Readable.from(Readable.fromWeb(response.body)
+                    .pipe(decodeStream)
+                    .pipe(encodeStream)))
+                : null;
             return {
-                response: encodedResponse,
+                response: new ResponseWithUrl(reencodedBody, response),
                 encoding: utf8,
             };
         }
@@ -470,15 +481,15 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Checks and extends supported mime types
      */
-    _extendSupportedMimeTypes(additionalMimeTypes) {
+    extendSupportedMimeTypes(additionalMimeTypes) {
         for (const mimeType of additionalMimeTypes) {
             if (mimeType === '*/*') {
-                this.supportedMimeTypes.add(mimeType);
+                this.#supportedMimeTypes.add(mimeType);
                 continue;
             }
             try {
                 const parsedType = contentTypeParser.parse(mimeType);
-                this.supportedMimeTypes.add(parsedType.type);
+                this.#supportedMimeTypes.add(parsedType.type);
             }
             catch (err) {
                 throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
@@ -488,106 +499,53 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Handles timeout request
      */
-    _handleRequestTimeout(session) {
-        session?.markBad();
-        throw new Error(`request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+    handleRequestTimeout(session) {
+        session.markBad();
+        throw new Error(`Request timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
     }
-    _abortDownloadOfBody(request, response) {
-        const { statusCode } = response;
+    abortDownloadOfBody(request, response) {
+        const { status } = response;
         const { type } = parseContentTypeFromResponse(response);
-        // eslint-disable-next-line dot-notation -- accessing private property
-        const blockedStatusCodes = this.sessionPool ? this.sessionPool['blockedStatusCodes'] : [];
-        // if we retry the request, can the Content-Type change?
-        const isTransientContentType = statusCode >= 500 || blockedStatusCodes.includes(statusCode);
-        if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && !isTransientContentType) {
+        const isTransientContentType = status >= 500 || this.blockedStatusCodes.has(status);
+        if (!this.#supportedMimeTypes.has(type) && !this.#supportedMimeTypes.has('*/*') && !isTransientContentType) {
             request.noRetry = true;
             throw new Error(`Resource ${request.url} served Content-Type ${type}, ` +
-                `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
+                `but only ${Array.from(this.#supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
         }
     }
     /**
      * @internal wraps public utility for mocking purposes
      */
-    _requestAsBrowser = async (options, session) => {
-        const response = await this.httpClient.stream(processHttpRequestOptions({
+    requestAsBrowser = async (options, session) => {
+        const opts = processHttpRequestOptions({
             ...options,
-            cookieJar: options.cookieJar, // HACK - the type of ToughCookieJar in got is wrong
             responseType: 'text',
-        }), (redirectResponse, updatedRequest) => {
-            if (this.persistCookiesPerSession) {
-                session.setCookiesFromResponse(redirectResponse);
-                const cookieString = session.getCookieString(updatedRequest.url.toString());
-                if (cookieString !== '') {
-                    updatedRequest.headers.Cookie = cookieString;
-                }
-            }
         });
-        return addResponsePropertiesToStream(response.stream, response);
+        // When saveResponseCookies is false, the response cookies must not mutate the
+        // session jar. Reads still go through the session (so session.setCookie() in pre-nav
+        // hooks keeps working) but a per-request clone is passed in so writes are discarded.
+        const cookieJar = this.#saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
+        // Bind the request to the shared navigation window instead of a fixed per-request timeout, so
+        // `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and
+        // kill a lazily-read body mid-extension. This aborts the socket only during the header phase; the body
+        // read is bounded separately at the promise level (see `processHttpResponse`), so a slow-streaming body
+        // still fails cleanly with a navigation timeout, though the socket is left to close on its own.
+        const cancelSignal = storage.getStore()?.cancelTask.signal;
+        const response = await this.httpClient.sendRequest(new Request(opts.url, {
+            body: opts.body ? Readable.toWeb(opts.body) : undefined,
+            headers: new Headers(opts.headers),
+            method: opts.method,
+            // Node-specific option to make the request body work with streams
+            duplex: 'half',
+        }), {
+            session,
+            cookieJar,
+            signal: cancelSignal,
+            timeoutMillis: cancelSignal ? undefined : opts.timeout,
+        });
+        return response;
     };
 }
-/**
- * The stream object returned from got does not have the below properties.
- * At the same time, you can't read data directly from the response stream,
- * because they won't get emitted unless you also read from the primary
- * got stream. To be able to work with only one stream, we move the expected props
- * from the response stream to the got stream.
- * @internal
- */
-function addResponsePropertiesToStream(stream, response) {
-    const properties = [
-        'statusCode',
-        'statusMessage',
-        'headers',
-        'complete',
-        'httpVersion',
-        'rawHeaders',
-        'rawTrailers',
-        'trailers',
-        'url',
-        'request',
-    ];
-    stream.on('end', () => {
-        // @ts-expect-error
-        if (stream.rawTrailers)
-            stream.rawTrailers = response.rawTrailers; // TODO BC with got - remove in 4.0
-        // @ts-expect-error
-        if (stream.trailers)
-            stream.trailers = response.trailers;
-        // @ts-expect-error
-        stream.complete = response.complete;
-    });
-    for (const prop of properties) {
-        if (!(prop in stream)) {
-            stream[prop] = response[prop];
-        }
-    }
-    return stream;
-}
-/**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link HttpCrawler}.
- * Defaults to the {@link HttpCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<HttpCrawlingContext>()`.
- *
- * ```ts
- * import { HttpCrawler, createHttpRouter } from 'crawlee';
- *
- * const router = createHttpRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
- *
- * const crawler = new HttpCrawler({
- *     requestHandler: router,
- * });
- * await crawler.run();
- * ```
- */
-export function createHttpRouter(routes) {
-    return Router.create(routes);
+export function createHttpRouter(routesOrSchemas) {
+    return Router.create(routesOrSchemas);
 }
-//# sourceMappingURL=http-crawler.js.map