@crawlee/http 4.0.0-beta.2 → 4.0.0-beta.200

package/internals/http-crawler.js

@@ -1,28 +1,38 @@
-import { extname } from 'node:path';
+import { Readable } from 'node:stream';
 import util from 'node:util';
-import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS, BasicCrawler, Configuration, CrawlerExtension, mergeCookies, processHttpRequestOptions, RequestState, Router, SessionError, validators, } from '@crawlee/basic';
-import { RETRY_CSS_SELECTORS } from '@crawlee/utils';
-import * as cheerio from 'cheerio';
+import { BasicCrawler, ContextPipeline, getCookiesFromResponse, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, RequestThrottledError, Router, SessionError, } from '@crawlee/basic';
+import { ResponseWithUrl } from '@crawlee/http-client';
+import { parseArgument, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
 import contentTypeParser from 'content-type';
 import iconv from 'iconv-lite';
-import mime from 'mime-types';
-import ow, { ObjectPredicate } from 'ow';
-import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
-import { concatStreamToBuffer, readStreamToString } from '@apify/utilities';
-let TimeoutError;
+import { z } from 'zod';
+import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
+import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
 /**
  * Default mime types, which HttpScraper supports.
  */
 const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml', 'application/xml'];
 const APPLICATION_JSON_MIME_TYPE = 'application/json';
-const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
+/**
+ * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
+ * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default, with your own
+ * concurrency shortcuts (`minConcurrency`, `maxConcurrency`, `maxRequestsPerMinute`) kept on top.
+ *
+ * A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
+ * wholesale, tuning included, so spread these options in if you want to keep it:
+ *
+ * ```typescript
+ * new ConcurrencySystem({ ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS, maxConcurrency: 50 });
+ * ```
+ */
+export const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS = {
     desiredConcurrency: 10,
-    snapshotterOptions: {
-        eventLoopSnapshotIntervalSecs: 2,
-        maxBlockedMillis: 100,
-    },
-    systemStatusOptions: {
-        maxEventLoopOverloadedRatio: 0.7,
+    loadSignals: {
+        eventLoop: {
+            snapshotIntervalSecs: 2,
+            maxBlockedMillis: 100,
+            overloadedRatio: 0.7,
+        },
     },
 };
 /**
@@ -36,38 +46,40 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  *
  * This crawler downloads each URL using a plain HTTP request and doesn't do any HTML parsing.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link HttpCrawlerOptions.requestList}
- * or {@link HttpCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link HttpCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
+ * result as `requestManager`.
  *
- * If both {@link HttpCrawlerOptions.requestList} and {@link HttpCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link HttpCrawlerOptions.requestList|`requestList`} and {@link HttpCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
  *
  * ```javascript
  * preNavigationHooks: [
- *     (crawlingContext, gotOptions) => {
+ *     (crawlingContext) => {
  *         // ...
  *     },
  * ]
  * ```
  *
- * By default, this crawler only processes web pages with the `text/html`
- * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),
+ * By default, this crawler only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
+ * and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),
  * and skips pages with other content types. If you want the crawler to process other content types,
  * use the {@link HttpCrawlerOptions.additionalMimeTypes} constructor option.
  * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
  * For details, see {@link HttpCrawlerOptions.requestHandler}.
  *
- * New requests are only dispatched when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
- * parameter of the constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPool} options are available directly in the constructor.
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
+ * {@link ConcurrencySystem}.
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
+ * constructor, or, for finer control, by injecting a pre-configured
+ * {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * **Example usage:**
  *
@@ -93,184 +105,240 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  * @category Crawlers
  */
 export class HttpCrawler extends BasicCrawler {
-    config;
+    // Internal storage uses the base (non-extended) context types. The public option types are
+    // extension-aware for consumer DX, but internally the pipeline composes hooks against the
+    // concrete crawling context, which does not statically carry `ContextExtension`. The members
+    // added by `extendContext` are present at runtime regardless.
+    #preNavigationHooks;
+    #postNavigationHooks;
+    #saveResponseCookies;
+    #navigationTimeoutMillis;
+    #ignoreTlsErrors;
+    #suggestResponseEncoding;
+    #forceResponseEncoding;
+    #supportedMimeTypes;
     /**
-     * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
-     * Only available if used by the crawler.
+     * @internal
      */
-    proxyConfiguration;
-    userRequestHandlerTimeoutMillis;
-    preNavigationHooks;
-    postNavigationHooks;
-    persistCookiesPerSession;
-    navigationTimeoutMillis;
-    ignoreSslErrors;
-    suggestResponseEncoding;
-    forceResponseEncoding;
-    additionalHttpErrorStatusCodes;
-    ignoreHttpErrorStatusCodes;
-    supportedMimeTypes;
     static optionsShape = {
         ...BasicCrawler.optionsShape,
-        navigationTimeoutSecs: ow.optional.number,
-        ignoreSslErrors: ow.optional.boolean,
-        additionalMimeTypes: ow.optional.array.ofType(ow.string),
-        suggestResponseEncoding: ow.optional.string,
-        forceResponseEncoding: ow.optional.string,
-        proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
-        persistCookiesPerSession: ow.optional.boolean,
-        additionalHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
-        ignoreHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
-        preNavigationHooks: ow.optional.array,
-        postNavigationHooks: ow.optional.array,
+        navigationTimeoutSecs: schemas.anyNumber.default(30),
+        ignoreTlsErrors: z.boolean().default(true),
+        additionalMimeTypes: schemas.arrayOf(z.string(), 'strings').default(() => []),
+        suggestResponseEncoding: z.string().optional(),
+        forceResponseEncoding: z.string().optional(),
+        saveResponseCookies: z.boolean().default(true),
+        preNavigationHooks: schemas.anyArray.default(() => []),
+        postNavigationHooks: schemas.anyArray.default(() => []),
     };
+    /** @internal */
+    static optionsSchema = z.strictObject(HttpCrawler.optionsShape);
     /**
      * All `HttpCrawlerOptions` parameters are passed via an options object.
      */
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
-        ow(options, 'HttpCrawlerOptions', ow.object.exactShape(HttpCrawler.optionsShape));
-        const { requestHandler, requestHandlerTimeoutSecs = 60, navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, proxyConfiguration, persistCookiesPerSession, preNavigationHooks = [], postNavigationHooks = [], additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [], 
+    constructor(options = {}) {
+        const { navigationTimeoutSecs, ignoreTlsErrors, additionalMimeTypes, suggestResponseEncoding, forceResponseEncoding, saveResponseCookies, preNavigationHooks, postNavigationHooks, 
         // BasicCrawler
-        autoscaledPoolOptions = HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS, ...basicCrawlerOptions } = options;
+        contextPipelineBuilder, ...basicCrawlerOptions } = parseArgument(options, HttpCrawler.optionsSchema, 'HttpCrawlerOptions');
         super({
             ...basicCrawlerOptions,
-            requestHandler,
-            autoscaledPoolOptions,
-            // We need to add some time for internal functions to finish,
-            // but not too much so that we would stall the crawler.
-            requestHandlerTimeoutSecs: navigationTimeoutSecs + requestHandlerTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
-        }, config);
-        this.config = config;
-        // FIXME any
-        this.requestHandler = requestHandler ?? this.router;
-        // Cookies should be persisted per session only if session pool is used
-        if (!this.useSessionPool && persistCookiesPerSession) {
-            throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
-        }
-        this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
+            contextPipelineBuilder: contextPipelineBuilder ??
+                (() => this.buildContextPipeline()),
+        });
+        this.#supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
         if (additionalMimeTypes.length)
-            this._extendSupportedMimeTypes(additionalMimeTypes);
+            this.extendSupportedMimeTypes(additionalMimeTypes);
         if (suggestResponseEncoding && forceResponseEncoding) {
             this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
         }
-        this.userRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
-        this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
-        this.ignoreSslErrors = ignoreSslErrors;
-        this.suggestResponseEncoding = suggestResponseEncoding;
-        this.forceResponseEncoding = forceResponseEncoding;
-        this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
-        this.ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
-        this.proxyConfiguration = proxyConfiguration;
-        this.preNavigationHooks = preNavigationHooks;
-        this.postNavigationHooks = [
-            ({ request, response }) => this._abortDownloadOfBody(request, response),
+        this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
+        this.#ignoreTlsErrors = ignoreTlsErrors;
+        this.#suggestResponseEncoding = suggestResponseEncoding;
+        this.#forceResponseEncoding = forceResponseEncoding;
+        // Cast away the extension-aware option types to the base internal storage types (see the field
+        // declarations above). This is sound - the hooks only ever receive the base context plus the
+        // members `extendContext` added at runtime.
+        this.#preNavigationHooks = preNavigationHooks;
+        this.#postNavigationHooks = [
+            ({ request, response }) => this.abortDownloadOfBody(request, response),
             ...postNavigationHooks,
         ];
-        if (this.useSessionPool) {
-            this.persistCookiesPerSession = persistCookiesPerSession ?? true;
+        this.#saveResponseCookies = saveResponseCookies;
+    }
+    /** @internal */
+    getNavigationTimeoutMillis() {
+        return this.#navigationTimeoutMillis;
+    }
+    /** @internal */
+    createDefaultConcurrencySystem(options) {
+        return super.createDefaultConcurrencySystem({
+            ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS,
+            ...options,
+        });
+    }
+    buildContextPipeline() {
+        // When navigation is skipped, `prepareHttpRequest` has already installed throwing getters for
+        // the response-derived members, so the guarded action is bypassed and the context left untouched.
+        const skipGuard = (action) => async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {}));
+        // A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation
+        // hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same
+        // window the navigation uses instead of each step being timed on its own.
+        const navigationTimedOut = `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`;
+        const windowGuard = (step) => skipGuard(async (ctx) => {
+            const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);
+            if (remaining <= 0) {
+                throw new TimeoutError(navigationTimedOut);
+            }
+            return addTimeoutToPromise(async () => step(ctx), remaining, navigationTimedOut);
+        });
+        let pipeline = ContextPipeline.create().compose(this.prepareHttpRequest.bind(this));
+        for (const hook of this.#preNavigationHooks) {
+            pipeline = pipeline.compose(windowGuard(hook));
         }
-        else {
-            this.persistCookiesPerSession = false;
+        let pipelineWithNavigation = pipeline.compose(skipGuard(this.makeHttpRequest.bind(this)));
+        for (const hook of this.#postNavigationHooks) {
+            pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));
         }
+        return pipelineWithNavigation
+            .compose(this.processHttpResponse.bind(this))
+            .compose(this.handleBlockedRequestByContent.bind(this));
     }
-    /**
-     * **EXPERIMENTAL**
-     * Function for attaching CrawlerExtensions such as the Unblockers.
-     * @param extension Crawler extension that overrides the crawler configuration.
-     */
-    use(extension) {
-        ow(extension, ow.object.instanceOf(CrawlerExtension));
-        const className = this.constructor.name;
-        const extensionOptions = extension.getCrawlerOptions();
-        for (const [key, value] of Object.entries(extensionOptions)) {
-            const isConfigurable = Object.hasOwn(this, key);
-            const originalType = typeof this[key];
-            const extensionType = typeof value; // What if we want to null something? It is really needed?
-            const isSameType = originalType === extensionType || value == null; // fast track for deleting keys
-            const exists = this[key] != null;
-            if (!isConfigurable) {
-                // Test if the property can be configured on the crawler
-                throw new Error(`${extension.name} tries to set property "${key}" that is not configurable on ${className} instance.`);
-            }
-            if (!isSameType && exists) {
-                // Assuming that extensions will only add up configuration
-                throw new Error(`${extension.name} tries to set property of different type "${extensionType}". "${className}.${key}: ${originalType}".`);
-            }
-            this.log.warning(`${extension.name} is overriding "${className}.${key}: ${originalType}" with ${value}.`);
-            this[key] = value;
+    async prepareHttpRequest(crawlingContext) {
+        const { request } = crawlingContext;
+        if (request.skipNavigation) {
+            return {
+                request: new Proxy(request, {
+                    get(target, propertyName, receiver) {
+                        if (propertyName === 'loadedUrl') {
+                            throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
+                        }
+                        return Reflect.get(target, propertyName, receiver);
+                    },
+                }),
+                get response() {
+                    throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
+                },
+            };
         }
+        request.state = RequestState.BEFORE_NAV;
+        return {};
     }
-    /**
-     * Wrapper around requestHandler that opens and closes pages etc.
-     */
-    async _runRequestHandler(crawlingContext) {
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
+    async makeHttpRequest(crawlingContext) {
+        tryCancel();
         const { request, session } = crawlingContext;
-        if (this.proxyConfiguration) {
-            const sessionId = session ? session.id : undefined;
-            crawlingContext.proxyInfo = await this.proxyConfiguration.newProxyInfo(sessionId, { request });
-        }
-        if (!request.skipNavigation) {
-            await this._handleNavigation(crawlingContext);
-            tryCancel();
-            const parsed = await this._parseResponse(request, crawlingContext.response, crawlingContext);
-            const response = parsed.response;
-            const contentType = parsed.contentType;
-            tryCancel();
-            // `??=` because descendant classes may already set optimized version
-            crawlingContext.waitForSelector ??= async (selector, _timeoutMs) => {
-                const $ = cheerio.load(parsed.body.toString());
-                if ($(selector).get().length === 0) {
-                    throw new Error(`Selector '${selector}' not found.`);
-                }
-            };
-            crawlingContext.parseWithCheerio ??= async (selector, timeoutMs) => {
-                const $ = cheerio.load(parsed.body.toString());
-                if (selector) {
-                    await crawlingContext.waitForSelector(selector, timeoutMs);
-                }
-                return $;
+        const proxyUrl = crawlingContext.proxyInfo?.url;
+        // Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
+        // have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
+        // client abort.
+        const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
+        tryCancel();
+        request.loadedUrl = httpResponse?.url;
+        request.state = RequestState.AFTER_NAV;
+        return { request: request, response: httpResponse };
+    }
+    async processHttpResponse(crawlingContext) {
+        if (crawlingContext.request.skipNavigation) {
+            return {
+                get contentType() {
+                    throw new NavigationSkippedError('The `contentType` property is not available - `skipNavigation` was used');
+                },
+                get body() {
+                    throw new NavigationSkippedError('The `body` property is not available - `skipNavigation` was used');
+                },
+                get json() {
+                    throw new NavigationSkippedError('The `json` property is not available - `skipNavigation` was used');
+                },
+                get waitForSelector() {
+                    throw new NavigationSkippedError('The `waitForSelector` method is not available - `skipNavigation` was used');
+                },
+                get parseWithCheerio() {
+                    throw new NavigationSkippedError('The `parseWithCheerio` method is not available - `skipNavigation` was used');
+                },
             };
-            if (this.useSessionPool) {
-                this._throwOnBlockedRequest(crawlingContext.session, response.statusCode);
+        }
+        tryCancel();
+        // Before `parseResponse`, which throws for error status codes - a 429 the user opted into treating as an
+        // error is still a rate limit the domain should back off from.
+        if (crawlingContext.response.status === 429) {
+            const retryAfter = crawlingContext.response.headers.get('retry-after');
+            if (this.recordDomainRateLimit(crawlingContext.request.url, retryAfter)) {
+                // This is the one path that never reads the body, so cancel it to release the connection
+                // rather than leaving it to the garbage collector.
+                await crawlingContext.response.body?.cancel().catch(() => { });
+                throw new RequestThrottledError(`${crawlingContext.request.url} responded with 429.`);
+            }
+        }
+        // Reading the body is still part of the navigation, so it draws from the same shared window: on a server
+        // that streams the body slowly the request completes (headers arrive) but the body read would otherwise
+        // run unbounded. `extendTimeout` from a post-navigation hook has already pushed this deadline out if asked.
+        const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);
+        if (remaining <= 0) {
+            throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
+        }
+        const parsed = await addTimeoutToPromise(async () => this.parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
+        tryCancel();
+        const response = parsed.response;
+        const contentType = parsed.contentType;
+        const loadBody = async () => {
+            const { load } = await import('cheerio/slim');
+            return load(parsed.body.toString(), { xmlMode: contentType.type.includes('xml') });
+        };
+        const waitForSelector = async (selector, _timeoutMs) => {
+            const $ = await loadBody();
+            if ($(selector).get().length === 0) {
+                throw new Error(`Selector '${selector}' not found.`);
+            }
+        };
+        const parseWithCheerio = async (selector, timeoutMs) => {
+            const $ = await loadBody();
+            if (selector) {
+                await crawlingContext.waitForSelector(selector, timeoutMs);
             }
-            if (this.persistCookiesPerSession) {
-                crawlingContext.session.setCookiesFromResponse(response);
+            return $;
+        };
+        this.throwOnBlockedRequest(response.status);
+        if (this.#saveResponseCookies) {
+            try {
+                for (const cookie of getCookiesFromResponse(response)) {
+                    if (!cookie)
+                        continue;
+                    try {
+                        await crawlingContext.session.cookieJar.setCookie(cookie, response.url, {
+                            ignoreError: false,
+                        });
+                    }
+                    catch (e) {
+                        this.log.debug(`Could not set cookie: ${e.message}`);
+                    }
+                }
             }
-            request.loadedUrl = response.url;
-            if (!this.requestMatchesEnqueueStrategy(request)) {
-                this.log.debug(
-                // eslint-disable-next-line dot-notation
-                `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`);
-                request.noRetry = true;
-                request.state = RequestState.SKIPPED;
-                return;
+            catch (e) {
+                this.log.exception(e, 'Could not get cookies from response');
             }
-            Object.assign(crawlingContext, parsed);
-            Object.defineProperty(crawlingContext, 'json', {
-                get() {
-                    if (contentType.type !== APPLICATION_JSON_MIME_TYPE)
-                        return null;
-                    const jsonString = parsed.body.toString(contentType.encoding);
-                    return JSON.parse(jsonString);
-                },
-            });
         }
+        return {
+            get json() {
+                if (contentType.type !== APPLICATION_JSON_MIME_TYPE)
+                    return null;
+                const jsonString = parsed.body.toString(contentType.encoding);
+                return JSON.parse(jsonString);
+            },
+            waitForSelector,
+            parseWithCheerio,
+            contentType,
+            body: parsed.body,
+        };
+    }
+    async handleBlockedRequestByContent(crawlingContext) {
         if (this.retryOnBlocked) {
-            const error = await this.isRequestBlocked(crawlingContext);
+            const error = await this.#isRequestBlocked(crawlingContext);
             if (error)
                 throw new SessionError(error);
         }
-        request.state = RequestState.REQUEST_HANDLER;
-        try {
-            await addTimeoutToPromise(async () => Promise.resolve(this.requestHandler(crawlingContext)), this.userRequestHandlerTimeoutMillis, `requestHandler timed out after ${this.userRequestHandlerTimeoutMillis / 1000} seconds.`);
-            request.state = RequestState.DONE;
-        }
-        catch (e) {
-            request.state = RequestState.ERROR;
-            throw e;
-        }
+        return {};
     }
-    async isRequestBlocked(crawlingContext) {
+    async #isRequestBlocked(crawlingContext) {
         if (HTML_AND_XML_MIME_TYPES.includes(crawlingContext.contentType.type)) {
             const $ = await crawlingContext.parseWithCheerio();
             const foundSelectors = RETRY_CSS_SELECTORS.filter((selector) => $(selector).length > 0);
@@ -278,87 +346,28 @@ export class HttpCrawler extends BasicCrawler {
                 return `Found selectors: ${foundSelectors.join(', ')}`;
             }
         }
-        return false;
-    }
-    async _handleNavigation(crawlingContext) {
-        const gotOptions = {};
-        const { request, session } = crawlingContext;
-        const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
-        request.state = RequestState.BEFORE_NAV;
-        // Execute pre navigation hooks before applying session pool cookies,
-        // as they may also set cookies in the session
-        await this._executeHooks(this.preNavigationHooks, crawlingContext, gotOptions);
-        tryCancel();
-        const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
-        this._applyCookies(crawlingContext, gotOptions, preNavigationHooksCookies, postNavigationHooksCookies);
-        const proxyUrl = crawlingContext.proxyInfo?.url;
-        crawlingContext.response = await addTimeoutToPromise(async () => this._requestFunction({ request, session, proxyUrl, gotOptions }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
-        tryCancel();
-        request.state = RequestState.AFTER_NAV;
-        await this._executeHooks(this.postNavigationHooks, crawlingContext, gotOptions);
-        tryCancel();
-    }
-    /**
-     * Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks.
-     */
-    _applyCookies({ session, request }, gotOptions, preHookCookies, postHookCookies) {
-        const sessionCookie = session?.getCookieString(request.url) ?? '';
-        let alteredGotOptionsCookies = gotOptions.headers?.Cookie || gotOptions.headers?.cookie || '';
-        if (gotOptions.headers?.Cookie && gotOptions.headers?.cookie) {
-            const { Cookie: upperCaseHeader, cookie: lowerCaseHeader } = gotOptions.headers;
-            this.log.warning(`Encountered mixed casing for the cookie headers in the got options for request ${request.url} (${request.id}). Their values will be merged`);
-            const sourceCookies = [];
-            if (Array.isArray(lowerCaseHeader)) {
-                sourceCookies.push(...lowerCaseHeader);
-            }
-            else {
-                sourceCookies.push(lowerCaseHeader);
-            }
-            if (Array.isArray(upperCaseHeader)) {
-                sourceCookies.push(...upperCaseHeader);
-            }
-            else {
-                sourceCookies.push(upperCaseHeader);
-            }
-            alteredGotOptionsCookies = mergeCookies(request.url, sourceCookies);
-        }
-        const sourceCookies = [sessionCookie, preHookCookies];
-        if (Array.isArray(alteredGotOptionsCookies)) {
-            sourceCookies.push(...alteredGotOptionsCookies);
-        }
-        else {
-            sourceCookies.push(alteredGotOptionsCookies);
-        }
-        sourceCookies.push(postHookCookies);
-        const mergedCookie = mergeCookies(request.url, sourceCookies);
-        gotOptions.headers ??= {};
-        Reflect.deleteProperty(gotOptions.headers, 'Cookie');
-        Reflect.deleteProperty(gotOptions.headers, 'cookie');
-        if (mergedCookie !== '') {
-            gotOptions.headers.Cookie = mergedCookie;
+        if (this.blockedStatusCodes.has(crawlingContext.response.status)) {
+            return `Blocked by status code ${crawlingContext.response.status}`;
         }
+        return false;
     }
     /**
      * Function to make the HTTP request. It performs optimizations
      * on the request such as only downloading the request body if the
      * received content type matches text/html, application/xml, application/xhtml+xml.
      */
-    async _requestFunction({ request, session, proxyUrl, gotOptions, }) {
-        if (!TimeoutError) {
-            // @ts-ignore
-            ({ TimeoutError } = await import('got-scraping'));
-        }
-        const opts = this._getRequestOptions(request, session, proxyUrl, gotOptions);
+    async requestFunction({ request, session, proxyUrl }) {
+        const opts = this.getRequestOptions(request, proxyUrl);
         try {
-            return await this._requestAsBrowser(opts, session);
+            return await this.requestAsBrowser(opts, session);
         }
         catch (e) {
-            if (e instanceof TimeoutError) {
-                this._handleRequestTimeout(session);
-                return undefined;
+            if (e instanceof Error && e.constructor.name === 'TimeoutError') {
+                this.handleRequestTimeout(session);
+                return new Response(); // this will never happen, as handleRequestTimeout always throws
             }
             if (this.isProxyError(e)) {
-                throw new SessionError(this._getMessageFromError(e));
+                throw new SessionError(this.getMessageFromError(e));
             }
             else {
                 throw e;
@@ -368,18 +377,15 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Encodes and parses response according to the provided content type
      */
-    async _parseResponse(request, responseStream, crawlingContext) {
-        const { statusCode } = responseStream;
-        const { type, charset } = parseContentTypeFromResponse(responseStream);
-        const { response, encoding } = this._encodeResponse(request, responseStream, charset);
-        const contentType = { type, encoding };
-        if (statusCode >= 400 && statusCode <= 599) {
-            this.stats.registerStatusCode(statusCode);
+    async parseResponse(request, response) {
+        const { status } = response;
+        const { type, charset } = parseContentTypeFromResponse(response);
+        if (status >= 400 && status <= 599) {
+            this.statistics.registerStatusCode(status);
         }
-        const excludeError = this.ignoreHttpErrorStatusCodes.has(statusCode);
-        const includeError = this.additionalHttpErrorStatusCodes.has(statusCode);
-        if ((statusCode >= 500 && !excludeError) || includeError) {
-            const body = await readStreamToString(response, encoding);
+        if (this.isErrorStatusCode(status)) {
+            const { response: decoded } = this.encodeResponse(request, response, charset);
+            const body = await decoded.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
             // Errors are often sent as JSON, so attempt to parse them,
             // despite Accept header being set to text/html.
             if (type === APPLICATION_JSON_MIME_TYPE) {
@@ -387,74 +393,62 @@ export class HttpCrawler extends BasicCrawler {
                 let { message } = errorResponse;
                 if (!message)
                     message = util.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
-                throw new Error(`${statusCode} - ${message}`);
+                throw new Error(`${status} - ${message}`);
             }
-            if (includeError) {
-                throw new Error(`${statusCode} - Error status code was set by user.`);
+            if (this.additionalHttpErrorStatusCodes.has(status)) {
+                throw new Error(`${status} - Error status code was set by user.`);
             }
             // It's not a JSON, so it's probably some text. Get the first 100 chars of it.
-            throw new Error(`${statusCode} - Internal Server Error: ${body.slice(0, 100)}`);
+            throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
         }
-        else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
-            const isXml = type.includes('xml');
-            const parsed = await this._parseHTML(response, isXml, crawlingContext);
-            return { ...parsed, isXml, response, contentType };
+        if (HTML_AND_XML_MIME_TYPES.includes(type) && !charset && !this.#forceResponseEncoding) {
+            // The charset comes from the document itself, so the raw bytes are what we need -
+            // decoding them through `encodeResponse` first would consume the body for nothing.
+            const rawBytes = Buffer.from(await response.arrayBuffer());
+            const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
+            const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';
+            const body = iconv.encodingExists(charsetToUse)
+                ? iconv.decode(rawBytes, charsetToUse)
+                : rawBytes.toString('utf8');
+            return { response, contentType: { type, encoding: 'utf-8' }, body };
         }
-        else {
-            const body = await concatStreamToBuffer(response);
-            return {
-                body,
-                response,
-                contentType,
-                enqueueLinks: async () => Promise.resolve({ processedRequests: [], unprocessedRequests: [] }),
-            };
+        const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);
+        const contentType = { type, encoding };
+        if (HTML_AND_XML_MIME_TYPES.includes(type)) {
+            return { response, contentType, body: await reencodedResponse.text() };
         }
-    }
-    async _parseHTML(response, _isXml, _crawlingContext) {
         return {
-            body: await concatStreamToBuffer(response),
+            body: Buffer.from(await reencodedResponse.bytes()),
+            response,
+            contentType,
         };
     }
     /**
      * Combines the provided `requestOptions` with mandatory (non-overridable) values.
      */
-    _getRequestOptions(request, session, proxyUrl, gotOptions) {
+    getRequestOptions(request, proxyUrl) {
         const requestOptions = {
             url: request.url,
             method: request.method,
             proxyUrl,
-            timeout: { request: this.navigationTimeoutMillis },
-            sessionToken: session,
-            ...gotOptions,
-            headers: { ...request.headers, ...gotOptions?.headers },
-            https: {
-                ...gotOptions?.https,
-                rejectUnauthorized: !this.ignoreSslErrors,
-            },
-            isStream: true,
+            timeout: this.#navigationTimeoutMillis,
+            headers: request.headers,
+            body: undefined,
         };
-        // Delete any possible lowercased header for cookie as they are merged in _applyCookies under the uppercase Cookie header
-        Reflect.deleteProperty(requestOptions.headers, 'cookie');
-        // TODO this is incorrect, the check for man in the middle needs to be done
-        //   on individual proxy level, not on the `proxyConfiguration` level,
-        //   because users can use normal + MITM proxies in a single configuration.
-        // Disable SSL verification for MITM proxies
-        if (this.proxyConfiguration && this.proxyConfiguration.isManInTheMiddle) {
-            requestOptions.https = {
-                ...requestOptions.https,
-                rejectUnauthorized: false,
-            };
+        if (requestOptions.headers?.cookie || requestOptions.headers?.Cookie) {
+            requestOptions.headers.Cookie = this.getCookieHeaderFromRequest(request);
+            delete requestOptions.headers.cookie;
         }
         if (/PATCH|POST|PUT/.test(request.method))
             requestOptions.body = request.payload ?? '';
         return requestOptions;
     }
-    _encodeResponse(request, response, encoding) {
-        if (this.forceResponseEncoding) {
-            encoding = this.forceResponseEncoding;
+    encodeResponse(request, response, encoding) {
+        if (this.#forceResponseEncoding) {
+            encoding = this.#forceResponseEncoding;
         }
-        else if (!encoding && this.suggestResponseEncoding) {
-            encoding = this.suggestResponseEncoding;
+        else if (!encoding && this.#suggestResponseEncoding) {
+            encoding = this.#suggestResponseEncoding;
         }
         // Fall back to utf-8 if we still don't have encoding.
         const utf8 = 'utf8';
@@ -467,14 +461,16 @@ export class HttpCrawler extends BasicCrawler {
         // Try to re-encode a variety of unsupported encodings to utf-8
         if (iconv.encodingExists(encoding)) {
             const encodeStream = iconv.encodeStream(utf8);
-            const decodeStream = iconv.decodeStream(encoding).on('error', (err) => encodeStream.emit('error', err));
-            response.on('error', (err) => decodeStream.emit('error', err));
-            const encodedResponse = response.pipe(decodeStream).pipe(encodeStream);
-            encodedResponse.statusCode = response.statusCode;
-            encodedResponse.headers = response.headers;
-            encodedResponse.url = response.url;
+            const decodeStream = iconv
+                .decodeStream(encoding)
+                .on('error', (err) => encodeStream.emit('error', err));
+            const reencodedBody = response.body
+                ? Readable.toWeb(Readable.from(Readable.fromWeb(response.body)
+                    .pipe(decodeStream)
+                    .pipe(encodeStream)))
+                : null;
             return {
-                response: encodedResponse,
+                response: new ResponseWithUrl(reencodedBody, response),
                 encoding: utf8,
             };
         }
@@ -483,15 +479,15 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Checks and extends supported mime types
      */
-    _extendSupportedMimeTypes(additionalMimeTypes) {
+    extendSupportedMimeTypes(additionalMimeTypes) {
         for (const mimeType of additionalMimeTypes) {
             if (mimeType === '*/*') {
-                this.supportedMimeTypes.add(mimeType);
+                this.#supportedMimeTypes.add(mimeType);
                 continue;
             }
             try {
                 const parsedType = contentTypeParser.parse(mimeType);
-                this.supportedMimeTypes.add(parsedType.type);
+                this.#supportedMimeTypes.add(parsedType.type);
             }
             catch (err) {
                 throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
@@ -501,136 +497,55 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Handles timeout request
      */
-    _handleRequestTimeout(session) {
-        session?.markBad();
-        throw new Error(`request timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds.`);
+    handleRequestTimeout(session) {
+        session.markBad();
+        throw new Error(`Request timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
     }
-    _abortDownloadOfBody(request, response) {
-        const { statusCode } = response;
+    abortDownloadOfBody(request, response) {
+        const { status } = response;
         const { type } = parseContentTypeFromResponse(response);
-        // eslint-disable-next-line dot-notation -- accessing private property
-        const blockedStatusCodes = this.sessionPool ? this.sessionPool['blockedStatusCodes'] : [];
-        // if we retry the request, can the Content-Type change?
-        const isTransientContentType = statusCode >= 500 || blockedStatusCodes.includes(statusCode);
-        if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && !isTransientContentType) {
+        const isTransientContentType = status >= 500 || this.blockedStatusCodes.has(status);
+        if (!this.#supportedMimeTypes.has(type) && !this.#supportedMimeTypes.has('*/*') && !isTransientContentType) {
             request.noRetry = true;
             throw new Error(`Resource ${request.url} served Content-Type ${type}, ` +
-                `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
+                `but only ${Array.from(this.#supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
         }
     }
     /**
      * @internal wraps public utility for mocking purposes
      */
-    _requestAsBrowser = async (options, session) => {
-        const response = await this.httpClient.stream(processHttpRequestOptions({
+    requestAsBrowser = async (options, session) => {
+        const opts = processHttpRequestOptions({
             ...options,
-            cookieJar: options.cookieJar, // HACK - the type of ToughCookieJar in got is wrong
             responseType: 'text',
-        }), (redirectResponse, updatedRequest) => {
-            if (this.persistCookiesPerSession) {
-                session.setCookiesFromResponse(redirectResponse);
-                const cookieString = session.getCookieString(updatedRequest.url.toString());
-                if (cookieString !== '') {
-                    updatedRequest.headers.Cookie = cookieString;
-                }
-            }
         });
-        return addResponsePropertiesToStream(response.stream, response);
-    };
-}
-/**
- * The stream object returned from got does not have the below properties.
- * At the same time, you can't read data directly from the response stream,
- * because they won't get emitted unless you also read from the primary
- * got stream. To be able to work with only one stream, we move the expected props
- * from the response stream to the got stream.
- * @internal
- */
-function addResponsePropertiesToStream(stream, response) {
-    const properties = [
-        'statusCode',
-        'statusMessage',
-        'headers',
-        'complete',
-        'httpVersion',
-        'rawHeaders',
-        'rawTrailers',
-        'trailers',
-        'url',
-        'request',
-    ];
-    stream.on('end', () => {
-        // @ts-expect-error
-        if (stream.rawTrailers)
-            stream.rawTrailers = response.rawTrailers; // TODO BC with got - remove in 4.0
-        // @ts-expect-error
-        if (stream.trailers)
-            stream.trailers = response.trailers;
-        // @ts-expect-error
-        stream.complete = response.complete;
-    });
-    for (const prop of properties) {
-        if (!(prop in stream)) {
-            stream[prop] = response[prop];
-        }
-    }
-    return stream;
-}
-/**
- * Gets parsed content type from response object
- * @param response HTTP response object
- */
-function parseContentTypeFromResponse(response) {
-    ow(response, ow.object.partialShape({
-        url: ow.string.url,
-        headers: new ObjectPredicate(),
-    }));
-    const { url, headers } = response;
-    let parsedContentType;
-    if (headers['content-type']) {
-        try {
-            parsedContentType = contentTypeParser.parse(headers['content-type']);
-        }
-        catch {
-            // Can not parse content type from Content-Type header. Try to parse it from file extension.
-        }
-    }
-    // Parse content type from file extension as fallback
-    if (!parsedContentType) {
-        const parsedUrl = new URL(url);
-        const contentTypeFromExtname = mime.contentType(extname(parsedUrl.pathname)) || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5
-        parsedContentType = contentTypeParser.parse(contentTypeFromExtname);
-    }
-    return {
-        type: parsedContentType.type,
-        charset: parsedContentType.parameters.charset,
+        // When saveResponseCookies is false, the response cookies must not mutate the
+        // session jar. Reads still go through the session (so session.setCookie() in pre-nav
+        // hooks keeps working) but a per-request clone is passed in so writes are discarded.
+        const cookieJar = this.#saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
+        // Bind the request to the shared navigation window instead of a fixed per-request timeout, so
+        // `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and
+        // kill a lazily-read body mid-extension. This aborts the socket only during the header phase; the body
+        // read is bounded separately at the promise level (see `processHttpResponse`), so a slow-streaming body
+        // still fails cleanly with a navigation timeout, though the socket is left to close on its own.
+        const cancelSignal = storage.getStore()?.cancelTask.signal;
+        const response = await this.httpClient.sendRequest(new Request(opts.url, {
+            body: opts.body ? Readable.toWeb(opts.body) : undefined,
+            headers: new Headers(opts.headers),
+            method: opts.method,
+            // Node-specific option to make the request body work with streams
+            duplex: 'half',
+        }), {
+            session,
+            cookieJar,
+            proxyUrl: opts.proxyUrl,
+            signal: cancelSignal,
+            timeoutMillis: cancelSignal ? undefined : opts.timeout,
+            ignoreTlsErrors: this.#ignoreTlsErrors,
+        });
+        return response;
     };
 }
-/**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link HttpCrawler}.
- * Defaults to the {@link HttpCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<HttpCrawlingContext>()`.
- *
- * ```ts
- * import { HttpCrawler, createHttpRouter } from 'crawlee';
- *
- * const router = createHttpRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
- *
- * const crawler = new HttpCrawler({
- *     requestHandler: router,
- * });
- * await crawler.run();
- * ```
- */
-export function createHttpRouter(routes) {
-    return Router.create(routes);
+export function createHttpRouter(routesOrSchemas) {
+    return Router.create(routesOrSchemas);
 }
-//# sourceMappingURL=http-crawler.js.map