@crawlee/http 4.0.0-rc.0 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/index.d.ts

@@ -1,3 +1,4 @@
 export * from '@crawlee/basic';
 export * from './internals/http-crawler.js';
+export * from './internals/dom-crawler.js';
 export * from './internals/file-download.js';

package/index.js

@@ -1,3 +1,4 @@
 export * from '@crawlee/basic';
 export * from './internals/http-crawler.js';
+export * from './internals/dom-crawler.js';
 export * from './internals/file-download.js';

package/internals/dom-crawler.d.ts

@@ -0,0 +1,131 @@
+import type { AddRequestsBatchedResult, ContextPipeline, CrawlingContext, EnqueueLinksOptions, ExtractLinksOptions, GetUserDataFromRequest } from '@crawlee/basic';
+import type { Awaitable, Dictionary } from '@crawlee/types';
+import type { CheerioAPI } from 'cheerio';
+import type { HttpCrawlerOptions, InternalHttpCrawlingContext } from './http-crawler.js';
+import { HttpCrawler } from './http-crawler.js';
+/**
+ * The minimum a {@link DOMParser} has to contribute to the crawling context - the serialized document, used by
+ * the {@link DOMCrawlingContext.parseWithCheerio|`parseWithCheerio`} helper.
+ */
+export interface DOMParseResult {
+    body: string;
+}
+/**
+ * Turns a response body into a DOM representation and knows how to query it. Passing one to {@link DOMCrawler}
+ * is what makes the crawler jsdom-based, linkedom-based, or based on a DOM implementation of your own.
+ *
+ * **Example usage:**
+ * ```ts
+ * import { DOMCrawler } from 'crawlee';
+ * import type { DOMParser } from 'crawlee';
+ *
+ * const myParser: DOMParser<{ body: string }> = {
+ *     // ...
+ * };
+ *
+ * const crawler = new DOMCrawler({
+ *     parser: myParser,
+ *     async requestHandler({ body }) {
+ *         // ...
+ *     },
+ * });
+ * ```
+ */
+export interface DOMParser<Parsed extends DOMParseResult> {
+    /**
+     * The context members {@link DOMParser.parse|`parse`} contributes, mapped to `true`. Used to build the
+     * placeholders that report a helpful error when the members are accessed after `skipNavigation` - the `Record`
+     * type forces every key of `Parsed` to be listed, so the compiler catches an omission that would otherwise
+     * yield `undefined` (rather than throwing) after `skipNavigation`.
+     */
+    readonly placeholderMembers: Record<keyof Parsed & string, true>;
+    parse(context: InternalHttpCrawlingContext): Awaitable<Parsed>;
+    /**
+     * Returns the URLs the `selector` matches, resolved against `baseUrl`.
+     */
+    extractLinks(parsed: Parsed, selector: string, baseUrl: string): Awaitable<string[]>;
+    /**
+     * Returns the current matches of `selector`. Only the count is used, by
+     * {@link DOMCrawlingContext.waitForSelector|`waitForSelector`}.
+     */
+    select(parsed: Parsed, selector: string): Awaitable<ArrayLike<unknown>>;
+    /**
+     * Whether the parse result can change after {@link DOMParser.parse|`parse`} returned - the case when the DOM
+     * implementation runs the page scripts. If it can, {@link DOMCrawlingContext.waitForSelector|`waitForSelector`}
+     * polls until the timeout elapses; otherwise it fails as soon as the selector does not match.
+     */
+    readonly mutable?: boolean;
+    /**
+     * Returns a Cheerio handle over the parse result, for parsers that are backed by Cheerio anyway. Without it,
+     * {@link DOMCrawlingContext.parseWithCheerio|`parseWithCheerio`} parses
+     * {@link DOMParseResult.body|`body`} again.
+     */
+    toCheerio?(parsed: Parsed): Awaitable<CheerioAPI>;
+    /**
+     * Releases whatever {@link DOMParser.parse|`parse`} allocated. Called after the request handler finishes or
+     * fails, and skipped entirely when navigation was skipped.
+     */
+    cleanup?(parsed: Parsed): Awaitable<void>;
+}
+export interface DOMCrawlingHelpers {
+    /**
+     * Extracts URLs from the parsed DOM, without adding them to the request queue.
+     */
+    extractLinks(options?: ExtractLinksOptions): Promise<string[]>;
+    /**
+     * Helper function for extracting URLs from the parsed DOM and adding them to the request queue.
+     */
+    enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;
+    /**
+     * Wait for an element matching the selector to appear. The `timeoutMs` only has an effect when the parser is
+     * {@link DOMParser.mutable|`mutable`} (e.g. {@link JSDOMCrawler} with `runScripts: true`); otherwise the
+     * selector is checked once and the call resolves or throws immediately.
+     * Timeout defaults to 5s.
+     *
+     * **Example usage:**
+     * ```ts
+     * async requestHandler({ waitForSelector, parseWithCheerio }) {
+     *     await waitForSelector('article h1');
+     *     const $ = await parseWithCheerio();
+     *     const title = $('title').text();
+     * });
+     * ```
+     */
+    waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+    /**
+     * Returns Cheerio handle, allowing to work with the data same way as with {@link CheerioCrawler}.
+     * When provided with the `selector` argument, it will throw if it's not available.
+     *
+     * **Example usage:**
+     * ```javascript
+     * async requestHandler({ parseWithCheerio }) {
+     *     const $ = await parseWithCheerio();
+     *     const title = $('title').text();
+     * });
+     * ```
+     */
+    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
+}
+export type DOMCrawlingContext<Parsed extends DOMParseResult = DOMParseResult, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> = InternalHttpCrawlingContext<UserData, JSONData> & Parsed & DOMCrawlingHelpers;
+export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, any>, StatisticStateExtension extends object = {}> extends Omit<HttpCrawlerOptions<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'contextPipelineBuilder'> {
+    /**
+     * The DOM implementation to parse the response bodies with. Its parse result becomes part of the crawling
+     * context, so the members the request handler receives follow from the parser you pass.
+     */
+    parser: DOMParser<Parsed>;
+}
+/**
+ * An {@link HttpCrawler} that parses each response into a DOM using the {@link DOMCrawlerOptions.parser|`parser`}
+ * it is given, and exposes the parse result plus the {@link DOMCrawlingContext.enqueueLinks|`enqueueLinks`} and
+ * {@link DOMCrawlingContext.extractLinks|`extractLinks`} helpers on the crawling context.
+ *
+ * {@link JSDOMCrawler} and {@link LinkeDOMCrawler} are this crawler with a parser already chosen.
+ *
+ * @category Crawlers
+ */
+export declare class DOMCrawler<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<DOMCrawlingContext<Parsed>['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+    #private;
+    constructor(options: DOMCrawlerOptions<Parsed, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
+    protected buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed>>;
+}

package/internals/dom-crawler.js

@@ -0,0 +1,93 @@
+import { EnqueueStrategy, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering } from '@crawlee/basic';
+import { sleep } from '@crawlee/utils';
+import { HttpCrawler } from './http-crawler.js';
+/**
+ * An {@link HttpCrawler} that parses each response into a DOM using the {@link DOMCrawlerOptions.parser|`parser`}
+ * it is given, and exposes the parse result plus the {@link DOMCrawlingContext.enqueueLinks|`enqueueLinks`} and
+ * {@link DOMCrawlingContext.extractLinks|`extractLinks`} helpers on the crawling context.
+ *
+ * {@link JSDOMCrawler} and {@link LinkeDOMCrawler} are this crawler with a parser already chosen.
+ *
+ * @category Crawlers
+ */
+export class DOMCrawler extends HttpCrawler {
+    #parser;
+    constructor(options) {
+        const { parser, ...rest } = options;
+        super({ ...rest, contextPipelineBuilder: () => this.buildContextPipeline() });
+        this.#parser = parser;
+    }
+    buildContextPipeline() {
+        return super
+            .buildContextPipeline()
+            .compose(this.#parseContent.bind(this))
+            .compose(async (context) => this.#addHelpers(context));
+    }
+    async #parseContent(context, onCleanup) {
+        try {
+            const parsed = await this.#parser.parse(context);
+            onCleanup(() => this.#parser.cleanup?.(parsed));
+            return parsed;
+        }
+        catch (err) {
+            // The `skipNavigation` placeholders below throw on access, so there is nothing to clean up.
+            if (err instanceof NavigationSkippedError) {
+                return Object.defineProperties({}, Object.fromEntries(Object.keys(this.#parser.placeholderMembers).map((member) => [
+                    member,
+                    {
+                        configurable: true,
+                        enumerable: true,
+                        get() {
+                            throw new NavigationSkippedError(`The \`${member}\` property is not available - \`skipNavigation\` was used`, { cause: err });
+                        },
+                    },
+                ])));
+            }
+            throw err;
+        }
+    }
+    async #addHelpers(context) {
+        const { addRequests } = context;
+        const parser = this.#parser;
+        const extractLinks = async (options) => parser.extractLinks(context, options?.selector ?? 'a', options?.baseUrl ?? context.request.loadedUrl ?? context.request.url);
+        const waitForSelector = async (selector, timeoutMs = 5_000) => {
+            let remaining = parser.mutable ? timeoutMs : 0;
+            while ((await parser.select(context, selector)).length === 0) {
+                if (remaining <= 0) {
+                    throw new Error(`Selector '${selector}' not found.`);
+                }
+                await sleep(50);
+                remaining -= 50;
+            }
+        };
+        return {
+            extractLinks,
+            waitForSelector,
+            enqueueLinks: async (options = {}) => {
+                const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
+                    enqueueStrategy: options.strategy,
+                    finalRequestUrl: context.request.loadedUrl,
+                    originalRequestUrl: context.request.url,
+                    userProvidedBaseUrl: options.baseUrl,
+                });
+                const urls = await extractLinks(options);
+                return addRequests(urls, {
+                    ...options,
+                    baseUrl,
+                    strategy: options.strategy ?? EnqueueStrategy.SameHostname,
+                });
+            },
+            async parseWithCheerio(selector, _timeoutMs = 5_000) {
+                // Import full cheerio (not cheerio/slim) to be browser-compliant for DOM Crawlers (not CheerioCrawler).
+                const $ = (await parser.toCheerio?.(context)) ??
+                    (await import('cheerio')).load(context.body, {
+                        xmlMode: context.contentType.type.includes('xml'),
+                    });
+                if (selector && $(selector).get().length === 0) {
+                    throw new Error(`Selector '${selector}' not found.`);
+                }
+                return $;
+            },
+        };
+    }
+}

package/internals/file-download.d.ts

@@ -1,16 +1,11 @@
-import { Transform } from 'node:stream';
-import type { BasicCrawlerOptions } from '@crawlee/basic';
+import type { BasicCrawlerOptions, CrawlingContext, CrawlingRequest, LoadedRequest } from '@crawlee/basic';
 import { BasicCrawler } from '@crawlee/basic';
-import type { CrawlingContext, LoadedRequest, Request } from '@crawlee/core';
-import { ResponseWithUrl } from '@crawlee/http-client';
 import type { Dictionary } from '@crawlee/types';
-import type { ErrorHandler, GetUserDataFromRequest, InternalHttpHook, RequestHandler, RouterRoutes } from '../index.js';
-declare const kBodyDrained: unique symbol;
+import type { ErrorHandler, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '../index.js';
 export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, FileDownloadCrawlingContext<UserData> & ContextExtension>;
-export type FileDownloadHook<UserData extends Dictionary = any> = InternalHttpHook<FileDownloadCrawlingContext<UserData>>;
 export interface FileDownloadCrawlingContext<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
-    request: LoadedRequest<Request<UserData>>;
+    request: LoadedRequest<CrawlingRequest<UserData>>;
     response: Response;
     contentType: {
         type: string;
@@ -18,31 +13,6 @@ export interface FileDownloadCrawlingContext<UserData extends Dictionary = any>
     };
 }
 export type FileDownloadRequestHandler<UserData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData>>;
-/**
- * Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
- * This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
- * If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
- * it will throw an error.
- *
- * Can be used e.g. to abort a download if the network speed is too slow.
- * @returns Transform stream that monitors the speed of the incoming data.
- */
-export declare function MinimumSpeedStream({ minSpeedKbps, historyLengthMs, checkProgressInterval: checkProgressIntervalMs, }: {
-    minSpeedKbps: number;
-    historyLengthMs?: number;
-    checkProgressInterval?: number;
-}): Transform;
-/**
- * Creates a transform stream that logs the progress of the incoming data.
- * This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
- *
- * Can be used e.g. to log the progress of a download.
- * @returns Transform stream logging the progress of the incoming data.
- */
-export declare function ByteCounterStream({ logTransferredBytes, loggingInterval, }: {
-    logTransferredBytes: (transferredBytes: number) => void;
-    loggingInterval?: number;
-}): Transform;
 /**
  * Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
  *
@@ -50,24 +20,14 @@ export declare function ByteCounterStream({ logTransferredBytes, loggingInterval
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
  *
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link FileDownloadOptions.requestHandler} where the user can specify what to do with the downloaded data.
+ * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
  *
- * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link FileDownloadOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
+ * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
  *
- * > The {@link FileDownloadOptions.requestList|`requestList`} and {@link FileDownloadOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
+ * > The {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
- * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
- *
- * ```
- * preNavigationHooks: [
- *     (crawlingContext) => {
- *         // ...
- *     },
- * ]
- * ```
- *
  * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * ## Example usage
@@ -87,18 +47,8 @@ export declare function ByteCounterStream({ logTransferredBytes, loggingInterval
  * ```
  */
 export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
-    constructor(options?: BasicCrawlerOptions<FileDownloadCrawlingContext>);
-// @ts-ignore optional peer dependency or compatibility with es2022
-    protected buildContextPipeline(): import("@crawlee/basic").ContextPipeline<CrawlingContext<Dictionary>, CrawlingContext<Dictionary> & {
-        request: LoadedRequest<Request>;
-        response: ResponseWithUrl;
-        contentType: {
-            type: string;
-            encoding: BufferEncoding;
-        };
-        [kBodyDrained]: Promise<void>;
-    }>;
-    private initiateDownload;
+    #private;
+    constructor(options?: Omit<BasicCrawlerOptions<FileDownloadCrawlingContext>, 'contextPipelineBuilder'>);
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.
@@ -124,6 +74,6 @@ export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingConte
  * await crawler.run();
  * ```
  */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/basic").RouterHandler<Context, Record<string, GetUserDataFromRequest<Context["request"]>>>;
-export {};
+export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;

package/internals/file-download.js

@@ -1,67 +1,7 @@
-import { Transform } from 'node:stream';
-import { BasicCrawler } from '@crawlee/basic';
+import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
 import { ResponseWithUrl } from '@crawlee/http-client';
 import { Router } from '../index.js';
 import { parseContentTypeFromResponse } from './utils.js';
-const kBodyDrained = Symbol('bodyDrained');
-/**
- * Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
- * This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
- * If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
- * it will throw an error.
- *
- * Can be used e.g. to abort a download if the network speed is too slow.
- * @returns Transform stream that monitors the speed of the incoming data.
- */
-export function MinimumSpeedStream({ minSpeedKbps, historyLengthMs = 10e3, checkProgressInterval: checkProgressIntervalMs = 5e3, }) {
-    let snapshots = [];
-    const checkInterval = setInterval(() => {
-        const now = Date.now();
-        snapshots = snapshots.filter((snapshot) => now - snapshot.timestamp < historyLengthMs);
-        const totalBytes = snapshots.reduce((acc, snapshot) => acc + snapshot.bytes, 0);
-        const elapsed = (now - (snapshots[0]?.timestamp ?? 0)) / 1000;
-        if (totalBytes / 1024 / elapsed < minSpeedKbps) {
-            clearInterval(checkInterval);
-            stream.emit('error', new Error(`Stream speed too slow, aborting...`));
-        }
-    }, checkProgressIntervalMs);
-    const stream = new Transform({
-        transform: (chunk, _, callback) => {
-            snapshots.push({ timestamp: Date.now(), bytes: chunk.length });
-            callback(null, chunk);
-        },
-        final: (callback) => {
-            clearInterval(checkInterval);
-            callback();
-        },
-    });
-    return stream;
-}
-/**
- * Creates a transform stream that logs the progress of the incoming data.
- * This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
- *
- * Can be used e.g. to log the progress of a download.
- * @returns Transform stream logging the progress of the incoming data.
- */
-export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000, }) {
-    let transferredBytes = 0;
-    let lastLogTime = Date.now();
-    return new Transform({
-        transform: (chunk, _, callback) => {
-            transferredBytes += chunk.length;
-            if (Date.now() - lastLogTime > loggingInterval) {
-                lastLogTime = Date.now();
-                logTransferredBytes(transferredBytes);
-            }
-            callback(null, chunk);
-        },
-        flush: (callback) => {
-            logTransferredBytes(transferredBytes);
-            callback();
-        },
-    });
-}
 /**
  * Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
  *
@@ -69,24 +9,14 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
  *
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link FileDownloadOptions.requestHandler} where the user can specify what to do with the downloaded data.
+ * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
  *
- * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link FileDownloadOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
+ * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
  *
- * > The {@link FileDownloadOptions.requestList|`requestList`} and {@link FileDownloadOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
+ * > The {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
- * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
- *
- * ```
- * preNavigationHooks: [
- *     (crawlingContext) => {
- *         // ...
- *     },
- * ]
- * ```
- *
  * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * ## Example usage
@@ -110,36 +40,32 @@ export class FileDownload extends BasicCrawler {
     constructor(options = {}) {
         super({
             ...options,
-            contextPipelineBuilder: () => this.buildContextPipeline(),
+            contextPipelineBuilder: () => this.#buildContextPipeline(),
         });
     }
-    buildContextPipeline() {
-        return super.buildContextPipeline().compose({
-            action: async (context) => this.initiateDownload(context),
-            cleanup: async (context) => {
-                if (!context.response.bodyUsed) {
-                    // Nobody consumed the body — cancel it so the
-                    // underlying connection can be released.
-                    await context.response.body?.cancel();
-                }
-                await context[kBodyDrained];
-            },
-        });
+    #buildContextPipeline() {
+        return ContextPipeline.create().compose(this.#initiateDownload.bind(this));
     }
-    async initiateDownload(context) {
+    async #initiateDownload(context, onCleanup) {
         const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
             session: context.session,
         });
         const { type, charset: encoding } = parseContentTypeFromResponse(response);
         context.request.url = response.url;
         const { response: trackedResponse, bodyDrained } = trackBodyConsumption(response);
-        const contextExtension = {
+        onCleanup(async () => {
+            if (!trackedResponse.bodyUsed) {
+                // Nobody consumed the body — cancel it so the
+                // underlying connection can be released.
+                await trackedResponse.body?.cancel();
+            }
+            await bodyDrained;
+        });
+        return {
             request: context.request,
             response: trackedResponse,
             contentType: { type, encoding },
-            [kBodyDrained]: bodyDrained,
         };
-        return contextExtension;
     }
 }
 /**
@@ -162,30 +88,6 @@ function trackBodyConsumption(response) {
     });
     return { response: trackedResponse, bodyDrained };
 }
-/**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link FileDownload}.
- * Defaults to the {@link FileDownloadCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<FileDownloadCrawlingContext>()`.
- *
- * ```ts
- * import { FileDownload, createFileRouter } from 'crawlee';
- *
- * const router = createFileRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
- *
- * const crawler = new FileDownload({
- *     requestHandler: router,
- * });
- * await crawler.run();
- * ```
- */
-export function createFileRouter(routes) {
-    return Router.create(routes);
+export function createFileRouter(routesOrSchemas) {
+    return Router.create(routesOrSchemas);
 }

package/internals/http-crawler.d.ts

@@ -1,13 +1,13 @@
-import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, Request as CrawleeRequest, RequestHandler, RequireContextPipeline, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/basic';
+import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequireContextPipeline, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/basic';
 import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
-import { type LoadedRequest } from '@crawlee/core';
 import type { Awaitable, Dictionary } from '@crawlee/types';
-import { type CheerioRoot } from '@crawlee/utils/internal';
+import type { CheerioAPI } from 'cheerio';
 import type { JsonValue } from 'type-fest';
 import { z } from 'zod';
 /**
  * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
- * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default.
+ * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default, with your own
+ * concurrency shortcuts (`minConcurrency`, `maxConcurrency`, `maxRequestsPerMinute`) kept on top.
  *
  * A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
  * wholesale, tuning included, so spread these options in if you want to keep it:
@@ -77,7 +77,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
      * ]
      * ```
      */
-    postNavigationHooks?: ((crawlingContext: CrawlingContextWithResponse & ContextExtension) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
+    postNavigationHooks?: InternalHttpHook<CrawlingContextWithResponse, ContextExtension>[];
     /**
      * An array of [MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types)
      * you want the crawler to load and process. By default, only `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
@@ -115,13 +115,11 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
     saveResponseCookies?: boolean;
 }
 export type InternalHttpHook<Context, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
-export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends JsonValue = any> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
 interface CrawlingContextWithResponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
     /**
      * The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
      */
-    request: LoadedRequest<CrawleeRequest<UserData>>;
+    request: LoadedRequest<CrawlingRequest<UserData>>;
     /**
      * The HTTP response object containing status code, headers, and other response metadata.
      */
@@ -148,7 +146,11 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
         encoding: BufferEncoding;
     };
     /**
-     * Wait for an element matching the selector to appear. Timeout is ignored.
+     * Wait for an element matching the selector to appear.
+     *
+     * `HttpCrawler` and {@link CheerioCrawler} parse a response that is already fully downloaded, so there is
+     * nothing to wait for and `timeoutMs` is ignored. {@link JSDOMCrawler} and {@link LinkeDOMCrawler} poll for
+     * the selector instead, with `timeoutMs` defaulting to 5s.
      *
      * **Example usage:**
      * ```ts
@@ -162,7 +164,12 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
     waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
     /**
      * Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@link CheerioCrawler}.
-     * When provided with the `selector` argument, it will throw if it's not available.
+     * This is here to unify the crawler API, so they all have this handy method - in {@link CheerioCrawler} it has
+     * the same return type as the `$` context property, so use it only if you are abstracting your workflow to
+     * support different context types in one handler.
+     *
+     * When provided with the `selector` argument, it will throw if it's not available. {@link JSDOMCrawler} and
+     * {@link LinkeDOMCrawler} wait for the selector first, with `timeoutMs` defaulting to 5s.
      *
      * **Example usage:**
      * ```ts
@@ -172,7 +179,7 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
      * });
      * ```
      */
-    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
 }
 export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
 }
@@ -249,15 +256,10 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
  */
 export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     #private;
+    /**
+     * @internal
+     */
     protected static optionsShape: {
-        navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
-        ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
-        additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
-        suggestResponseEncoding: z.ZodOptional<z.ZodString>;
-        forceResponseEncoding: z.ZodOptional<z.ZodString>;
-        saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
-        preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
-        postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
         requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -290,21 +292,20 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
         }, z.core.$strict>]>>;
         onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
+        httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
+        configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
         storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
+        eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
         logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
         minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
         keepAlive: z.ZodOptional<z.ZodBoolean>;
         statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         id: z.ZodOptional<z.ZodString>;
-    };
-    protected static optionsSchema: z.ZodObject<{
         navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
         ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
         additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
@@ -313,6 +314,9 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
         saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
         preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+    };
+    /** @internal */
+    protected static optionsSchema: z.ZodObject<{
         contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
         requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -345,66 +349,39 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
         }, z.core.$strict>]>>;
         onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
+        httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
+        configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
         storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
+        eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
         logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
         minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
         keepAlive: z.ZodOptional<z.ZodBoolean>;
         statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         id: z.ZodOptional<z.ZodString>;
+        navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
+        additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        suggestResponseEncoding: z.ZodOptional<z.ZodString>;
+        forceResponseEncoding: z.ZodOptional<z.ZodString>;
+        saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
+        preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+        postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
     }, z.core.$strict>;
     /**
      * All `HttpCrawlerOptions` parameters are passed via an options object.
      */
     constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
+    /** @internal */
     protected getNavigationTimeoutMillis(): number;
-    /**
-     * Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
-     * concurrency shortcuts on top. Not called for a supplied
-     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} — spread the constant into it yourself to
-     * keep the tuning.
-     */
+    /** @internal */
     protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
     protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;
-    private prepareHttpRequest;
     private makeHttpRequest;
-    private processHttpResponse;
-    private handleBlockedRequestByContent;
-    protected isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise<string | false>;
-    /**
-     * Function to make the HTTP request. It performs optimizations
-     * on the request such as only downloading the request body if the
-     * received content type matches text/html, application/xml, application/xhtml+xml.
-     */
-    private requestFunction;
-    /**
-     * Encodes and parses response according to the provided content type
-     */
-    private parseResponse;
-    /**
-     * Combines the provided `requestOptions` with mandatory (non-overridable) values.
-     */
-    private getRequestOptions;
-    private encodeResponse;
-    /**
-     * Checks and extends supported mime types
-     */
-    private extendSupportedMimeTypes;
-    /**
-     * Handles timeout request
-     */
-    private handleRequestTimeout;
-    private abortDownloadOfBody;
-    /**
-     * @internal wraps public utility for mocking purposes
-     */
-    private requestAsBrowser;
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.

package/internals/http-crawler.js

@@ -1,10 +1,9 @@
 import { Readable } from 'node:stream';
 import util from 'node:util';
-import { BasicCrawler, ContextPipeline, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, Router, SessionError, } from '@crawlee/basic';
-import { RequestThrottledError, getCookiesFromResponse, parseArgument, schemas, } from '@crawlee/core';
+import { BasicCrawler, ContextPipeline, getCookiesFromResponse, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, RequestThrottledError, Router, SessionError, } from '@crawlee/basic';
 import { ResponseWithUrl } from '@crawlee/http-client';
-import { RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
-import contentTypeParser from 'content-type';
+import { parseArgument, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
+import { isTypeValid, parse as parseContentType } from 'content-type';
 import iconv from 'iconv-lite';
 import { z } from 'zod';
 import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
@@ -16,7 +15,8 @@ const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml
 const APPLICATION_JSON_MIME_TYPE = 'application/json';
 /**
  * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
- * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default.
+ * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default, with your own
+ * concurrency shortcuts (`minConcurrency`, `maxConcurrency`, `maxRequestsPerMinute`) kept on top.
  *
  * A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
  * wholesale, tuning included, so spread these options in if you want to keep it:
@@ -117,6 +117,9 @@ export class HttpCrawler extends BasicCrawler {
     #suggestResponseEncoding;
     #forceResponseEncoding;
     #supportedMimeTypes;
+    /**
+     * @internal
+     */
     static optionsShape = {
         ...BasicCrawler.optionsShape,
         navigationTimeoutSecs: schemas.anyNumber.default(30),
@@ -128,6 +131,7 @@ export class HttpCrawler extends BasicCrawler {
         preNavigationHooks: schemas.anyArray.default(() => []),
         postNavigationHooks: schemas.anyArray.default(() => []),
     };
+    /** @internal */
     static optionsSchema = z.strictObject(HttpCrawler.optionsShape);
     /**
      * All `HttpCrawlerOptions` parameters are passed via an options object.
@@ -143,7 +147,7 @@ export class HttpCrawler extends BasicCrawler {
         });
         this.#supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
         if (additionalMimeTypes.length)
-            this.extendSupportedMimeTypes(additionalMimeTypes);
+            this.#extendSupportedMimeTypes(additionalMimeTypes);
         if (suggestResponseEncoding && forceResponseEncoding) {
             this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
         }
@@ -156,20 +160,16 @@ export class HttpCrawler extends BasicCrawler {
         // members `extendContext` added at runtime.
         this.#preNavigationHooks = preNavigationHooks;
         this.#postNavigationHooks = [
-            ({ request, response }) => this.abortDownloadOfBody(request, response),
+            ({ request, response }) => this.#abortDownloadOfBody(request, response),
             ...postNavigationHooks,
         ];
         this.#saveResponseCookies = saveResponseCookies;
     }
+    /** @internal */
     getNavigationTimeoutMillis() {
         return this.#navigationTimeoutMillis;
     }
-    /**
-     * Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
-     * concurrency shortcuts on top. Not called for a supplied
-     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} — spread the constant into it yourself to
-     * keep the tuning.
-     */
+    /** @internal */
     createDefaultConcurrencySystem(options) {
         return super.createDefaultConcurrencySystem({
             ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS,
@@ -179,9 +179,7 @@ export class HttpCrawler extends BasicCrawler {
     buildContextPipeline() {
         // When navigation is skipped, `prepareHttpRequest` has already installed throwing getters for
         // the response-derived members, so the guarded action is bypassed and the context left untouched.
-        const skipGuard = (action) => ({
-            action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
-        });
+        const skipGuard = (action) => async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {}));
         // A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation
         // hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same
         // window the navigation uses instead of each step being timed on its own.
@@ -193,9 +191,7 @@ export class HttpCrawler extends BasicCrawler {
             }
             return addTimeoutToPromise(async () => step(ctx), remaining, navigationTimedOut);
         });
-        let pipeline = ContextPipeline.create().compose({
-            action: this.prepareHttpRequest.bind(this),
-        });
+        let pipeline = ContextPipeline.create().compose(this.#prepareHttpRequest.bind(this));
         for (const hook of this.#preNavigationHooks) {
             pipeline = pipeline.compose(windowGuard(hook));
         }
@@ -204,20 +200,23 @@ export class HttpCrawler extends BasicCrawler {
             pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));
         }
         return pipelineWithNavigation
-            .compose({ action: this.processHttpResponse.bind(this) })
-            .compose({ action: this.handleBlockedRequestByContent.bind(this) });
+            .compose(this.#processHttpResponse.bind(this))
+            .compose(this.#handleBlockedRequestByContent.bind(this));
     }
-    async prepareHttpRequest(crawlingContext) {
+    async #prepareHttpRequest(crawlingContext) {
         const { request } = crawlingContext;
         if (request.skipNavigation) {
             return {
                 request: new Proxy(request, {
-                    get(target, propertyName, receiver) {
+                    get(target, propertyName) {
                         if (propertyName === 'loadedUrl') {
                             throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
                         }
-                        return Reflect.get(target, propertyName, receiver);
+                        // `target` as receiver and bound methods, or `#` members throw on the proxy
+                        const value = Reflect.get(target, propertyName, target);
+                        return typeof value === 'function' ? value.bind(target) : value;
                     },
+                    set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
                 }),
                 get response() {
                     throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
@@ -227,6 +226,7 @@ export class HttpCrawler extends BasicCrawler {
         request.state = RequestState.BEFORE_NAV;
         return {};
     }
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async makeHttpRequest(crawlingContext) {
         tryCancel();
         const { request, session } = crawlingContext;
@@ -234,13 +234,13 @@ export class HttpCrawler extends BasicCrawler {
         // Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
         // have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
         // client abort.
-        const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
+        const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
         request.loadedUrl = httpResponse?.url;
         request.state = RequestState.AFTER_NAV;
         return { request: request, response: httpResponse };
     }
-    async processHttpResponse(crawlingContext) {
+    async #processHttpResponse(crawlingContext) {
         if (crawlingContext.request.skipNavigation) {
             return {
                 get contentType() {
@@ -279,20 +279,22 @@ export class HttpCrawler extends BasicCrawler {
         if (remaining <= 0) {
             throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         }
-        const parsed = await addTimeoutToPromise(async () => this.parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
+        const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
         const response = parsed.response;
         const contentType = parsed.contentType;
+        const loadBody = async () => {
+            const { load } = await import('cheerio/slim');
+            return load(parsed.body.toString(), { xmlMode: contentType.type.includes('xml') });
+        };
         const waitForSelector = async (selector, _timeoutMs) => {
-            const cheerio = await import('cheerio');
-            const $ = cheerio.load(parsed.body.toString());
+            const $ = await loadBody();
             if ($(selector).get().length === 0) {
                 throw new Error(`Selector '${selector}' not found.`);
             }
         };
         const parseWithCheerio = async (selector, timeoutMs) => {
-            const cheerio = await import('cheerio');
-            const $ = cheerio.load(parsed.body.toString());
+            const $ = await loadBody();
             if (selector) {
                 await crawlingContext.waitForSelector(selector, timeoutMs);
             }
@@ -331,15 +333,15 @@ export class HttpCrawler extends BasicCrawler {
             body: parsed.body,
         };
     }
-    async handleBlockedRequestByContent(crawlingContext) {
+    async #handleBlockedRequestByContent(crawlingContext) {
         if (this.retryOnBlocked) {
-            const error = await this.isRequestBlocked(crawlingContext);
+            const error = await this.#isRequestBlocked(crawlingContext);
             if (error)
                 throw new SessionError(error);
         }
         return {};
     }
-    async isRequestBlocked(crawlingContext) {
+    async #isRequestBlocked(crawlingContext) {
         if (HTML_AND_XML_MIME_TYPES.includes(crawlingContext.contentType.type)) {
             const $ = await crawlingContext.parseWithCheerio();
             const foundSelectors = RETRY_CSS_SELECTORS.filter((selector) => $(selector).length > 0);
@@ -357,14 +359,14 @@ export class HttpCrawler extends BasicCrawler {
      * on the request such as only downloading the request body if the
      * received content type matches text/html, application/xml, application/xhtml+xml.
      */
-    async requestFunction({ request, session, proxyUrl }) {
-        const opts = this.getRequestOptions(request, session, proxyUrl);
+    async #requestFunction({ request, session, proxyUrl }) {
+        const opts = this.#getRequestOptions(request, proxyUrl);
         try {
-            return await this.requestAsBrowser(opts, session);
+            return await this.#requestAsBrowser(opts, session);
         }
         catch (e) {
             if (e instanceof Error && e.constructor.name === 'TimeoutError') {
-                this.handleRequestTimeout(session);
+                this.#handleRequestTimeout(session);
                 return new Response(); // this will never happen, as handleRequestTimeout always throws
             }
             if (this.isProxyError(e)) {
@@ -378,16 +380,15 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Encodes and parses response according to the provided content type
      */
-    async parseResponse(request, response) {
+    async #parseResponse(request, response) {
         const { status } = response;
         const { type, charset } = parseContentTypeFromResponse(response);
-        const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);
-        const contentType = { type, encoding };
         if (status >= 400 && status <= 599) {
             this.statistics.registerStatusCode(status);
         }
         if (this.isErrorStatusCode(status)) {
-            const body = await reencodedResponse.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
+            const { response: decoded } = this.#encodeResponse(request, response, charset);
+            const body = await decoded.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
             // Errors are often sent as JSON, so attempt to parse them,
             // despite Accept header being set to text/html.
             if (type === APPLICATION_JSON_MIME_TYPE) {
@@ -403,37 +404,37 @@ export class HttpCrawler extends BasicCrawler {
             // It's not a JSON, so it's probably some text. Get the first 100 chars of it.
             throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
         }
-        else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
-            if (!charset && !this.#forceResponseEncoding) {
-                const rawBytes = Buffer.from(await response.arrayBuffer());
-                const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
-                const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';
-                const body = iconv.encodingExists(charsetToUse)
-                    ? iconv.decode(rawBytes, charsetToUse)
-                    : rawBytes.toString('utf8');
-                return { response, contentType: { type, encoding: 'utf-8' }, body };
-            }
-            return { response, contentType, body: await reencodedResponse.text() };
+        if (HTML_AND_XML_MIME_TYPES.includes(type) && !charset && !this.#forceResponseEncoding) {
+            // The charset comes from the document itself, so the raw bytes are what we need -
+            // decoding them through `encodeResponse` first would consume the body for nothing.
+            const rawBytes = Buffer.from(await response.arrayBuffer());
+            const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
+            const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';
+            const body = iconv.encodingExists(charsetToUse)
+                ? iconv.decode(rawBytes, charsetToUse)
+                : rawBytes.toString('utf8');
+            return { response, contentType: { type, encoding: 'utf-8' }, body };
         }
-        else {
-            const body = Buffer.from(await reencodedResponse.bytes());
-            return {
-                body,
-                response,
-                contentType,
-            };
+        const { response: reencodedResponse, encoding } = this.#encodeResponse(request, response, charset);
+        const contentType = { type, encoding };
+        if (HTML_AND_XML_MIME_TYPES.includes(type)) {
+            return { response, contentType, body: await reencodedResponse.text() };
         }
+        return {
+            body: Buffer.from(await reencodedResponse.bytes()),
+            response,
+            contentType,
+        };
     }
     /**
      * Combines the provided `requestOptions` with mandatory (non-overridable) values.
      */
-    getRequestOptions(request, session, proxyUrl) {
+    #getRequestOptions(request, proxyUrl) {
         const requestOptions = {
             url: request.url,
             method: request.method,
             proxyUrl,
             timeout: this.#navigationTimeoutMillis,
-            sessionToken: session,
             headers: request.headers,
             body: undefined,
         };
@@ -445,7 +446,7 @@ export class HttpCrawler extends BasicCrawler {
             requestOptions.body = request.payload ?? '';
         return requestOptions;
     }
-    encodeResponse(request, response, encoding) {
+    #encodeResponse(request, response, encoding) {
         if (this.#forceResponseEncoding) {
             encoding = this.#forceResponseEncoding;
         }
@@ -481,29 +482,27 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Checks and extends supported mime types
      */
-    extendSupportedMimeTypes(additionalMimeTypes) {
+    #extendSupportedMimeTypes(additionalMimeTypes) {
         for (const mimeType of additionalMimeTypes) {
             if (mimeType === '*/*') {
                 this.#supportedMimeTypes.add(mimeType);
                 continue;
             }
-            try {
-                const parsedType = contentTypeParser.parse(mimeType);
-                this.#supportedMimeTypes.add(parsedType.type);
-            }
-            catch (err) {
+            const parsedType = parseContentType(mimeType);
+            if (!isTypeValid(parsedType.type)) {
                 throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
             }
+            this.#supportedMimeTypes.add(parsedType.type);
         }
     }
     /**
      * Handles timeout request
      */
-    handleRequestTimeout(session) {
+    #handleRequestTimeout(session) {
         session.markBad();
         throw new Error(`Request timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
     }
-    abortDownloadOfBody(request, response) {
+    #abortDownloadOfBody(request, response) {
         const { status } = response;
         const { type } = parseContentTypeFromResponse(response);
         const isTransientContentType = status >= 500 || this.blockedStatusCodes.has(status);
@@ -516,7 +515,7 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * @internal wraps public utility for mocking purposes
      */
-    requestAsBrowser = async (options, session) => {
+    #requestAsBrowser = async (options, session) => {
         const opts = processHttpRequestOptions({
             ...options,
             responseType: 'text',
@@ -540,6 +539,7 @@ export class HttpCrawler extends BasicCrawler {
         }), {
             session,
             cookieJar,
+            proxyUrl: opts.proxyUrl,
             signal: cancelSignal,
             timeoutMillis: cancelSignal ? undefined : opts.timeout,
             ignoreTlsErrors: this.#ignoreTlsErrors,

package/internals/utils.js

@@ -1,7 +1,7 @@
 import { extname } from 'node:path';
 import { Readable } from 'node:stream';
 import { applySearchParams, parseArgument } from '@crawlee/utils/internal';
-import contentTypeParser from 'content-type';
+import { isTypeValid, parse as parseContentType } from 'content-type';
 import mime from 'mime-types';
 import { z } from 'zod';
 const responseWithUrlSchema = z.looseObject({ url: z.url(), headers: z.looseObject({}) });
@@ -59,18 +59,16 @@ export function parseContentTypeFromResponse(response) {
     const { url, headers } = response;
     let parsedContentType;
     if (headers.get('content-type')) {
-        try {
-            parsedContentType = contentTypeParser.parse(headers.get('content-type'));
-        }
-        catch {
-            // Can not parse content type from Content-Type header. Try to parse it from file extension.
-        }
+        const parsed = parseContentType(headers.get('content-type'));
+        // Can not parse content type from Content-Type header. Try to parse it from file extension.
+        if (isTypeValid(parsed.type))
+            parsedContentType = parsed;
     }
     // Parse content type from file extension as fallback
     if (!parsedContentType) {
         const parsedUrl = new URL(url);
         const contentTypeFromExtname = mime.contentType(extname(parsedUrl.pathname)) || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5
-        parsedContentType = contentTypeParser.parse(contentTypeFromExtname);
+        parsedContentType = parseContentType(contentTypeFromExtname);
     }
     return {
         type: parsedContentType.type,

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-rc.0",
+    "version": "4.0.0-rc.1",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
@@ -47,21 +47,18 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/timeout": "^0.4.4",
-        "@apify/utilities": "^2.15.5",
-        "@crawlee/basic": "4.0.0-rc.0",
-        "@crawlee/core": "4.0.0-rc.0",
-        "@crawlee/http-client": "4.0.0-rc.0",
-        "@crawlee/types": "4.0.0-rc.0",
-        "@crawlee/utils": "4.0.0-rc.0",
-        "@types/content-type": "^1.1.8",
+        "@apify/timeout": "^1.0.1",
+        "@crawlee/basic": "4.0.0-rc.1",
+        "@crawlee/http-client": "4.0.0-rc.1",
+        "@crawlee/types": "4.0.0-rc.1",
+        "@crawlee/utils": "4.0.0-rc.1",
         "cheerio": "^1.0.0",
-        "content-type": "^1.0.5",
+        "content-type": "^3.1.1",
         "iconv-lite": "^0.7.2",
         "mime-types": "^3.0.1",
         "tslib": "^2.8.1",
         "type-fest": "^4.41.0",
-        "zod": "^4.4.3"
+        "zod": "^4.5.4"
     },
     "lerna": {
         "command": {
@@ -70,5 +67,5 @@
             }
         }
     },
-    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }