@crawlee/http 4.0.0-beta.99 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/index.d.ts

@@ -1,3 +1,4 @@
 export * from '@crawlee/basic';
 export * from './internals/http-crawler.js';
+export * from './internals/dom-crawler.js';
 export * from './internals/file-download.js';

package/index.js

@@ -1,3 +1,4 @@
 export * from '@crawlee/basic';
 export * from './internals/http-crawler.js';
+export * from './internals/dom-crawler.js';
 export * from './internals/file-download.js';

package/internals/dom-crawler.d.ts

@@ -0,0 +1,131 @@
+import type { AddRequestsBatchedResult, ContextPipeline, CrawlingContext, EnqueueLinksOptions, ExtractLinksOptions, GetUserDataFromRequest } from '@crawlee/basic';
+import type { Awaitable, Dictionary } from '@crawlee/types';
+import type { CheerioAPI } from 'cheerio';
+import type { HttpCrawlerOptions, InternalHttpCrawlingContext } from './http-crawler.js';
+import { HttpCrawler } from './http-crawler.js';
+/**
+ * The minimum a {@link DOMParser} has to contribute to the crawling context - the serialized document, used by
+ * the {@link DOMCrawlingContext.parseWithCheerio|`parseWithCheerio`} helper.
+ */
+export interface DOMParseResult {
+    body: string;
+}
+/**
+ * Turns a response body into a DOM representation and knows how to query it. Passing one to {@link DOMCrawler}
+ * is what makes the crawler jsdom-based, linkedom-based, or based on a DOM implementation of your own.
+ *
+ * **Example usage:**
+ * ```ts
+ * import { DOMCrawler } from 'crawlee';
+ * import type { DOMParser } from 'crawlee';
+ *
+ * const myParser: DOMParser<{ body: string }> = {
+ *     // ...
+ * };
+ *
+ * const crawler = new DOMCrawler({
+ *     parser: myParser,
+ *     async requestHandler({ body }) {
+ *         // ...
+ *     },
+ * });
+ * ```
+ */
+export interface DOMParser<Parsed extends DOMParseResult> {
+    /**
+     * The context members {@link DOMParser.parse|`parse`} contributes, mapped to `true`. Used to build the
+     * placeholders that report a helpful error when the members are accessed after `skipNavigation` - the `Record`
+     * type forces every key of `Parsed` to be listed, so the compiler catches an omission that would otherwise
+     * yield `undefined` (rather than throwing) after `skipNavigation`.
+     */
+    readonly placeholderMembers: Record<keyof Parsed & string, true>;
+    parse(context: InternalHttpCrawlingContext): Awaitable<Parsed>;
+    /**
+     * Returns the URLs the `selector` matches, resolved against `baseUrl`.
+     */
+    extractLinks(parsed: Parsed, selector: string, baseUrl: string): Awaitable<string[]>;
+    /**
+     * Returns the current matches of `selector`. Only the count is used, by
+     * {@link DOMCrawlingContext.waitForSelector|`waitForSelector`}.
+     */
+    select(parsed: Parsed, selector: string): Awaitable<ArrayLike<unknown>>;
+    /**
+     * Whether the parse result can change after {@link DOMParser.parse|`parse`} returned - the case when the DOM
+     * implementation runs the page scripts. If it can, {@link DOMCrawlingContext.waitForSelector|`waitForSelector`}
+     * polls until the timeout elapses; otherwise it fails as soon as the selector does not match.
+     */
+    readonly mutable?: boolean;
+    /**
+     * Returns a Cheerio handle over the parse result, for parsers that are backed by Cheerio anyway. Without it,
+     * {@link DOMCrawlingContext.parseWithCheerio|`parseWithCheerio`} parses
+     * {@link DOMParseResult.body|`body`} again.
+     */
+    toCheerio?(parsed: Parsed): Awaitable<CheerioAPI>;
+    /**
+     * Releases whatever {@link DOMParser.parse|`parse`} allocated. Called after the request handler finishes or
+     * fails, and skipped entirely when navigation was skipped.
+     */
+    cleanup?(parsed: Parsed): Awaitable<void>;
+}
+export interface DOMCrawlingHelpers {
+    /**
+     * Extracts URLs from the parsed DOM, without adding them to the request queue.
+     */
+    extractLinks(options?: ExtractLinksOptions): Promise<string[]>;
+    /**
+     * Helper function for extracting URLs from the parsed DOM and adding them to the request queue.
+     */
+    enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;
+    /**
+     * Wait for an element matching the selector to appear. The `timeoutMs` only has an effect when the parser is
+     * {@link DOMParser.mutable|`mutable`} (e.g. {@link JSDOMCrawler} with `runScripts: true`); otherwise the
+     * selector is checked once and the call resolves or throws immediately.
+     * Timeout defaults to 5s.
+     *
+     * **Example usage:**
+     * ```ts
+     * async requestHandler({ waitForSelector, parseWithCheerio }) {
+     *     await waitForSelector('article h1');
+     *     const $ = await parseWithCheerio();
+     *     const title = $('title').text();
+     * });
+     * ```
+     */
+    waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+    /**
+     * Returns Cheerio handle, allowing to work with the data same way as with {@link CheerioCrawler}.
+     * When provided with the `selector` argument, it will throw if it's not available.
+     *
+     * **Example usage:**
+     * ```javascript
+     * async requestHandler({ parseWithCheerio }) {
+     *     const $ = await parseWithCheerio();
+     *     const title = $('title').text();
+     * });
+     * ```
+     */
+    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
+}
+export type DOMCrawlingContext<Parsed extends DOMParseResult = DOMParseResult, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> = InternalHttpCrawlingContext<UserData, JSONData> & Parsed & DOMCrawlingHelpers;
+export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, any>, StatisticStateExtension extends object = {}> extends Omit<HttpCrawlerOptions<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'contextPipelineBuilder'> {
+    /**
+     * The DOM implementation to parse the response bodies with. Its parse result becomes part of the crawling
+     * context, so the members the request handler receives follow from the parser you pass.
+     */
+    parser: DOMParser<Parsed>;
+}
+/**
+ * An {@link HttpCrawler} that parses each response into a DOM using the {@link DOMCrawlerOptions.parser|`parser`}
+ * it is given, and exposes the parse result plus the {@link DOMCrawlingContext.enqueueLinks|`enqueueLinks`} and
+ * {@link DOMCrawlingContext.extractLinks|`extractLinks`} helpers on the crawling context.
+ *
+ * {@link JSDOMCrawler} and {@link LinkeDOMCrawler} are this crawler with a parser already chosen.
+ *
+ * @category Crawlers
+ */
+export declare class DOMCrawler<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<DOMCrawlingContext<Parsed>['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+    #private;
+    constructor(options: DOMCrawlerOptions<Parsed, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
+    protected buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed>>;
+}

package/internals/dom-crawler.js

@@ -0,0 +1,93 @@
+import { EnqueueStrategy, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering } from '@crawlee/basic';
+import { sleep } from '@crawlee/utils';
+import { HttpCrawler } from './http-crawler.js';
+/**
+ * An {@link HttpCrawler} that parses each response into a DOM using the {@link DOMCrawlerOptions.parser|`parser`}
+ * it is given, and exposes the parse result plus the {@link DOMCrawlingContext.enqueueLinks|`enqueueLinks`} and
+ * {@link DOMCrawlingContext.extractLinks|`extractLinks`} helpers on the crawling context.
+ *
+ * {@link JSDOMCrawler} and {@link LinkeDOMCrawler} are this crawler with a parser already chosen.
+ *
+ * @category Crawlers
+ */
+export class DOMCrawler extends HttpCrawler {
+    #parser;
+    constructor(options) {
+        const { parser, ...rest } = options;
+        super({ ...rest, contextPipelineBuilder: () => this.buildContextPipeline() });
+        this.#parser = parser;
+    }
+    buildContextPipeline() {
+        return super
+            .buildContextPipeline()
+            .compose(this.#parseContent.bind(this))
+            .compose(async (context) => this.#addHelpers(context));
+    }
+    async #parseContent(context, onCleanup) {
+        try {
+            const parsed = await this.#parser.parse(context);
+            onCleanup(() => this.#parser.cleanup?.(parsed));
+            return parsed;
+        }
+        catch (err) {
+            // The `skipNavigation` placeholders below throw on access, so there is nothing to clean up.
+            if (err instanceof NavigationSkippedError) {
+                return Object.defineProperties({}, Object.fromEntries(Object.keys(this.#parser.placeholderMembers).map((member) => [
+                    member,
+                    {
+                        configurable: true,
+                        enumerable: true,
+                        get() {
+                            throw new NavigationSkippedError(`The \`${member}\` property is not available - \`skipNavigation\` was used`, { cause: err });
+                        },
+                    },
+                ])));
+            }
+            throw err;
+        }
+    }
+    async #addHelpers(context) {
+        const { addRequests } = context;
+        const parser = this.#parser;
+        const extractLinks = async (options) => parser.extractLinks(context, options?.selector ?? 'a', options?.baseUrl ?? context.request.loadedUrl ?? context.request.url);
+        const waitForSelector = async (selector, timeoutMs = 5_000) => {
+            let remaining = parser.mutable ? timeoutMs : 0;
+            while ((await parser.select(context, selector)).length === 0) {
+                if (remaining <= 0) {
+                    throw new Error(`Selector '${selector}' not found.`);
+                }
+                await sleep(50);
+                remaining -= 50;
+            }
+        };
+        return {
+            extractLinks,
+            waitForSelector,
+            enqueueLinks: async (options = {}) => {
+                const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
+                    enqueueStrategy: options.strategy,
+                    finalRequestUrl: context.request.loadedUrl,
+                    originalRequestUrl: context.request.url,
+                    userProvidedBaseUrl: options.baseUrl,
+                });
+                const urls = await extractLinks(options);
+                return addRequests(urls, {
+                    ...options,
+                    baseUrl,
+                    strategy: options.strategy ?? EnqueueStrategy.SameHostname,
+                });
+            },
+            async parseWithCheerio(selector, _timeoutMs = 5_000) {
+                // Import full cheerio (not cheerio/slim) to be browser-compliant for DOM Crawlers (not CheerioCrawler).
+                const $ = (await parser.toCheerio?.(context)) ??
+                    (await import('cheerio')).load(context.body, {
+                        xmlMode: context.contentType.type.includes('xml'),
+                    });
+                if (selector && $(selector).get().length === 0) {
+                    throw new Error(`Selector '${selector}' not found.`);
+                }
+                return $;
+            },
+        };
+    }
+}

package/internals/file-download.d.ts

@@ -1,16 +1,11 @@
-import { Transform } from 'node:stream';
-import type { BasicCrawlerOptions } from '@crawlee/basic';
+import type { BasicCrawlerOptions, CrawlingContext, CrawlingRequest, LoadedRequest } from '@crawlee/basic';
 import { BasicCrawler } from '@crawlee/basic';
-import type { CrawlingContext, LoadedRequest, Request } from '@crawlee/core';
-import { ResponseWithUrl } from '@crawlee/http-client';
 import type { Dictionary } from '@crawlee/types';
-import type { ErrorHandler, GetUserDataFromRequest, InternalHttpHook, RequestHandler, RouterRoutes } from '../index.js';
-declare const kBodyDrained: unique symbol;
+import type { ErrorHandler, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '../index.js';
 export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, FileDownloadCrawlingContext<UserData> & ContextExtension>;
-export type FileDownloadHook<UserData extends Dictionary = any> = InternalHttpHook<FileDownloadCrawlingContext<UserData>>;
 export interface FileDownloadCrawlingContext<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
-    request: LoadedRequest<Request<UserData>>;
+    request: LoadedRequest<CrawlingRequest<UserData>>;
     response: Response;
     contentType: {
         type: string;
@@ -18,31 +13,6 @@ export interface FileDownloadCrawlingContext<UserData extends Dictionary = any>
     };
 }
 export type FileDownloadRequestHandler<UserData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData>>;
-/**
- * Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
- * This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
- * If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
- * it will throw an error.
- *
- * Can be used e.g. to abort a download if the network speed is too slow.
- * @returns Transform stream that monitors the speed of the incoming data.
- */
-export declare function MinimumSpeedStream({ minSpeedKbps, historyLengthMs, checkProgressInterval: checkProgressIntervalMs, }: {
-    minSpeedKbps: number;
-    historyLengthMs?: number;
-    checkProgressInterval?: number;
-}): Transform;
-/**
- * Creates a transform stream that logs the progress of the incoming data.
- * This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
- *
- * Can be used e.g. to log the progress of a download.
- * @returns Transform stream logging the progress of the incoming data.
- */
-export declare function ByteCounterStream({ logTransferredBytes, loggingInterval, }: {
-    logTransferredBytes: (transferredBytes: number) => void;
-    loggingInterval?: number;
-}): Transform;
 /**
  * Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
  *
@@ -50,24 +20,14 @@ export declare function ByteCounterStream({ logTransferredBytes, loggingInterval
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
  *
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link FileDownloadOptions.requestHandler} where the user can specify what to do with the downloaded data.
+ * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
  *
- * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link FileDownloadOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
+ * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
  *
- * > The {@link FileDownloadOptions.requestList|`requestList`} and {@link FileDownloadOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
+ * > The {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
- * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
- *
- * ```
- * preNavigationHooks: [
- *     (crawlingContext) => {
- *         // ...
- *     },
- * ]
- * ```
- *
  * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * ## Example usage
@@ -87,18 +47,8 @@ export declare function ByteCounterStream({ logTransferredBytes, loggingInterval
  * ```
  */
 export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
-    constructor(options?: BasicCrawlerOptions<FileDownloadCrawlingContext>);
-// @ts-ignore optional peer dependency or compatibility with es2022
-    protected buildContextPipeline(): import("@crawlee/basic").ContextPipeline<CrawlingContext<Dictionary>, CrawlingContext<Dictionary> & {
-        request: LoadedRequest<Request>;
-        response: ResponseWithUrl;
-        contentType: {
-            type: string;
-            encoding: BufferEncoding;
-        };
-        [kBodyDrained]: Promise<void>;
-    }>;
-    private initiateDownload;
+    #private;
+    constructor(options?: Omit<BasicCrawlerOptions<FileDownloadCrawlingContext>, 'contextPipelineBuilder'>);
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.
@@ -124,6 +74,6 @@ export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingConte
  * await crawler.run();
  * ```
  */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/basic").RouterHandler<Context, Record<string, GetUserDataFromRequest<Context["request"]>>>;
-export {};
+export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;

package/internals/file-download.js

@@ -1,67 +1,7 @@
-import { Transform } from 'node:stream';
-import { BasicCrawler } from '@crawlee/basic';
+import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
 import { ResponseWithUrl } from '@crawlee/http-client';
 import { Router } from '../index.js';
 import { parseContentTypeFromResponse } from './utils.js';
-const kBodyDrained = Symbol('bodyDrained');
-/**
- * Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
- * This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
- * If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
- * it will throw an error.
- *
- * Can be used e.g. to abort a download if the network speed is too slow.
- * @returns Transform stream that monitors the speed of the incoming data.
- */
-export function MinimumSpeedStream({ minSpeedKbps, historyLengthMs = 10e3, checkProgressInterval: checkProgressIntervalMs = 5e3, }) {
-    let snapshots = [];
-    const checkInterval = setInterval(() => {
-        const now = Date.now();
-        snapshots = snapshots.filter((snapshot) => now - snapshot.timestamp < historyLengthMs);
-        const totalBytes = snapshots.reduce((acc, snapshot) => acc + snapshot.bytes, 0);
-        const elapsed = (now - (snapshots[0]?.timestamp ?? 0)) / 1000;
-        if (totalBytes / 1024 / elapsed < minSpeedKbps) {
-            clearInterval(checkInterval);
-            stream.emit('error', new Error(`Stream speed too slow, aborting...`));
-        }
-    }, checkProgressIntervalMs);
-    const stream = new Transform({
-        transform: (chunk, _, callback) => {
-            snapshots.push({ timestamp: Date.now(), bytes: chunk.length });
-            callback(null, chunk);
-        },
-        final: (callback) => {
-            clearInterval(checkInterval);
-            callback();
-        },
-    });
-    return stream;
-}
-/**
- * Creates a transform stream that logs the progress of the incoming data.
- * This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
- *
- * Can be used e.g. to log the progress of a download.
- * @returns Transform stream logging the progress of the incoming data.
- */
-export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000, }) {
-    let transferredBytes = 0;
-    let lastLogTime = Date.now();
-    return new Transform({
-        transform: (chunk, _, callback) => {
-            transferredBytes += chunk.length;
-            if (Date.now() - lastLogTime > loggingInterval) {
-                lastLogTime = Date.now();
-                logTransferredBytes(transferredBytes);
-            }
-            callback(null, chunk);
-        },
-        flush: (callback) => {
-            logTransferredBytes(transferredBytes);
-            callback();
-        },
-    });
-}
 /**
  * Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
  *
@@ -69,24 +9,14 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
  *
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link FileDownloadOptions.requestHandler} where the user can specify what to do with the downloaded data.
+ * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
  *
- * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link FileDownloadOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
+ * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
  *
- * > The {@link FileDownloadOptions.requestList|`requestList`} and {@link FileDownloadOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
+ * > The {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
- * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
- *
- * ```
- * preNavigationHooks: [
- *     (crawlingContext) => {
- *         // ...
- *     },
- * ]
- * ```
- *
  * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * ## Example usage
@@ -110,36 +40,32 @@ export class FileDownload extends BasicCrawler {
     constructor(options = {}) {
         super({
             ...options,
-            contextPipelineBuilder: () => this.buildContextPipeline(),
+            contextPipelineBuilder: () => this.#buildContextPipeline(),
         });
     }
-    buildContextPipeline() {
-        return super.buildContextPipeline().compose({
-            action: async (context) => this.initiateDownload(context),
-            cleanup: async (context) => {
-                if (!context.response.bodyUsed) {
-                    // Nobody consumed the body — cancel it so the
-                    // underlying connection can be released.
-                    await context.response.body?.cancel();
-                }
-                await context[kBodyDrained];
-            },
-        });
+    #buildContextPipeline() {
+        return ContextPipeline.create().compose(this.#initiateDownload.bind(this));
     }
-    async initiateDownload(context) {
+    async #initiateDownload(context, onCleanup) {
         const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
             session: context.session,
         });
         const { type, charset: encoding } = parseContentTypeFromResponse(response);
         context.request.url = response.url;
         const { response: trackedResponse, bodyDrained } = trackBodyConsumption(response);
-        const contextExtension = {
+        onCleanup(async () => {
+            if (!trackedResponse.bodyUsed) {
+                // Nobody consumed the body — cancel it so the
+                // underlying connection can be released.
+                await trackedResponse.body?.cancel();
+            }
+            await bodyDrained;
+        });
+        return {
             request: context.request,
             response: trackedResponse,
             contentType: { type, encoding },
-            [kBodyDrained]: bodyDrained,
         };
-        return contextExtension;
     }
 }
 /**
@@ -162,30 +88,6 @@ function trackBodyConsumption(response) {
     });
     return { response: trackedResponse, bodyDrained };
 }
-/**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link FileDownload}.
- * Defaults to the {@link FileDownloadCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<FileDownloadCrawlingContext>()`.
- *
- * ```ts
- * import { FileDownload, createFileRouter } from 'crawlee';
- *
- * const router = createFileRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
- *
- * const crawler = new FileDownload({
- *     requestHandler: router,
- * });
- * await crawler.run();
- * ```
- */
-export function createFileRouter(routes) {
-    return Router.create(routes);
+export function createFileRouter(routesOrSchemas) {
+    return Router.create(routesOrSchemas);
 }