@crawlee/http 4.0.0-beta.20 → 4.0.0-beta.201

package/README.md

@@ -1,23 +1,23 @@
 <h1 align="center">
     <a href="https://crawlee.dev">
         <picture>
-          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
-          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
         </picture>
     </a>
-    <br>
+    <br />
     <small>A web scraping and browser automation library</small>
 </h1>
 
-<p align=center>
-    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+<p align="center">
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
 </p>
 
-<p align=center>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
-    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+<p align="center">
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
 We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
 
 ```bash
-npm install crawlee@3.12.3-beta.13
+npm install crawlee@next
 ```
 
 If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
 {
     "overrides": {
        "apify": {
-           "@crawlee/core": "3.12.3-beta.13",
-           "@crawlee/types": "3.12.3-beta.13",
-           "@crawlee/utils": "3.12.3-beta.13"
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
        }
     }
 }

package/index.d.ts

@@ -1,4 +1,4 @@
 export * from '@crawlee/basic';
 export * from './internals/http-crawler.js';
+export * from './internals/dom-crawler.js';
 export * from './internals/file-download.js';
-//# sourceMappingURL=index.d.ts.map

package/index.js

@@ -1,4 +1,4 @@
 export * from '@crawlee/basic';
 export * from './internals/http-crawler.js';
+export * from './internals/dom-crawler.js';
 export * from './internals/file-download.js';
-//# sourceMappingURL=index.js.map

package/internals/dom-crawler.d.ts

@@ -0,0 +1,131 @@
+import type { AddRequestsBatchedResult, ContextPipeline, CrawlingContext, EnqueueLinksOptions, ExtractLinksOptions, GetUserDataFromRequest } from '@crawlee/basic';
+import type { Awaitable, Dictionary } from '@crawlee/types';
+import type { CheerioAPI } from 'cheerio';
+import type { HttpCrawlerOptions, InternalHttpCrawlingContext } from './http-crawler.js';
+import { HttpCrawler } from './http-crawler.js';
+/**
+ * The minimum a {@link DOMParser} has to contribute to the crawling context - the serialized document, used by
+ * the {@link DOMCrawlingContext.parseWithCheerio|`parseWithCheerio`} helper.
+ */
+export interface DOMParseResult {
+    body: string;
+}
+/**
+ * Turns a response body into a DOM representation and knows how to query it. Passing one to {@link DOMCrawler}
+ * is what makes the crawler jsdom-based, linkedom-based, or based on a DOM implementation of your own.
+ *
+ * **Example usage:**
+ * ```ts
+ * import { DOMCrawler } from 'crawlee';
+ * import type { DOMParser } from 'crawlee';
+ *
+ * const myParser: DOMParser<{ body: string }> = {
+ *     // ...
+ * };
+ *
+ * const crawler = new DOMCrawler({
+ *     parser: myParser,
+ *     async requestHandler({ body }) {
+ *         // ...
+ *     },
+ * });
+ * ```
+ */
+export interface DOMParser<Parsed extends DOMParseResult> {
+    /**
+     * The context members {@link DOMParser.parse|`parse`} contributes, mapped to `true`. Used to build the
+     * placeholders that report a helpful error when the members are accessed after `skipNavigation` - the `Record`
+     * type forces every key of `Parsed` to be listed, so the compiler catches an omission that would otherwise
+     * yield `undefined` (rather than throwing) after `skipNavigation`.
+     */
+    readonly placeholderMembers: Record<keyof Parsed & string, true>;
+    parse(context: InternalHttpCrawlingContext): Awaitable<Parsed>;
+    /**
+     * Returns the URLs the `selector` matches, resolved against `baseUrl`.
+     */
+    extractLinks(parsed: Parsed, selector: string, baseUrl: string): Awaitable<string[]>;
+    /**
+     * Returns the current matches of `selector`. Only the count is used, by
+     * {@link DOMCrawlingContext.waitForSelector|`waitForSelector`}.
+     */
+    select(parsed: Parsed, selector: string): Awaitable<ArrayLike<unknown>>;
+    /**
+     * Whether the parse result can change after {@link DOMParser.parse|`parse`} returned - the case when the DOM
+     * implementation runs the page scripts. If it can, {@link DOMCrawlingContext.waitForSelector|`waitForSelector`}
+     * polls until the timeout elapses; otherwise it fails as soon as the selector does not match.
+     */
+    readonly mutable?: boolean;
+    /**
+     * Returns a Cheerio handle over the parse result, for parsers that are backed by Cheerio anyway. Without it,
+     * {@link DOMCrawlingContext.parseWithCheerio|`parseWithCheerio`} parses
+     * {@link DOMParseResult.body|`body`} again.
+     */
+    toCheerio?(parsed: Parsed): Awaitable<CheerioAPI>;
+    /**
+     * Releases whatever {@link DOMParser.parse|`parse`} allocated. Called after the request handler finishes or
+     * fails, and skipped entirely when navigation was skipped.
+     */
+    cleanup?(parsed: Parsed): Awaitable<void>;
+}
+export interface DOMCrawlingHelpers {
+    /**
+     * Extracts URLs from the parsed DOM, without adding them to the request queue.
+     */
+    extractLinks(options?: ExtractLinksOptions): Promise<string[]>;
+    /**
+     * Helper function for extracting URLs from the parsed DOM and adding them to the request queue.
+     */
+    enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;
+    /**
+     * Wait for an element matching the selector to appear. The `timeoutMs` only has an effect when the parser is
+     * {@link DOMParser.mutable|`mutable`} (e.g. {@link JSDOMCrawler} with `runScripts: true`); otherwise the
+     * selector is checked once and the call resolves or throws immediately.
+     * Timeout defaults to 5s.
+     *
+     * **Example usage:**
+     * ```ts
+     * async requestHandler({ waitForSelector, parseWithCheerio }) {
+     *     await waitForSelector('article h1');
+     *     const $ = await parseWithCheerio();
+     *     const title = $('title').text();
+     * });
+     * ```
+     */
+    waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+    /**
+     * Returns Cheerio handle, allowing to work with the data same way as with {@link CheerioCrawler}.
+     * When provided with the `selector` argument, it will throw if it's not available.
+     *
+     * **Example usage:**
+     * ```javascript
+     * async requestHandler({ parseWithCheerio }) {
+     *     const $ = await parseWithCheerio();
+     *     const title = $('title').text();
+     * });
+     * ```
+     */
+    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
+}
+export type DOMCrawlingContext<Parsed extends DOMParseResult = DOMParseResult, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> = InternalHttpCrawlingContext<UserData, JSONData> & Parsed & DOMCrawlingHelpers;
+export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, any>, StatisticStateExtension extends object = {}> extends Omit<HttpCrawlerOptions<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'contextPipelineBuilder'> {
+    /**
+     * The DOM implementation to parse the response bodies with. Its parse result becomes part of the crawling
+     * context, so the members the request handler receives follow from the parser you pass.
+     */
+    parser: DOMParser<Parsed>;
+}
+/**
+ * An {@link HttpCrawler} that parses each response into a DOM using the {@link DOMCrawlerOptions.parser|`parser`}
+ * it is given, and exposes the parse result plus the {@link DOMCrawlingContext.enqueueLinks|`enqueueLinks`} and
+ * {@link DOMCrawlingContext.extractLinks|`extractLinks`} helpers on the crawling context.
+ *
+ * {@link JSDOMCrawler} and {@link LinkeDOMCrawler} are this crawler with a parser already chosen.
+ *
+ * @category Crawlers
+ */
+export declare class DOMCrawler<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<DOMCrawlingContext<Parsed>['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+    #private;
+    constructor(options: DOMCrawlerOptions<Parsed, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
+    protected buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed>>;
+}

package/internals/dom-crawler.js

@@ -0,0 +1,93 @@
+import { EnqueueStrategy, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering } from '@crawlee/basic';
+import { sleep } from '@crawlee/utils';
+import { HttpCrawler } from './http-crawler.js';
+/**
+ * An {@link HttpCrawler} that parses each response into a DOM using the {@link DOMCrawlerOptions.parser|`parser`}
+ * it is given, and exposes the parse result plus the {@link DOMCrawlingContext.enqueueLinks|`enqueueLinks`} and
+ * {@link DOMCrawlingContext.extractLinks|`extractLinks`} helpers on the crawling context.
+ *
+ * {@link JSDOMCrawler} and {@link LinkeDOMCrawler} are this crawler with a parser already chosen.
+ *
+ * @category Crawlers
+ */
+export class DOMCrawler extends HttpCrawler {
+    #parser;
+    constructor(options) {
+        const { parser, ...rest } = options;
+        super({ ...rest, contextPipelineBuilder: () => this.buildContextPipeline() });
+        this.#parser = parser;
+    }
+    buildContextPipeline() {
+        return super
+            .buildContextPipeline()
+            .compose(this.#parseContent.bind(this))
+            .compose(async (context) => this.#addHelpers(context));
+    }
+    async #parseContent(context, onCleanup) {
+        try {
+            const parsed = await this.#parser.parse(context);
+            onCleanup(() => this.#parser.cleanup?.(parsed));
+            return parsed;
+        }
+        catch (err) {
+            // The `skipNavigation` placeholders below throw on access, so there is nothing to clean up.
+            if (err instanceof NavigationSkippedError) {
+                return Object.defineProperties({}, Object.fromEntries(Object.keys(this.#parser.placeholderMembers).map((member) => [
+                    member,
+                    {
+                        configurable: true,
+                        enumerable: true,
+                        get() {
+                            throw new NavigationSkippedError(`The \`${member}\` property is not available - \`skipNavigation\` was used`, { cause: err });
+                        },
+                    },
+                ])));
+            }
+            throw err;
+        }
+    }
+    async #addHelpers(context) {
+        const { addRequests } = context;
+        const parser = this.#parser;
+        const extractLinks = async (options) => parser.extractLinks(context, options?.selector ?? 'a', options?.baseUrl ?? context.request.loadedUrl ?? context.request.url);
+        const waitForSelector = async (selector, timeoutMs = 5_000) => {
+            let remaining = parser.mutable ? timeoutMs : 0;
+            while ((await parser.select(context, selector)).length === 0) {
+                if (remaining <= 0) {
+                    throw new Error(`Selector '${selector}' not found.`);
+                }
+                await sleep(50);
+                remaining -= 50;
+            }
+        };
+        return {
+            extractLinks,
+            waitForSelector,
+            enqueueLinks: async (options = {}) => {
+                const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
+                    enqueueStrategy: options.strategy,
+                    finalRequestUrl: context.request.loadedUrl,
+                    originalRequestUrl: context.request.url,
+                    userProvidedBaseUrl: options.baseUrl,
+                });
+                const urls = await extractLinks(options);
+                return addRequests(urls, {
+                    ...options,
+                    baseUrl,
+                    strategy: options.strategy ?? EnqueueStrategy.SameHostname,
+                });
+            },
+            async parseWithCheerio(selector, _timeoutMs = 5_000) {
+                // Import full cheerio (not cheerio/slim) to be browser-compliant for DOM Crawlers (not CheerioCrawler).
+                const $ = (await parser.toCheerio?.(context)) ??
+                    (await import('cheerio')).load(context.body, {
+                        xmlMode: context.contentType.type.includes('xml'),
+                    });
+                if (selector && $(selector).get().length === 0) {
+                    throw new Error(`Selector '${selector}' not found.`);
+                }
+                return $;
+            },
+        };
+    }
+}

package/internals/file-download.d.ts

@@ -1,13 +1,11 @@
-import { Transform } from 'node:stream';
-import type { BasicCrawlerOptions } from '@crawlee/basic';
+import type { BasicCrawlerOptions, CrawlingContext, CrawlingRequest, LoadedRequest } from '@crawlee/basic';
 import { BasicCrawler } from '@crawlee/basic';
-import type { CrawlingContext, LoadedRequest, Request } from '@crawlee/core';
 import type { Dictionary } from '@crawlee/types';
-import type { ErrorHandler, GetUserDataFromRequest, InternalHttpHook, RequestHandler, RouterRoutes } from '../index.js';
-export type FileDownloadErrorHandler<UserData extends Dictionary = any> = ErrorHandler<FileDownloadCrawlingContext<UserData>>;
-export type FileDownloadHook<UserData extends Dictionary = any> = InternalHttpHook<FileDownloadCrawlingContext<UserData>>;
+import type { ErrorHandler, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '../index.js';
+export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, FileDownloadCrawlingContext<UserData> & ContextExtension>;
 export interface FileDownloadCrawlingContext<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
-    request: LoadedRequest<Request<UserData>>;
+    request: LoadedRequest<CrawlingRequest<UserData>>;
     response: Response;
     contentType: {
         type: string;
@@ -15,31 +13,6 @@ export interface FileDownloadCrawlingContext<UserData extends Dictionary = any>
     };
 }
 export type FileDownloadRequestHandler<UserData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData>>;
-/**
- * Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
- * This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
- * If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
- * it will throw an error.
- *
- * Can be used e.g. to abort a download if the network speed is too slow.
- * @returns Transform stream that monitors the speed of the incoming data.
- */
-export declare function MinimumSpeedStream({ minSpeedKbps, historyLengthMs, checkProgressInterval: checkProgressIntervalMs, }: {
-    minSpeedKbps: number;
-    historyLengthMs?: number;
-    checkProgressInterval?: number;
-}): Transform;
-/**
- * Creates a transform stream that logs the progress of the incoming data.
- * This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
- *
- * Can be used e.g. to log the progress of a download.
- * @returns Transform stream logging the progress of the incoming data.
- */
-export declare function ByteCounterStream({ logTransferredBytes, loggingInterval, }: {
-    logTransferredBytes: (transferredBytes: number) => void;
-    loggingInterval?: number;
-}): Transform;
 /**
  * Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
  *
@@ -47,25 +20,15 @@ export declare function ByteCounterStream({ logTransferredBytes, loggingInterval
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
  *
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link FileDownloadOptions.requestHandler} where the user can specify what to do with the downloaded data.
+ * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
  *
- * The source URLs are represented using {@link Request} objects that are fed from {@link RequestList} or {@link RequestQueue} instances provided by the {@link FileDownloadOptions.requestList} or {@link FileDownloadOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
  *
- * If both {@link FileDownloadOptions.requestList} and {@link FileDownloadOptions.requestQueue} are used, the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
- * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
- *
- * ```
- * preNavigationHooks: [
- *     (crawlingContext) => {
- *         // ...
- *     },
- * ]
- * ```
- *
- * New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the {@link AutoscaledPool} class. All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions` parameter of the `FileCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency` {@link AutoscaledPool} options are available directly in the `FileCrawler` constructor.
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * ## Example usage
  *
@@ -84,7 +47,8 @@ export declare function ByteCounterStream({ logTransferredBytes, loggingInterval
  * ```
  */
 export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
-    constructor(options?: BasicCrawlerOptions<FileDownloadCrawlingContext>);
+    #private;
+    constructor(options?: Omit<BasicCrawlerOptions<FileDownloadCrawlingContext>, 'contextPipelineBuilder'>);
     private initiateDownload;
 }
 /**
@@ -111,6 +75,6 @@ export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingConte
  * await crawler.run();
  * ```
  */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/basic").RouterHandler<Context>;
-//# sourceMappingURL=file-download.d.ts.map
+export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;

package/internals/file-download.js

@@ -1,66 +1,7 @@
-import { Transform } from 'node:stream';
-import { finished } from 'node:stream/promises';
 import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
+import { ResponseWithUrl } from '@crawlee/http-client';
 import { Router } from '../index.js';
 import { parseContentTypeFromResponse } from './utils.js';
-/**
- * Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
- * This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
- * If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
- * it will throw an error.
- *
- * Can be used e.g. to abort a download if the network speed is too slow.
- * @returns Transform stream that monitors the speed of the incoming data.
- */
-export function MinimumSpeedStream({ minSpeedKbps, historyLengthMs = 10e3, checkProgressInterval: checkProgressIntervalMs = 5e3, }) {
-    let snapshots = [];
-    const checkInterval = setInterval(() => {
-        const now = Date.now();
-        snapshots = snapshots.filter((snapshot) => now - snapshot.timestamp < historyLengthMs);
-        const totalBytes = snapshots.reduce((acc, snapshot) => acc + snapshot.bytes, 0);
-        const elapsed = (now - (snapshots[0]?.timestamp ?? 0)) / 1000;
-        if (totalBytes / 1024 / elapsed < minSpeedKbps) {
-            clearInterval(checkInterval);
-            stream.emit('error', new Error(`Stream speed too slow, aborting...`));
-        }
-    }, checkProgressIntervalMs);
-    const stream = new Transform({
-        transform: (chunk, _, callback) => {
-            snapshots.push({ timestamp: Date.now(), bytes: chunk.length });
-            callback(null, chunk);
-        },
-        final: (callback) => {
-            clearInterval(checkInterval);
-            callback();
-        },
-    });
-    return stream;
-}
-/**
- * Creates a transform stream that logs the progress of the incoming data.
- * This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
- *
- * Can be used e.g. to log the progress of a download.
- * @returns Transform stream logging the progress of the incoming data.
- */
-export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000, }) {
-    let transferredBytes = 0;
-    let lastLogTime = Date.now();
-    return new Transform({
-        transform: (chunk, _, callback) => {
-            transferredBytes += chunk.length;
-            if (Date.now() - lastLogTime > loggingInterval) {
-                lastLogTime = Date.now();
-                logTransferredBytes(transferredBytes);
-            }
-            callback(null, chunk);
-        },
-        flush: (callback) => {
-            logTransferredBytes(transferredBytes);
-            callback();
-        },
-    });
-}
 /**
  * Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
  *
@@ -68,25 +9,15 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
  *
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link FileDownloadOptions.requestHandler} where the user can specify what to do with the downloaded data.
+ * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
  *
- * The source URLs are represented using {@link Request} objects that are fed from {@link RequestList} or {@link RequestQueue} instances provided by the {@link FileDownloadOptions.requestList} or {@link FileDownloadOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
  *
- * If both {@link FileDownloadOptions.requestList} and {@link FileDownloadOptions.requestQueue} are used, the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
- * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
- *
- * ```
- * preNavigationHooks: [
- *     (crawlingContext) => {
- *         // ...
- *     },
- * ]
- * ```
- *
- * New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the {@link AutoscaledPool} class. All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions` parameter of the `FileCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency` {@link AutoscaledPool} options are available directly in the `FileCrawler` constructor.
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * ## Example usage
  *
@@ -109,53 +40,54 @@ export class FileDownload extends BasicCrawler {
     constructor(options = {}) {
         super({
             ...options,
-            contextPipelineBuilder: () => ContextPipeline.create().compose({
-                action: async (context) => this.initiateDownload(context),
-                cleanup: async (context) => {
-                    await (context.response.body ? finished(context.response.body) : Promise.resolve());
-                },
-            }),
+            contextPipelineBuilder: () => this.#buildContextPipeline(),
         });
     }
-    async initiateDownload(context) {
-        const response = await this.httpClient.stream(context.request.intoFetchAPIRequest(), {
+    #buildContextPipeline() {
+        return ContextPipeline.create().compose(this.initiateDownload.bind(this));
+    }
+    async initiateDownload(context, onCleanup) {
+        const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
             session: context.session,
         });
         const { type, charset: encoding } = parseContentTypeFromResponse(response);
         context.request.url = response.url;
-        const contextExtension = {
+        const { response: trackedResponse, bodyDrained } = trackBodyConsumption(response);
+        onCleanup(async () => {
+            if (!trackedResponse.bodyUsed) {
+                // Nobody consumed the body — cancel it so the
+                // underlying connection can be released.
+                await trackedResponse.body?.cancel();
+            }
+            await bodyDrained;
+        });
+        return {
             request: context.request,
-            response,
+            response: trackedResponse,
             contentType: { type, encoding },
         };
-        return contextExtension;
     }
 }
 /**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link FileDownload}.
- * Defaults to the {@link FileDownloadCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<FileDownloadCrawlingContext>()`.
- *
- * ```ts
- * import { FileDownload, createFileRouter } from 'crawlee';
- *
- * const router = createFileRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
- *
- * const crawler = new FileDownload({
- *     requestHandler: router,
- * });
- * await crawler.run();
- * ```
+ * Wraps a Response so that we can track when the body stream has been fully
+ * consumed (or errored). Pipes the original body through a TransformStream;
+ * the readable side becomes the new Response body, and `pipeTo` gives us a
+ * promise that resolves once the body is fully read or cancelled.
  */
-export function createFileRouter(routes) {
-    return Router.create(routes);
+function trackBodyConsumption(response) {
+    if (!response.body) {
+        return { response, bodyDrained: Promise.resolve() };
+    }
+    const passthrough = new TransformStream();
+    const bodyDrained = response.body.pipeTo(passthrough.writable).catch(() => { });
+    const trackedResponse = new ResponseWithUrl(passthrough.readable, {
+        headers: response.headers,
+        status: response.status,
+        statusText: response.statusText,
+        url: response.url,
+    });
+    return { response: trackedResponse, bodyDrained };
+}
+export function createFileRouter(routesOrSchemas) {
+    return Router.create(routesOrSchemas);
 }
-//# sourceMappingURL=file-download.js.map