@crawlee/http 4.0.0-beta.2 → 4.0.0-beta.200

package/README.md

@@ -1,19 +1,23 @@
 <h1 align="center">
     <a href="https://crawlee.dev">
         <picture>
-          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
-          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
         </picture>
     </a>
-    <br>
+    <br />
     <small>A web scraping and browser automation library</small>
 </h1>
 
-<p align=center>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
-    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+<p align="center">
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
+</p>
+
+<p align="center">
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
 
-> Crawlee for Python is open for early adopters. 🐍  [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
+> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
 
 ## Installation
 
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
 We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
 
 ```bash
-npm install crawlee@3.12.3-beta.13
+npm install crawlee@next
 ```
 
 If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
 {
     "overrides": {
        "apify": {
-           "@crawlee/core": "3.12.3-beta.13",
-           "@crawlee/types": "3.12.3-beta.13",
-           "@crawlee/utils": "3.12.3-beta.13"
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
        }
     }
 }

package/index.d.ts

@@ -1,4 +1,4 @@
 export * from '@crawlee/basic';
 export * from './internals/http-crawler.js';
+export * from './internals/dom-crawler.js';
 export * from './internals/file-download.js';
-//# sourceMappingURL=index.d.ts.map

package/index.js

@@ -1,4 +1,4 @@
 export * from '@crawlee/basic';
 export * from './internals/http-crawler.js';
+export * from './internals/dom-crawler.js';
 export * from './internals/file-download.js';
-//# sourceMappingURL=index.js.map

package/internals/dom-crawler.d.ts

@@ -0,0 +1,131 @@
+import type { AddRequestsBatchedResult, ContextPipeline, CrawlingContext, EnqueueLinksOptions, ExtractLinksOptions, GetUserDataFromRequest } from '@crawlee/basic';
+import type { Awaitable, Dictionary } from '@crawlee/types';
+import type { CheerioAPI } from 'cheerio';
+import type { HttpCrawlerOptions, InternalHttpCrawlingContext } from './http-crawler.js';
+import { HttpCrawler } from './http-crawler.js';
+/**
+ * The minimum a {@link DOMParser} has to contribute to the crawling context - the serialized document, used by
+ * the {@link DOMCrawlingContext.parseWithCheerio|`parseWithCheerio`} helper.
+ */
+export interface DOMParseResult {
+    body: string;
+}
+/**
+ * Turns a response body into a DOM representation and knows how to query it. Passing one to {@link DOMCrawler}
+ * is what makes the crawler jsdom-based, linkedom-based, or based on a DOM implementation of your own.
+ *
+ * **Example usage:**
+ * ```ts
+ * import { DOMCrawler } from 'crawlee';
+ * import type { DOMParser } from 'crawlee';
+ *
+ * const myParser: DOMParser<{ body: string }> = {
+ *     // ...
+ * };
+ *
+ * const crawler = new DOMCrawler({
+ *     parser: myParser,
+ *     async requestHandler({ body }) {
+ *         // ...
+ *     },
+ * });
+ * ```
+ */
+export interface DOMParser<Parsed extends DOMParseResult> {
+    /**
+     * The context members {@link DOMParser.parse|`parse`} contributes, mapped to `true`. Used to build the
+     * placeholders that report a helpful error when the members are accessed after `skipNavigation` - the `Record`
+     * type forces every key of `Parsed` to be listed, so the compiler catches an omission that would otherwise
+     * yield `undefined` (rather than throwing) after `skipNavigation`.
+     */
+    readonly placeholderMembers: Record<keyof Parsed & string, true>;
+    parse(context: InternalHttpCrawlingContext): Awaitable<Parsed>;
+    /**
+     * Returns the URLs the `selector` matches, resolved against `baseUrl`.
+     */
+    extractLinks(parsed: Parsed, selector: string, baseUrl: string): Awaitable<string[]>;
+    /**
+     * Returns the current matches of `selector`. Only the count is used, by
+     * {@link DOMCrawlingContext.waitForSelector|`waitForSelector`}.
+     */
+    select(parsed: Parsed, selector: string): Awaitable<ArrayLike<unknown>>;
+    /**
+     * Whether the parse result can change after {@link DOMParser.parse|`parse`} returned - the case when the DOM
+     * implementation runs the page scripts. If it can, {@link DOMCrawlingContext.waitForSelector|`waitForSelector`}
+     * polls until the timeout elapses; otherwise it fails as soon as the selector does not match.
+     */
+    readonly mutable?: boolean;
+    /**
+     * Returns a Cheerio handle over the parse result, for parsers that are backed by Cheerio anyway. Without it,
+     * {@link DOMCrawlingContext.parseWithCheerio|`parseWithCheerio`} parses
+     * {@link DOMParseResult.body|`body`} again.
+     */
+    toCheerio?(parsed: Parsed): Awaitable<CheerioAPI>;
+    /**
+     * Releases whatever {@link DOMParser.parse|`parse`} allocated. Called after the request handler finishes or
+     * fails, and skipped entirely when navigation was skipped.
+     */
+    cleanup?(parsed: Parsed): Awaitable<void>;
+}
+export interface DOMCrawlingHelpers {
+    /**
+     * Extracts URLs from the parsed DOM, without adding them to the request queue.
+     */
+    extractLinks(options?: ExtractLinksOptions): Promise<string[]>;
+    /**
+     * Helper function for extracting URLs from the parsed DOM and adding them to the request queue.
+     */
+    enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;
+    /**
+     * Wait for an element matching the selector to appear. The `timeoutMs` only has an effect when the parser is
+     * {@link DOMParser.mutable|`mutable`} (e.g. {@link JSDOMCrawler} with `runScripts: true`); otherwise the
+     * selector is checked once and the call resolves or throws immediately.
+     * Timeout defaults to 5s.
+     *
+     * **Example usage:**
+     * ```ts
+     * async requestHandler({ waitForSelector, parseWithCheerio }) {
+     *     await waitForSelector('article h1');
+     *     const $ = await parseWithCheerio();
+     *     const title = $('title').text();
+     * });
+     * ```
+     */
+    waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+    /**
+     * Returns Cheerio handle, allowing to work with the data same way as with {@link CheerioCrawler}.
+     * When provided with the `selector` argument, it will throw if it's not available.
+     *
+     * **Example usage:**
+     * ```javascript
+     * async requestHandler({ parseWithCheerio }) {
+     *     const $ = await parseWithCheerio();
+     *     const title = $('title').text();
+     * });
+     * ```
+     */
+    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
+}
+export type DOMCrawlingContext<Parsed extends DOMParseResult = DOMParseResult, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> = InternalHttpCrawlingContext<UserData, JSONData> & Parsed & DOMCrawlingHelpers;
+export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, any>, StatisticStateExtension extends object = {}> extends Omit<HttpCrawlerOptions<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'contextPipelineBuilder'> {
+    /**
+     * The DOM implementation to parse the response bodies with. Its parse result becomes part of the crawling
+     * context, so the members the request handler receives follow from the parser you pass.
+     */
+    parser: DOMParser<Parsed>;
+}
+/**
+ * An {@link HttpCrawler} that parses each response into a DOM using the {@link DOMCrawlerOptions.parser|`parser`}
+ * it is given, and exposes the parse result plus the {@link DOMCrawlingContext.enqueueLinks|`enqueueLinks`} and
+ * {@link DOMCrawlingContext.extractLinks|`extractLinks`} helpers on the crawling context.
+ *
+ * {@link JSDOMCrawler} and {@link LinkeDOMCrawler} are this crawler with a parser already chosen.
+ *
+ * @category Crawlers
+ */
+export declare class DOMCrawler<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<DOMCrawlingContext<Parsed>['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+    #private;
+    constructor(options: DOMCrawlerOptions<Parsed, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
+    protected buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed>>;
+}

package/internals/dom-crawler.js

@@ -0,0 +1,93 @@
+import { EnqueueStrategy, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering } from '@crawlee/basic';
+import { sleep } from '@crawlee/utils';
+import { HttpCrawler } from './http-crawler.js';
+/**
+ * An {@link HttpCrawler} that parses each response into a DOM using the {@link DOMCrawlerOptions.parser|`parser`}
+ * it is given, and exposes the parse result plus the {@link DOMCrawlingContext.enqueueLinks|`enqueueLinks`} and
+ * {@link DOMCrawlingContext.extractLinks|`extractLinks`} helpers on the crawling context.
+ *
+ * {@link JSDOMCrawler} and {@link LinkeDOMCrawler} are this crawler with a parser already chosen.
+ *
+ * @category Crawlers
+ */
+export class DOMCrawler extends HttpCrawler {
+    #parser;
+    constructor(options) {
+        const { parser, ...rest } = options;
+        super({ ...rest, contextPipelineBuilder: () => this.buildContextPipeline() });
+        this.#parser = parser;
+    }
+    buildContextPipeline() {
+        return super
+            .buildContextPipeline()
+            .compose(this.#parseContent.bind(this))
+            .compose(async (context) => this.#addHelpers(context));
+    }
+    async #parseContent(context, onCleanup) {
+        try {
+            const parsed = await this.#parser.parse(context);
+            onCleanup(() => this.#parser.cleanup?.(parsed));
+            return parsed;
+        }
+        catch (err) {
+            // The `skipNavigation` placeholders below throw on access, so there is nothing to clean up.
+            if (err instanceof NavigationSkippedError) {
+                return Object.defineProperties({}, Object.fromEntries(Object.keys(this.#parser.placeholderMembers).map((member) => [
+                    member,
+                    {
+                        configurable: true,
+                        enumerable: true,
+                        get() {
+                            throw new NavigationSkippedError(`The \`${member}\` property is not available - \`skipNavigation\` was used`, { cause: err });
+                        },
+                    },
+                ])));
+            }
+            throw err;
+        }
+    }
+    async #addHelpers(context) {
+        const { addRequests } = context;
+        const parser = this.#parser;
+        const extractLinks = async (options) => parser.extractLinks(context, options?.selector ?? 'a', options?.baseUrl ?? context.request.loadedUrl ?? context.request.url);
+        const waitForSelector = async (selector, timeoutMs = 5_000) => {
+            let remaining = parser.mutable ? timeoutMs : 0;
+            while ((await parser.select(context, selector)).length === 0) {
+                if (remaining <= 0) {
+                    throw new Error(`Selector '${selector}' not found.`);
+                }
+                await sleep(50);
+                remaining -= 50;
+            }
+        };
+        return {
+            extractLinks,
+            waitForSelector,
+            enqueueLinks: async (options = {}) => {
+                const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
+                    enqueueStrategy: options.strategy,
+                    finalRequestUrl: context.request.loadedUrl,
+                    originalRequestUrl: context.request.url,
+                    userProvidedBaseUrl: options.baseUrl,
+                });
+                const urls = await extractLinks(options);
+                return addRequests(urls, {
+                    ...options,
+                    baseUrl,
+                    strategy: options.strategy ?? EnqueueStrategy.SameHostname,
+                });
+            },
+            async parseWithCheerio(selector, _timeoutMs = 5_000) {
+                // Import full cheerio (not cheerio/slim) to be browser-compliant for DOM Crawlers (not CheerioCrawler).
+                const $ = (await parser.toCheerio?.(context)) ??
+                    (await import('cheerio')).load(context.body, {
+                        xmlMode: context.contentType.type.includes('xml'),
+                    });
+                if (selector && $(selector).get().length === 0) {
+                    throw new Error(`Selector '${selector}' not found.`);
+                }
+                return $;
+            },
+        };
+    }
+}

package/internals/file-download.d.ts

@@ -1,29 +1,18 @@
+import type { BasicCrawlerOptions, CrawlingContext, CrawlingRequest, LoadedRequest } from '@crawlee/basic';
+import { BasicCrawler } from '@crawlee/basic';
 import type { Dictionary } from '@crawlee/types';
-// @ts-ignore optional peer dependency or compatibility with es2022
-import type { Request } from 'got-scraping';
-import type { ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RouterRoutes } from '../index.js';
-import { HttpCrawler } from '../index.js';
+import type { ErrorHandler, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '../index.js';
 export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> = ErrorHandler<FileDownloadCrawlingContext<UserData, JSONData>>;
-export type StreamHandlerContext = Omit<FileDownloadCrawlingContext, 'body' | 'parseWithCheerio' | 'json' | 'addRequests' | 'contentType'> & {
-    stream: Request;
-};
-type StreamHandler = (context: StreamHandlerContext) => void | Promise<void>;
-export type FileDownloadOptions<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> = (Omit<HttpCrawlerOptions<FileDownloadCrawlingContext<UserData, JSONData>>, 'requestHandler'> & {
-    requestHandler?: never;
-    streamHandler?: StreamHandler;
-}) | (Omit<HttpCrawlerOptions<FileDownloadCrawlingContext<UserData, JSONData>>, 'requestHandler'> & {
-    requestHandler: FileDownloadRequestHandler;
-    streamHandler?: never;
-});
-export type FileDownloadHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> = InternalHttpHook<FileDownloadCrawlingContext<UserData, JSONData>>;
-export interface FileDownloadCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData, FileDownload> {
+ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, FileDownloadCrawlingContext<UserData> & ContextExtension>;
+export interface FileDownloadCrawlingContext<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
+    request: LoadedRequest<CrawlingRequest<UserData>>;
+    response: Response;
+    contentType: {
+        type: string;
+        encoding: BufferEncoding;
+    };
 }
-export type FileDownloadRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData, JSONData>>;
+export type FileDownloadRequestHandler<UserData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData>>;
 /**
  * Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
  *
@@ -31,25 +20,15 @@ JSONData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
  *
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link FileDownloadOptions.requestHandler} where the user can specify what to do with the downloaded data.
+ * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
  *
- * The source URLs are represented using {@link Request} objects that are fed from {@link RequestList} or {@link RequestQueue} instances provided by the {@link FileDownloadOptions.requestList} or {@link FileDownloadOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
  *
- * If both {@link FileDownloadOptions.requestList} and {@link FileDownloadOptions.requestQueue} are used, the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
- *
- * ```
- * preNavigationHooks: [
- *     (crawlingContext, gotOptions) => {
- *         // ...
- *     },
- * ]
- * ```
- *
- * New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the {@link AutoscaledPool} class. All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions` parameter of the `FileCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency` {@link AutoscaledPool} options are available directly in the `FileCrawler` constructor.
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * ## Example usage
  *
@@ -67,11 +46,10 @@ JSONData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<
  * ]);
  * ```
  */
-export declare class FileDownload extends HttpCrawler<FileDownloadCrawlingContext> {
-    private streamHandler?;
-    constructor(options?: FileDownloadOptions);
-    protected _runRequestHandler(context: FileDownloadCrawlingContext): Promise<void>;
-    private streamRequestHandler;
+export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
+    #private;
+    constructor(options?: Omit<BasicCrawlerOptions<FileDownloadCrawlingContext>, 'contextPipelineBuilder'>);
+    private initiateDownload;
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.
@@ -97,7 +75,6 @@ export declare class FileDownload extends HttpCrawler<FileDownloadCrawlingContex
  * await crawler.run();
  * ```
  */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("packages/core/dist/router.js").RouterHandler<Context>;
-export {};
-//# sourceMappingURL=file-download.d.ts.map
+export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;

package/internals/file-download.js

@@ -1,6 +1,7 @@
-import { finished } from 'node:stream/promises';
-import { isPromise } from 'node:util/types';
-import { HttpCrawler, Router } from '../index.js';
+import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
+import { ResponseWithUrl } from '@crawlee/http-client';
+import { Router } from '../index.js';
+import { parseContentTypeFromResponse } from './utils.js';
 /**
  * Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
  *
@@ -8,25 +9,15 @@ import { HttpCrawler, Router } from '../index.js';
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
  *
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link FileDownloadOptions.requestHandler} where the user can specify what to do with the downloaded data.
+ * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
  *
- * The source URLs are represented using {@link Request} objects that are fed from {@link RequestList} or {@link RequestQueue} instances provided by the {@link FileDownloadOptions.requestList} or {@link FileDownloadOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
  *
- * If both {@link FileDownloadOptions.requestList} and {@link FileDownloadOptions.requestQueue} are used, the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
- *
- * ```
- * preNavigationHooks: [
- *     (crawlingContext, gotOptions) => {
- *         // ...
- *     },
- * ]
- * ```
- *
- * New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the {@link AutoscaledPool} class. All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions` parameter of the `FileCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency` {@link AutoscaledPool} options are available directly in the `FileCrawler` constructor.
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * ## Example usage
  *
@@ -44,106 +35,59 @@ import { HttpCrawler, Router } from '../index.js';
  * ]);
  * ```
  */
-export class FileDownload extends HttpCrawler {
-    streamHandler;
+export class FileDownload extends BasicCrawler {
+    // TODO hooks
     constructor(options = {}) {
-        const { streamHandler } = options;
-        delete options.streamHandler;
-        if (streamHandler) {
-            // For streams, the navigation is done in the request handler.
-            options.requestHandlerTimeoutSecs = options.navigationTimeoutSecs ?? 120;
-        }
-        super(options);
-        this.streamHandler = streamHandler;
-        if (this.streamHandler) {
-            this.requestHandler = this.streamRequestHandler;
-        }
-        // The base HttpCrawler class only supports a handful of text based mime types.
-        // With the FileDownload crawler, we want to download any file type.
-        this.supportedMimeTypes = new Set(['*/*']);
+        super({
+            ...options,
+            contextPipelineBuilder: () => this.#buildContextPipeline(),
+        });
     }
-    async _runRequestHandler(context) {
-        if (this.streamHandler) {
-            context.request.skipNavigation = true;
-        }
-        await super._runRequestHandler(context);
+    #buildContextPipeline() {
+        return ContextPipeline.create().compose(this.initiateDownload.bind(this));
     }
-    async streamRequestHandler(context) {
-        const { log, request: { url }, } = context;
-        const response = await this.httpClient.stream({
-            url,
-            timeout: { request: undefined },
-            proxyUrl: context.proxyInfo?.url,
+    async initiateDownload(context, onCleanup) {
+        const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
+            session: context.session,
         });
-        let pollingInterval;
-        const cleanUp = () => {
-            clearInterval(pollingInterval);
-            response.stream.destroy();
-        };
-        const downloadPromise = new Promise((resolve, reject) => {
-            pollingInterval = setInterval(() => {
-                const { total, transferred } = response.downloadProgress;
-                if (transferred > 0) {
-                    log.debug(`Downloaded ${transferred} bytes of ${total ?? 0} bytes from ${url}.`);
-                }
-            }, 5000);
-            response.stream.on('error', async (error) => {
-                cleanUp();
-                reject(error);
-            });
-            let streamHandlerResult;
-            try {
-                context.stream = response.stream;
-                context.response = response;
-                streamHandlerResult = this.streamHandler(context);
-            }
-            catch (e) {
-                cleanUp();
-                reject(e);
-            }
-            if (isPromise(streamHandlerResult)) {
-                streamHandlerResult
-                    .then(() => {
-                    resolve();
-                })
-                    .catch((e) => {
-                    cleanUp();
-                    reject(e);
-                });
-            }
-            else {
-                resolve();
+        const { type, charset: encoding } = parseContentTypeFromResponse(response);
+        context.request.url = response.url;
+        const { response: trackedResponse, bodyDrained } = trackBodyConsumption(response);
+        onCleanup(async () => {
+            if (!trackedResponse.bodyUsed) {
+                // Nobody consumed the body — cancel it so the
+                // underlying connection can be released.
+                await trackedResponse.body?.cancel();
             }
+            await bodyDrained;
         });
-        await Promise.all([downloadPromise, finished(response.stream)]);
-        cleanUp();
+        return {
+            request: context.request,
+            response: trackedResponse,
+            contentType: { type, encoding },
+        };
     }
 }
 /**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link FileDownload}.
- * Defaults to the {@link FileDownloadCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<FileDownloadCrawlingContext>()`.
- *
- * ```ts
- * import { FileDownload, createFileRouter } from 'crawlee';
- *
- * const router = createFileRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
- *
- * const crawler = new FileDownload({
- *     requestHandler: router,
- * });
- * await crawler.run();
- * ```
+ * Wraps a Response so that we can track when the body stream has been fully
+ * consumed (or errored). Pipes the original body through a TransformStream;
+ * the readable side becomes the new Response body, and `pipeTo` gives us a
+ * promise that resolves once the body is fully read or cancelled.
  */
-export function createFileRouter(routes) {
-    return Router.create(routes);
+function trackBodyConsumption(response) {
+    if (!response.body) {
+        return { response, bodyDrained: Promise.resolve() };
+    }
+    const passthrough = new TransformStream();
+    const bodyDrained = response.body.pipeTo(passthrough.writable).catch(() => { });
+    const trackedResponse = new ResponseWithUrl(passthrough.readable, {
+        headers: response.headers,
+        status: response.status,
+        statusText: response.statusText,
+        url: response.url,
+    });
+    return { response: trackedResponse, bodyDrained };
+}
+export function createFileRouter(routesOrSchemas) {
+    return Router.create(routesOrSchemas);
 }
-//# sourceMappingURL=file-download.js.map