@crawlee/http 4.0.0-beta.27 → 4.0.0-beta.270

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,23 +1,23 @@
1
1
  <h1 align="center">
2
2
  <a href="https://crawlee.dev">
3
3
  <picture>
4
- <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
5
- <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
4
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
5
+ <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
6
6
  </picture>
7
7
  </a>
8
- <br>
8
+ <br />
9
9
  <small>A web scraping and browser automation library</small>
10
10
  </h1>
11
11
 
12
- <p align=center>
13
- <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
12
+ <p align="center">
13
+ <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
14
14
  </p>
15
15
 
16
- <p align=center>
17
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
18
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
19
- <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
20
- <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
16
+ <p align="center">
17
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
18
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
19
+ <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
20
+ <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
21
21
  </p>
22
22
 
23
23
  Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
34
34
 
35
35
  We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
36
36
 
37
- > Crawlee requires **Node.js 16 or higher**.
37
+ > Crawlee requires **Node.js 22.13 or higher**.
38
38
 
39
39
  ### With Crawlee CLI
40
40
 
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
89
89
  We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
90
90
 
91
91
  ```bash
92
- npm install crawlee@3.12.3-beta.13
92
+ npm install crawlee@next
93
93
  ```
94
94
 
95
95
  If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
98
98
  {
99
99
  "overrides": {
100
100
  "apify": {
101
- "@crawlee/core": "3.12.3-beta.13",
102
- "@crawlee/types": "3.12.3-beta.13",
103
- "@crawlee/utils": "3.12.3-beta.13"
101
+ "@crawlee/core": "$crawlee",
102
+ "@crawlee/types": "$crawlee",
103
+ "@crawlee/utils": "$crawlee"
104
104
  }
105
105
  }
106
106
  }
package/index.d.ts CHANGED
@@ -1,4 +1,4 @@
1
1
  export * from '@crawlee/basic';
2
2
  export * from './internals/http-crawler.js';
3
+ export * from './internals/dom-crawler.js';
3
4
  export * from './internals/file-download.js';
4
- //# sourceMappingURL=index.d.ts.map
package/index.js CHANGED
@@ -1,4 +1,4 @@
1
1
  export * from '@crawlee/basic';
2
2
  export * from './internals/http-crawler.js';
3
+ export * from './internals/dom-crawler.js';
3
4
  export * from './internals/file-download.js';
4
- //# sourceMappingURL=index.js.map
@@ -0,0 +1,145 @@
1
+ import type { AddRequestsBatchedResult, ContextPipeline, CrawlingContext, EnqueueLinksOptions, ExtractLinksOptions, GetUserDataFromRequest } from '@crawlee/basic';
2
+ import type { Awaitable, Dictionary } from '@crawlee/types';
3
+ import type { CheerioAPI } from 'cheerio';
4
+ import type { HttpCrawlerOptions, InternalHttpCrawlingContext } from './http-crawler.js';
5
+ import { HttpCrawler } from './http-crawler.js';
6
+ /**
7
+ * The minimum a {@link DOMParser} has to contribute to the crawling context - the serialized document, used by
8
+ * the {@link DOMCrawlingContext.parseWithCheerio|`parseWithCheerio`} helper.
9
+ */
10
+ export interface DOMParseResult {
11
+ body: string;
12
+ }
13
+ /**
14
+ * Turns a response body into a DOM representation and knows how to query it. Passing one to {@link DOMCrawler}
15
+ * is what makes the crawler jsdom-based, linkedom-based, or based on a DOM implementation of your own.
16
+ *
17
+ * **Example usage:**
18
+ * ```ts
19
+ * import { DOMCrawler } from 'crawlee';
20
+ * import type { DOMParser } from 'crawlee';
21
+ *
22
+ * const myParser: DOMParser<{ body: string }> = {
23
+ * // ...
24
+ * };
25
+ *
26
+ * const crawler = new DOMCrawler({
27
+ * parser: myParser,
28
+ * async requestHandler({ body }) {
29
+ * // ...
30
+ * },
31
+ * });
32
+ * ```
33
+ */
34
+ export interface DOMParser<Parsed extends DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown> {
35
+ /**
36
+ * The context members {@link DOMParser.parse|`parse`} contributes, mapped to `true`. Used to build the
37
+ * placeholders that report a helpful error when the members are accessed after `skipNavigation` - the `Record`
38
+ * type forces every key of `Parsed` to be listed, so the compiler catches an omission that would otherwise
39
+ * yield `undefined` (rather than throwing) after `skipNavigation`.
40
+ */
41
+ readonly placeholderMembers: Record<keyof Parsed & string, true>;
42
+ parse(context: InternalHttpCrawlingContext): Awaitable<Parsed>;
43
+ /**
44
+ * Returns the URLs the `selector` matches, resolved against `baseUrl`.
45
+ */
46
+ extractLinks(parsed: Parsed, selector: string, baseUrl: string): Awaitable<string[]>;
47
+ /**
48
+ * Returns the current matches of `selector`, in whatever form the DOM implementation uses - this is what
49
+ * {@link DOMCrawlingContext.querySelectorAll|`querySelectorAll`} hands to the request handler.
50
+ */
51
+ querySelectorAll(parsed: Parsed, selector: string): Awaitable<Selection>;
52
+ /**
53
+ * Returns the first match of `selector`, or `undefined` when there is none - this is what
54
+ * {@link DOMCrawlingContext.querySelectorOne|`querySelectorOne`} hands to the request handler.
55
+ */
56
+ querySelectorOne(parsed: Parsed, selector: string): Awaitable<Single | undefined>;
57
+ /**
58
+ * Whether the parse result can change after {@link DOMParser.parse|`parse`} returned - the case when the DOM
59
+ * implementation runs the page scripts. If it can, {@link DOMCrawlingContext.waitForSelector|`waitForSelector`}
60
+ * polls until the timeout elapses; otherwise it fails as soon as the selector does not match.
61
+ */
62
+ readonly mutable?: boolean;
63
+ /**
64
+ * Returns a Cheerio handle over the parse result, for parsers that are backed by Cheerio anyway. Without it,
65
+ * {@link DOMCrawlingContext.parseWithCheerio|`parseWithCheerio`} parses
66
+ * {@link DOMParseResult.body|`body`} again.
67
+ */
68
+ toCheerio?(parsed: Parsed): Awaitable<CheerioAPI>;
69
+ /**
70
+ * Releases whatever {@link DOMParser.parse|`parse`} allocated. Called after the request handler finishes or
71
+ * fails, and skipped entirely when navigation was skipped.
72
+ */
73
+ cleanup?(parsed: Parsed): Awaitable<void>;
74
+ }
75
+ export interface DOMCrawlingHelpers<Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown> {
76
+ /**
77
+ * Extracts URLs from the parsed DOM, without adding them to the request queue.
78
+ */
79
+ extractLinks(options?: ExtractLinksOptions): Promise<string[]>;
80
+ /**
81
+ * Helper function for extracting URLs from the parsed DOM and adding them to the request queue.
82
+ */
83
+ enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;
84
+ /**
85
+ * Wait for an element matching the selector to appear. The `timeoutMs` only has an effect when the parser is
86
+ * {@link DOMParser.mutable|`mutable`} (e.g. {@link JSDOMCrawler} with `runScripts: true`); otherwise the
87
+ * selector is checked once and the call resolves or throws immediately.
88
+ * Timeout defaults to 5s.
89
+ *
90
+ * **Example usage:**
91
+ * ```ts
92
+ * async requestHandler({ waitForSelector, parseWithCheerio }) {
93
+ * await waitForSelector('article h1');
94
+ * const $ = await parseWithCheerio();
95
+ * const title = $('title').text();
96
+ * });
97
+ * ```
98
+ */
99
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
100
+ /**
101
+ * Like {@link DOMCrawlingHelpers.waitForSelector|`waitForSelector`}, but returns the matches in the form
102
+ * the parser produces them - a Cheerio object in {@link CheerioCrawler}. Throws when nothing matches.
103
+ */
104
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Selection>;
105
+ /**
106
+ * Like {@link DOMCrawlingHelpers.querySelectorAll|`querySelectorAll`}, but returns only the first match.
107
+ */
108
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Single>;
109
+ /**
110
+ * Returns Cheerio handle, allowing to work with the data same way as with {@link CheerioCrawler}.
111
+ * When provided with the `selector` argument, it will throw if it's not available.
112
+ *
113
+ * **Example usage:**
114
+ * ```javascript
115
+ * async requestHandler({ parseWithCheerio }) {
116
+ * const $ = await parseWithCheerio();
117
+ * const title = $('title').text();
118
+ * });
119
+ * ```
120
+ */
121
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
122
+ }
123
+ export type DOMCrawlingContext<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
124
+ JSONData extends Dictionary = any> = Omit<InternalHttpCrawlingContext<UserData, JSONData>, keyof DOMCrawlingHelpers> & Parsed & DOMCrawlingHelpers<Selection, Single>;
125
+ export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<Parsed, Selection, Single> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, any>, StatisticStateExtension extends object = {}> extends Omit<HttpCrawlerOptions<DOMCrawlingContext<Parsed, Selection, Single>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'contextPipelineBuilder'> {
126
+ /**
127
+ * The DOM implementation to parse the response bodies with. Its parse result becomes part of the crawling
128
+ * context, so the members the request handler receives follow from the parser you pass.
129
+ */
130
+ parser: DOMParser<Parsed, Selection, Single>;
131
+ }
132
+ /**
133
+ * An {@link HttpCrawler} that parses each response into a DOM using the {@link DOMCrawlerOptions.parser|`parser`}
134
+ * it is given, and exposes the parse result plus the {@link DOMCrawlingContext.enqueueLinks|`enqueueLinks`} and
135
+ * {@link DOMCrawlingContext.extractLinks|`extractLinks`} helpers on the crawling context.
136
+ *
137
+ * {@link JSDOMCrawler} and {@link LinkeDOMCrawler} are this crawler with a parser already chosen.
138
+ *
139
+ * @category Crawlers
140
+ */
141
+ export declare class DOMCrawler<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<Parsed, Selection, Single> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<DOMCrawlingContext<Parsed, Selection, Single>['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<DOMCrawlingContext<Parsed, Selection, Single>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
142
+ #private;
143
+ constructor(options: DOMCrawlerOptions<Parsed, Selection, Single, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
144
+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed, Selection, Single>>;
145
+ }
@@ -0,0 +1,102 @@
1
+ import { EnqueueStrategy, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering } from '@crawlee/basic';
2
+ import { sleep } from '@crawlee/utils';
3
+ import { HttpCrawler } from './http-crawler.js';
4
+ /**
5
+ * An {@link HttpCrawler} that parses each response into a DOM using the {@link DOMCrawlerOptions.parser|`parser`}
6
+ * it is given, and exposes the parse result plus the {@link DOMCrawlingContext.enqueueLinks|`enqueueLinks`} and
7
+ * {@link DOMCrawlingContext.extractLinks|`extractLinks`} helpers on the crawling context.
8
+ *
9
+ * {@link JSDOMCrawler} and {@link LinkeDOMCrawler} are this crawler with a parser already chosen.
10
+ *
11
+ * @category Crawlers
12
+ */
13
+ export class DOMCrawler extends HttpCrawler {
14
+ #parser;
15
+ constructor(options) {
16
+ const { parser, ...rest } = options;
17
+ super({ ...rest, contextPipelineBuilder: () => this.buildContextPipeline() });
18
+ this.#parser = parser;
19
+ }
20
+ buildContextPipeline() {
21
+ return super
22
+ .buildContextPipeline()
23
+ .compose(this.#parseContent.bind(this))
24
+ .compose(async (context) => this.#addHelpers(context));
25
+ }
26
+ async #parseContent(context, onCleanup) {
27
+ try {
28
+ const parsed = await this.#parser.parse(context);
29
+ onCleanup(() => this.#parser.cleanup?.(parsed));
30
+ return parsed;
31
+ }
32
+ catch (err) {
33
+ // The `skipNavigation` placeholders below throw on access, so there is nothing to clean up.
34
+ if (err instanceof NavigationSkippedError) {
35
+ return Object.defineProperties({}, Object.fromEntries(Object.keys(this.#parser.placeholderMembers).map((member) => [
36
+ member,
37
+ {
38
+ configurable: true,
39
+ enumerable: true,
40
+ get() {
41
+ throw new NavigationSkippedError(`The \`${member}\` property is not available - \`skipNavigation\` was used`, { cause: err });
42
+ },
43
+ },
44
+ ])));
45
+ }
46
+ throw err;
47
+ }
48
+ }
49
+ async #addHelpers(context) {
50
+ const { addRequests } = context;
51
+ const parser = this.#parser;
52
+ const extractLinks = async (options) => parser.extractLinks(context, options?.selector ?? 'a', options?.baseUrl ?? context.request.loadedUrl ?? context.request.url);
53
+ // Polls `query` while `isEmpty` holds, for as long as a mutable parser allows; a static one gets one shot.
54
+ const poll = async (selector, timeoutMs, query, isEmpty) => {
55
+ let remaining = parser.mutable ? timeoutMs : 0;
56
+ let result = await query();
57
+ while (isEmpty(result)) {
58
+ if (remaining <= 0) {
59
+ throw new Error(`Selector '${selector}' not found.`);
60
+ }
61
+ await sleep(50);
62
+ remaining -= 50;
63
+ result = await query();
64
+ }
65
+ return result;
66
+ };
67
+ const querySelectorAll = async (selector, timeoutMs = 5_000) => poll(selector, timeoutMs, () => parser.querySelectorAll(context, selector), (matches) => matches.length === 0);
68
+ const waitForSelector = async (selector, timeoutMs) => {
69
+ await querySelectorAll(selector, timeoutMs);
70
+ };
71
+ return {
72
+ extractLinks,
73
+ waitForSelector,
74
+ querySelectorAll,
75
+ querySelectorOne: async (selector, timeoutMs = 5_000) => poll(selector, timeoutMs, () => parser.querySelectorOne(context, selector), (match) => match === undefined),
76
+ enqueueLinks: async (options = {}) => {
77
+ const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
78
+ enqueueStrategy: options.strategy,
79
+ finalRequestUrl: context.request.loadedUrl,
80
+ originalRequestUrl: context.request.url,
81
+ userProvidedBaseUrl: options.baseUrl,
82
+ });
83
+ const urls = await extractLinks(options);
84
+ return addRequests(urls, {
85
+ ...options,
86
+ baseUrl,
87
+ strategy: options.strategy ?? EnqueueStrategy.SameHostname,
88
+ });
89
+ },
90
+ async parseWithCheerio(selector, timeoutMs) {
91
+ if (selector) {
92
+ await waitForSelector(selector, timeoutMs);
93
+ }
94
+ // Import full cheerio (not cheerio/slim) to be browser-compliant for DOM Crawlers (not CheerioCrawler).
95
+ return ((await parser.toCheerio?.(context)) ??
96
+ (await import('cheerio')).load(context.body, {
97
+ xmlMode: context.contentType.type.includes('xml'),
98
+ }));
99
+ },
100
+ };
101
+ }
102
+ }
@@ -1,13 +1,11 @@
1
- import { Transform } from 'node:stream';
2
- import type { BasicCrawlerOptions } from '@crawlee/basic';
1
+ import type { BasicCrawlerOptions, CrawlingContext, CrawlingRequest, LoadedRequest } from '@crawlee/basic';
3
2
  import { BasicCrawler } from '@crawlee/basic';
4
- import type { CrawlingContext, LoadedRequest, Request } from '@crawlee/core';
5
3
  import type { Dictionary } from '@crawlee/types';
6
- import type { ErrorHandler, GetUserDataFromRequest, InternalHttpHook, RequestHandler, RouterRoutes } from '../index.js';
7
- export type FileDownloadErrorHandler<UserData extends Dictionary = any> = ErrorHandler<FileDownloadCrawlingContext<UserData>>;
8
- export type FileDownloadHook<UserData extends Dictionary = any> = InternalHttpHook<FileDownloadCrawlingContext<UserData>>;
4
+ import type { ErrorHandler, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '../index.js';
5
+ export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
6
+ ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, FileDownloadCrawlingContext<UserData> & ContextExtension>;
9
7
  export interface FileDownloadCrawlingContext<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
10
- request: LoadedRequest<Request<UserData>>;
8
+ request: LoadedRequest<CrawlingRequest<UserData>>;
11
9
  response: Response;
12
10
  contentType: {
13
11
  type: string;
@@ -15,31 +13,6 @@ export interface FileDownloadCrawlingContext<UserData extends Dictionary = any>
15
13
  };
16
14
  }
17
15
  export type FileDownloadRequestHandler<UserData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData>>;
18
- /**
19
- * Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
20
- * This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
21
- * If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
22
- * it will throw an error.
23
- *
24
- * Can be used e.g. to abort a download if the network speed is too slow.
25
- * @returns Transform stream that monitors the speed of the incoming data.
26
- */
27
- export declare function MinimumSpeedStream({ minSpeedKbps, historyLengthMs, checkProgressInterval: checkProgressIntervalMs, }: {
28
- minSpeedKbps: number;
29
- historyLengthMs?: number;
30
- checkProgressInterval?: number;
31
- }): Transform;
32
- /**
33
- * Creates a transform stream that logs the progress of the incoming data.
34
- * This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
35
- *
36
- * Can be used e.g. to log the progress of a download.
37
- * @returns Transform stream logging the progress of the incoming data.
38
- */
39
- export declare function ByteCounterStream({ logTransferredBytes, loggingInterval, }: {
40
- logTransferredBytes: (transferredBytes: number) => void;
41
- loggingInterval?: number;
42
- }): Transform;
43
16
  /**
44
17
  * Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
45
18
  *
@@ -47,25 +20,15 @@ export declare function ByteCounterStream({ logTransferredBytes, loggingInterval
47
20
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
48
21
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
49
22
  *
50
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link FileDownloadOptions.requestHandler} where the user can specify what to do with the downloaded data.
23
+ * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
51
24
  *
52
- * The source URLs are represented using {@link Request} objects that are fed from {@link RequestList} or {@link RequestQueue} instances provided by the {@link FileDownloadOptions.requestList} or {@link FileDownloadOptions.requestQueue} constructor options, respectively.
25
+ * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
53
26
  *
54
- * If both {@link FileDownloadOptions.requestList} and {@link FileDownloadOptions.requestQueue} are used, the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
27
+ * > The {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
55
28
  *
56
29
  * The crawler finishes when there are no more {@link Request} objects to crawl.
57
30
  *
58
- * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
59
- *
60
- * ```
61
- * preNavigationHooks: [
62
- * (crawlingContext) => {
63
- * // ...
64
- * },
65
- * ]
66
- * ```
67
- *
68
- * New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the {@link AutoscaledPool} class. All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions` parameter of the `FileCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency` {@link AutoscaledPool} options are available directly in the `FileCrawler` constructor.
31
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
69
32
  *
70
33
  * ## Example usage
71
34
  *
@@ -84,8 +47,8 @@ export declare function ByteCounterStream({ logTransferredBytes, loggingInterval
84
47
  * ```
85
48
  */
86
49
  export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
87
- constructor(options?: BasicCrawlerOptions<FileDownloadCrawlingContext>);
88
- private initiateDownload;
50
+ #private;
51
+ constructor(options?: Omit<BasicCrawlerOptions<FileDownloadCrawlingContext>, 'contextPipelineBuilder'>);
89
52
  }
90
53
  /**
91
54
  * Creates new {@link Router} instance that works based on request labels.
@@ -111,6 +74,6 @@ export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingConte
111
74
  * await crawler.run();
112
75
  * ```
113
76
  */
114
- // @ts-ignore optional peer dependency or compatibility with es2022
115
- export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/basic").RouterHandler<Context>;
116
- //# sourceMappingURL=file-download.d.ts.map
77
+ export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
78
+ export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
79
+ export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
@@ -1,67 +1,7 @@
1
- import { Transform } from 'node:stream';
2
1
  import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
3
2
  import { ResponseWithUrl } from '@crawlee/http-client';
4
3
  import { Router } from '../index.js';
5
4
  import { parseContentTypeFromResponse } from './utils.js';
6
- const kBodyDrained = Symbol('bodyDrained');
7
- /**
8
- * Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
9
- * This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
10
- * If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
11
- * it will throw an error.
12
- *
13
- * Can be used e.g. to abort a download if the network speed is too slow.
14
- * @returns Transform stream that monitors the speed of the incoming data.
15
- */
16
- export function MinimumSpeedStream({ minSpeedKbps, historyLengthMs = 10e3, checkProgressInterval: checkProgressIntervalMs = 5e3, }) {
17
- let snapshots = [];
18
- const checkInterval = setInterval(() => {
19
- const now = Date.now();
20
- snapshots = snapshots.filter((snapshot) => now - snapshot.timestamp < historyLengthMs);
21
- const totalBytes = snapshots.reduce((acc, snapshot) => acc + snapshot.bytes, 0);
22
- const elapsed = (now - (snapshots[0]?.timestamp ?? 0)) / 1000;
23
- if (totalBytes / 1024 / elapsed < minSpeedKbps) {
24
- clearInterval(checkInterval);
25
- stream.emit('error', new Error(`Stream speed too slow, aborting...`));
26
- }
27
- }, checkProgressIntervalMs);
28
- const stream = new Transform({
29
- transform: (chunk, _, callback) => {
30
- snapshots.push({ timestamp: Date.now(), bytes: chunk.length });
31
- callback(null, chunk);
32
- },
33
- final: (callback) => {
34
- clearInterval(checkInterval);
35
- callback();
36
- },
37
- });
38
- return stream;
39
- }
40
- /**
41
- * Creates a transform stream that logs the progress of the incoming data.
42
- * This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
43
- *
44
- * Can be used e.g. to log the progress of a download.
45
- * @returns Transform stream logging the progress of the incoming data.
46
- */
47
- export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000, }) {
48
- let transferredBytes = 0;
49
- let lastLogTime = Date.now();
50
- return new Transform({
51
- transform: (chunk, _, callback) => {
52
- transferredBytes += chunk.length;
53
- if (Date.now() - lastLogTime > loggingInterval) {
54
- lastLogTime = Date.now();
55
- logTransferredBytes(transferredBytes);
56
- }
57
- callback(null, chunk);
58
- },
59
- flush: (callback) => {
60
- logTransferredBytes(transferredBytes);
61
- callback();
62
- },
63
- });
64
- }
65
5
  /**
66
6
  * Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
67
7
  *
@@ -69,25 +9,15 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
69
9
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
70
10
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
71
11
  *
72
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link FileDownloadOptions.requestHandler} where the user can specify what to do with the downloaded data.
12
+ * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
73
13
  *
74
- * The source URLs are represented using {@link Request} objects that are fed from {@link RequestList} or {@link RequestQueue} instances provided by the {@link FileDownloadOptions.requestList} or {@link FileDownloadOptions.requestQueue} constructor options, respectively.
14
+ * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
75
15
  *
76
- * If both {@link FileDownloadOptions.requestList} and {@link FileDownloadOptions.requestQueue} are used, the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
16
+ * > The {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
77
17
  *
78
18
  * The crawler finishes when there are no more {@link Request} objects to crawl.
79
19
  *
80
- * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
81
- *
82
- * ```
83
- * preNavigationHooks: [
84
- * (crawlingContext) => {
85
- * // ...
86
- * },
87
- * ]
88
- * ```
89
- *
90
- * New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the {@link AutoscaledPool} class. All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions` parameter of the `FileCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency` {@link AutoscaledPool} options are available directly in the `FileCrawler` constructor.
20
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
91
21
  *
92
22
  * ## Example usage
93
23
  *
@@ -110,33 +40,32 @@ export class FileDownload extends BasicCrawler {
110
40
  constructor(options = {}) {
111
41
  super({
112
42
  ...options,
113
- contextPipelineBuilder: () => ContextPipeline.create().compose({
114
- action: async (context) => this.initiateDownload(context),
115
- cleanup: async (context) => {
116
- if (!context.response.bodyUsed) {
117
- // Nobody consumed the body — cancel it so the
118
- // underlying connection can be released.
119
- await context.response.body?.cancel();
120
- }
121
- await context[kBodyDrained];
122
- },
123
- }),
43
+ contextPipelineBuilder: () => this.#buildContextPipeline(),
124
44
  });
125
45
  }
126
- async initiateDownload(context) {
46
+ #buildContextPipeline() {
47
+ return ContextPipeline.create().compose(this.#initiateDownload.bind(this));
48
+ }
49
+ async #initiateDownload(context, onCleanup) {
127
50
  const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
128
51
  session: context.session,
129
52
  });
130
53
  const { type, charset: encoding } = parseContentTypeFromResponse(response);
131
54
  context.request.url = response.url;
132
55
  const { response: trackedResponse, bodyDrained } = trackBodyConsumption(response);
133
- const contextExtension = {
56
+ onCleanup(async () => {
57
+ if (!trackedResponse.bodyUsed) {
58
+ // Nobody consumed the body — cancel it so the
59
+ // underlying connection can be released.
60
+ await trackedResponse.body?.cancel();
61
+ }
62
+ await bodyDrained;
63
+ });
64
+ return {
134
65
  request: context.request,
135
66
  response: trackedResponse,
136
67
  contentType: { type, encoding },
137
- [kBodyDrained]: bodyDrained,
138
68
  };
139
- return contextExtension;
140
69
  }
141
70
  }
142
71
  /**
@@ -159,31 +88,6 @@ function trackBodyConsumption(response) {
159
88
  });
160
89
  return { response: trackedResponse, bodyDrained };
161
90
  }
162
- /**
163
- * Creates new {@link Router} instance that works based on request labels.
164
- * This instance can then serve as a `requestHandler` of your {@link FileDownload}.
165
- * Defaults to the {@link FileDownloadCrawlingContext}.
166
- *
167
- * > Serves as a shortcut for using `Router.create<FileDownloadCrawlingContext>()`.
168
- *
169
- * ```ts
170
- * import { FileDownload, createFileRouter } from 'crawlee';
171
- *
172
- * const router = createFileRouter();
173
- * router.addHandler('label-a', async (ctx) => {
174
- * ctx.log.info('...');
175
- * });
176
- * router.addDefaultHandler(async (ctx) => {
177
- * ctx.log.info('...');
178
- * });
179
- *
180
- * const crawler = new FileDownload({
181
- * requestHandler: router,
182
- * });
183
- * await crawler.run();
184
- * ```
185
- */
186
- export function createFileRouter(routes) {
187
- return Router.create(routes);
91
+ export function createFileRouter(routesOrSchemas) {
92
+ return Router.create(routesOrSchemas);
188
93
  }
189
- //# sourceMappingURL=file-download.js.map