@crawlee/http 4.0.0-beta.28 → 4.0.0-beta.280

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,67 +1,9 @@
1
- import { Transform } from 'node:stream';
2
1
  import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
3
2
  import { ResponseWithUrl } from '@crawlee/http-client';
3
+ import { parseArgument } from '@crawlee/utils/internal';
4
+ import { z } from 'zod';
4
5
  import { Router } from '../index.js';
5
6
  import { parseContentTypeFromResponse } from './utils.js';
6
- const kBodyDrained = Symbol('bodyDrained');
7
- /**
8
- * Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
9
- * This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
10
- * If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
11
- * it will throw an error.
12
- *
13
- * Can be used e.g. to abort a download if the network speed is too slow.
14
- * @returns Transform stream that monitors the speed of the incoming data.
15
- */
16
- export function MinimumSpeedStream({ minSpeedKbps, historyLengthMs = 10e3, checkProgressInterval: checkProgressIntervalMs = 5e3, }) {
17
- let snapshots = [];
18
- const checkInterval = setInterval(() => {
19
- const now = Date.now();
20
- snapshots = snapshots.filter((snapshot) => now - snapshot.timestamp < historyLengthMs);
21
- const totalBytes = snapshots.reduce((acc, snapshot) => acc + snapshot.bytes, 0);
22
- const elapsed = (now - (snapshots[0]?.timestamp ?? 0)) / 1000;
23
- if (totalBytes / 1024 / elapsed < minSpeedKbps) {
24
- clearInterval(checkInterval);
25
- stream.emit('error', new Error(`Stream speed too slow, aborting...`));
26
- }
27
- }, checkProgressIntervalMs);
28
- const stream = new Transform({
29
- transform: (chunk, _, callback) => {
30
- snapshots.push({ timestamp: Date.now(), bytes: chunk.length });
31
- callback(null, chunk);
32
- },
33
- final: (callback) => {
34
- clearInterval(checkInterval);
35
- callback();
36
- },
37
- });
38
- return stream;
39
- }
40
- /**
41
- * Creates a transform stream that logs the progress of the incoming data.
42
- * This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
43
- *
44
- * Can be used e.g. to log the progress of a download.
45
- * @returns Transform stream logging the progress of the incoming data.
46
- */
47
- export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000, }) {
48
- let transferredBytes = 0;
49
- let lastLogTime = Date.now();
50
- return new Transform({
51
- transform: (chunk, _, callback) => {
52
- transferredBytes += chunk.length;
53
- if (Date.now() - lastLogTime > loggingInterval) {
54
- lastLogTime = Date.now();
55
- logTransferredBytes(transferredBytes);
56
- }
57
- callback(null, chunk);
58
- },
59
- flush: (callback) => {
60
- logTransferredBytes(transferredBytes);
61
- callback();
62
- },
63
- });
64
- }
65
7
  /**
66
8
  * Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
67
9
  *
@@ -69,32 +11,22 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
69
11
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
70
12
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
71
13
  *
72
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link FileDownloadOptions.requestHandler} where the user can specify what to do with the downloaded data.
14
+ * `FileDownload` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
73
15
  *
74
- * The source URLs are represented using {@link Request} objects that are fed from {@link RequestList} or {@link RequestQueue} instances provided by the {@link FileDownloadOptions.requestList} or {@link FileDownloadOptions.requestQueue} constructor options, respectively.
16
+ * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
75
17
  *
76
- * If both {@link FileDownloadOptions.requestList} and {@link FileDownloadOptions.requestQueue} are used, the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
18
+ * > The {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
77
19
  *
78
20
  * The crawler finishes when there are no more {@link Request} objects to crawl.
79
21
  *
80
- * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
81
- *
82
- * ```
83
- * preNavigationHooks: [
84
- * (crawlingContext) => {
85
- * // ...
86
- * },
87
- * ]
88
- * ```
89
- *
90
- * New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the {@link AutoscaledPool} class. All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions` parameter of the `FileCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency` {@link AutoscaledPool} options are available directly in the `FileCrawler` constructor.
22
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileDownload` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
91
23
  *
92
24
  * ## Example usage
93
25
  *
94
26
  * ```ts
95
- * const crawler = new FileDownloader({
96
- * requestHandler({ body, request }) {
97
- * writeFileSync(request.url.replace(/[^a-z0-9\.]/gi, '_'), body);
27
+ * const crawler = new FileDownload({
28
+ * async requestHandler({ response, request }) {
29
+ * await writeFile(request.url.replace(/[^a-z0-9\.]/gi, '_'), Buffer.from(await response.arrayBuffer()));
98
30
  * },
99
31
  * });
100
32
  *
@@ -106,37 +38,49 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
106
38
  * ```
107
39
  */
108
40
  export class FileDownload extends BasicCrawler {
41
+ #ignoreTlsErrors;
42
+ /**
43
+ * @internal
44
+ */
45
+ static optionsShape = {
46
+ ...BasicCrawler.optionsShape,
47
+ ignoreTlsErrors: z.boolean().default(false),
48
+ };
49
+ /** @internal */
50
+ static optionsSchema = z.strictObject(FileDownload.optionsShape);
109
51
  // TODO hooks
110
52
  constructor(options = {}) {
53
+ const { ignoreTlsErrors, ...basicCrawlerOptions } = parseArgument(options, FileDownload.optionsSchema, 'FileDownloadOptions');
111
54
  super({
112
- ...options,
113
- contextPipelineBuilder: () => ContextPipeline.create().compose({
114
- action: async (context) => this.initiateDownload(context),
115
- cleanup: async (context) => {
116
- if (!context.response.bodyUsed) {
117
- // Nobody consumed the body — cancel it so the
118
- // underlying connection can be released.
119
- await context.response.body?.cancel();
120
- }
121
- await context[kBodyDrained];
122
- },
123
- }),
55
+ ...basicCrawlerOptions,
56
+ contextPipelineBuilder: () => this.#buildContextPipeline(),
124
57
  });
58
+ this.#ignoreTlsErrors = ignoreTlsErrors;
59
+ }
60
+ #buildContextPipeline() {
61
+ return ContextPipeline.create().compose(this.#initiateDownload.bind(this));
125
62
  }
126
- async initiateDownload(context) {
63
+ async #initiateDownload(context, onCleanup) {
127
64
  const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
128
65
  session: context.session,
66
+ ignoreTlsErrors: this.#ignoreTlsErrors,
129
67
  });
130
68
  const { type, charset: encoding } = parseContentTypeFromResponse(response);
131
69
  context.request.url = response.url;
132
70
  const { response: trackedResponse, bodyDrained } = trackBodyConsumption(response);
133
- const contextExtension = {
71
+ onCleanup(async () => {
72
+ if (!trackedResponse.bodyUsed) {
73
+ // Nobody consumed the body — cancel it so the
74
+ // underlying connection can be released.
75
+ await trackedResponse.body?.cancel();
76
+ }
77
+ await bodyDrained;
78
+ });
79
+ return {
134
80
  request: context.request,
135
81
  response: trackedResponse,
136
82
  contentType: { type, encoding },
137
- [kBodyDrained]: bodyDrained,
138
83
  };
139
- return contextExtension;
140
84
  }
141
85
  }
142
86
  /**
@@ -159,31 +103,6 @@ function trackBodyConsumption(response) {
159
103
  });
160
104
  return { response: trackedResponse, bodyDrained };
161
105
  }
162
- /**
163
- * Creates new {@link Router} instance that works based on request labels.
164
- * This instance can then serve as a `requestHandler` of your {@link FileDownload}.
165
- * Defaults to the {@link FileDownloadCrawlingContext}.
166
- *
167
- * > Serves as a shortcut for using `Router.create<FileDownloadCrawlingContext>()`.
168
- *
169
- * ```ts
170
- * import { FileDownload, createFileRouter } from 'crawlee';
171
- *
172
- * const router = createFileRouter();
173
- * router.addHandler('label-a', async (ctx) => {
174
- * ctx.log.info('...');
175
- * });
176
- * router.addDefaultHandler(async (ctx) => {
177
- * ctx.log.info('...');
178
- * });
179
- *
180
- * const crawler = new FileDownload({
181
- * requestHandler: router,
182
- * });
183
- * await crawler.run();
184
- * ```
185
- */
186
- export function createFileRouter(routes) {
187
- return Router.create(routes);
106
+ export function createFileRouter(routesOrSchemas) {
107
+ return Router.create(routesOrSchemas);
188
108
  }
189
- //# sourceMappingURL=file-download.js.map