@crawlee/http 4.0.0-beta.28 → 4.0.0-beta.280
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +1 -1
- package/index.js +1 -1
- package/internals/dom-crawler.d.ts +145 -0
- package/internals/dom-crawler.js +102 -0
- package/internals/file-download.d.ts +134 -54
- package/internals/file-download.js +39 -120
- package/internals/http-crawler.d.ts +220 -232
- package/internals/http-crawler.js +280 -229
- package/internals/utils.d.ts +4 -4
- package/internals/utils.js +18 -47
- package/package.json +17 -18
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/file-download.d.ts.map +0 -1
- package/internals/file-download.js.map +0 -1
- package/internals/http-crawler.d.ts.map +0 -1
- package/internals/http-crawler.js.map +0 -1
- package/internals/utils.d.ts.map +0 -1
- package/internals/utils.js.map +0 -1
|
@@ -1,67 +1,9 @@
|
|
|
1
|
-
import { Transform } from 'node:stream';
|
|
2
1
|
import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
|
|
3
2
|
import { ResponseWithUrl } from '@crawlee/http-client';
|
|
3
|
+
import { parseArgument } from '@crawlee/utils/internal';
|
|
4
|
+
import { z } from 'zod';
|
|
4
5
|
import { Router } from '../index.js';
|
|
5
6
|
import { parseContentTypeFromResponse } from './utils.js';
|
|
6
|
-
const kBodyDrained = Symbol('bodyDrained');
|
|
7
|
-
/**
|
|
8
|
-
* Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
|
|
9
|
-
* This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
|
|
10
|
-
* If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
|
|
11
|
-
* it will throw an error.
|
|
12
|
-
*
|
|
13
|
-
* Can be used e.g. to abort a download if the network speed is too slow.
|
|
14
|
-
* @returns Transform stream that monitors the speed of the incoming data.
|
|
15
|
-
*/
|
|
16
|
-
export function MinimumSpeedStream({ minSpeedKbps, historyLengthMs = 10e3, checkProgressInterval: checkProgressIntervalMs = 5e3, }) {
|
|
17
|
-
let snapshots = [];
|
|
18
|
-
const checkInterval = setInterval(() => {
|
|
19
|
-
const now = Date.now();
|
|
20
|
-
snapshots = snapshots.filter((snapshot) => now - snapshot.timestamp < historyLengthMs);
|
|
21
|
-
const totalBytes = snapshots.reduce((acc, snapshot) => acc + snapshot.bytes, 0);
|
|
22
|
-
const elapsed = (now - (snapshots[0]?.timestamp ?? 0)) / 1000;
|
|
23
|
-
if (totalBytes / 1024 / elapsed < minSpeedKbps) {
|
|
24
|
-
clearInterval(checkInterval);
|
|
25
|
-
stream.emit('error', new Error(`Stream speed too slow, aborting...`));
|
|
26
|
-
}
|
|
27
|
-
}, checkProgressIntervalMs);
|
|
28
|
-
const stream = new Transform({
|
|
29
|
-
transform: (chunk, _, callback) => {
|
|
30
|
-
snapshots.push({ timestamp: Date.now(), bytes: chunk.length });
|
|
31
|
-
callback(null, chunk);
|
|
32
|
-
},
|
|
33
|
-
final: (callback) => {
|
|
34
|
-
clearInterval(checkInterval);
|
|
35
|
-
callback();
|
|
36
|
-
},
|
|
37
|
-
});
|
|
38
|
-
return stream;
|
|
39
|
-
}
|
|
40
|
-
/**
|
|
41
|
-
* Creates a transform stream that logs the progress of the incoming data.
|
|
42
|
-
* This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
|
|
43
|
-
*
|
|
44
|
-
* Can be used e.g. to log the progress of a download.
|
|
45
|
-
* @returns Transform stream logging the progress of the incoming data.
|
|
46
|
-
*/
|
|
47
|
-
export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000, }) {
|
|
48
|
-
let transferredBytes = 0;
|
|
49
|
-
let lastLogTime = Date.now();
|
|
50
|
-
return new Transform({
|
|
51
|
-
transform: (chunk, _, callback) => {
|
|
52
|
-
transferredBytes += chunk.length;
|
|
53
|
-
if (Date.now() - lastLogTime > loggingInterval) {
|
|
54
|
-
lastLogTime = Date.now();
|
|
55
|
-
logTransferredBytes(transferredBytes);
|
|
56
|
-
}
|
|
57
|
-
callback(null, chunk);
|
|
58
|
-
},
|
|
59
|
-
flush: (callback) => {
|
|
60
|
-
logTransferredBytes(transferredBytes);
|
|
61
|
-
callback();
|
|
62
|
-
},
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
7
|
/**
|
|
66
8
|
* Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
|
|
67
9
|
*
|
|
@@ -69,32 +11,22 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
|
|
|
69
11
|
* However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
|
|
70
12
|
* you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
|
|
71
13
|
*
|
|
72
|
-
* `
|
|
14
|
+
* `FileDownload` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
|
|
73
15
|
*
|
|
74
|
-
* The source URLs are represented using {@link Request} objects that are fed from {@link
|
|
16
|
+
* The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
|
|
75
17
|
*
|
|
76
|
-
*
|
|
18
|
+
* > The {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
77
19
|
*
|
|
78
20
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
79
21
|
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
* ```
|
|
83
|
-
* preNavigationHooks: [
|
|
84
|
-
* (crawlingContext) => {
|
|
85
|
-
* // ...
|
|
86
|
-
* },
|
|
87
|
-
* ]
|
|
88
|
-
* ```
|
|
89
|
-
*
|
|
90
|
-
* New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the {@link AutoscaledPool} class. All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions` parameter of the `FileCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency` {@link AutoscaledPool} options are available directly in the `FileCrawler` constructor.
|
|
22
|
+
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileDownload` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
|
|
91
23
|
*
|
|
92
24
|
* ## Example usage
|
|
93
25
|
*
|
|
94
26
|
* ```ts
|
|
95
|
-
* const crawler = new
|
|
96
|
-
* requestHandler({
|
|
97
|
-
*
|
|
27
|
+
* const crawler = new FileDownload({
|
|
28
|
+
* async requestHandler({ response, request }) {
|
|
29
|
+
* await writeFile(request.url.replace(/[^a-z0-9\.]/gi, '_'), Buffer.from(await response.arrayBuffer()));
|
|
98
30
|
* },
|
|
99
31
|
* });
|
|
100
32
|
*
|
|
@@ -106,37 +38,49 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
|
|
|
106
38
|
* ```
|
|
107
39
|
*/
|
|
108
40
|
export class FileDownload extends BasicCrawler {
|
|
41
|
+
#ignoreTlsErrors;
|
|
42
|
+
/**
|
|
43
|
+
* @internal
|
|
44
|
+
*/
|
|
45
|
+
static optionsShape = {
|
|
46
|
+
...BasicCrawler.optionsShape,
|
|
47
|
+
ignoreTlsErrors: z.boolean().default(false),
|
|
48
|
+
};
|
|
49
|
+
/** @internal */
|
|
50
|
+
static optionsSchema = z.strictObject(FileDownload.optionsShape);
|
|
109
51
|
// TODO hooks
|
|
110
52
|
constructor(options = {}) {
|
|
53
|
+
const { ignoreTlsErrors, ...basicCrawlerOptions } = parseArgument(options, FileDownload.optionsSchema, 'FileDownloadOptions');
|
|
111
54
|
super({
|
|
112
|
-
...
|
|
113
|
-
contextPipelineBuilder: () =>
|
|
114
|
-
action: async (context) => this.initiateDownload(context),
|
|
115
|
-
cleanup: async (context) => {
|
|
116
|
-
if (!context.response.bodyUsed) {
|
|
117
|
-
// Nobody consumed the body — cancel it so the
|
|
118
|
-
// underlying connection can be released.
|
|
119
|
-
await context.response.body?.cancel();
|
|
120
|
-
}
|
|
121
|
-
await context[kBodyDrained];
|
|
122
|
-
},
|
|
123
|
-
}),
|
|
55
|
+
...basicCrawlerOptions,
|
|
56
|
+
contextPipelineBuilder: () => this.#buildContextPipeline(),
|
|
124
57
|
});
|
|
58
|
+
this.#ignoreTlsErrors = ignoreTlsErrors;
|
|
59
|
+
}
|
|
60
|
+
#buildContextPipeline() {
|
|
61
|
+
return ContextPipeline.create().compose(this.#initiateDownload.bind(this));
|
|
125
62
|
}
|
|
126
|
-
async initiateDownload(context) {
|
|
63
|
+
async #initiateDownload(context, onCleanup) {
|
|
127
64
|
const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
|
|
128
65
|
session: context.session,
|
|
66
|
+
ignoreTlsErrors: this.#ignoreTlsErrors,
|
|
129
67
|
});
|
|
130
68
|
const { type, charset: encoding } = parseContentTypeFromResponse(response);
|
|
131
69
|
context.request.url = response.url;
|
|
132
70
|
const { response: trackedResponse, bodyDrained } = trackBodyConsumption(response);
|
|
133
|
-
|
|
71
|
+
onCleanup(async () => {
|
|
72
|
+
if (!trackedResponse.bodyUsed) {
|
|
73
|
+
// Nobody consumed the body — cancel it so the
|
|
74
|
+
// underlying connection can be released.
|
|
75
|
+
await trackedResponse.body?.cancel();
|
|
76
|
+
}
|
|
77
|
+
await bodyDrained;
|
|
78
|
+
});
|
|
79
|
+
return {
|
|
134
80
|
request: context.request,
|
|
135
81
|
response: trackedResponse,
|
|
136
82
|
contentType: { type, encoding },
|
|
137
|
-
[kBodyDrained]: bodyDrained,
|
|
138
83
|
};
|
|
139
|
-
return contextExtension;
|
|
140
84
|
}
|
|
141
85
|
}
|
|
142
86
|
/**
|
|
@@ -159,31 +103,6 @@ function trackBodyConsumption(response) {
|
|
|
159
103
|
});
|
|
160
104
|
return { response: trackedResponse, bodyDrained };
|
|
161
105
|
}
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
* This instance can then serve as a `requestHandler` of your {@link FileDownload}.
|
|
165
|
-
* Defaults to the {@link FileDownloadCrawlingContext}.
|
|
166
|
-
*
|
|
167
|
-
* > Serves as a shortcut for using `Router.create<FileDownloadCrawlingContext>()`.
|
|
168
|
-
*
|
|
169
|
-
* ```ts
|
|
170
|
-
* import { FileDownload, createFileRouter } from 'crawlee';
|
|
171
|
-
*
|
|
172
|
-
* const router = createFileRouter();
|
|
173
|
-
* router.addHandler('label-a', async (ctx) => {
|
|
174
|
-
* ctx.log.info('...');
|
|
175
|
-
* });
|
|
176
|
-
* router.addDefaultHandler(async (ctx) => {
|
|
177
|
-
* ctx.log.info('...');
|
|
178
|
-
* });
|
|
179
|
-
*
|
|
180
|
-
* const crawler = new FileDownload({
|
|
181
|
-
* requestHandler: router,
|
|
182
|
-
* });
|
|
183
|
-
* await crawler.run();
|
|
184
|
-
* ```
|
|
185
|
-
*/
|
|
186
|
-
export function createFileRouter(routes) {
|
|
187
|
-
return Router.create(routes);
|
|
106
|
+
export function createFileRouter(routesOrSchemas) {
|
|
107
|
+
return Router.create(routesOrSchemas);
|
|
188
108
|
}
|
|
189
|
-
//# sourceMappingURL=file-download.js.map
|