@crawlee/basic 4.0.0-rc.1 → 4.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +6 -6
- package/index.js +6 -6
- package/internals/autoscaling/{autoscaled_pool.d.ts → autoscaled-pool.d.ts} +6 -6
- package/internals/autoscaling/{autoscaled_pool.js → autoscaled-pool.js} +3 -1
- package/internals/autoscaling/{concurrency_system.js → concurrency-system.js} +1 -1
- package/internals/autoscaling/{cpu_load_signal.d.ts → cpu-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{cpu_load_signal.js → cpu-load-signal.js} +1 -1
- package/internals/autoscaling/{event_loop_load_signal.d.ts → event-loop-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{event_loop_load_signal.js → event-loop-load-signal.js} +1 -1
- package/internals/autoscaling/index.d.ts +8 -8
- package/internals/autoscaling/index.js +8 -8
- package/internals/autoscaling/{load_signal.js → load-signal.js} +1 -1
- package/internals/autoscaling/{memory_load_signal.d.ts → memory-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{memory_load_signal.js → memory-load-signal.js} +1 -1
- package/internals/autoscaling/snapshotter.d.ts +5 -5
- package/internals/autoscaling/snapshotter.js +4 -4
- package/internals/autoscaling/{storage_backend_load_signal.d.ts → storage-backend-load-signal.d.ts} +1 -1
- package/internals/autoscaling/{storage_backend_load_signal.js → storage-backend-load-signal.js} +1 -1
- package/internals/autoscaling/{system_status.d.ts → system-status.d.ts} +1 -1
- package/internals/autoscaling/{system_status.js → system-status.js} +1 -1
- package/internals/basic-crawler.d.ts +16 -11
- package/internals/basic-crawler.js +61 -66
- package/internals/{cookie_utils.js → cookie-utils.js} +1 -1
- package/internals/crawler-run.d.ts +2 -2
- package/internals/crawler-run.js +3 -1
- package/internals/crawlers/{crawler_commons.d.ts → crawler-commons.d.ts} +2 -2
- package/internals/crawlers/{error_snapshotter.d.ts → error-snapshotter.d.ts} +2 -2
- package/internals/crawlers/{error_tracker.d.ts → error-tracker.d.ts} +1 -1
- package/internals/crawlers/{error_tracker.js → error-tracker.js} +1 -1
- package/internals/crawlers/index.d.ts +3 -3
- package/internals/crawlers/index.js +2 -2
- package/internals/crawlers/statistics.d.ts +11 -1
- package/internals/crawlers/statistics.js +39 -5
- package/internals/{crawling_request.js → crawling-request.js} +1 -1
- package/internals/{enqueue_links/enqueue_links.d.ts → enqueue-links/enqueue-links.d.ts} +1 -6
- package/internals/{enqueue_links/enqueue_links.js → enqueue-links/enqueue-links.js} +0 -41
- package/internals/{enqueue_links → enqueue-links}/index.d.ts +1 -1
- package/internals/{enqueue_links → enqueue-links}/index.js +1 -1
- package/internals/{enqueue_links → enqueue-links}/shared.d.ts +5 -4
- package/internals/{enqueue_links → enqueue-links}/shared.js +15 -45
- package/internals/errors.d.ts +6 -1
- package/internals/errors.js +25 -0
- package/internals/router.d.ts +2 -2
- package/internals/send-request.js +17 -8
- package/internals/{session_pool → session-pool}/index.d.ts +1 -1
- package/internals/{session_pool → session-pool}/index.js +1 -1
- package/internals/{sitemap_request_loader.d.ts → sitemap-request-loader.d.ts} +8 -1
- package/internals/{sitemap_request_loader.js → sitemap-request-loader.js} +116 -56
- package/internals/{throttling_request_manager.d.ts → throttling-request-manager.d.ts} +7 -7
- package/internals/{throttling_request_manager.js → throttling-request-manager.js} +61 -44
- package/package.json +15 -15
- /package/internals/autoscaling/{concurrency_system.d.ts → concurrency-system.d.ts} +0 -0
- /package/internals/autoscaling/{load_signal.d.ts → load-signal.d.ts} +0 -0
- /package/internals/autoscaling/{weighted_avg.d.ts → weighted-avg.d.ts} +0 -0
- /package/internals/autoscaling/{weighted_avg.js → weighted-avg.js} +0 -0
- /package/internals/{cookie_utils.d.ts → cookie-utils.d.ts} +0 -0
- /package/internals/crawlers/{context_pipeline.d.ts → context-pipeline.d.ts} +0 -0
- /package/internals/crawlers/{context_pipeline.js → context-pipeline.js} +0 -0
- /package/internals/crawlers/{crawler_commons.js → crawler-commons.js} +0 -0
- /package/internals/crawlers/{error_snapshotter.js → error-snapshotter.js} +0 -0
- /package/internals/{crawling_request.d.ts → crawling-request.d.ts} +0 -0
- /package/internals/{session_pool → session-pool}/consts.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/consts.js +0 -0
- /package/internals/{session_pool → session-pool}/errors.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/errors.js +0 -0
- /package/internals/{session_pool → session-pool}/fingerprint.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/fingerprint.js +0 -0
- /package/internals/{session_pool/session_pool.d.ts → session-pool/session-pool.d.ts} +0 -0
- /package/internals/{session_pool/session_pool.js → session-pool/session-pool.js} +0 -0
- /package/internals/{session_pool → session-pool}/session.d.ts +0 -0
- /package/internals/{session_pool → session-pool}/session.js +0 -0
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import { Transform } from 'node:stream';
|
|
2
2
|
import { EnqueueStrategy } from '@crawlee/utils';
|
|
3
3
|
import { parseArgument, parseSitemap, schemas } from '@crawlee/utils/internal';
|
|
4
|
-
import { minimatch } from 'minimatch';
|
|
5
4
|
import { z } from 'zod';
|
|
6
5
|
import { EventType, KeyValueStore, purgeDefaultStorages, Request, RequestManagerTandem, RequestQueue, serviceLocator, } from '@crawlee/core';
|
|
7
|
-
import { constructUrlPatternObjects, urlPatternSchema } from './
|
|
6
|
+
import { constructUrlPatternObjects, urlPatternSchema } from './enqueue-links/index.js';
|
|
8
7
|
const sitemapRequestLoaderOptionsSchema = z.strictObject({
|
|
9
8
|
sitemapUrls: schemas.arrayOf(z.string(), 'strings'),
|
|
10
9
|
proxyUrl: z.string().optional(),
|
|
@@ -18,6 +17,7 @@ const sitemapRequestLoaderOptionsSchema = z.strictObject({
|
|
|
18
17
|
exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
|
|
19
18
|
persistenceOptions: z.looseObject({}).optional(),
|
|
20
19
|
httpClient: schemas.httpClient.optional(),
|
|
20
|
+
concurrency: z.number().int().gte(1).default(1),
|
|
21
21
|
});
|
|
22
22
|
/** @internal */
|
|
23
23
|
const STATE_PERSISTENCE_KEY = 'SITEMAP_REQUEST_LOADER_STATE';
|
|
@@ -42,19 +42,24 @@ export class SitemapRequestLoader {
|
|
|
42
42
|
* Object for keeping track of the sitemap parsing progress.
|
|
43
43
|
*/
|
|
44
44
|
#sitemapParsingProgress = {
|
|
45
|
+
inProgressSitemapUrls: new Set(),
|
|
45
46
|
/**
|
|
46
|
-
*
|
|
47
|
+
* URLs already emitted from each in-progress sitemap. Used for tracking partially loaded sitemaps across migrations.
|
|
47
48
|
*/
|
|
48
|
-
|
|
49
|
+
inProgressEntriesPerSitemap: new Map(),
|
|
49
50
|
/**
|
|
50
|
-
*
|
|
51
|
+
* Set of sitemap URLs that have not been parsed yet. If this and `inProgressSitemapUrls` are empty, the sitemap loading is finished.
|
|
51
52
|
*/
|
|
52
|
-
|
|
53
|
+
pendingSitemapUrls: new Set(),
|
|
53
54
|
/**
|
|
54
|
-
* Set of sitemap URLs that have
|
|
55
|
+
* Set of sitemap URLs that have already been parsed. Keeps sitemap indexes that reference each other from being loaded again.
|
|
55
56
|
*/
|
|
56
|
-
|
|
57
|
+
processedSitemapUrls: new Set(),
|
|
57
58
|
};
|
|
59
|
+
/**
|
|
60
|
+
* Maximum number of sitemaps to load in parallel.
|
|
61
|
+
*/
|
|
62
|
+
#concurrency;
|
|
58
63
|
/**
|
|
59
64
|
* Object stream of URLs parsed from the sitemaps.
|
|
60
65
|
* Using `highWaterMark`, this can manage the speed of the sitemap loading.
|
|
@@ -95,7 +100,7 @@ export class SitemapRequestLoader {
|
|
|
95
100
|
#persistenceOptions;
|
|
96
101
|
/** @internal */
|
|
97
102
|
constructor(options) {
|
|
98
|
-
const { include, exclude, persistStateKey, persistenceOptions, proxyUrl, maxBufferSize, sitemapUrls, enqueueStrategy, } = parseArgument(options, sitemapRequestLoaderOptionsSchema, 'SitemapRequestLoaderOptions');
|
|
103
|
+
const { include, exclude, persistStateKey, persistenceOptions, proxyUrl, maxBufferSize, sitemapUrls, enqueueStrategy, concurrency, } = parseArgument(options, sitemapRequestLoaderOptionsSchema, 'SitemapRequestLoaderOptions');
|
|
99
104
|
this.#log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestLoader' });
|
|
100
105
|
if (exclude?.length) {
|
|
101
106
|
this.#urlExcludePatternObjects.push(...constructUrlPatternObjects(exclude));
|
|
@@ -107,6 +112,7 @@ export class SitemapRequestLoader {
|
|
|
107
112
|
this.#persistenceOptions = { enable: true, ...persistenceOptions };
|
|
108
113
|
this.#proxyUrl = proxyUrl;
|
|
109
114
|
this.#enqueueStrategy = enqueueStrategy;
|
|
115
|
+
this.#concurrency = concurrency;
|
|
110
116
|
this.#urlQueueStream = this.#createNewStream(maxBufferSize);
|
|
111
117
|
this.#sitemapParsingProgress.pendingSitemapUrls = new Set(sitemapUrls);
|
|
112
118
|
this.#events = serviceLocator.getEventManager();
|
|
@@ -123,27 +129,14 @@ export class SitemapRequestLoader {
|
|
|
123
129
|
highWaterMark,
|
|
124
130
|
}).pause();
|
|
125
131
|
}
|
|
126
|
-
/**
|
|
127
|
-
* Returns a function that checks whether the provided pattern matches the closure URL.
|
|
128
|
-
* @param url URL to be checked.
|
|
129
|
-
* @returns A matcher function that checks whether the pattern matches the closure URL.
|
|
130
|
-
*/
|
|
131
|
-
#matchesUrl(url) {
|
|
132
|
-
return (patternObject) => {
|
|
133
|
-
const { regexp, glob } = patternObject;
|
|
134
|
-
const matchesRegex = (regexp && url.match(regexp)) || false;
|
|
135
|
-
const matchesGlob = (glob && minimatch(url, glob, { nocase: true })) || false;
|
|
136
|
-
return Boolean(matchesRegex || matchesGlob);
|
|
137
|
-
};
|
|
138
|
-
}
|
|
139
132
|
/**
|
|
140
133
|
* Checks whether the URL matches the `include` / `exclude` patterns provided in the `options`.
|
|
141
134
|
* @param url URL to be checked.
|
|
142
135
|
* @returns `true` if the URL matches the patterns, `false` otherwise.
|
|
143
136
|
*/
|
|
144
137
|
#isUrlMatchingPatterns(url) {
|
|
145
|
-
return (!this.#urlExcludePatternObjects.some(
|
|
146
|
-
(this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(
|
|
138
|
+
return (!this.#urlExcludePatternObjects.some(({ match }) => match(url)) &&
|
|
139
|
+
(this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(({ match }) => match(url))));
|
|
147
140
|
}
|
|
148
141
|
/**
|
|
149
142
|
* Adds a URL to the queue of parsed URLs.
|
|
@@ -198,43 +191,89 @@ export class SitemapRequestLoader {
|
|
|
198
191
|
* If this is `false`, the background processing is either still in progress or was aborted.
|
|
199
192
|
*/
|
|
200
193
|
isSitemapFullyLoaded() {
|
|
201
|
-
return (this.#sitemapParsingProgress.
|
|
194
|
+
return (this.#sitemapParsingProgress.inProgressSitemapUrls.size === 0 &&
|
|
202
195
|
this.#sitemapParsingProgress.pendingSitemapUrls.size === 0);
|
|
203
196
|
}
|
|
197
|
+
#isUrlAlreadyEmitted(url) {
|
|
198
|
+
for (const entries of this.#sitemapParsingProgress.inProgressEntriesPerSitemap.values()) {
|
|
199
|
+
if (entries.has(url)) {
|
|
200
|
+
return true;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return false;
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Processes a single sitemap URL and emits its URLs to the queue.
|
|
207
|
+
* Nested sitemaps are added to the pending queue for later processing.
|
|
208
|
+
*/
|
|
209
|
+
async #loadSingleSitemap(sitemapUrl, parseSitemapOptions) {
|
|
210
|
+
const progress = this.#sitemapParsingProgress;
|
|
211
|
+
progress.inProgressSitemapUrls.add(sitemapUrl);
|
|
212
|
+
if (!progress.inProgressEntriesPerSitemap.has(sitemapUrl)) {
|
|
213
|
+
progress.inProgressEntriesPerSitemap.set(sitemapUrl, new Set());
|
|
214
|
+
}
|
|
215
|
+
const entries = progress.inProgressEntriesPerSitemap.get(sitemapUrl);
|
|
216
|
+
try {
|
|
217
|
+
for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.#proxyUrl, {
|
|
218
|
+
...parseSitemapOptions,
|
|
219
|
+
maxDepth: 0,
|
|
220
|
+
emitNestedSitemaps: true,
|
|
221
|
+
enqueueStrategy: this.#enqueueStrategy,
|
|
222
|
+
})) {
|
|
223
|
+
// Leave the sitemap in progress so `isSitemapFullyLoaded()` keeps reporting the abort.
|
|
224
|
+
if (this.#abortLoading)
|
|
225
|
+
return;
|
|
226
|
+
if (!item.originSitemapUrl) {
|
|
227
|
+
// This is a nested sitemap
|
|
228
|
+
if (!progress.processedSitemapUrls.has(item.loc) && !progress.inProgressSitemapUrls.has(item.loc)) {
|
|
229
|
+
progress.pendingSitemapUrls.add(item.loc);
|
|
230
|
+
}
|
|
231
|
+
continue;
|
|
232
|
+
}
|
|
233
|
+
if (!entries.has(item.loc) && !this.#isUrlAlreadyEmitted(item.loc)) {
|
|
234
|
+
await this.#pushNextUrl(item.loc);
|
|
235
|
+
entries.add(item.loc);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
catch (e) {
|
|
240
|
+
this.#log.error(`Error loading sitemap contents from ${sitemapUrl}:`, e);
|
|
241
|
+
}
|
|
242
|
+
progress.inProgressSitemapUrls.delete(sitemapUrl);
|
|
243
|
+
progress.pendingSitemapUrls.delete(sitemapUrl);
|
|
244
|
+
progress.inProgressEntriesPerSitemap.delete(sitemapUrl);
|
|
245
|
+
progress.processedSitemapUrls.add(sitemapUrl);
|
|
246
|
+
}
|
|
204
247
|
/**
|
|
205
248
|
* Start processing the sitemaps and loading the URLs.
|
|
206
249
|
*
|
|
207
250
|
* Resolves once all the sitemaps URLs have been fully loaded (sets `isSitemapFullyLoaded` to `true`).
|
|
208
251
|
*/
|
|
209
252
|
async #load({ parseSitemapOptions, }) {
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
continue;
|
|
224
|
-
}
|
|
225
|
-
if (!this.#sitemapParsingProgress.inProgressEntries.has(item.loc)) {
|
|
226
|
-
await this.#pushNextUrl(item.loc);
|
|
227
|
-
this.#sitemapParsingProgress.inProgressEntries.add(item.loc);
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
catch (e) {
|
|
232
|
-
this.#log.error('Error loading sitemap contents:', e);
|
|
253
|
+
const activeLoaders = [];
|
|
254
|
+
const startNewLoaders = () => {
|
|
255
|
+
while (!this.#abortLoading &&
|
|
256
|
+
activeLoaders.length < this.#concurrency &&
|
|
257
|
+
this.#sitemapParsingProgress.pendingSitemapUrls.size > 0) {
|
|
258
|
+
const sitemapUrl = this.#sitemapParsingProgress.pendingSitemapUrls.values().next().value;
|
|
259
|
+
this.#sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
|
|
260
|
+
const loaderPromise = this.#loadSingleSitemap(sitemapUrl, parseSitemapOptions).finally(() => {
|
|
261
|
+
const index = activeLoaders.indexOf(loaderPromise);
|
|
262
|
+
if (index !== -1)
|
|
263
|
+
void activeLoaders.splice(index, 1);
|
|
264
|
+
});
|
|
265
|
+
activeLoaders.push(loaderPromise);
|
|
233
266
|
}
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
267
|
+
};
|
|
268
|
+
startNewLoaders();
|
|
269
|
+
while ((activeLoaders.length > 0 || this.#sitemapParsingProgress.pendingSitemapUrls.size > 0) &&
|
|
270
|
+
!this.#abortLoading) {
|
|
271
|
+
if (activeLoaders.length > 0)
|
|
272
|
+
await Promise.race(activeLoaders);
|
|
273
|
+
startNewLoaders();
|
|
237
274
|
}
|
|
275
|
+
if (activeLoaders.length > 0)
|
|
276
|
+
await Promise.all(activeLoaders);
|
|
238
277
|
this.#urlQueueStream.end();
|
|
239
278
|
}
|
|
240
279
|
/**
|
|
@@ -342,11 +381,16 @@ export class SitemapRequestLoader {
|
|
|
342
381
|
// Re-emit `readdata` on the previous stream to release any such pending waiter (its URL has already
|
|
343
382
|
// been transferred to the new stream above).
|
|
344
383
|
previousStream.emit('readdata');
|
|
384
|
+
const inProgressEntriesPerSitemap = {};
|
|
385
|
+
for (const [sitemapUrl, entries] of this.#sitemapParsingProgress.inProgressEntriesPerSitemap) {
|
|
386
|
+
inProgressEntriesPerSitemap[sitemapUrl] = Array.from(entries);
|
|
387
|
+
}
|
|
345
388
|
await this.#store.setValue(this.#persistStateKey, {
|
|
346
389
|
sitemapParsingProgress: {
|
|
347
390
|
pendingSitemapUrls: Array.from(this.#sitemapParsingProgress.pendingSitemapUrls),
|
|
348
|
-
|
|
349
|
-
|
|
391
|
+
inProgressSitemapUrls: Array.from(this.#sitemapParsingProgress.inProgressSitemapUrls),
|
|
392
|
+
inProgressEntriesPerSitemap,
|
|
393
|
+
processedSitemapUrls: Array.from(this.#sitemapParsingProgress.processedSitemapUrls),
|
|
350
394
|
},
|
|
351
395
|
// Re-queue in-progress requests to the front so they are retried if the state is restored.
|
|
352
396
|
urlQueue: [...this.inProgress, ...urlQueue],
|
|
@@ -365,10 +409,26 @@ export class SitemapRequestLoader {
|
|
|
365
409
|
if (state === null) {
|
|
366
410
|
return;
|
|
367
411
|
}
|
|
412
|
+
const progress = state.sitemapParsingProgress;
|
|
413
|
+
const inProgressSitemapUrls = new Set(progress.inProgressSitemapUrls ?? []);
|
|
414
|
+
if (progress.inProgressSitemapUrl && !progress.inProgressSitemapUrls) {
|
|
415
|
+
inProgressSitemapUrls.add(progress.inProgressSitemapUrl);
|
|
416
|
+
}
|
|
417
|
+
const inProgressEntriesPerSitemap = new Map();
|
|
418
|
+
if (progress.inProgressEntriesPerSitemap) {
|
|
419
|
+
for (const [sitemapUrl, entries] of Object.entries(progress.inProgressEntriesPerSitemap)) {
|
|
420
|
+
inProgressEntriesPerSitemap.set(sitemapUrl, new Set(entries));
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
else if (progress.inProgressEntries && progress.inProgressSitemapUrl) {
|
|
424
|
+
inProgressEntriesPerSitemap.set(progress.inProgressSitemapUrl, new Set(progress.inProgressEntries));
|
|
425
|
+
}
|
|
426
|
+
// Interrupted sitemaps are re-parsed; `inProgressEntriesPerSitemap` skips the URLs they already emitted.
|
|
368
427
|
this.#sitemapParsingProgress = {
|
|
369
|
-
pendingSitemapUrls: new Set(
|
|
370
|
-
|
|
371
|
-
|
|
428
|
+
pendingSitemapUrls: new Set([...progress.pendingSitemapUrls, ...inProgressSitemapUrls]),
|
|
429
|
+
inProgressSitemapUrls: new Set(),
|
|
430
|
+
inProgressEntriesPerSitemap,
|
|
431
|
+
processedSitemapUrls: new Set(progress.processedSitemapUrls ?? []),
|
|
372
432
|
};
|
|
373
433
|
this.#requestData = new Map(state.requestData ?? []);
|
|
374
434
|
for (const url of state.urlQueue) {
|
|
@@ -153,7 +153,7 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
|
|
|
153
153
|
*/
|
|
154
154
|
export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager {
|
|
155
155
|
#private;
|
|
156
|
-
constructor(options: ThrottlingRequestManagerOptions<T>,
|
|
156
|
+
constructor(options: ThrottlingRequestManagerOptions<T>, configuration?: Configuration);
|
|
157
157
|
/**
|
|
158
158
|
* The wrapped manager, holding every request whose domain is not throttled. `undefined` until an `inner`
|
|
159
159
|
* passed as a factory is resolved - reading this never forces it, because a getter should not open a queue
|
|
@@ -188,13 +188,13 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
|
|
|
188
188
|
/**
|
|
189
189
|
* Reports whether anything can be dispatched right now, and if not, when — or why never.
|
|
190
190
|
*
|
|
191
|
-
*
|
|
192
|
-
*
|
|
193
|
-
* crawler gated on this idles for the backoff instead of concluding it is done.
|
|
191
|
+
* Dispatchable domains are probed first. When nothing can be dispatched, cooling domains contribute their
|
|
192
|
+
* deadlines only while they still have outstanding work, so a finished crawl does not wait out a cooldown.
|
|
194
193
|
*
|
|
195
194
|
* `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,
|
|
196
195
|
* so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -
|
|
197
|
-
* see the traversal.
|
|
196
|
+
* see the traversal. A domain that is only cooling down holds off a stalling one the same way, because the
|
|
197
|
+
* crawl can continue once its cooldown ends.
|
|
198
198
|
*/
|
|
199
199
|
checkReadiness(): Promise<RequestSourceStatus>;
|
|
200
200
|
/**
|
|
@@ -202,13 +202,13 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
|
|
|
202
202
|
* site rather than of the run, so it survives.
|
|
203
203
|
*/
|
|
204
204
|
purge(): Promise<void>;
|
|
205
|
-
|
|
205
|
+
setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void>;
|
|
206
206
|
/**
|
|
207
207
|
* @inheritdoc
|
|
208
208
|
* Extending a lock must not consume the in-flight marker that later routes
|
|
209
209
|
* {@link ThrottlingRequestManager.markRequestAsHandled} / {@link ThrottlingRequestManager.reclaimRequest}.
|
|
210
210
|
*/
|
|
211
|
-
|
|
211
|
+
extendRequestProcessingTime(request: Request, extensionSecs: number): Promise<boolean>;
|
|
212
212
|
/**
|
|
213
213
|
* Returns the next request from a domain that is not backing off, or from the inner manager.
|
|
214
214
|
*
|
|
@@ -22,6 +22,15 @@ const throttlingRequestManagerOptionsSchema = z.strictObject({
|
|
|
22
22
|
function throttledUntil(state) {
|
|
23
23
|
return Math.max(state.backoffUntil, state.crawlDelayUntil);
|
|
24
24
|
}
|
|
25
|
+
/**
|
|
26
|
+
* Still refusing, and has been for longer than the window. Not dispatched to, so this lapses a window after the
|
|
27
|
+
* last 429 and the domain gets one burst to prove itself with.
|
|
28
|
+
*/
|
|
29
|
+
function isStalling(state, now, maxStallMs) {
|
|
30
|
+
return (state.rateLimitedSince !== 0 &&
|
|
31
|
+
now - state.lastRateLimitedAt <= maxStallMs &&
|
|
32
|
+
now - state.rateLimitedSince > maxStallMs);
|
|
33
|
+
}
|
|
25
34
|
/** How long a domain must be left alone after a dispatch: what it asked for, or our floor, whichever is longer. */
|
|
26
35
|
function crawlDelayMs(state, minCrawlDelayMs) {
|
|
27
36
|
return Math.max(state.declaredCrawlDelayMs ?? 0, minCrawlDelayMs);
|
|
@@ -103,7 +112,7 @@ export class ThrottlingRequestManager {
|
|
|
103
112
|
#throttleBy;
|
|
104
113
|
#maxThrottledDomains;
|
|
105
114
|
#persistStateKey;
|
|
106
|
-
#
|
|
115
|
+
#configuration;
|
|
107
116
|
#subManagers = new Map();
|
|
108
117
|
#domainStates = new Map();
|
|
109
118
|
#log;
|
|
@@ -134,20 +143,20 @@ export class ThrottlingRequestManager {
|
|
|
134
143
|
#inProgressBatchCount = 0;
|
|
135
144
|
/** Requests moved into a sub-queue, which both managers count. Subtracted below; not persisted. */
|
|
136
145
|
#migratedFromInner = 0;
|
|
137
|
-
/** The latest {@link
|
|
146
|
+
/** The latest {@link setExpectedRequestProcessingTime} hint, kept for a wrapped manager resolved later. */
|
|
138
147
|
#expectedRequestProcessingSecs;
|
|
139
148
|
#warnedAbout = new Set();
|
|
140
149
|
/** Whether any domain at all may end up throttled - listed up front, or discovered as the crawl runs. */
|
|
141
150
|
get #throttlingEnabled() {
|
|
142
151
|
return this.#listedDomains.size > 0 || this.#throttlesEveryDomain;
|
|
143
152
|
}
|
|
144
|
-
constructor(options,
|
|
145
|
-
this.#
|
|
153
|
+
constructor(options, configuration = serviceLocator.getConfiguration()) {
|
|
154
|
+
this.#configuration = configuration;
|
|
146
155
|
parseArgument(options, throttlingRequestManagerOptionsSchema, 'ThrottlingRequestManagerOptions');
|
|
147
156
|
if (options.inner === undefined) {
|
|
148
157
|
// Deferred like any other factory, so a manager nobody fetches from opens no storage. The opener is
|
|
149
158
|
// assigned below and read only when this runs.
|
|
150
|
-
this.#innerFactory = () => this.#requestManagerOpener(null, { configuration: this.#
|
|
159
|
+
this.#innerFactory = () => this.#requestManagerOpener(null, { configuration: this.#configuration });
|
|
151
160
|
}
|
|
152
161
|
else if (typeof options.inner === 'function') {
|
|
153
162
|
this.#innerFactory = options.inner;
|
|
@@ -220,7 +229,7 @@ export class ThrottlingRequestManager {
|
|
|
220
229
|
this.#innerPromise ??= Promise.resolve(this.#innerFactory());
|
|
221
230
|
this.#resolvedInner = await this.#innerPromise;
|
|
222
231
|
if (this.#expectedRequestProcessingSecs !== undefined) {
|
|
223
|
-
await this.#resolvedInner.
|
|
232
|
+
await this.#resolvedInner.setExpectedRequestProcessingTime?.(this.#expectedRequestProcessingSecs);
|
|
224
233
|
}
|
|
225
234
|
}
|
|
226
235
|
return this.#resolvedInner;
|
|
@@ -301,7 +310,7 @@ export class ThrottlingRequestManager {
|
|
|
301
310
|
subManager = this.#requestManagerOpener(
|
|
302
311
|
// Backends use the alias as a directory name, and an IPv6 literal is full of characters
|
|
303
312
|
// Windows will not accept. Ordinary hostnames survive this untouched.
|
|
304
|
-
{ alias: `throttled-${encodeURIComponent(domain)}` }, { configuration: this.#
|
|
313
|
+
{ alias: `throttled-${encodeURIComponent(domain)}` }, { configuration: this.#configuration });
|
|
305
314
|
this.#subManagers.set(domain, subManager);
|
|
306
315
|
this.#ensureDomainState(domain);
|
|
307
316
|
}
|
|
@@ -333,7 +342,7 @@ export class ThrottlingRequestManager {
|
|
|
333
342
|
async #ensureSubManagers() {
|
|
334
343
|
this.#subManagersReady ??= (async () => {
|
|
335
344
|
if (this.#throttlesEveryDomain) {
|
|
336
|
-
this.#domainListStore = await KeyValueStore.open(null, { configuration: this.#
|
|
345
|
+
this.#domainListStore = await KeyValueStore.open(null, { configuration: this.#configuration });
|
|
337
346
|
for (const domain of (await this.#domainListStore.getValue(this.#persistStateKey)) ?? []) {
|
|
338
347
|
this.#discoveredDomains.add(domain);
|
|
339
348
|
}
|
|
@@ -355,7 +364,9 @@ export class ThrottlingRequestManager {
|
|
|
355
364
|
#fetchableDomains() {
|
|
356
365
|
const now = Date.now();
|
|
357
366
|
return Array.from(this.#domainStates.values())
|
|
358
|
-
.filter((state) => now >= throttledUntil(state) &&
|
|
367
|
+
.filter((state) => now >= throttledUntil(state) &&
|
|
368
|
+
!isStalling(state, now, this.#maxDomainStallMs) &&
|
|
369
|
+
this.#subManagers.has(state.domain))
|
|
359
370
|
.sort((a, b) => throttledUntil(a) - throttledUntil(b))
|
|
360
371
|
.map((state) => state.domain);
|
|
361
372
|
}
|
|
@@ -492,11 +503,15 @@ export class ThrottlingRequestManager {
|
|
|
492
503
|
}
|
|
493
504
|
return true;
|
|
494
505
|
}
|
|
495
|
-
/**
|
|
506
|
+
/**
|
|
507
|
+
* Records that a domain let a request through, which ends any rate-limit run stall detection was timing
|
|
508
|
+
* and the run of 429s the backoff was doubling for.
|
|
509
|
+
*/
|
|
496
510
|
#recordProgress(url) {
|
|
497
511
|
const state = this.#getDomainState(url);
|
|
498
512
|
if (state) {
|
|
499
513
|
state.rateLimitedSince = 0;
|
|
514
|
+
state.consecutive429Count = 0;
|
|
500
515
|
}
|
|
501
516
|
}
|
|
502
517
|
// --- IRequestManager Implementation ---
|
|
@@ -526,6 +541,7 @@ export class ThrottlingRequestManager {
|
|
|
526
541
|
waitBetweenBatchesMillis: options.waitBetweenBatchesMillis ?? 1000,
|
|
527
542
|
waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded ?? false,
|
|
528
543
|
maxNewRequests: options.maxNewRequests,
|
|
544
|
+
log: this.#log,
|
|
529
545
|
// Routing is the only thing this manager adds; the targets do the batching, validation and
|
|
530
546
|
// deduplication themselves.
|
|
531
547
|
processChunk: async (chunk) => {
|
|
@@ -602,43 +618,37 @@ export class ThrottlingRequestManager {
|
|
|
602
618
|
/**
|
|
603
619
|
* Reports whether anything can be dispatched right now, and if not, when — or why never.
|
|
604
620
|
*
|
|
605
|
-
*
|
|
606
|
-
*
|
|
607
|
-
* crawler gated on this idles for the backoff instead of concluding it is done.
|
|
621
|
+
* Dispatchable domains are probed first. When nothing can be dispatched, cooling domains contribute their
|
|
622
|
+
* deadlines only while they still have outstanding work, so a finished crawl does not wait out a cooldown.
|
|
608
623
|
*
|
|
609
624
|
* `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,
|
|
610
625
|
* so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -
|
|
611
|
-
* see the traversal.
|
|
626
|
+
* see the traversal. A domain that is only cooling down holds off a stalling one the same way, because the
|
|
627
|
+
* crawl can continue once its cooldown ends.
|
|
612
628
|
*/
|
|
613
629
|
async checkReadiness() {
|
|
614
630
|
await this.#ensureSubManagers();
|
|
615
631
|
const now = Date.now();
|
|
616
632
|
const dispatchable = [];
|
|
617
|
-
|
|
633
|
+
const cooling = [];
|
|
618
634
|
let stallCandidates;
|
|
619
635
|
for (const state of this.#domainStates.values()) {
|
|
620
636
|
// A `Crawl-delay` can give a domain a clock before its first request gives it a queue - nothing
|
|
621
637
|
// to fetch from and nothing to wait for until then.
|
|
622
|
-
|
|
638
|
+
const subManager = this.#subManagers.get(state.domain);
|
|
639
|
+
if (!subManager) {
|
|
623
640
|
continue;
|
|
624
641
|
}
|
|
625
|
-
if (
|
|
626
|
-
// Together: still turning us away, and has been without a break for longer than the window - a
|
|
627
|
-
// domain that has merely been idle starts this clock at its first 429, not with idle time on it.
|
|
628
|
-
state.rateLimitedSince !== 0 &&
|
|
629
|
-
now - state.lastRateLimitedAt <= this.#maxDomainStallMs &&
|
|
630
|
-
now - state.rateLimitedSince > this.#maxDomainStallMs) {
|
|
631
|
-
// Deliberately not dispatchable: a domain that has refused every request for the whole window
|
|
632
|
-
// is not progress just because its backoff momentarily lapsed - that is what it does between 429s.
|
|
642
|
+
if (isStalling(state, now, this.#maxDomainStallMs)) {
|
|
633
643
|
(stallCandidates ??= []).push(state);
|
|
634
644
|
continue;
|
|
635
645
|
}
|
|
636
|
-
const
|
|
637
|
-
if (now >=
|
|
638
|
-
dispatchable.push(
|
|
646
|
+
const readyAt = throttledUntil(state);
|
|
647
|
+
if (now >= readyAt) {
|
|
648
|
+
dispatchable.push(subManager);
|
|
639
649
|
}
|
|
640
|
-
else
|
|
641
|
-
readyAt
|
|
650
|
+
else {
|
|
651
|
+
cooling.push({ subManager, readyAt });
|
|
642
652
|
}
|
|
643
653
|
}
|
|
644
654
|
const inner = await this.#getInner();
|
|
@@ -650,6 +660,19 @@ export class ThrottlingRequestManager {
|
|
|
650
660
|
if (probed.status === 'ready') {
|
|
651
661
|
return probed;
|
|
652
662
|
}
|
|
663
|
+
// A cooling domain with nothing left to dispatch is nothing to wait for; in deadline order, the first
|
|
664
|
+
// with work left is the earliest wake-up.
|
|
665
|
+
cooling.sort((a, b) => a.readyAt - b.readyAt);
|
|
666
|
+
for (const { subManager, readyAt } of cooling) {
|
|
667
|
+
const { status } = await (await subManager).checkReadiness();
|
|
668
|
+
if (status !== 'finished') {
|
|
669
|
+
return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
// Batches still landing in the background are work nobody can see in a queue yet.
|
|
673
|
+
if (this.#inProgressBatchCount > 0 && probed.status === 'finished') {
|
|
674
|
+
return { status: 'waiting' };
|
|
675
|
+
}
|
|
653
676
|
if (stallCandidates !== undefined) {
|
|
654
677
|
const stalled = (await Promise.all(stallCandidates.map(async (state) => (await (await this.#subManagers.get(state.domain)).checkReadiness()).status === 'ready'
|
|
655
678
|
? state
|
|
@@ -658,22 +681,15 @@ export class ThrottlingRequestManager {
|
|
|
658
681
|
const summary = stalled
|
|
659
682
|
.map((state) => `"${state.domain}" (${((now - state.rateLimitedSince) / 1000).toFixed(0)}s)`)
|
|
660
683
|
.join(', ');
|
|
661
|
-
return {
|
|
684
|
+
return joinRequestSourceStatuses(probed, {
|
|
662
685
|
status: 'stalled',
|
|
663
686
|
reason: `${summary} rate-limited every request for longer than maxDomainStallSecs ` +
|
|
664
687
|
`(${(this.#maxDomainStallMs / 1000).toFixed(0)}s). Waiting longer will not help - lower the ` +
|
|
665
688
|
`crawler's concurrency, or drop these domains. Their requests are still queued, so ` +
|
|
666
689
|
`re-running with \`purgeOnStart\` disabled will resume them if the rate limit lifts.`,
|
|
667
|
-
};
|
|
690
|
+
});
|
|
668
691
|
}
|
|
669
692
|
}
|
|
670
|
-
if (readyAt !== undefined) {
|
|
671
|
-
return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });
|
|
672
|
-
}
|
|
673
|
-
// Batches still landing in the background are work nobody can see in a queue yet.
|
|
674
|
-
if (this.#inProgressBatchCount > 0 && probed.status === 'finished') {
|
|
675
|
-
return { status: 'waiting' };
|
|
676
|
-
}
|
|
677
693
|
return probed;
|
|
678
694
|
}
|
|
679
695
|
/**
|
|
@@ -701,22 +717,22 @@ export class ThrottlingRequestManager {
|
|
|
701
717
|
state.lastRateLimitedAt = 0;
|
|
702
718
|
}
|
|
703
719
|
}
|
|
704
|
-
async
|
|
720
|
+
async setExpectedRequestProcessingTime(processingTimeSecs) {
|
|
705
721
|
// Remembered so a manager opened later still gets the hint, without opening one now just to pass it along.
|
|
706
|
-
this.#expectedRequestProcessingSecs =
|
|
707
|
-
await this.#forEachManager((manager) => manager.
|
|
722
|
+
this.#expectedRequestProcessingSecs = processingTimeSecs;
|
|
723
|
+
await this.#forEachManager((manager) => manager.setExpectedRequestProcessingTime?.(processingTimeSecs));
|
|
708
724
|
}
|
|
709
725
|
/**
|
|
710
726
|
* @inheritdoc
|
|
711
727
|
* Extending a lock must not consume the in-flight marker that later routes
|
|
712
728
|
* {@link ThrottlingRequestManager.markRequestAsHandled} / {@link ThrottlingRequestManager.reclaimRequest}.
|
|
713
729
|
*/
|
|
714
|
-
async
|
|
730
|
+
async extendRequestProcessingTime(request, extensionSecs) {
|
|
715
731
|
const key = request.id ?? request.uniqueKey;
|
|
716
732
|
const manager = this.#inFlightFromInner.has(key)
|
|
717
733
|
? this.#resolvedInner
|
|
718
734
|
: await this.#selectManagerOrThrow(request.url);
|
|
719
|
-
return (await manager.
|
|
735
|
+
return (await manager.extendRequestProcessingTime?.(request, extensionSecs)) ?? false;
|
|
720
736
|
}
|
|
721
737
|
/**
|
|
722
738
|
* Runs `fn` over the sub-queues and, if it has been resolved, the wrapped manager - bookkeeping never forces
|
|
@@ -795,7 +811,8 @@ export class ThrottlingRequestManager {
|
|
|
795
811
|
return null;
|
|
796
812
|
}
|
|
797
813
|
const state = this.#getDomainState(request.url);
|
|
798
|
-
|
|
814
|
+
const now = Date.now();
|
|
815
|
+
if (state === null || (now >= throttledUntil(state) && !isStalling(state, now, this.#maxDomainStallMs))) {
|
|
799
816
|
return request;
|
|
800
817
|
}
|
|
801
818
|
await this.#migrateToSubQueue(request, inner);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-rc.
|
|
3
|
+
"version": "4.0.0-rc.2",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -41,23 +41,23 @@
|
|
|
41
41
|
"dependencies": {
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
|
-
"@apify/utilities": "^3.0
|
|
45
|
-
"@crawlee/core": "4.0.0-rc.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-rc.
|
|
47
|
-
"@crawlee/types": "4.0.0-rc.
|
|
48
|
-
"@crawlee/utils": "4.0.0-rc.
|
|
44
|
+
"@apify/utilities": "^3.1.0",
|
|
45
|
+
"@crawlee/core": "4.0.0-rc.2",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-rc.2",
|
|
47
|
+
"@crawlee/types": "4.0.0-rc.2",
|
|
48
|
+
"@crawlee/utils": "4.0.0-rc.2",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
|
-
"@standard-schema/spec": "^1.
|
|
51
|
-
"csv-stringify": "^6.
|
|
52
|
-
"minimatch": "^
|
|
53
|
-
"tldts": "^7.
|
|
54
|
-
"tough-cookie": "^6.
|
|
55
|
-
"tslib": "^2.
|
|
50
|
+
"@standard-schema/spec": "^1.1.0",
|
|
51
|
+
"csv-stringify": "^6.9.0",
|
|
52
|
+
"minimatch": "^11.0.5",
|
|
53
|
+
"tldts": "^7.4.19",
|
|
54
|
+
"tough-cookie": "^6.1.0",
|
|
55
|
+
"tslib": "^2.9.1",
|
|
56
56
|
"type-fest": "^4.41.0",
|
|
57
|
-
"zod": "^4.
|
|
57
|
+
"zod": "^4.6.7"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-rc.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-rc.2"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "a9b623229cd7d1bff5063fc82940efebaea47165"
|
|
70
70
|
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|