@crawlee/basic 4.0.0-rc.1 → 4.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/index.d.ts +6 -6
  2. package/index.js +6 -6
  3. package/internals/autoscaling/{autoscaled_pool.d.ts → autoscaled-pool.d.ts} +6 -6
  4. package/internals/autoscaling/{autoscaled_pool.js → autoscaled-pool.js} +3 -1
  5. package/internals/autoscaling/{concurrency_system.js → concurrency-system.js} +1 -1
  6. package/internals/autoscaling/{cpu_load_signal.d.ts → cpu-load-signal.d.ts} +1 -1
  7. package/internals/autoscaling/{cpu_load_signal.js → cpu-load-signal.js} +1 -1
  8. package/internals/autoscaling/{event_loop_load_signal.d.ts → event-loop-load-signal.d.ts} +1 -1
  9. package/internals/autoscaling/{event_loop_load_signal.js → event-loop-load-signal.js} +1 -1
  10. package/internals/autoscaling/index.d.ts +8 -8
  11. package/internals/autoscaling/index.js +8 -8
  12. package/internals/autoscaling/{load_signal.js → load-signal.js} +1 -1
  13. package/internals/autoscaling/{memory_load_signal.d.ts → memory-load-signal.d.ts} +1 -1
  14. package/internals/autoscaling/{memory_load_signal.js → memory-load-signal.js} +1 -1
  15. package/internals/autoscaling/snapshotter.d.ts +5 -5
  16. package/internals/autoscaling/snapshotter.js +4 -4
  17. package/internals/autoscaling/{storage_backend_load_signal.d.ts → storage-backend-load-signal.d.ts} +1 -1
  18. package/internals/autoscaling/{storage_backend_load_signal.js → storage-backend-load-signal.js} +1 -1
  19. package/internals/autoscaling/{system_status.d.ts → system-status.d.ts} +1 -1
  20. package/internals/autoscaling/{system_status.js → system-status.js} +1 -1
  21. package/internals/basic-crawler.d.ts +16 -11
  22. package/internals/basic-crawler.js +61 -66
  23. package/internals/{cookie_utils.js → cookie-utils.js} +1 -1
  24. package/internals/crawler-run.d.ts +2 -2
  25. package/internals/crawler-run.js +3 -1
  26. package/internals/crawlers/{crawler_commons.d.ts → crawler-commons.d.ts} +2 -2
  27. package/internals/crawlers/{error_snapshotter.d.ts → error-snapshotter.d.ts} +2 -2
  28. package/internals/crawlers/{error_tracker.d.ts → error-tracker.d.ts} +1 -1
  29. package/internals/crawlers/{error_tracker.js → error-tracker.js} +1 -1
  30. package/internals/crawlers/index.d.ts +3 -3
  31. package/internals/crawlers/index.js +2 -2
  32. package/internals/crawlers/statistics.d.ts +11 -1
  33. package/internals/crawlers/statistics.js +39 -5
  34. package/internals/{crawling_request.js → crawling-request.js} +1 -1
  35. package/internals/{enqueue_links/enqueue_links.d.ts → enqueue-links/enqueue-links.d.ts} +1 -6
  36. package/internals/{enqueue_links/enqueue_links.js → enqueue-links/enqueue-links.js} +0 -41
  37. package/internals/{enqueue_links → enqueue-links}/index.d.ts +1 -1
  38. package/internals/{enqueue_links → enqueue-links}/index.js +1 -1
  39. package/internals/{enqueue_links → enqueue-links}/shared.d.ts +5 -4
  40. package/internals/{enqueue_links → enqueue-links}/shared.js +15 -45
  41. package/internals/errors.d.ts +6 -1
  42. package/internals/errors.js +25 -0
  43. package/internals/router.d.ts +2 -2
  44. package/internals/send-request.js +17 -8
  45. package/internals/{session_pool → session-pool}/index.d.ts +1 -1
  46. package/internals/{session_pool → session-pool}/index.js +1 -1
  47. package/internals/{sitemap_request_loader.d.ts → sitemap-request-loader.d.ts} +8 -1
  48. package/internals/{sitemap_request_loader.js → sitemap-request-loader.js} +116 -56
  49. package/internals/{throttling_request_manager.d.ts → throttling-request-manager.d.ts} +7 -7
  50. package/internals/{throttling_request_manager.js → throttling-request-manager.js} +61 -44
  51. package/package.json +15 -15
  52. /package/internals/autoscaling/{concurrency_system.d.ts → concurrency-system.d.ts} +0 -0
  53. /package/internals/autoscaling/{load_signal.d.ts → load-signal.d.ts} +0 -0
  54. /package/internals/autoscaling/{weighted_avg.d.ts → weighted-avg.d.ts} +0 -0
  55. /package/internals/autoscaling/{weighted_avg.js → weighted-avg.js} +0 -0
  56. /package/internals/{cookie_utils.d.ts → cookie-utils.d.ts} +0 -0
  57. /package/internals/crawlers/{context_pipeline.d.ts → context-pipeline.d.ts} +0 -0
  58. /package/internals/crawlers/{context_pipeline.js → context-pipeline.js} +0 -0
  59. /package/internals/crawlers/{crawler_commons.js → crawler-commons.js} +0 -0
  60. /package/internals/crawlers/{error_snapshotter.js → error-snapshotter.js} +0 -0
  61. /package/internals/{crawling_request.d.ts → crawling-request.d.ts} +0 -0
  62. /package/internals/{session_pool → session-pool}/consts.d.ts +0 -0
  63. /package/internals/{session_pool → session-pool}/consts.js +0 -0
  64. /package/internals/{session_pool → session-pool}/errors.d.ts +0 -0
  65. /package/internals/{session_pool → session-pool}/errors.js +0 -0
  66. /package/internals/{session_pool → session-pool}/fingerprint.d.ts +0 -0
  67. /package/internals/{session_pool → session-pool}/fingerprint.js +0 -0
  68. /package/internals/{session_pool/session_pool.d.ts → session-pool/session-pool.d.ts} +0 -0
  69. /package/internals/{session_pool/session_pool.js → session-pool/session-pool.js} +0 -0
  70. /package/internals/{session_pool → session-pool}/session.d.ts +0 -0
  71. /package/internals/{session_pool → session-pool}/session.js +0 -0
@@ -1,10 +1,9 @@
1
1
  import { Transform } from 'node:stream';
2
2
  import { EnqueueStrategy } from '@crawlee/utils';
3
3
  import { parseArgument, parseSitemap, schemas } from '@crawlee/utils/internal';
4
- import { minimatch } from 'minimatch';
5
4
  import { z } from 'zod';
6
5
  import { EventType, KeyValueStore, purgeDefaultStorages, Request, RequestManagerTandem, RequestQueue, serviceLocator, } from '@crawlee/core';
7
- import { constructUrlPatternObjects, urlPatternSchema } from './enqueue_links/index.js';
6
+ import { constructUrlPatternObjects, urlPatternSchema } from './enqueue-links/index.js';
8
7
  const sitemapRequestLoaderOptionsSchema = z.strictObject({
9
8
  sitemapUrls: schemas.arrayOf(z.string(), 'strings'),
10
9
  proxyUrl: z.string().optional(),
@@ -18,6 +17,7 @@ const sitemapRequestLoaderOptionsSchema = z.strictObject({
18
17
  exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
19
18
  persistenceOptions: z.looseObject({}).optional(),
20
19
  httpClient: schemas.httpClient.optional(),
20
+ concurrency: z.number().int().gte(1).default(1),
21
21
  });
22
22
  /** @internal */
23
23
  const STATE_PERSISTENCE_KEY = 'SITEMAP_REQUEST_LOADER_STATE';
@@ -42,19 +42,24 @@ export class SitemapRequestLoader {
42
42
  * Object for keeping track of the sitemap parsing progress.
43
43
  */
44
44
  #sitemapParsingProgress = {
45
+ inProgressSitemapUrls: new Set(),
45
46
  /**
46
- * URL of the sitemap that is currently being parsed. `null` if no sitemap is being parsed.
47
+ * URLs already emitted from each in-progress sitemap. Used for tracking partially loaded sitemaps across migrations.
47
48
  */
48
- inProgressSitemapUrl: null,
49
+ inProgressEntriesPerSitemap: new Map(),
49
50
  /**
50
- * Buffer for URLs from the currently parsed sitemap. Used for tracking partially loaded sitemaps across migrations.
51
+ * Set of sitemap URLs that have not been parsed yet. If this and `inProgressSitemapUrls` are empty, the sitemap loading is finished.
51
52
  */
52
- inProgressEntries: new Set(),
53
+ pendingSitemapUrls: new Set(),
53
54
  /**
54
- * Set of sitemap URLs that have not been parsed yet. If the set is empty and `inProgressSitemapUrl` is `null`, the sitemap loading is finished.
55
+ * Set of sitemap URLs that have already been parsed. Keeps sitemap indexes that reference each other from being loaded again.
55
56
  */
56
- pendingSitemapUrls: new Set(),
57
+ processedSitemapUrls: new Set(),
57
58
  };
59
+ /**
60
+ * Maximum number of sitemaps to load in parallel.
61
+ */
62
+ #concurrency;
58
63
  /**
59
64
  * Object stream of URLs parsed from the sitemaps.
60
65
  * Using `highWaterMark`, this can manage the speed of the sitemap loading.
@@ -95,7 +100,7 @@ export class SitemapRequestLoader {
95
100
  #persistenceOptions;
96
101
  /** @internal */
97
102
  constructor(options) {
98
- const { include, exclude, persistStateKey, persistenceOptions, proxyUrl, maxBufferSize, sitemapUrls, enqueueStrategy, } = parseArgument(options, sitemapRequestLoaderOptionsSchema, 'SitemapRequestLoaderOptions');
103
+ const { include, exclude, persistStateKey, persistenceOptions, proxyUrl, maxBufferSize, sitemapUrls, enqueueStrategy, concurrency, } = parseArgument(options, sitemapRequestLoaderOptionsSchema, 'SitemapRequestLoaderOptions');
99
104
  this.#log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestLoader' });
100
105
  if (exclude?.length) {
101
106
  this.#urlExcludePatternObjects.push(...constructUrlPatternObjects(exclude));
@@ -107,6 +112,7 @@ export class SitemapRequestLoader {
107
112
  this.#persistenceOptions = { enable: true, ...persistenceOptions };
108
113
  this.#proxyUrl = proxyUrl;
109
114
  this.#enqueueStrategy = enqueueStrategy;
115
+ this.#concurrency = concurrency;
110
116
  this.#urlQueueStream = this.#createNewStream(maxBufferSize);
111
117
  this.#sitemapParsingProgress.pendingSitemapUrls = new Set(sitemapUrls);
112
118
  this.#events = serviceLocator.getEventManager();
@@ -123,27 +129,14 @@ export class SitemapRequestLoader {
123
129
  highWaterMark,
124
130
  }).pause();
125
131
  }
126
- /**
127
- * Returns a function that checks whether the provided pattern matches the closure URL.
128
- * @param url URL to be checked.
129
- * @returns A matcher function that checks whether the pattern matches the closure URL.
130
- */
131
- #matchesUrl(url) {
132
- return (patternObject) => {
133
- const { regexp, glob } = patternObject;
134
- const matchesRegex = (regexp && url.match(regexp)) || false;
135
- const matchesGlob = (glob && minimatch(url, glob, { nocase: true })) || false;
136
- return Boolean(matchesRegex || matchesGlob);
137
- };
138
- }
139
132
  /**
140
133
  * Checks whether the URL matches the `include` / `exclude` patterns provided in the `options`.
141
134
  * @param url URL to be checked.
142
135
  * @returns `true` if the URL matches the patterns, `false` otherwise.
143
136
  */
144
137
  #isUrlMatchingPatterns(url) {
145
- return (!this.#urlExcludePatternObjects.some(this.#matchesUrl(url)) &&
146
- (this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(this.#matchesUrl(url))));
138
+ return (!this.#urlExcludePatternObjects.some(({ match }) => match(url)) &&
139
+ (this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(({ match }) => match(url))));
147
140
  }
148
141
  /**
149
142
  * Adds a URL to the queue of parsed URLs.
@@ -198,43 +191,89 @@ export class SitemapRequestLoader {
198
191
  * If this is `false`, the background processing is either still in progress or was aborted.
199
192
  */
200
193
  isSitemapFullyLoaded() {
201
- return (this.#sitemapParsingProgress.inProgressSitemapUrl === null &&
194
+ return (this.#sitemapParsingProgress.inProgressSitemapUrls.size === 0 &&
202
195
  this.#sitemapParsingProgress.pendingSitemapUrls.size === 0);
203
196
  }
197
+ #isUrlAlreadyEmitted(url) {
198
+ for (const entries of this.#sitemapParsingProgress.inProgressEntriesPerSitemap.values()) {
199
+ if (entries.has(url)) {
200
+ return true;
201
+ }
202
+ }
203
+ return false;
204
+ }
205
+ /**
206
+ * Processes a single sitemap URL and emits its URLs to the queue.
207
+ * Nested sitemaps are added to the pending queue for later processing.
208
+ */
209
+ async #loadSingleSitemap(sitemapUrl, parseSitemapOptions) {
210
+ const progress = this.#sitemapParsingProgress;
211
+ progress.inProgressSitemapUrls.add(sitemapUrl);
212
+ if (!progress.inProgressEntriesPerSitemap.has(sitemapUrl)) {
213
+ progress.inProgressEntriesPerSitemap.set(sitemapUrl, new Set());
214
+ }
215
+ const entries = progress.inProgressEntriesPerSitemap.get(sitemapUrl);
216
+ try {
217
+ for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.#proxyUrl, {
218
+ ...parseSitemapOptions,
219
+ maxDepth: 0,
220
+ emitNestedSitemaps: true,
221
+ enqueueStrategy: this.#enqueueStrategy,
222
+ })) {
223
+ // Leave the sitemap in progress so `isSitemapFullyLoaded()` keeps reporting the abort.
224
+ if (this.#abortLoading)
225
+ return;
226
+ if (!item.originSitemapUrl) {
227
+ // This is a nested sitemap
228
+ if (!progress.processedSitemapUrls.has(item.loc) && !progress.inProgressSitemapUrls.has(item.loc)) {
229
+ progress.pendingSitemapUrls.add(item.loc);
230
+ }
231
+ continue;
232
+ }
233
+ if (!entries.has(item.loc) && !this.#isUrlAlreadyEmitted(item.loc)) {
234
+ await this.#pushNextUrl(item.loc);
235
+ entries.add(item.loc);
236
+ }
237
+ }
238
+ }
239
+ catch (e) {
240
+ this.#log.error(`Error loading sitemap contents from ${sitemapUrl}:`, e);
241
+ }
242
+ progress.inProgressSitemapUrls.delete(sitemapUrl);
243
+ progress.pendingSitemapUrls.delete(sitemapUrl);
244
+ progress.inProgressEntriesPerSitemap.delete(sitemapUrl);
245
+ progress.processedSitemapUrls.add(sitemapUrl);
246
+ }
204
247
  /**
205
248
  * Start processing the sitemaps and loading the URLs.
206
249
  *
207
250
  * Resolves once all the sitemaps URLs have been fully loaded (sets `isSitemapFullyLoaded` to `true`).
208
251
  */
209
252
  async #load({ parseSitemapOptions, }) {
210
- while (!this.isSitemapFullyLoaded() && !this.#abortLoading) {
211
- const sitemapUrl = this.#sitemapParsingProgress.inProgressSitemapUrl ??
212
- this.#sitemapParsingProgress.pendingSitemapUrls.values().next().value;
213
- try {
214
- for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.#proxyUrl, {
215
- ...parseSitemapOptions,
216
- maxDepth: 0,
217
- emitNestedSitemaps: true,
218
- enqueueStrategy: this.#enqueueStrategy,
219
- })) {
220
- if (!item.originSitemapUrl) {
221
- // This is a nested sitemap
222
- this.#sitemapParsingProgress.pendingSitemapUrls.add(item.loc);
223
- continue;
224
- }
225
- if (!this.#sitemapParsingProgress.inProgressEntries.has(item.loc)) {
226
- await this.#pushNextUrl(item.loc);
227
- this.#sitemapParsingProgress.inProgressEntries.add(item.loc);
228
- }
229
- }
230
- }
231
- catch (e) {
232
- this.#log.error('Error loading sitemap contents:', e);
253
+ const activeLoaders = [];
254
+ const startNewLoaders = () => {
255
+ while (!this.#abortLoading &&
256
+ activeLoaders.length < this.#concurrency &&
257
+ this.#sitemapParsingProgress.pendingSitemapUrls.size > 0) {
258
+ const sitemapUrl = this.#sitemapParsingProgress.pendingSitemapUrls.values().next().value;
259
+ this.#sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
260
+ const loaderPromise = this.#loadSingleSitemap(sitemapUrl, parseSitemapOptions).finally(() => {
261
+ const index = activeLoaders.indexOf(loaderPromise);
262
+ if (index !== -1)
263
+ void activeLoaders.splice(index, 1);
264
+ });
265
+ activeLoaders.push(loaderPromise);
233
266
  }
234
- this.#sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
235
- this.#sitemapParsingProgress.inProgressEntries.clear();
236
- this.#sitemapParsingProgress.inProgressSitemapUrl = null;
267
+ };
268
+ startNewLoaders();
269
+ while ((activeLoaders.length > 0 || this.#sitemapParsingProgress.pendingSitemapUrls.size > 0) &&
270
+ !this.#abortLoading) {
271
+ if (activeLoaders.length > 0)
272
+ await Promise.race(activeLoaders);
273
+ startNewLoaders();
237
274
  }
275
+ if (activeLoaders.length > 0)
276
+ await Promise.all(activeLoaders);
238
277
  this.#urlQueueStream.end();
239
278
  }
240
279
  /**
@@ -342,11 +381,16 @@ export class SitemapRequestLoader {
342
381
  // Re-emit `readdata` on the previous stream to release any such pending waiter (its URL has already
343
382
  // been transferred to the new stream above).
344
383
  previousStream.emit('readdata');
384
+ const inProgressEntriesPerSitemap = {};
385
+ for (const [sitemapUrl, entries] of this.#sitemapParsingProgress.inProgressEntriesPerSitemap) {
386
+ inProgressEntriesPerSitemap[sitemapUrl] = Array.from(entries);
387
+ }
345
388
  await this.#store.setValue(this.#persistStateKey, {
346
389
  sitemapParsingProgress: {
347
390
  pendingSitemapUrls: Array.from(this.#sitemapParsingProgress.pendingSitemapUrls),
348
- inProgressSitemapUrl: this.#sitemapParsingProgress.inProgressSitemapUrl,
349
- inProgressEntries: Array.from(this.#sitemapParsingProgress.inProgressEntries),
391
+ inProgressSitemapUrls: Array.from(this.#sitemapParsingProgress.inProgressSitemapUrls),
392
+ inProgressEntriesPerSitemap,
393
+ processedSitemapUrls: Array.from(this.#sitemapParsingProgress.processedSitemapUrls),
350
394
  },
351
395
  // Re-queue in-progress requests to the front so they are retried if the state is restored.
352
396
  urlQueue: [...this.inProgress, ...urlQueue],
@@ -365,10 +409,26 @@ export class SitemapRequestLoader {
365
409
  if (state === null) {
366
410
  return;
367
411
  }
412
+ const progress = state.sitemapParsingProgress;
413
+ const inProgressSitemapUrls = new Set(progress.inProgressSitemapUrls ?? []);
414
+ if (progress.inProgressSitemapUrl && !progress.inProgressSitemapUrls) {
415
+ inProgressSitemapUrls.add(progress.inProgressSitemapUrl);
416
+ }
417
+ const inProgressEntriesPerSitemap = new Map();
418
+ if (progress.inProgressEntriesPerSitemap) {
419
+ for (const [sitemapUrl, entries] of Object.entries(progress.inProgressEntriesPerSitemap)) {
420
+ inProgressEntriesPerSitemap.set(sitemapUrl, new Set(entries));
421
+ }
422
+ }
423
+ else if (progress.inProgressEntries && progress.inProgressSitemapUrl) {
424
+ inProgressEntriesPerSitemap.set(progress.inProgressSitemapUrl, new Set(progress.inProgressEntries));
425
+ }
426
+ // Interrupted sitemaps are re-parsed; `inProgressEntriesPerSitemap` skips the URLs they already emitted.
368
427
  this.#sitemapParsingProgress = {
369
- pendingSitemapUrls: new Set(state.sitemapParsingProgress.pendingSitemapUrls),
370
- inProgressSitemapUrl: state.sitemapParsingProgress.inProgressSitemapUrl,
371
- inProgressEntries: new Set(state.sitemapParsingProgress.inProgressEntries),
428
+ pendingSitemapUrls: new Set([...progress.pendingSitemapUrls, ...inProgressSitemapUrls]),
429
+ inProgressSitemapUrls: new Set(),
430
+ inProgressEntriesPerSitemap,
431
+ processedSitemapUrls: new Set(progress.processedSitemapUrls ?? []),
372
432
  };
373
433
  this.#requestData = new Map(state.requestData ?? []);
374
434
  for (const url of state.urlQueue) {
@@ -153,7 +153,7 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
153
153
  */
154
154
  export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager {
155
155
  #private;
156
- constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);
156
+ constructor(options: ThrottlingRequestManagerOptions<T>, configuration?: Configuration);
157
157
  /**
158
158
  * The wrapped manager, holding every request whose domain is not throttled. `undefined` until an `inner`
159
159
  * passed as a factory is resolved - reading this never forces it, because a getter should not open a queue
@@ -188,13 +188,13 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
188
188
  /**
189
189
  * Reports whether anything can be dispatched right now, and if not, when — or why never.
190
190
  *
191
- * One traversal of the domain clocks answers all of it: only domains whose delays have run out are probed,
192
- * the rest merely contribute the moment they come due. Throttled requests count as outstanding work, so a
193
- * crawler gated on this idles for the backoff instead of concluding it is done.
191
+ * Dispatchable domains are probed first. When nothing can be dispatched, cooling domains contribute their
192
+ * deadlines only while they still have outstanding work, so a finished crawl does not wait out a cooldown.
194
193
  *
195
194
  * `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,
196
195
  * so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -
197
- * see the traversal.
196
+ * see the traversal. A domain that is only cooling down holds off a stalling one the same way, because the
197
+ * crawl can continue once its cooldown ends.
198
198
  */
199
199
  checkReadiness(): Promise<RequestSourceStatus>;
200
200
  /**
@@ -202,13 +202,13 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
202
202
  * site rather than of the run, so it survives.
203
203
  */
204
204
  purge(): Promise<void>;
205
- setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
205
+ setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void>;
206
206
  /**
207
207
  * @inheritdoc
208
208
  * Extending a lock must not consume the in-flight marker that later routes
209
209
  * {@link ThrottlingRequestManager.markRequestAsHandled} / {@link ThrottlingRequestManager.reclaimRequest}.
210
210
  */
211
- extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean>;
211
+ extendRequestProcessingTime(request: Request, extensionSecs: number): Promise<boolean>;
212
212
  /**
213
213
  * Returns the next request from a domain that is not backing off, or from the inner manager.
214
214
  *
@@ -22,6 +22,15 @@ const throttlingRequestManagerOptionsSchema = z.strictObject({
22
22
  function throttledUntil(state) {
23
23
  return Math.max(state.backoffUntil, state.crawlDelayUntil);
24
24
  }
25
+ /**
26
+ * Still refusing, and has been for longer than the window. Not dispatched to, so this lapses a window after the
27
+ * last 429 and the domain gets one burst to prove itself with.
28
+ */
29
+ function isStalling(state, now, maxStallMs) {
30
+ return (state.rateLimitedSince !== 0 &&
31
+ now - state.lastRateLimitedAt <= maxStallMs &&
32
+ now - state.rateLimitedSince > maxStallMs);
33
+ }
25
34
  /** How long a domain must be left alone after a dispatch: what it asked for, or our floor, whichever is longer. */
26
35
  function crawlDelayMs(state, minCrawlDelayMs) {
27
36
  return Math.max(state.declaredCrawlDelayMs ?? 0, minCrawlDelayMs);
@@ -103,7 +112,7 @@ export class ThrottlingRequestManager {
103
112
  #throttleBy;
104
113
  #maxThrottledDomains;
105
114
  #persistStateKey;
106
- #config;
115
+ #configuration;
107
116
  #subManagers = new Map();
108
117
  #domainStates = new Map();
109
118
  #log;
@@ -134,20 +143,20 @@ export class ThrottlingRequestManager {
134
143
  #inProgressBatchCount = 0;
135
144
  /** Requests moved into a sub-queue, which both managers count. Subtracted below; not persisted. */
136
145
  #migratedFromInner = 0;
137
- /** The latest {@link setExpectedRequestProcessingTimeSecs} hint, kept for a wrapped manager resolved later. */
146
+ /** The latest {@link setExpectedRequestProcessingTime} hint, kept for a wrapped manager resolved later. */
138
147
  #expectedRequestProcessingSecs;
139
148
  #warnedAbout = new Set();
140
149
  /** Whether any domain at all may end up throttled - listed up front, or discovered as the crawl runs. */
141
150
  get #throttlingEnabled() {
142
151
  return this.#listedDomains.size > 0 || this.#throttlesEveryDomain;
143
152
  }
144
- constructor(options, config = serviceLocator.getConfiguration()) {
145
- this.#config = config;
153
+ constructor(options, configuration = serviceLocator.getConfiguration()) {
154
+ this.#configuration = configuration;
146
155
  parseArgument(options, throttlingRequestManagerOptionsSchema, 'ThrottlingRequestManagerOptions');
147
156
  if (options.inner === undefined) {
148
157
  // Deferred like any other factory, so a manager nobody fetches from opens no storage. The opener is
149
158
  // assigned below and read only when this runs.
150
- this.#innerFactory = () => this.#requestManagerOpener(null, { configuration: this.#config });
159
+ this.#innerFactory = () => this.#requestManagerOpener(null, { configuration: this.#configuration });
151
160
  }
152
161
  else if (typeof options.inner === 'function') {
153
162
  this.#innerFactory = options.inner;
@@ -220,7 +229,7 @@ export class ThrottlingRequestManager {
220
229
  this.#innerPromise ??= Promise.resolve(this.#innerFactory());
221
230
  this.#resolvedInner = await this.#innerPromise;
222
231
  if (this.#expectedRequestProcessingSecs !== undefined) {
223
- await this.#resolvedInner.setExpectedRequestProcessingTimeSecs?.(this.#expectedRequestProcessingSecs);
232
+ await this.#resolvedInner.setExpectedRequestProcessingTime?.(this.#expectedRequestProcessingSecs);
224
233
  }
225
234
  }
226
235
  return this.#resolvedInner;
@@ -301,7 +310,7 @@ export class ThrottlingRequestManager {
301
310
  subManager = this.#requestManagerOpener(
302
311
  // Backends use the alias as a directory name, and an IPv6 literal is full of characters
303
312
  // Windows will not accept. Ordinary hostnames survive this untouched.
304
- { alias: `throttled-${encodeURIComponent(domain)}` }, { configuration: this.#config });
313
+ { alias: `throttled-${encodeURIComponent(domain)}` }, { configuration: this.#configuration });
305
314
  this.#subManagers.set(domain, subManager);
306
315
  this.#ensureDomainState(domain);
307
316
  }
@@ -333,7 +342,7 @@ export class ThrottlingRequestManager {
333
342
  async #ensureSubManagers() {
334
343
  this.#subManagersReady ??= (async () => {
335
344
  if (this.#throttlesEveryDomain) {
336
- this.#domainListStore = await KeyValueStore.open(null, { configuration: this.#config });
345
+ this.#domainListStore = await KeyValueStore.open(null, { configuration: this.#configuration });
337
346
  for (const domain of (await this.#domainListStore.getValue(this.#persistStateKey)) ?? []) {
338
347
  this.#discoveredDomains.add(domain);
339
348
  }
@@ -355,7 +364,9 @@ export class ThrottlingRequestManager {
355
364
  #fetchableDomains() {
356
365
  const now = Date.now();
357
366
  return Array.from(this.#domainStates.values())
358
- .filter((state) => now >= throttledUntil(state) && this.#subManagers.has(state.domain))
367
+ .filter((state) => now >= throttledUntil(state) &&
368
+ !isStalling(state, now, this.#maxDomainStallMs) &&
369
+ this.#subManagers.has(state.domain))
359
370
  .sort((a, b) => throttledUntil(a) - throttledUntil(b))
360
371
  .map((state) => state.domain);
361
372
  }
@@ -492,11 +503,15 @@ export class ThrottlingRequestManager {
492
503
  }
493
504
  return true;
494
505
  }
495
- /** Records that a domain let a request through, which ends any rate-limit run stall detection was timing. */
506
+ /**
507
+ * Records that a domain let a request through, which ends any rate-limit run stall detection was timing
508
+ * and the run of 429s the backoff was doubling for.
509
+ */
496
510
  #recordProgress(url) {
497
511
  const state = this.#getDomainState(url);
498
512
  if (state) {
499
513
  state.rateLimitedSince = 0;
514
+ state.consecutive429Count = 0;
500
515
  }
501
516
  }
502
517
  // --- IRequestManager Implementation ---
@@ -526,6 +541,7 @@ export class ThrottlingRequestManager {
526
541
  waitBetweenBatchesMillis: options.waitBetweenBatchesMillis ?? 1000,
527
542
  waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded ?? false,
528
543
  maxNewRequests: options.maxNewRequests,
544
+ log: this.#log,
529
545
  // Routing is the only thing this manager adds; the targets do the batching, validation and
530
546
  // deduplication themselves.
531
547
  processChunk: async (chunk) => {
@@ -602,43 +618,37 @@ export class ThrottlingRequestManager {
602
618
  /**
603
619
  * Reports whether anything can be dispatched right now, and if not, when — or why never.
604
620
  *
605
- * One traversal of the domain clocks answers all of it: only domains whose delays have run out are probed,
606
- * the rest merely contribute the moment they come due. Throttled requests count as outstanding work, so a
607
- * crawler gated on this idles for the backoff instead of concluding it is done.
621
+ * Dispatchable domains are probed first. When nothing can be dispatched, cooling domains contribute their
622
+ * deadlines only while they still have outstanding work, so a finished crawl does not wait out a cooldown.
608
623
  *
609
624
  * `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,
610
625
  * so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -
611
- * see the traversal.
626
+ * see the traversal. A domain that is only cooling down holds off a stalling one the same way, because the
627
+ * crawl can continue once its cooldown ends.
612
628
  */
613
629
  async checkReadiness() {
614
630
  await this.#ensureSubManagers();
615
631
  const now = Date.now();
616
632
  const dispatchable = [];
617
- let readyAt;
633
+ const cooling = [];
618
634
  let stallCandidates;
619
635
  for (const state of this.#domainStates.values()) {
620
636
  // A `Crawl-delay` can give a domain a clock before its first request gives it a queue - nothing
621
637
  // to fetch from and nothing to wait for until then.
622
- if (!this.#subManagers.has(state.domain)) {
638
+ const subManager = this.#subManagers.get(state.domain);
639
+ if (!subManager) {
623
640
  continue;
624
641
  }
625
- if (
626
- // Together: still turning us away, and has been without a break for longer than the window - a
627
- // domain that has merely been idle starts this clock at its first 429, not with idle time on it.
628
- state.rateLimitedSince !== 0 &&
629
- now - state.lastRateLimitedAt <= this.#maxDomainStallMs &&
630
- now - state.rateLimitedSince > this.#maxDomainStallMs) {
631
- // Deliberately not dispatchable: a domain that has refused every request for the whole window
632
- // is not progress just because its backoff momentarily lapsed - that is what it does between 429s.
642
+ if (isStalling(state, now, this.#maxDomainStallMs)) {
633
643
  (stallCandidates ??= []).push(state);
634
644
  continue;
635
645
  }
636
- const until = throttledUntil(state);
637
- if (now >= until) {
638
- dispatchable.push(this.#subManagers.get(state.domain));
646
+ const readyAt = throttledUntil(state);
647
+ if (now >= readyAt) {
648
+ dispatchable.push(subManager);
639
649
  }
640
- else if (readyAt === undefined || until < readyAt) {
641
- readyAt = until;
650
+ else {
651
+ cooling.push({ subManager, readyAt });
642
652
  }
643
653
  }
644
654
  const inner = await this.#getInner();
@@ -650,6 +660,19 @@ export class ThrottlingRequestManager {
650
660
  if (probed.status === 'ready') {
651
661
  return probed;
652
662
  }
663
+ // A cooling domain with nothing left to dispatch is nothing to wait for; in deadline order, the first
664
+ // with work left is the earliest wake-up.
665
+ cooling.sort((a, b) => a.readyAt - b.readyAt);
666
+ for (const { subManager, readyAt } of cooling) {
667
+ const { status } = await (await subManager).checkReadiness();
668
+ if (status !== 'finished') {
669
+ return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });
670
+ }
671
+ }
672
+ // Batches still landing in the background are work nobody can see in a queue yet.
673
+ if (this.#inProgressBatchCount > 0 && probed.status === 'finished') {
674
+ return { status: 'waiting' };
675
+ }
653
676
  if (stallCandidates !== undefined) {
654
677
  const stalled = (await Promise.all(stallCandidates.map(async (state) => (await (await this.#subManagers.get(state.domain)).checkReadiness()).status === 'ready'
655
678
  ? state
@@ -658,22 +681,15 @@ export class ThrottlingRequestManager {
658
681
  const summary = stalled
659
682
  .map((state) => `"${state.domain}" (${((now - state.rateLimitedSince) / 1000).toFixed(0)}s)`)
660
683
  .join(', ');
661
- return {
684
+ return joinRequestSourceStatuses(probed, {
662
685
  status: 'stalled',
663
686
  reason: `${summary} rate-limited every request for longer than maxDomainStallSecs ` +
664
687
  `(${(this.#maxDomainStallMs / 1000).toFixed(0)}s). Waiting longer will not help - lower the ` +
665
688
  `crawler's concurrency, or drop these domains. Their requests are still queued, so ` +
666
689
  `re-running with \`purgeOnStart\` disabled will resume them if the rate limit lifts.`,
667
- };
690
+ });
668
691
  }
669
692
  }
670
- if (readyAt !== undefined) {
671
- return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });
672
- }
673
- // Batches still landing in the background are work nobody can see in a queue yet.
674
- if (this.#inProgressBatchCount > 0 && probed.status === 'finished') {
675
- return { status: 'waiting' };
676
- }
677
693
  return probed;
678
694
  }
679
695
  /**
@@ -701,22 +717,22 @@ export class ThrottlingRequestManager {
701
717
  state.lastRateLimitedAt = 0;
702
718
  }
703
719
  }
704
- async setExpectedRequestProcessingTimeSecs(secs) {
720
+ async setExpectedRequestProcessingTime(processingTimeSecs) {
705
721
  // Remembered so a manager opened later still gets the hint, without opening one now just to pass it along.
706
- this.#expectedRequestProcessingSecs = secs;
707
- await this.#forEachManager((manager) => manager.setExpectedRequestProcessingTimeSecs?.(secs));
722
+ this.#expectedRequestProcessingSecs = processingTimeSecs;
723
+ await this.#forEachManager((manager) => manager.setExpectedRequestProcessingTime?.(processingTimeSecs));
708
724
  }
709
725
  /**
710
726
  * @inheritdoc
711
727
  * Extending a lock must not consume the in-flight marker that later routes
712
728
  * {@link ThrottlingRequestManager.markRequestAsHandled} / {@link ThrottlingRequestManager.reclaimRequest}.
713
729
  */
714
- async extendRequestProcessingTimeSecs(request, secs) {
730
+ async extendRequestProcessingTime(request, extensionSecs) {
715
731
  const key = request.id ?? request.uniqueKey;
716
732
  const manager = this.#inFlightFromInner.has(key)
717
733
  ? this.#resolvedInner
718
734
  : await this.#selectManagerOrThrow(request.url);
719
- return (await manager.extendRequestProcessingTimeSecs?.(request, secs)) ?? false;
735
+ return (await manager.extendRequestProcessingTime?.(request, extensionSecs)) ?? false;
720
736
  }
721
737
  /**
722
738
  * Runs `fn` over the sub-queues and, if it has been resolved, the wrapped manager - bookkeeping never forces
@@ -795,7 +811,8 @@ export class ThrottlingRequestManager {
795
811
  return null;
796
812
  }
797
813
  const state = this.#getDomainState(request.url);
798
- if (state === null || Date.now() >= throttledUntil(state)) {
814
+ const now = Date.now();
815
+ if (state === null || (now >= throttledUntil(state) && !isStalling(state, now, this.#maxDomainStallMs))) {
799
816
  return request;
800
817
  }
801
818
  await this.#migrateToSubQueue(request, inner);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/basic",
3
- "version": "4.0.0-rc.1",
3
+ "version": "4.0.0-rc.2",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -41,23 +41,23 @@
41
41
  "dependencies": {
42
42
  "@apify/datastructures": "^3.0.1",
43
43
  "@apify/timeout": "^1.0.1",
44
- "@apify/utilities": "^3.0.1",
45
- "@crawlee/core": "4.0.0-rc.1",
46
- "@crawlee/http-client": "4.0.0-rc.1",
47
- "@crawlee/types": "4.0.0-rc.1",
48
- "@crawlee/utils": "4.0.0-rc.1",
44
+ "@apify/utilities": "^3.1.0",
45
+ "@crawlee/core": "4.0.0-rc.2",
46
+ "@crawlee/http-client": "4.0.0-rc.2",
47
+ "@crawlee/types": "4.0.0-rc.2",
48
+ "@crawlee/utils": "4.0.0-rc.2",
49
49
  "@sapphire/async-queue": "^1.5.5",
50
- "@standard-schema/spec": "^1.0.0",
51
- "csv-stringify": "^6.5.2",
52
- "minimatch": "^10.2.6",
53
- "tldts": "^7.0.6",
54
- "tough-cookie": "^6.0.0",
55
- "tslib": "^2.8.1",
50
+ "@standard-schema/spec": "^1.1.0",
51
+ "csv-stringify": "^6.9.0",
52
+ "minimatch": "^11.0.5",
53
+ "tldts": "^7.4.19",
54
+ "tough-cookie": "^6.1.0",
55
+ "tslib": "^2.9.1",
56
56
  "type-fest": "^4.41.0",
57
- "zod": "^4.5.4"
57
+ "zod": "^4.6.7"
58
58
  },
59
59
  "optionalDependencies": {
60
- "@crawlee/impit-client": "4.0.0-rc.1"
60
+ "@crawlee/impit-client": "4.0.0-rc.2"
61
61
  },
62
62
  "lerna": {
63
63
  "command": {
@@ -66,5 +66,5 @@
66
66
  }
67
67
  }
68
68
  },
69
- "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
69
+ "gitHead": "a9b623229cd7d1bff5063fc82940efebaea47165"
70
70
  }