@crawlee/basic 4.0.0-beta.266 → 4.0.0-beta.267

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -194,7 +194,8 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
194
194
  *
195
195
  * `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,
196
196
  * so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -
197
- * see the traversal.
197
+ * see the traversal. A domain that is only cooling down holds off a stalling one the same way, because the
198
+ * crawl can continue once its cooldown ends.
198
199
  */
199
200
  checkReadiness(): Promise<RequestSourceStatus>;
200
201
  /**
@@ -22,6 +22,15 @@ const throttlingRequestManagerOptionsSchema = z.strictObject({
22
22
  function throttledUntil(state) {
23
23
  return Math.max(state.backoffUntil, state.crawlDelayUntil);
24
24
  }
25
+ /**
26
+ * Still refusing, and has been for longer than the window. Not dispatched to, so this lapses a window after the
27
+ * last 429 and the domain gets one burst to prove itself with.
28
+ */
29
+ function isStalling(state, now, maxStallMs) {
30
+ return (state.rateLimitedSince !== 0 &&
31
+ now - state.lastRateLimitedAt <= maxStallMs &&
32
+ now - state.rateLimitedSince > maxStallMs);
33
+ }
25
34
  /** How long a domain must be left alone after a dispatch: what it asked for, or our floor, whichever is longer. */
26
35
  function crawlDelayMs(state, minCrawlDelayMs) {
27
36
  return Math.max(state.declaredCrawlDelayMs ?? 0, minCrawlDelayMs);
@@ -355,7 +364,9 @@ export class ThrottlingRequestManager {
355
364
  #fetchableDomains() {
356
365
  const now = Date.now();
357
366
  return Array.from(this.#domainStates.values())
358
- .filter((state) => now >= throttledUntil(state) && this.#subManagers.has(state.domain))
367
+ .filter((state) => now >= throttledUntil(state) &&
368
+ !isStalling(state, now, this.#maxDomainStallMs) &&
369
+ this.#subManagers.has(state.domain))
359
370
  .sort((a, b) => throttledUntil(a) - throttledUntil(b))
360
371
  .map((state) => state.domain);
361
372
  }
@@ -492,11 +503,15 @@ export class ThrottlingRequestManager {
492
503
  }
493
504
  return true;
494
505
  }
495
- /** Records that a domain let a request through, which ends any rate-limit run stall detection was timing. */
506
+ /**
507
+ * Records that a domain let a request through, which ends any rate-limit run stall detection was timing
508
+ * and the run of 429s the backoff was doubling for.
509
+ */
496
510
  #recordProgress(url) {
497
511
  const state = this.#getDomainState(url);
498
512
  if (state) {
499
513
  state.rateLimitedSince = 0;
514
+ state.consecutive429Count = 0;
500
515
  }
501
516
  }
502
517
  // --- IRequestManager Implementation ---
@@ -609,7 +624,8 @@ export class ThrottlingRequestManager {
609
624
  *
610
625
  * `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,
611
626
  * so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -
612
- * see the traversal.
627
+ * see the traversal. A domain that is only cooling down holds off a stalling one the same way, because the
628
+ * crawl can continue once its cooldown ends.
613
629
  */
614
630
  async checkReadiness() {
615
631
  await this.#ensureSubManagers();
@@ -623,14 +639,7 @@ export class ThrottlingRequestManager {
623
639
  if (!this.#subManagers.has(state.domain)) {
624
640
  continue;
625
641
  }
626
- if (
627
- // Together: still turning us away, and has been without a break for longer than the window - a
628
- // domain that has merely been idle starts this clock at its first 429, not with idle time on it.
629
- state.rateLimitedSince !== 0 &&
630
- now - state.lastRateLimitedAt <= this.#maxDomainStallMs &&
631
- now - state.rateLimitedSince > this.#maxDomainStallMs) {
632
- // Deliberately not dispatchable: a domain that has refused every request for the whole window
633
- // is not progress just because its backoff momentarily lapsed - that is what it does between 429s.
642
+ if (isStalling(state, now, this.#maxDomainStallMs)) {
634
643
  (stallCandidates ??= []).push(state);
635
644
  continue;
636
645
  }
@@ -651,6 +660,13 @@ export class ThrottlingRequestManager {
651
660
  if (probed.status === 'ready') {
652
661
  return probed;
653
662
  }
663
+ if (readyAt !== undefined) {
664
+ return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });
665
+ }
666
+ // Batches still landing in the background are work nobody can see in a queue yet.
667
+ if (this.#inProgressBatchCount > 0 && probed.status === 'finished') {
668
+ return { status: 'waiting' };
669
+ }
654
670
  if (stallCandidates !== undefined) {
655
671
  const stalled = (await Promise.all(stallCandidates.map(async (state) => (await (await this.#subManagers.get(state.domain)).checkReadiness()).status === 'ready'
656
672
  ? state
@@ -659,22 +675,15 @@ export class ThrottlingRequestManager {
659
675
  const summary = stalled
660
676
  .map((state) => `"${state.domain}" (${((now - state.rateLimitedSince) / 1000).toFixed(0)}s)`)
661
677
  .join(', ');
662
- return {
678
+ return joinRequestSourceStatuses(probed, {
663
679
  status: 'stalled',
664
680
  reason: `${summary} rate-limited every request for longer than maxDomainStallSecs ` +
665
681
  `(${(this.#maxDomainStallMs / 1000).toFixed(0)}s). Waiting longer will not help - lower the ` +
666
682
  `crawler's concurrency, or drop these domains. Their requests are still queued, so ` +
667
683
  `re-running with \`purgeOnStart\` disabled will resume them if the rate limit lifts.`,
668
- };
684
+ });
669
685
  }
670
686
  }
671
- if (readyAt !== undefined) {
672
- return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });
673
- }
674
- // Batches still landing in the background are work nobody can see in a queue yet.
675
- if (this.#inProgressBatchCount > 0 && probed.status === 'finished') {
676
- return { status: 'waiting' };
677
- }
678
687
  return probed;
679
688
  }
680
689
  /**
@@ -796,7 +805,8 @@ export class ThrottlingRequestManager {
796
805
  return null;
797
806
  }
798
807
  const state = this.#getDomainState(request.url);
799
- if (state === null || Date.now() >= throttledUntil(state)) {
808
+ const now = Date.now();
809
+ if (state === null || (now >= throttledUntil(state) && !isStalling(state, now, this.#maxDomainStallMs))) {
800
810
  return request;
801
811
  }
802
812
  await this.#migrateToSubQueue(request, inner);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/basic",
3
- "version": "4.0.0-beta.266",
3
+ "version": "4.0.0-beta.267",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -42,10 +42,10 @@
42
42
  "@apify/datastructures": "^3.0.1",
43
43
  "@apify/timeout": "^1.0.1",
44
44
  "@apify/utilities": "^3.1.0",
45
- "@crawlee/core": "4.0.0-beta.266",
46
- "@crawlee/http-client": "4.0.0-beta.266",
47
- "@crawlee/types": "4.0.0-beta.266",
48
- "@crawlee/utils": "4.0.0-beta.266",
45
+ "@crawlee/core": "4.0.0-beta.267",
46
+ "@crawlee/http-client": "4.0.0-beta.267",
47
+ "@crawlee/types": "4.0.0-beta.267",
48
+ "@crawlee/utils": "4.0.0-beta.267",
49
49
  "@sapphire/async-queue": "^1.5.5",
50
50
  "@standard-schema/spec": "^1.1.0",
51
51
  "csv-stringify": "^6.9.0",
@@ -57,7 +57,7 @@
57
57
  "zod": "^4.6.7"
58
58
  },
59
59
  "optionalDependencies": {
60
- "@crawlee/impit-client": "4.0.0-beta.266"
60
+ "@crawlee/impit-client": "4.0.0-beta.267"
61
61
  },
62
62
  "lerna": {
63
63
  "command": {
@@ -66,5 +66,5 @@
66
66
  }
67
67
  }
68
68
  },
69
- "gitHead": "1b0e67fa736823f6ffd42537c87e41fe373394fd"
69
+ "gitHead": "a4e49e8a8117acdb808c6cba5b05c3d56bf2b123"
70
70
  }