@crawlee/basic 4.0.0-beta.266 → 4.0.0-beta.268
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1191,7 +1191,7 @@ export class BasicCrawler {
|
|
|
1191
1191
|
}
|
|
1192
1192
|
const result = await this.requestManager.addRequestsBatched(filteredRequests(), {
|
|
1193
1193
|
forefront: options.forefront,
|
|
1194
|
-
waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded,
|
|
1194
|
+
waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded || requestLimit !== undefined,
|
|
1195
1195
|
batchSize: options.batchSize,
|
|
1196
1196
|
waitBetweenBatchesMillis: options.waitBetweenBatchesMillis,
|
|
1197
1197
|
maxNewRequests: requestLimit,
|
|
@@ -188,13 +188,13 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
|
|
|
188
188
|
/**
|
|
189
189
|
* Reports whether anything can be dispatched right now, and if not, when — or why never.
|
|
190
190
|
*
|
|
191
|
-
*
|
|
192
|
-
*
|
|
193
|
-
* crawler gated on this idles for the backoff instead of concluding it is done.
|
|
191
|
+
* Dispatchable domains are probed first. When nothing can be dispatched, cooling domains contribute their
|
|
192
|
+
* deadlines only while they still have outstanding work, so a finished crawl does not wait out a cooldown.
|
|
194
193
|
*
|
|
195
194
|
* `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,
|
|
196
195
|
* so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -
|
|
197
|
-
* see the traversal.
|
|
196
|
+
* see the traversal. A domain that is only cooling down holds off a stalling one the same way, because the
|
|
197
|
+
* crawl can continue once its cooldown ends.
|
|
198
198
|
*/
|
|
199
199
|
checkReadiness(): Promise<RequestSourceStatus>;
|
|
200
200
|
/**
|
|
@@ -22,6 +22,15 @@ const throttlingRequestManagerOptionsSchema = z.strictObject({
|
|
|
22
22
|
function throttledUntil(state) {
|
|
23
23
|
return Math.max(state.backoffUntil, state.crawlDelayUntil);
|
|
24
24
|
}
|
|
25
|
+
/**
|
|
26
|
+
* Still refusing, and has been for longer than the window. Not dispatched to, so this lapses a window after the
|
|
27
|
+
* last 429 and the domain gets one burst to prove itself with.
|
|
28
|
+
*/
|
|
29
|
+
function isStalling(state, now, maxStallMs) {
|
|
30
|
+
return (state.rateLimitedSince !== 0 &&
|
|
31
|
+
now - state.lastRateLimitedAt <= maxStallMs &&
|
|
32
|
+
now - state.rateLimitedSince > maxStallMs);
|
|
33
|
+
}
|
|
25
34
|
/** How long a domain must be left alone after a dispatch: what it asked for, or our floor, whichever is longer. */
|
|
26
35
|
function crawlDelayMs(state, minCrawlDelayMs) {
|
|
27
36
|
return Math.max(state.declaredCrawlDelayMs ?? 0, minCrawlDelayMs);
|
|
@@ -355,7 +364,9 @@ export class ThrottlingRequestManager {
|
|
|
355
364
|
#fetchableDomains() {
|
|
356
365
|
const now = Date.now();
|
|
357
366
|
return Array.from(this.#domainStates.values())
|
|
358
|
-
.filter((state) => now >= throttledUntil(state) &&
|
|
367
|
+
.filter((state) => now >= throttledUntil(state) &&
|
|
368
|
+
!isStalling(state, now, this.#maxDomainStallMs) &&
|
|
369
|
+
this.#subManagers.has(state.domain))
|
|
359
370
|
.sort((a, b) => throttledUntil(a) - throttledUntil(b))
|
|
360
371
|
.map((state) => state.domain);
|
|
361
372
|
}
|
|
@@ -492,11 +503,15 @@ export class ThrottlingRequestManager {
|
|
|
492
503
|
}
|
|
493
504
|
return true;
|
|
494
505
|
}
|
|
495
|
-
/**
|
|
506
|
+
/**
|
|
507
|
+
* Records that a domain let a request through, which ends any rate-limit run stall detection was timing
|
|
508
|
+
* and the run of 429s the backoff was doubling for.
|
|
509
|
+
*/
|
|
496
510
|
#recordProgress(url) {
|
|
497
511
|
const state = this.#getDomainState(url);
|
|
498
512
|
if (state) {
|
|
499
513
|
state.rateLimitedSince = 0;
|
|
514
|
+
state.consecutive429Count = 0;
|
|
500
515
|
}
|
|
501
516
|
}
|
|
502
517
|
// --- IRequestManager Implementation ---
|
|
@@ -603,43 +618,37 @@ export class ThrottlingRequestManager {
|
|
|
603
618
|
/**
|
|
604
619
|
* Reports whether anything can be dispatched right now, and if not, when — or why never.
|
|
605
620
|
*
|
|
606
|
-
*
|
|
607
|
-
*
|
|
608
|
-
* crawler gated on this idles for the backoff instead of concluding it is done.
|
|
621
|
+
* Dispatchable domains are probed first. When nothing can be dispatched, cooling domains contribute their
|
|
622
|
+
* deadlines only while they still have outstanding work, so a finished crawl does not wait out a cooldown.
|
|
609
623
|
*
|
|
610
624
|
* `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,
|
|
611
625
|
* so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -
|
|
612
|
-
* see the traversal.
|
|
626
|
+
* see the traversal. A domain that is only cooling down holds off a stalling one the same way, because the
|
|
627
|
+
* crawl can continue once its cooldown ends.
|
|
613
628
|
*/
|
|
614
629
|
async checkReadiness() {
|
|
615
630
|
await this.#ensureSubManagers();
|
|
616
631
|
const now = Date.now();
|
|
617
632
|
const dispatchable = [];
|
|
618
|
-
|
|
633
|
+
const cooling = [];
|
|
619
634
|
let stallCandidates;
|
|
620
635
|
for (const state of this.#domainStates.values()) {
|
|
621
636
|
// A `Crawl-delay` can give a domain a clock before its first request gives it a queue - nothing
|
|
622
637
|
// to fetch from and nothing to wait for until then.
|
|
623
|
-
|
|
638
|
+
const subManager = this.#subManagers.get(state.domain);
|
|
639
|
+
if (!subManager) {
|
|
624
640
|
continue;
|
|
625
641
|
}
|
|
626
|
-
if (
|
|
627
|
-
// Together: still turning us away, and has been without a break for longer than the window - a
|
|
628
|
-
// domain that has merely been idle starts this clock at its first 429, not with idle time on it.
|
|
629
|
-
state.rateLimitedSince !== 0 &&
|
|
630
|
-
now - state.lastRateLimitedAt <= this.#maxDomainStallMs &&
|
|
631
|
-
now - state.rateLimitedSince > this.#maxDomainStallMs) {
|
|
632
|
-
// Deliberately not dispatchable: a domain that has refused every request for the whole window
|
|
633
|
-
// is not progress just because its backoff momentarily lapsed - that is what it does between 429s.
|
|
642
|
+
if (isStalling(state, now, this.#maxDomainStallMs)) {
|
|
634
643
|
(stallCandidates ??= []).push(state);
|
|
635
644
|
continue;
|
|
636
645
|
}
|
|
637
|
-
const
|
|
638
|
-
if (now >=
|
|
639
|
-
dispatchable.push(
|
|
646
|
+
const readyAt = throttledUntil(state);
|
|
647
|
+
if (now >= readyAt) {
|
|
648
|
+
dispatchable.push(subManager);
|
|
640
649
|
}
|
|
641
|
-
else
|
|
642
|
-
readyAt
|
|
650
|
+
else {
|
|
651
|
+
cooling.push({ subManager, readyAt });
|
|
643
652
|
}
|
|
644
653
|
}
|
|
645
654
|
const inner = await this.#getInner();
|
|
@@ -651,6 +660,19 @@ export class ThrottlingRequestManager {
|
|
|
651
660
|
if (probed.status === 'ready') {
|
|
652
661
|
return probed;
|
|
653
662
|
}
|
|
663
|
+
// A cooling domain with nothing left to dispatch is nothing to wait for; in deadline order, the first
|
|
664
|
+
// with work left is the earliest wake-up.
|
|
665
|
+
cooling.sort((a, b) => a.readyAt - b.readyAt);
|
|
666
|
+
for (const { subManager, readyAt } of cooling) {
|
|
667
|
+
const { status } = await (await subManager).checkReadiness();
|
|
668
|
+
if (status !== 'finished') {
|
|
669
|
+
return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
// Batches still landing in the background are work nobody can see in a queue yet.
|
|
673
|
+
if (this.#inProgressBatchCount > 0 && probed.status === 'finished') {
|
|
674
|
+
return { status: 'waiting' };
|
|
675
|
+
}
|
|
654
676
|
if (stallCandidates !== undefined) {
|
|
655
677
|
const stalled = (await Promise.all(stallCandidates.map(async (state) => (await (await this.#subManagers.get(state.domain)).checkReadiness()).status === 'ready'
|
|
656
678
|
? state
|
|
@@ -659,22 +681,15 @@ export class ThrottlingRequestManager {
|
|
|
659
681
|
const summary = stalled
|
|
660
682
|
.map((state) => `"${state.domain}" (${((now - state.rateLimitedSince) / 1000).toFixed(0)}s)`)
|
|
661
683
|
.join(', ');
|
|
662
|
-
return {
|
|
684
|
+
return joinRequestSourceStatuses(probed, {
|
|
663
685
|
status: 'stalled',
|
|
664
686
|
reason: `${summary} rate-limited every request for longer than maxDomainStallSecs ` +
|
|
665
687
|
`(${(this.#maxDomainStallMs / 1000).toFixed(0)}s). Waiting longer will not help - lower the ` +
|
|
666
688
|
`crawler's concurrency, or drop these domains. Their requests are still queued, so ` +
|
|
667
689
|
`re-running with \`purgeOnStart\` disabled will resume them if the rate limit lifts.`,
|
|
668
|
-
};
|
|
690
|
+
});
|
|
669
691
|
}
|
|
670
692
|
}
|
|
671
|
-
if (readyAt !== undefined) {
|
|
672
|
-
return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });
|
|
673
|
-
}
|
|
674
|
-
// Batches still landing in the background are work nobody can see in a queue yet.
|
|
675
|
-
if (this.#inProgressBatchCount > 0 && probed.status === 'finished') {
|
|
676
|
-
return { status: 'waiting' };
|
|
677
|
-
}
|
|
678
693
|
return probed;
|
|
679
694
|
}
|
|
680
695
|
/**
|
|
@@ -796,7 +811,8 @@ export class ThrottlingRequestManager {
|
|
|
796
811
|
return null;
|
|
797
812
|
}
|
|
798
813
|
const state = this.#getDomainState(request.url);
|
|
799
|
-
|
|
814
|
+
const now = Date.now();
|
|
815
|
+
if (state === null || (now >= throttledUntil(state) && !isStalling(state, now, this.#maxDomainStallMs))) {
|
|
800
816
|
return request;
|
|
801
817
|
}
|
|
802
818
|
await this.#migrateToSubQueue(request, inner);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.268",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.1.0",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.268",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.268",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.268",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.268",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.1.0",
|
|
51
51
|
"csv-stringify": "^6.9.0",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.6.7"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.268"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "9462257a02015e7ef40ebc16d8fdd0665c9554fe"
|
|
70
70
|
}
|