@crawlee/basic 4.0.0-beta.267 → 4.0.0-beta.268
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1191,7 +1191,7 @@ export class BasicCrawler {
|
|
|
1191
1191
|
}
|
|
1192
1192
|
const result = await this.requestManager.addRequestsBatched(filteredRequests(), {
|
|
1193
1193
|
forefront: options.forefront,
|
|
1194
|
-
waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded,
|
|
1194
|
+
waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded || requestLimit !== undefined,
|
|
1195
1195
|
batchSize: options.batchSize,
|
|
1196
1196
|
waitBetweenBatchesMillis: options.waitBetweenBatchesMillis,
|
|
1197
1197
|
maxNewRequests: requestLimit,
|
|
@@ -188,9 +188,8 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
|
|
|
188
188
|
/**
|
|
189
189
|
* Reports whether anything can be dispatched right now, and if not, when — or why never.
|
|
190
190
|
*
|
|
191
|
-
*
|
|
192
|
-
*
|
|
193
|
-
* crawler gated on this idles for the backoff instead of concluding it is done.
|
|
191
|
+
* Dispatchable domains are probed first. When nothing can be dispatched, cooling domains contribute their
|
|
192
|
+
* deadlines only while they still have outstanding work, so a finished crawl does not wait out a cooldown.
|
|
194
193
|
*
|
|
195
194
|
* `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,
|
|
196
195
|
* so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -
|
|
@@ -618,9 +618,8 @@ export class ThrottlingRequestManager {
|
|
|
618
618
|
/**
|
|
619
619
|
* Reports whether anything can be dispatched right now, and if not, when — or why never.
|
|
620
620
|
*
|
|
621
|
-
*
|
|
622
|
-
*
|
|
623
|
-
* crawler gated on this idles for the backoff instead of concluding it is done.
|
|
621
|
+
* Dispatchable domains are probed first. When nothing can be dispatched, cooling domains contribute their
|
|
622
|
+
* deadlines only while they still have outstanding work, so a finished crawl does not wait out a cooldown.
|
|
624
623
|
*
|
|
625
624
|
* `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,
|
|
626
625
|
* so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -
|
|
@@ -631,24 +630,25 @@ export class ThrottlingRequestManager {
|
|
|
631
630
|
await this.#ensureSubManagers();
|
|
632
631
|
const now = Date.now();
|
|
633
632
|
const dispatchable = [];
|
|
634
|
-
|
|
633
|
+
const cooling = [];
|
|
635
634
|
let stallCandidates;
|
|
636
635
|
for (const state of this.#domainStates.values()) {
|
|
637
636
|
// A `Crawl-delay` can give a domain a clock before its first request gives it a queue - nothing
|
|
638
637
|
// to fetch from and nothing to wait for until then.
|
|
639
|
-
|
|
638
|
+
const subManager = this.#subManagers.get(state.domain);
|
|
639
|
+
if (!subManager) {
|
|
640
640
|
continue;
|
|
641
641
|
}
|
|
642
642
|
if (isStalling(state, now, this.#maxDomainStallMs)) {
|
|
643
643
|
(stallCandidates ??= []).push(state);
|
|
644
644
|
continue;
|
|
645
645
|
}
|
|
646
|
-
const
|
|
647
|
-
if (now >=
|
|
648
|
-
dispatchable.push(
|
|
646
|
+
const readyAt = throttledUntil(state);
|
|
647
|
+
if (now >= readyAt) {
|
|
648
|
+
dispatchable.push(subManager);
|
|
649
649
|
}
|
|
650
|
-
else
|
|
651
|
-
readyAt
|
|
650
|
+
else {
|
|
651
|
+
cooling.push({ subManager, readyAt });
|
|
652
652
|
}
|
|
653
653
|
}
|
|
654
654
|
const inner = await this.#getInner();
|
|
@@ -660,8 +660,14 @@ export class ThrottlingRequestManager {
|
|
|
660
660
|
if (probed.status === 'ready') {
|
|
661
661
|
return probed;
|
|
662
662
|
}
|
|
663
|
-
|
|
664
|
-
|
|
663
|
+
// A cooling domain with nothing left to dispatch is nothing to wait for; in deadline order, the first
|
|
664
|
+
// with work left is the earliest wake-up.
|
|
665
|
+
cooling.sort((a, b) => a.readyAt - b.readyAt);
|
|
666
|
+
for (const { subManager, readyAt } of cooling) {
|
|
667
|
+
const { status } = await (await subManager).checkReadiness();
|
|
668
|
+
if (status !== 'finished') {
|
|
669
|
+
return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });
|
|
670
|
+
}
|
|
665
671
|
}
|
|
666
672
|
// Batches still landing in the background are work nobody can see in a queue yet.
|
|
667
673
|
if (this.#inProgressBatchCount > 0 && probed.status === 'finished') {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.268",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.1.0",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.268",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.268",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.268",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.268",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.1.0",
|
|
51
51
|
"csv-stringify": "^6.9.0",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.6.7"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.268"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "9462257a02015e7ef40ebc16d8fdd0665c9554fe"
|
|
70
70
|
}
|