@crawlee/basic 4.0.0-beta.276 → 4.0.0-beta.278
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -624,7 +624,7 @@ export class BasicCrawler {
|
|
|
624
624
|
context[navigationDeadlineKey] += extraMillis;
|
|
625
625
|
}
|
|
626
626
|
// Extension failure must not fail the request that asked for more time.
|
|
627
|
-
this.requestManager?.
|
|
627
|
+
this.requestManager?.extendRequestProcessingTime?.(context.request, secs)?.catch((error) => {
|
|
628
628
|
this.log.debug('Extending the request processing time failed', { url: context.request.url, error });
|
|
629
629
|
});
|
|
630
630
|
},
|
|
@@ -969,7 +969,7 @@ export class BasicCrawler {
|
|
|
969
969
|
this.#requestManager = await this.#openOwnedRequestQueue();
|
|
970
970
|
}
|
|
971
971
|
// Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
|
|
972
|
-
// now that `
|
|
972
|
+
// now that `setExpectedRequestProcessingTime` is async. The hint is raise-only and idempotent,
|
|
973
973
|
// but guard so we do not re-issue it on every call.
|
|
974
974
|
if (!this.#requestManagerTimeoutsApplied) {
|
|
975
975
|
this.#requestManagerTimeoutsApplied = true;
|
|
@@ -1006,7 +1006,7 @@ export class BasicCrawler {
|
|
|
1006
1006
|
// raise-only, so erring high here is safe.
|
|
1007
1007
|
const maxRouteTimeoutSecs = this.requestHandler.getMaxTimeoutSecs?.() ?? 0;
|
|
1008
1008
|
const handlerTimeoutSecs = Math.max(this.#requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
|
|
1009
|
-
await requestManager.
|
|
1009
|
+
await requestManager.setExpectedRequestProcessingTime?.(Math.max(handlerTimeoutSecs + 5, 60));
|
|
1010
1010
|
}
|
|
1011
1011
|
/**
|
|
1012
1012
|
* Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
|
|
@@ -38,7 +38,7 @@ const errorTrackerConfig = {
|
|
|
38
38
|
* order here *is* the record's field order (guarded by a test).
|
|
39
39
|
*
|
|
40
40
|
* JSON has no infinity, so the three fields that are `Infinity` until the first request settles are written as
|
|
41
|
-
* `null`. Both
|
|
41
|
+
* `null`. Both the private serialize and deserialize steps of `Statistics` run through this,
|
|
42
42
|
* which is what keeps them describing the same record.
|
|
43
43
|
*
|
|
44
44
|
* Nothing is optional on purpose: the record has always carried every field, so one missing a field is not one of
|
|
@@ -153,7 +153,7 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
|
|
|
153
153
|
*/
|
|
154
154
|
export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager {
|
|
155
155
|
#private;
|
|
156
|
-
constructor(options: ThrottlingRequestManagerOptions<T>,
|
|
156
|
+
constructor(options: ThrottlingRequestManagerOptions<T>, configuration?: Configuration);
|
|
157
157
|
/**
|
|
158
158
|
* The wrapped manager, holding every request whose domain is not throttled. `undefined` until an `inner`
|
|
159
159
|
* passed as a factory is resolved - reading this never forces it, because a getter should not open a queue
|
|
@@ -202,13 +202,13 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
|
|
|
202
202
|
* site rather than of the run, so it survives.
|
|
203
203
|
*/
|
|
204
204
|
purge(): Promise<void>;
|
|
205
|
-
|
|
205
|
+
setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void>;
|
|
206
206
|
/**
|
|
207
207
|
* @inheritdoc
|
|
208
208
|
* Extending a lock must not consume the in-flight marker that later routes
|
|
209
209
|
* {@link ThrottlingRequestManager.markRequestAsHandled} / {@link ThrottlingRequestManager.reclaimRequest}.
|
|
210
210
|
*/
|
|
211
|
-
|
|
211
|
+
extendRequestProcessingTime(request: Request, extensionSecs: number): Promise<boolean>;
|
|
212
212
|
/**
|
|
213
213
|
* Returns the next request from a domain that is not backing off, or from the inner manager.
|
|
214
214
|
*
|
|
@@ -112,7 +112,7 @@ export class ThrottlingRequestManager {
|
|
|
112
112
|
#throttleBy;
|
|
113
113
|
#maxThrottledDomains;
|
|
114
114
|
#persistStateKey;
|
|
115
|
-
#
|
|
115
|
+
#configuration;
|
|
116
116
|
#subManagers = new Map();
|
|
117
117
|
#domainStates = new Map();
|
|
118
118
|
#log;
|
|
@@ -143,20 +143,20 @@ export class ThrottlingRequestManager {
|
|
|
143
143
|
#inProgressBatchCount = 0;
|
|
144
144
|
/** Requests moved into a sub-queue, which both managers count. Subtracted below; not persisted. */
|
|
145
145
|
#migratedFromInner = 0;
|
|
146
|
-
/** The latest {@link
|
|
146
|
+
/** The latest {@link setExpectedRequestProcessingTime} hint, kept for a wrapped manager resolved later. */
|
|
147
147
|
#expectedRequestProcessingSecs;
|
|
148
148
|
#warnedAbout = new Set();
|
|
149
149
|
/** Whether any domain at all may end up throttled - listed up front, or discovered as the crawl runs. */
|
|
150
150
|
get #throttlingEnabled() {
|
|
151
151
|
return this.#listedDomains.size > 0 || this.#throttlesEveryDomain;
|
|
152
152
|
}
|
|
153
|
-
constructor(options,
|
|
154
|
-
this.#
|
|
153
|
+
constructor(options, configuration = serviceLocator.getConfiguration()) {
|
|
154
|
+
this.#configuration = configuration;
|
|
155
155
|
parseArgument(options, throttlingRequestManagerOptionsSchema, 'ThrottlingRequestManagerOptions');
|
|
156
156
|
if (options.inner === undefined) {
|
|
157
157
|
// Deferred like any other factory, so a manager nobody fetches from opens no storage. The opener is
|
|
158
158
|
// assigned below and read only when this runs.
|
|
159
|
-
this.#innerFactory = () => this.#requestManagerOpener(null, { configuration: this.#
|
|
159
|
+
this.#innerFactory = () => this.#requestManagerOpener(null, { configuration: this.#configuration });
|
|
160
160
|
}
|
|
161
161
|
else if (typeof options.inner === 'function') {
|
|
162
162
|
this.#innerFactory = options.inner;
|
|
@@ -229,7 +229,7 @@ export class ThrottlingRequestManager {
|
|
|
229
229
|
this.#innerPromise ??= Promise.resolve(this.#innerFactory());
|
|
230
230
|
this.#resolvedInner = await this.#innerPromise;
|
|
231
231
|
if (this.#expectedRequestProcessingSecs !== undefined) {
|
|
232
|
-
await this.#resolvedInner.
|
|
232
|
+
await this.#resolvedInner.setExpectedRequestProcessingTime?.(this.#expectedRequestProcessingSecs);
|
|
233
233
|
}
|
|
234
234
|
}
|
|
235
235
|
return this.#resolvedInner;
|
|
@@ -310,7 +310,7 @@ export class ThrottlingRequestManager {
|
|
|
310
310
|
subManager = this.#requestManagerOpener(
|
|
311
311
|
// Backends use the alias as a directory name, and an IPv6 literal is full of characters
|
|
312
312
|
// Windows will not accept. Ordinary hostnames survive this untouched.
|
|
313
|
-
{ alias: `throttled-${encodeURIComponent(domain)}` }, { configuration: this.#
|
|
313
|
+
{ alias: `throttled-${encodeURIComponent(domain)}` }, { configuration: this.#configuration });
|
|
314
314
|
this.#subManagers.set(domain, subManager);
|
|
315
315
|
this.#ensureDomainState(domain);
|
|
316
316
|
}
|
|
@@ -342,7 +342,7 @@ export class ThrottlingRequestManager {
|
|
|
342
342
|
async #ensureSubManagers() {
|
|
343
343
|
this.#subManagersReady ??= (async () => {
|
|
344
344
|
if (this.#throttlesEveryDomain) {
|
|
345
|
-
this.#domainListStore = await KeyValueStore.open(null, { configuration: this.#
|
|
345
|
+
this.#domainListStore = await KeyValueStore.open(null, { configuration: this.#configuration });
|
|
346
346
|
for (const domain of (await this.#domainListStore.getValue(this.#persistStateKey)) ?? []) {
|
|
347
347
|
this.#discoveredDomains.add(domain);
|
|
348
348
|
}
|
|
@@ -717,22 +717,22 @@ export class ThrottlingRequestManager {
|
|
|
717
717
|
state.lastRateLimitedAt = 0;
|
|
718
718
|
}
|
|
719
719
|
}
|
|
720
|
-
async
|
|
720
|
+
async setExpectedRequestProcessingTime(processingTimeSecs) {
|
|
721
721
|
// Remembered so a manager opened later still gets the hint, without opening one now just to pass it along.
|
|
722
|
-
this.#expectedRequestProcessingSecs =
|
|
723
|
-
await this.#forEachManager((manager) => manager.
|
|
722
|
+
this.#expectedRequestProcessingSecs = processingTimeSecs;
|
|
723
|
+
await this.#forEachManager((manager) => manager.setExpectedRequestProcessingTime?.(processingTimeSecs));
|
|
724
724
|
}
|
|
725
725
|
/**
|
|
726
726
|
* @inheritdoc
|
|
727
727
|
* Extending a lock must not consume the in-flight marker that later routes
|
|
728
728
|
* {@link ThrottlingRequestManager.markRequestAsHandled} / {@link ThrottlingRequestManager.reclaimRequest}.
|
|
729
729
|
*/
|
|
730
|
-
async
|
|
730
|
+
async extendRequestProcessingTime(request, extensionSecs) {
|
|
731
731
|
const key = request.id ?? request.uniqueKey;
|
|
732
732
|
const manager = this.#inFlightFromInner.has(key)
|
|
733
733
|
? this.#resolvedInner
|
|
734
734
|
: await this.#selectManagerOrThrow(request.url);
|
|
735
|
-
return (await manager.
|
|
735
|
+
return (await manager.extendRequestProcessingTime?.(request, extensionSecs)) ?? false;
|
|
736
736
|
}
|
|
737
737
|
/**
|
|
738
738
|
* Runs `fn` over the sub-queues and, if it has been resolved, the wrapped manager - bookkeeping never forces
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.278",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.1.0",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.278",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.278",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.278",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.278",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.1.0",
|
|
51
51
|
"csv-stringify": "^6.9.0",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.6.7"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.278"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "745f27ebf5c8307d949416c9c884ab941eef1559"
|
|
70
70
|
}
|