@crawlee/basic 4.0.0-beta.276 → 4.0.0-beta.277
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -624,7 +624,7 @@ export class BasicCrawler {
|
|
|
624
624
|
context[navigationDeadlineKey] += extraMillis;
|
|
625
625
|
}
|
|
626
626
|
// Extension failure must not fail the request that asked for more time.
|
|
627
|
-
this.requestManager?.
|
|
627
|
+
this.requestManager?.extendRequestProcessingTime?.(context.request, secs)?.catch((error) => {
|
|
628
628
|
this.log.debug('Extending the request processing time failed', { url: context.request.url, error });
|
|
629
629
|
});
|
|
630
630
|
},
|
|
@@ -969,7 +969,7 @@ export class BasicCrawler {
|
|
|
969
969
|
this.#requestManager = await this.#openOwnedRequestQueue();
|
|
970
970
|
}
|
|
971
971
|
// Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
|
|
972
|
-
// now that `
|
|
972
|
+
// now that `setExpectedRequestProcessingTime` is async. The hint is raise-only and idempotent,
|
|
973
973
|
// but guard so we do not re-issue it on every call.
|
|
974
974
|
if (!this.#requestManagerTimeoutsApplied) {
|
|
975
975
|
this.#requestManagerTimeoutsApplied = true;
|
|
@@ -1006,7 +1006,7 @@ export class BasicCrawler {
|
|
|
1006
1006
|
// raise-only, so erring high here is safe.
|
|
1007
1007
|
const maxRouteTimeoutSecs = this.requestHandler.getMaxTimeoutSecs?.() ?? 0;
|
|
1008
1008
|
const handlerTimeoutSecs = Math.max(this.#requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
|
|
1009
|
-
await requestManager.
|
|
1009
|
+
await requestManager.setExpectedRequestProcessingTime?.(Math.max(handlerTimeoutSecs + 5, 60));
|
|
1010
1010
|
}
|
|
1011
1011
|
/**
|
|
1012
1012
|
* Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
|
|
@@ -202,13 +202,13 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
|
|
|
202
202
|
* site rather than of the run, so it survives.
|
|
203
203
|
*/
|
|
204
204
|
purge(): Promise<void>;
|
|
205
|
-
|
|
205
|
+
setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void>;
|
|
206
206
|
/**
|
|
207
207
|
* @inheritdoc
|
|
208
208
|
* Extending a lock must not consume the in-flight marker that later routes
|
|
209
209
|
* {@link ThrottlingRequestManager.markRequestAsHandled} / {@link ThrottlingRequestManager.reclaimRequest}.
|
|
210
210
|
*/
|
|
211
|
-
|
|
211
|
+
extendRequestProcessingTime(request: Request, extensionSecs: number): Promise<boolean>;
|
|
212
212
|
/**
|
|
213
213
|
* Returns the next request from a domain that is not backing off, or from the inner manager.
|
|
214
214
|
*
|
|
@@ -143,7 +143,7 @@ export class ThrottlingRequestManager {
|
|
|
143
143
|
#inProgressBatchCount = 0;
|
|
144
144
|
/** Requests moved into a sub-queue, which both managers count. Subtracted below; not persisted. */
|
|
145
145
|
#migratedFromInner = 0;
|
|
146
|
-
/** The latest {@link
|
|
146
|
+
/** The latest {@link setExpectedRequestProcessingTime} hint, kept for a wrapped manager resolved later. */
|
|
147
147
|
#expectedRequestProcessingSecs;
|
|
148
148
|
#warnedAbout = new Set();
|
|
149
149
|
/** Whether any domain at all may end up throttled - listed up front, or discovered as the crawl runs. */
|
|
@@ -229,7 +229,7 @@ export class ThrottlingRequestManager {
|
|
|
229
229
|
this.#innerPromise ??= Promise.resolve(this.#innerFactory());
|
|
230
230
|
this.#resolvedInner = await this.#innerPromise;
|
|
231
231
|
if (this.#expectedRequestProcessingSecs !== undefined) {
|
|
232
|
-
await this.#resolvedInner.
|
|
232
|
+
await this.#resolvedInner.setExpectedRequestProcessingTime?.(this.#expectedRequestProcessingSecs);
|
|
233
233
|
}
|
|
234
234
|
}
|
|
235
235
|
return this.#resolvedInner;
|
|
@@ -717,22 +717,22 @@ export class ThrottlingRequestManager {
|
|
|
717
717
|
state.lastRateLimitedAt = 0;
|
|
718
718
|
}
|
|
719
719
|
}
|
|
720
|
-
async
|
|
720
|
+
async setExpectedRequestProcessingTime(processingTimeSecs) {
|
|
721
721
|
// Remembered so a manager opened later still gets the hint, without opening one now just to pass it along.
|
|
722
|
-
this.#expectedRequestProcessingSecs =
|
|
723
|
-
await this.#forEachManager((manager) => manager.
|
|
722
|
+
this.#expectedRequestProcessingSecs = processingTimeSecs;
|
|
723
|
+
await this.#forEachManager((manager) => manager.setExpectedRequestProcessingTime?.(processingTimeSecs));
|
|
724
724
|
}
|
|
725
725
|
/**
|
|
726
726
|
* @inheritdoc
|
|
727
727
|
* Extending a lock must not consume the in-flight marker that later routes
|
|
728
728
|
* {@link ThrottlingRequestManager.markRequestAsHandled} / {@link ThrottlingRequestManager.reclaimRequest}.
|
|
729
729
|
*/
|
|
730
|
-
async
|
|
730
|
+
async extendRequestProcessingTime(request, extensionSecs) {
|
|
731
731
|
const key = request.id ?? request.uniqueKey;
|
|
732
732
|
const manager = this.#inFlightFromInner.has(key)
|
|
733
733
|
? this.#resolvedInner
|
|
734
734
|
: await this.#selectManagerOrThrow(request.url);
|
|
735
|
-
return (await manager.
|
|
735
|
+
return (await manager.extendRequestProcessingTime?.(request, extensionSecs)) ?? false;
|
|
736
736
|
}
|
|
737
737
|
/**
|
|
738
738
|
* Runs `fn` over the sub-queues and, if it has been resolved, the wrapped manager - bookkeeping never forces
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.277",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.1.0",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.277",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.277",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.277",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.277",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.1.0",
|
|
51
51
|
"csv-stringify": "^6.9.0",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.6.7"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.277"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "086a9f966a05dd1159f37cc25c4589b886fc5c93"
|
|
70
70
|
}
|