@crawlee/basic 4.0.0-beta.275 → 4.0.0-beta.277

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -624,7 +624,7 @@ export class BasicCrawler {
624
624
  context[navigationDeadlineKey] += extraMillis;
625
625
  }
626
626
  // Extension failure must not fail the request that asked for more time.
627
- this.requestManager?.extendRequestProcessingTimeSecs?.(context.request, secs)?.catch((error) => {
627
+ this.requestManager?.extendRequestProcessingTime?.(context.request, secs)?.catch((error) => {
628
628
  this.log.debug('Extending the request processing time failed', { url: context.request.url, error });
629
629
  });
630
630
  },
@@ -969,7 +969,7 @@ export class BasicCrawler {
969
969
  this.#requestManager = await this.#openOwnedRequestQueue();
970
970
  }
971
971
  // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
972
- // now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,
972
+ // now that `setExpectedRequestProcessingTime` is async. The hint is raise-only and idempotent,
973
973
  // but guard so we do not re-issue it on every call.
974
974
  if (!this.#requestManagerTimeoutsApplied) {
975
975
  this.#requestManagerTimeoutsApplied = true;
@@ -1006,7 +1006,7 @@ export class BasicCrawler {
1006
1006
  // raise-only, so erring high here is safe.
1007
1007
  const maxRouteTimeoutSecs = this.requestHandler.getMaxTimeoutSecs?.() ?? 0;
1008
1008
  const handlerTimeoutSecs = Math.max(this.#requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
1009
- await requestManager.setExpectedRequestProcessingTimeSecs?.(Math.max(handlerTimeoutSecs + 5, 60));
1009
+ await requestManager.setExpectedRequestProcessingTime?.(Math.max(handlerTimeoutSecs + 5, 60));
1010
1010
  }
1011
1011
  /**
1012
1012
  * Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
@@ -202,13 +202,13 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
202
202
  * site rather than of the run, so it survives.
203
203
  */
204
204
  purge(): Promise<void>;
205
- setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
205
+ setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void>;
206
206
  /**
207
207
  * @inheritdoc
208
208
  * Extending a lock must not consume the in-flight marker that later routes
209
209
  * {@link ThrottlingRequestManager.markRequestAsHandled} / {@link ThrottlingRequestManager.reclaimRequest}.
210
210
  */
211
- extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean>;
211
+ extendRequestProcessingTime(request: Request, extensionSecs: number): Promise<boolean>;
212
212
  /**
213
213
  * Returns the next request from a domain that is not backing off, or from the inner manager.
214
214
  *
@@ -143,7 +143,7 @@ export class ThrottlingRequestManager {
143
143
  #inProgressBatchCount = 0;
144
144
  /** Requests moved into a sub-queue, which both managers count. Subtracted below; not persisted. */
145
145
  #migratedFromInner = 0;
146
- /** The latest {@link setExpectedRequestProcessingTimeSecs} hint, kept for a wrapped manager resolved later. */
146
+ /** The latest {@link setExpectedRequestProcessingTime} hint, kept for a wrapped manager resolved later. */
147
147
  #expectedRequestProcessingSecs;
148
148
  #warnedAbout = new Set();
149
149
  /** Whether any domain at all may end up throttled - listed up front, or discovered as the crawl runs. */
@@ -229,7 +229,7 @@ export class ThrottlingRequestManager {
229
229
  this.#innerPromise ??= Promise.resolve(this.#innerFactory());
230
230
  this.#resolvedInner = await this.#innerPromise;
231
231
  if (this.#expectedRequestProcessingSecs !== undefined) {
232
- await this.#resolvedInner.setExpectedRequestProcessingTimeSecs?.(this.#expectedRequestProcessingSecs);
232
+ await this.#resolvedInner.setExpectedRequestProcessingTime?.(this.#expectedRequestProcessingSecs);
233
233
  }
234
234
  }
235
235
  return this.#resolvedInner;
@@ -717,22 +717,22 @@ export class ThrottlingRequestManager {
717
717
  state.lastRateLimitedAt = 0;
718
718
  }
719
719
  }
720
- async setExpectedRequestProcessingTimeSecs(secs) {
720
+ async setExpectedRequestProcessingTime(processingTimeSecs) {
721
721
  // Remembered so a manager opened later still gets the hint, without opening one now just to pass it along.
722
- this.#expectedRequestProcessingSecs = secs;
723
- await this.#forEachManager((manager) => manager.setExpectedRequestProcessingTimeSecs?.(secs));
722
+ this.#expectedRequestProcessingSecs = processingTimeSecs;
723
+ await this.#forEachManager((manager) => manager.setExpectedRequestProcessingTime?.(processingTimeSecs));
724
724
  }
725
725
  /**
726
726
  * @inheritdoc
727
727
  * Extending a lock must not consume the in-flight marker that later routes
728
728
  * {@link ThrottlingRequestManager.markRequestAsHandled} / {@link ThrottlingRequestManager.reclaimRequest}.
729
729
  */
730
- async extendRequestProcessingTimeSecs(request, secs) {
730
+ async extendRequestProcessingTime(request, extensionSecs) {
731
731
  const key = request.id ?? request.uniqueKey;
732
732
  const manager = this.#inFlightFromInner.has(key)
733
733
  ? this.#resolvedInner
734
734
  : await this.#selectManagerOrThrow(request.url);
735
- return (await manager.extendRequestProcessingTimeSecs?.(request, secs)) ?? false;
735
+ return (await manager.extendRequestProcessingTime?.(request, extensionSecs)) ?? false;
736
736
  }
737
737
  /**
738
738
  * Runs `fn` over the sub-queues and, if it has been resolved, the wrapped manager - bookkeeping never forces
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/basic",
3
- "version": "4.0.0-beta.275",
3
+ "version": "4.0.0-beta.277",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -42,10 +42,10 @@
42
42
  "@apify/datastructures": "^3.0.1",
43
43
  "@apify/timeout": "^1.0.1",
44
44
  "@apify/utilities": "^3.1.0",
45
- "@crawlee/core": "4.0.0-beta.275",
46
- "@crawlee/http-client": "4.0.0-beta.275",
47
- "@crawlee/types": "4.0.0-beta.275",
48
- "@crawlee/utils": "4.0.0-beta.275",
45
+ "@crawlee/core": "4.0.0-beta.277",
46
+ "@crawlee/http-client": "4.0.0-beta.277",
47
+ "@crawlee/types": "4.0.0-beta.277",
48
+ "@crawlee/utils": "4.0.0-beta.277",
49
49
  "@sapphire/async-queue": "^1.5.5",
50
50
  "@standard-schema/spec": "^1.1.0",
51
51
  "csv-stringify": "^6.9.0",
@@ -57,7 +57,7 @@
57
57
  "zod": "^4.6.7"
58
58
  },
59
59
  "optionalDependencies": {
60
- "@crawlee/impit-client": "4.0.0-beta.275"
60
+ "@crawlee/impit-client": "4.0.0-beta.277"
61
61
  },
62
62
  "lerna": {
63
63
  "command": {
@@ -66,5 +66,5 @@
66
66
  }
67
67
  }
68
68
  },
69
- "gitHead": "740f25703dfbc1f006c70a22488c816d6858bdce"
69
+ "gitHead": "086a9f966a05dd1159f37cc25c4589b886fc5c93"
70
70
  }