@crawlee/basic 4.0.0-beta.193 → 4.0.0-beta.195

package/internals/basic-crawler.js

@@ -659,6 +659,10 @@ export class BasicCrawler {
                 if (context[navigationDeadlineKey] !== undefined) {
                     context[navigationDeadlineKey] += extraMillis;
                 }
+                // Extension failure must not fail the request that asked for more time.
+                this.requestManager?.extendRequestProcessingTimeSecs?.(context.request, secs)?.catch((error) => {
+                    this.log.debug('Extending the request processing time failed', { url: context.request.url, error });
+                });
             },
         };
     }

package/internals/crawlers/crawler_commons.d.ts

@@ -186,8 +186,9 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
      * ```
      *
      * Extends the request handler's own timeout and the crawler's internal one together, so the extension
-     * is not immediately undone by the latter. Calling it from a handler that has already timed out does
-     * nothing.
+     * is not immediately undone by the latter. On a locking storage backend the request's lock is
+     * extended by the same amount, so the extra time is not spent while the queue considers the request
+     * free to hand out again. Calling it from a handler that has already timed out does nothing.
      */
     extendTimeout(secs: number): void;
 }

package/internals/throttling_request_manager.d.ts

@@ -199,6 +199,12 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
      */
     purge(): Promise<void>;
     setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+    /**
+     * @inheritdoc
+     * Extending a lock must not consume the in-flight marker that later routes
+     * {@link ThrottlingRequestManager.markRequestAsHandled} / {@link ThrottlingRequestManager.reclaimRequest}.
+     */
+    extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean>;
     /**
      * Returns the next request from a domain that is not backing off, or from the inner manager.
      *

package/internals/throttling_request_manager.js

@@ -704,6 +704,18 @@ export class ThrottlingRequestManager {
         this.#expectedRequestProcessingSecs = secs;
         await this.#forEachManager((manager) => manager.setExpectedRequestProcessingTimeSecs?.(secs));
     }
+    /**
+     * @inheritdoc
+     * Extending a lock must not consume the in-flight marker that later routes
+     * {@link ThrottlingRequestManager.markRequestAsHandled} / {@link ThrottlingRequestManager.reclaimRequest}.
+     */
+    async extendRequestProcessingTimeSecs(request, secs) {
+        const key = request.id ?? request.uniqueKey;
+        const manager = this.#inFlightFromInner.has(key)
+            ? this.#resolvedInner
+            : await this.#selectManagerOrThrow(request.url);
+        return (await manager.extendRequestProcessingTimeSecs?.(request, secs)) ?? false;
+    }
     /**
      * Runs `fn` over the sub-queues and, if it has been resolved, the wrapped manager - bookkeeping never forces
      * a lazily-opened `inner`, since there is no point opening a queue purely to tell it something.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.193",
+    "version": "4.0.0-beta.195",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/core": "4.0.0-beta.193",
-        "@crawlee/http-client": "4.0.0-beta.193",
-        "@crawlee/types": "4.0.0-beta.193",
-        "@crawlee/utils": "4.0.0-beta.193",
+        "@crawlee/core": "4.0.0-beta.195",
+        "@crawlee/http-client": "4.0.0-beta.195",
+        "@crawlee/types": "4.0.0-beta.195",
+        "@crawlee/utils": "4.0.0-beta.195",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "csv-stringify": "^6.5.2",
@@ -57,7 +57,7 @@
         "zod": "^4.5.4"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "4.0.0-beta.193"
+        "@crawlee/impit-client": "4.0.0-beta.195"
     },
     "lerna": {
         "command": {
@@ -66,5 +66,5 @@
             }
         }
     },
-    "gitHead": "db10ed8633118f3d53dcef58118f408ce578d5d5"
+    "gitHead": "7ef1df353c43614acc1275a53ef8ca96e850fe3e"
 }