@crawlee/basic 4.0.0-beta.96 → 4.0.0-beta.97

package/internals/basic-crawler.d.ts

@@ -6,6 +6,7 @@ import type { ReadonlyDeep, SetRequired } from 'type-fest';
 import { TimeoutError } from '@apify/timeout';
 export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
 }
+export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
 export type RequestHandler<Context extends CrawlingContext = CrawlingContext> = (inputs: Context) => Awaitable<void>;
 /**
  * An error handler receives the crawling context and the error that was thrown while processing the request.
@@ -769,6 +770,32 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * Initializes the crawler.
      */
     protected _init(): Promise<void>;
+    /**
+     * The navigation timeout (pre-navigation hooks, navigation, and post-navigation hooks) in milliseconds, used
+     * to size the internal request timeout. `BasicCrawler` has no navigation phase, so this is 0; the HTTP and
+     * browser crawlers override it with their `navigationTimeoutSecs`.
+     */
+    protected getNavigationTimeoutMillis(): number;
+    /**
+     * Races the request against the internal timeout (see {@link raceWithTimeout}), sized to outlast the phases
+     * that have their own timeout - the navigation, its hooks, and the request handler - so a legitimately slow
+     * request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes
+     * whichever is larger: the configured internal timeout, or this request's combined phase budget.
+     */
+    private withRequestTimeout;
+    /**
+     * The request handler timeout for a request with the given route label. A router route may override the
+     * crawler's own `requestHandlerTimeoutSecs`; anything else falls back to `fallbackMillis`.
+     *
+     * @param label The request's route label, or `undefined` for the default route / no specific request.
+     * @param fallbackMillis Timeout to use when no route overrides it.
+     */
+    private resolveRequestHandlerTimeoutMillis;
+    /**
+     * The timeout the router route with the given label asked for, or `undefined` when it did not override one
+     * (or the request handler is not a router at all).
+     */
+    private getRouteTimeoutMillis;
     protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
     /**
      * Handles blocked request
@@ -893,4 +920,3 @@ export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
  */
 export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
 export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
-export {};

package/internals/basic-crawler.js

@@ -8,8 +8,9 @@ import { ensureDir, writeJSON } from 'fs-extra/esm';
 import ow, { ArgumentError } from 'ow';
 import { getDomain } from 'tldts';
 import { LruCache } from '@apify/datastructures';
-import { addTimeoutToPromise, TimeoutError } from '@apify/timeout';
+import { addTimeoutToPromise, extendTimeout, TimeoutError } from '@apify/timeout';
 import { cryptoRandomObjectId } from '@apify/utilities';
+import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, timeoutExpiredKey, } from './request-timeout.js';
 import { createSendRequest } from './send-request.js';
 class LazyDefaultHttpClient {
     _delegatePromise;
@@ -37,6 +38,8 @@ class LazyDefaultHttpClient {
  */
 const SAFE_MIGRATION_WAIT_MILLIS = 20000;
 const deferredCleanupKey = Symbol('deferredCleanup');
+// The request timeout plumbing (the window helper, the context symbols, and the race) lives in its own module.
+export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
 export class BasicCrawler {
     static CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
     /**
@@ -314,10 +317,10 @@ export class BasicCrawler {
             this.retryOnBlocked = retryOnBlocked;
             this.respectRobotsTxtFile = respectRobotsTxtFile;
             this.onSkippedRequest = onSkippedRequest;
-            const tryEnv = (val) => (val == null ? null : +val);
             // allow at least 5min for internal timeouts
             this.internalTimeoutMillis =
-                tryEnv(process.env.CRAWLEE_INTERNAL_TIMEOUT) ?? Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
+                serviceLocator.getConfiguration().internalTimeoutMillis ??
+                    Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
             this.maxRequestRetries = maxRequestRetries;
             this.maxCrawlDepth = maxCrawlDepth;
             this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
@@ -370,9 +373,12 @@ export class BasicCrawler {
                     this.stats.startJob(request.id || request.uniqueKey);
                     const crawlingContext = { request };
                     try {
-                        await this.basicContextPipeline
+                        // Navigation, the navigation hooks and the request handler are timed individually, but the
+                        // phases between them are not, so a request could still get stuck indefinitely. This is the
+                        // catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
+                        await this.withRequestTimeout(crawlingContext, this.basicContextPipeline
                             .chain(this.contextPipeline)
-                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request));
+                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request)));
                     }
                     catch (error) {
                         // ContextPipelineInterruptedError means the request was intentionally skipped
@@ -383,8 +389,11 @@ export class BasicCrawler {
                             return;
                         }
                         // If the error happened during pipeline initialization (e.g., navigation timeout, session/proxy error,
-                        // i.e. not in user's requestHandler), handle it through the normal error flow.
-                        const isPipelineError = error instanceof ContextPipelineInitializationError || error instanceof SessionError;
+                        // i.e. not in user's requestHandler), handle it through the normal error flow. A bare `TimeoutError`
+                        // here is the internal timeout above firing - anything else thrown inside the pipeline arrives wrapped.
+                        const isPipelineError = error instanceof ContextPipelineInitializationError ||
+                            error instanceof SessionError ||
+                            error instanceof TimeoutError;
                         if (isPipelineError) {
                             const unwrappedError = this.unwrapError(error);
                             await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
@@ -481,7 +490,7 @@ export class BasicCrawler {
     buildBasicContextPipeline() {
         return ContextPipeline.create()
             .compose({ action: this.checkRobotsTxt.bind(this) })
-            .compose({ action: () => this.createBaseContext() })
+            .compose({ action: (context) => this.createBaseContext(context) })
             .compose({ action: this.resolveSession.bind(this) })
             .compose({ action: this.createContextHelpers.bind(this) });
     }
@@ -505,7 +514,7 @@ export class BasicCrawler {
     buildContextPipeline() {
         return ContextPipeline.create();
     }
-    createBaseContext() {
+    createBaseContext(context) {
         const deferredCleanup = [];
         return {
             id: cryptoRandomObjectId(10),
@@ -516,6 +525,18 @@ export class BasicCrawler {
             registerDeferredCleanup: (cleanup) => {
                 deferredCleanup.push(cleanup);
             },
+            extendTimeout: (secs) => {
+                const extraMillis = secs * 1000;
+                // the current `addTimeoutToPromise` window (the request handler, or a navigation hook)...
+                extendTimeout(extraMillis);
+                // ...the internal timeout around the whole request, which is not an `addTimeoutToPromise` frame...
+                context[extendTimeoutKey]?.(extraMillis);
+                // ...and, when called from within the navigation phase, its shared window, so extending a hook
+                // extends the whole navigation budget rather than just that hook's step.
+                if (context[navigationDeadlineKey] !== undefined) {
+                    context[navigationDeadlineKey] += extraMillis;
+                }
+            },
             [deferredCleanupKey]: deferredCleanup,
         };
     }
@@ -856,7 +877,12 @@ export class BasicCrawler {
      * regardless of whether the manager is a plain {@link RequestQueue} or a `RequestManagerTandem`.
      */
     async applyRequestManagerTimeouts(requestManager) {
-        await requestManager.setExpectedRequestProcessingTimeSecs?.(Math.max(this.requestHandlerTimeoutMillis / 1000 + 5, 60));
+        // A router route may hold a request for longer than the crawler's own timeout, and we cannot know
+        // which routes a run will hit, so reserve for the longest one any route asked for. The hint is
+        // raise-only, so erring high here is safe.
+        const maxRouteTimeoutSecs = this.requestHandler.getMaxTimeoutSecs?.() ?? 0;
+        const handlerTimeoutSecs = Math.max(this.requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
+        await requestManager.setExpectedRequestProcessingTimeSecs?.(Math.max(handlerTimeoutSecs + 5, 60));
     }
     /**
      * Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
@@ -1068,6 +1094,16 @@ export class BasicCrawler {
             await eventManager.init();
             this._closeEvents = true;
         }
+        // Warn once at startup if the internal timeout is shorter than the phases it is meant to outlast. It is
+        // floored per request so it will not actually cut them short, but the configured value is then effectively
+        // ignored, which is worth flagging. Checked here (not in the constructor) because a subclass sets its
+        // navigation timeout only after `super()`.
+        const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(undefined);
+        if (this.internalTimeoutMillis < phasesMillis) {
+            this.log.warning(`CRAWLEE_INTERNAL_TIMEOUT (${this.internalTimeoutMillis / 1000}s) is shorter than the navigation ` +
+                `and request handler timeouts combined (${phasesMillis / 1000}s); it will be raised per request ` +
+                `so it does not cut them short.`);
+        }
         // An owned governor is rebuilt (and started) for every run, so it always starts from a clean slate — stale
         // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this run's
         // scaling. An injected one is long-lived and its lifecycle belongs to the caller.
@@ -1080,8 +1116,51 @@ export class BasicCrawler {
         });
         await this.getRequestManager();
     }
+    /**
+     * The navigation timeout (pre-navigation hooks, navigation, and post-navigation hooks) in milliseconds, used
+     * to size the internal request timeout. `BasicCrawler` has no navigation phase, so this is 0; the HTTP and
+     * browser crawlers override it with their `navigationTimeoutSecs`.
+     */
+    getNavigationTimeoutMillis() {
+        return 0;
+    }
+    /**
+     * Races the request against the internal timeout (see {@link raceWithTimeout}), sized to outlast the phases
+     * that have their own timeout - the navigation, its hooks, and the request handler - so a legitimately slow
+     * request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes
+     * whichever is larger: the configured internal timeout, or this request's combined phase budget.
+     */
+    async withRequestTimeout(crawlingContext, work) {
+        const { request } = crawlingContext;
+        const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(request.label);
+        const timeoutMillis = Math.max(this.internalTimeoutMillis, phasesMillis);
+        await raceWithTimeout(crawlingContext, work, { timeoutMillis, requestId: request.id });
+    }
+    /**
+     * The request handler timeout for a request with the given route label. A router route may override the
+     * crawler's own `requestHandlerTimeoutSecs`; anything else falls back to `fallbackMillis`.
+     *
+     * @param label The request's route label, or `undefined` for the default route / no specific request.
+     * @param fallbackMillis Timeout to use when no route overrides it.
+     */
+    resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.requestHandlerTimeoutMillis) {
+        return this.getRouteTimeoutMillis(label) ?? fallbackMillis;
+    }
+    /**
+     * The timeout the router route with the given label asked for, or `undefined` when it did not override one
+     * (or the request handler is not a router at all).
+     */
+    getRouteTimeoutMillis(label) {
+        const getTimeoutSecs = this.requestHandler.getTimeoutSecs;
+        if (typeof getTimeoutSecs !== 'function') {
+            return undefined;
+        }
+        const timeoutSecs = getTimeoutSecs(label);
+        return timeoutSecs === undefined ? undefined : timeoutSecs * 1000;
+    }
     async runRequestHandler(crawlingContext) {
-        await addTimeoutToPromise(async () => this.requestHandler(crawlingContext), this.requestHandlerTimeoutMillis, `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds (${crawlingContext.request.id}).`);
+        const timeoutMillis = this.resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);
+        await addTimeoutToPromise(async () => this.requestHandler(crawlingContext), timeoutMillis, `requestHandler timed out after ${timeoutMillis / 1000} seconds (${crawlingContext.request.id}).`);
     }
     /**
      * Handles blocked request
@@ -1189,6 +1268,12 @@ export class BasicCrawler {
     }
     /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
     async handleRequest(crawlingContext, requestSource, request) {
+        // An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
+        // which already failed the request in `runTaskFunction`. Bail before running the handler so it does not
+        // execute (and re-report) on top of a request the crawler has already moved past.
+        if (crawlingContext[timeoutExpiredKey]?.()) {
+            return;
+        }
         const statisticsId = request.id || request.uniqueKey;
         let isRequestLocked = true;
         try {

package/internals/request-timeout.d.ts

@@ -0,0 +1,48 @@
+/**
+ * The shared navigation-window deadline (epoch millis), stored on the in-flight context so the pre- and
+ * post-navigation hooks and the navigation itself all draw from one budget - and so `context.extendTimeout`
+ * can push the whole window, not just the current step.
+ * @internal
+ */
+export declare const navigationDeadlineKey: unique symbol;
+/**
+ * Lets `context.extendTimeout` push back the internal request timeout as well. That timeout is a bare timer
+ * rather than an `addTimeoutToPromise` frame, so `extendTimeout` from `@apify/timeout` cannot reach it on its own.
+ */
+export declare const extendTimeoutKey: unique symbol;
+/**
+ * Returns `true` once the internal request timeout has fired for this request. It cannot cancel work stuck
+ * somewhere we do not control, but the phases we do run (e.g. the request handler) check this at their start and
+ * bail, so a request whose timeout already elapsed does not carry on after the crawler moved past it.
+ */
+export declare const timeoutExpiredKey: unique symbol;
+/** The slots the internal request timeout hangs on the in-flight crawling context. */
+export interface RequestTimeoutContext {
+    [extendTimeoutKey]?: (extraMillis: number) => void;
+    [timeoutExpiredKey]?: () => boolean;
+    [navigationDeadlineKey]?: number;
+}
+/**
+ * Milliseconds left in the shared navigation window for `ctx`, lazily starting the window on first use.
+ * @internal
+ */
+export declare function remainingNavigationWindowMillis(ctx: object, windowMillis: number): number;
+/**
+ * Races `work` against a `timeoutMillis` timeout, so a request stuck in a phase that has no timeout of its own
+ * (anything that is not the navigation, a navigation hook or the request handler) still fails instead of stalling
+ * the crawler.
+ *
+ * This is a bare timer, not an `addTimeoutToPromise` frame: nested `addTimeoutToPromise` calls share one
+ * `AbortController`, so wrapping the request in one would let the request handler timing out abort this outer
+ * context too, cancelling the error handling that follows it.
+ *
+ * `Promise.race` attaches a handler to both sides, so a late rejection from `work` cannot go unhandled. The losing
+ * side is not cancelled - by definition it is stuck somewhere we do not control - but the phases we do run check
+ * {@link timeoutExpiredKey} at their start and bail, so a request whose timeout already fired does not carry
+ * on (e.g. run the handler) after the crawler has moved past it. `context.extendTimeout` pushes the deadline back
+ * via {@link extendTimeoutKey}.
+ */
+export declare function raceWithTimeout(context: RequestTimeoutContext, work: Promise<void>, { timeoutMillis, requestId }: {
+    timeoutMillis: number;
+    requestId?: string;
+}): Promise<void>;

package/internals/request-timeout.js

@@ -0,0 +1,73 @@
+import { TimeoutError } from '@apify/timeout';
+/**
+ * The shared navigation-window deadline (epoch millis), stored on the in-flight context so the pre- and
+ * post-navigation hooks and the navigation itself all draw from one budget - and so `context.extendTimeout`
+ * can push the whole window, not just the current step.
+ * @internal
+ */
+export const navigationDeadlineKey = Symbol('navigationDeadline');
+/**
+ * Lets `context.extendTimeout` push back the internal request timeout as well. That timeout is a bare timer
+ * rather than an `addTimeoutToPromise` frame, so `extendTimeout` from `@apify/timeout` cannot reach it on its own.
+ */
+export const extendTimeoutKey = Symbol('extendTimeout');
+/**
+ * Returns `true` once the internal request timeout has fired for this request. It cannot cancel work stuck
+ * somewhere we do not control, but the phases we do run (e.g. the request handler) check this at their start and
+ * bail, so a request whose timeout already elapsed does not carry on after the crawler moved past it.
+ */
+export const timeoutExpiredKey = Symbol('timeoutExpired');
+/**
+ * Milliseconds left in the shared navigation window for `ctx`, lazily starting the window on first use.
+ * @internal
+ */
+export function remainingNavigationWindowMillis(ctx, windowMillis) {
+    const store = ctx;
+    store[navigationDeadlineKey] ??= Date.now() + windowMillis;
+    return store[navigationDeadlineKey] - Date.now();
+}
+/**
+ * Races `work` against a `timeoutMillis` timeout, so a request stuck in a phase that has no timeout of its own
+ * (anything that is not the navigation, a navigation hook or the request handler) still fails instead of stalling
+ * the crawler.
+ *
+ * This is a bare timer, not an `addTimeoutToPromise` frame: nested `addTimeoutToPromise` calls share one
+ * `AbortController`, so wrapping the request in one would let the request handler timing out abort this outer
+ * context too, cancelling the error handling that follows it.
+ *
+ * `Promise.race` attaches a handler to both sides, so a late rejection from `work` cannot go unhandled. The losing
+ * side is not cancelled - by definition it is stuck somewhere we do not control - but the phases we do run check
+ * {@link timeoutExpiredKey} at their start and bail, so a request whose timeout already fired does not carry
+ * on (e.g. run the handler) after the crawler has moved past it. `context.extendTimeout` pushes the deadline back
+ * via {@link extendTimeoutKey}.
+ */
+export async function raceWithTimeout(context, work, { timeoutMillis, requestId }) {
+    let timer;
+    let deadline = Date.now() + timeoutMillis;
+    let settled = false;
+    let firedByTimeout = false;
+    const timeout = new Promise((_, reject) => {
+        const fire = () => {
+            settled = true;
+            firedByTimeout = true;
+            reject(new TimeoutError(`Request timed out after ${timeoutMillis / 1e3} seconds (${requestId}).`));
+        };
+        timer = setTimeout(fire, timeoutMillis);
+        context[timeoutExpiredKey] = () => firedByTimeout;
+        context[extendTimeoutKey] = (extraMillis) => {
+            if (settled) {
+                return;
+            }
+            clearTimeout(timer);
+            deadline += extraMillis;
+            timer = setTimeout(fire, Math.max(deadline - Date.now(), 0));
+        };
+    });
+    try {
+        await Promise.race([work, timeout]);
+    }
+    finally {
+        settled = true;
+        clearTimeout(timer);
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.96",
+    "version": "4.0.0-beta.97",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -40,12 +40,12 @@
     },
     "dependencies": {
         "@apify/datastructures": "^2.0.0",
-        "@apify/timeout": "^0.3.2",
+        "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.96",
-        "@crawlee/http-client": "4.0.0-beta.96",
-        "@crawlee/types": "4.0.0-beta.96",
-        "@crawlee/utils": "4.0.0-beta.96",
+        "@crawlee/core": "4.0.0-beta.97",
+        "@crawlee/http-client": "4.0.0-beta.97",
+        "@crawlee/types": "4.0.0-beta.97",
+        "@crawlee/utils": "4.0.0-beta.97",
         "csv-stringify": "^6.5.2",
         "fs-extra": "^11.3.0",
         "ow": "^2.0.0",
@@ -54,7 +54,7 @@
         "type-fest": "^4.41.0"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.96"
+        "@crawlee/impit-client": "^4.0.0-beta.97"
     },
     "lerna": {
         "command": {
@@ -63,5 +63,5 @@
             }
         }
     },
-    "gitHead": "b5c20fe6cecad57a6e3dd26b4137776d4b62588a"
+    "gitHead": "43008dd43f4832d083353ba1b731c0b4607c9cfa"
 }