@crawlee/basic 4.0.0-beta.207 → 4.0.0-beta.209

package/internals/basic-crawler.d.ts

@@ -684,8 +684,8 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
     /**
      * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
-     * Deliberately does **not** commit on return - `handleRequest` swallows request handler failures, so
-     * a normal return says nothing about success. `handleRequest` owns the outcome.
+     * Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
+     * normal return says nothing about success. `handleRequest` commits on success.
      */
     private runInStorageTransaction;
     /**
@@ -710,7 +710,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * Fetches the next request to process from the underlying request provider.
      */
     private fetchNextRequest;
-    /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
+    /** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
     private handleRequest;
     /**
      * Generator function that yields requests injected with the given crawl depth.
@@ -729,7 +729,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     private isTaskReadyFunction;
     /**
      * Unwraps errors thrown by the context pipeline to get the actual user error.
-     * RequestHandlerError and ContextPipelineCleanupError wrap the actual error.
+     * RequestHandlerError, ContextPipelineInitializationError and ContextPipelineCleanupError wrap the actual error.
      */
     private unwrapError;
     /**

package/internals/basic-crawler.js

@@ -13,12 +13,12 @@ import { ConcurrencySystem } from './autoscaling/concurrency_system.js';
 import { mergeCookies } from './cookie_utils.js';
 import { ContextPipeline } from './crawlers/context_pipeline.js';
 import { CrawlingRequest, RequestState } from './crawling_request.js';
-import { ContextPipelineCleanupError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
+import { ContextPipelineCleanupError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
 import { Statistics } from './crawlers/statistics.js';
 import { applyRequestTransform, buildEnqueueStrategyPatterns, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue_links/index.js';
 import { parseRetryAfterHeader } from './http.js';
 import { CrawlerRun } from './crawler-run.js';
-import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, timeoutExpiredKey, } from './request-timeout.js';
+import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, suspendTimeoutKey, timeoutExpiredKey, } from './request-timeout.js';
 import { createSendRequest } from './send-request.js';
 import { Router, validateUserData } from './router.js';
 import { BLOCKED_STATUS_CODES } from './session_pool/consts.js';
@@ -462,44 +462,19 @@ export class BasicCrawler {
                     // matching `recordRequestStart`.
                     this.statistics.recordRequestStart(request.id || request.uniqueKey);
                     const crawlingContext = { request };
-                    // Runs inside the pipeline call, so the deferred cleanups registered by earlier middlewares
-                    // (e.g. closing the page) only fire after the error handler has seen the partial context.
-                    const handleInitializationError = async (rawError) => {
-                        // Roll back *before* the error handler runs, for the same reason `handleRequest` does.
-                        currentStorageTransaction()?.rollback();
-                        // ContextPipelineInterruptedError means the request was intentionally skipped
-                        // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
-                        if (rawError instanceof ContextPipelineInterruptedError) {
-                            this.statistics.discardRequestRecord(request.id || request.uniqueKey);
-                            await this.timeoutAndRetry(async () => this.requestManager?.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-                            return;
-                        }
-                        // An error during pipeline initialization (e.g., navigation timeout, session/proxy error,
-                        // i.e. not in user's requestHandler) goes through the normal error flow.
-                        const error = this.unwrapError(rawError);
-                        await this.requestFunctionErrorHandler(error, crawlingContext, request, this.requestManager);
-                        // SessionError already retired the session in `requestFunctionErrorHandler`;
-                        // skip `markBad` to avoid double-counting usage/error score.
-                        if (!this.errorAbsolvesSession(error)) {
-                            crawlingContext.session?.markBad();
-                        }
-                    };
                     try {
                         // The transaction spans the whole pipeline call, covering the navigation hooks
-                        // and `extendContext` too; `handleRequest` drives its outcome explicitly.
+                        // and `extendContext` too; `handleRequest` commits it and `handleRequestFailure` rolls it back.
                         await this.runInStorageTransaction(async () => 
                         // Navigation, the navigation hooks and the request handler are timed individually, but the
                         // phases between them are not, so a request could still get stuck indefinitely. This is the
                         // catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
-                        await this.withRequestTimeout(crawlingContext, this.#basicPipeline
-                            .chain(this.contextPipeline)
-                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request), handleInitializationError)));
+                        await this.withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request), (error) => this.#handleRequestFailure(error, crawlingContext, source, request))));
                     }
                     catch (error) {
-                        // A bare `TimeoutError` is the internal timeout above firing; a `SessionError` can only be a
-                        // secondary error from `handleRequest` - both take the normal error flow. Anything else thrown
-                        // inside the pipeline arrives wrapped.
-                        if (error instanceof SessionError || error instanceof TimeoutError) {
+                        // The internal timeout above fired, or `handleRequestFailure` threw a `SessionError` - both
+                        // take the normal error flow. Anything else is fatal.
+                        if (error instanceof SessionError || crawlingContext[timeoutExpiredKey]?.()) {
                             const unwrappedError = this.unwrapError(error);
                             await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
                             if (!this.errorAbsolvesSession(unwrappedError)) {
@@ -1373,8 +1348,8 @@ export class BasicCrawler {
     }
     /**
      * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
-     * Deliberately does **not** commit on return - `handleRequest` swallows request handler failures, so
-     * a normal return says nothing about success. `handleRequest` owns the outcome.
+     * Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
+     * normal return says nothing about success. `handleRequest` commits on success.
      */
     async runInStorageTransaction(callback) {
         if (!this.#transactionalStorageEnabled) {
@@ -1392,7 +1367,7 @@ export class BasicCrawler {
         }
         finally {
             if (transaction.state === 'open') {
-                // `handleRequest` commits or rolls back on every normal path, so an open transaction on
+                // `handleRequest` commits or `handleRequestFailure` rolls back on every normal path, so an open transaction on
                 // a normal return is a wiring bug; on a propagating throw (a pipeline-level failure) it is
                 // expected. Either way, discard the unvalidated writes; only the former is worth flagging.
                 if (!threw) {
@@ -1515,7 +1490,7 @@ export class BasicCrawler {
         }
         return this.requestManager.fetchNextRequest();
     }
-    /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
+    /** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
     // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async handleRequest(crawlingContext, requestSource, request) {
         // An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
@@ -1524,68 +1499,84 @@ export class BasicCrawler {
         if (crawlingContext[timeoutExpiredKey]?.()) {
             return;
         }
-        const statisticsId = request.id || request.uniqueKey;
+        request.state = RequestState.REQUEST_HANDLER;
+        await this.runRequestHandler(crawlingContext);
+        // Commit *before* marking the request as handled, so a commit failure fails the request and
+        // it is retried. This also closes the transaction, so everything below passes through.
         // Opened by `runInStorageTransaction`; absent when disabled or when the subclass opens its own.
-        const transaction = currentStorageTransaction();
-        let isRequestLocked = true;
+        await currentStorageTransaction()?.commit();
+        await this.timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+        this.statistics.recordRequestSuccess(request.id || request.uniqueKey, request.retryCount);
+        // reclaim session if request finishes successfully
+        request.state = RequestState.DONE;
+        crawlingContext.session.markGood();
+    }
+    /**
+     * Handles a request that failed anywhere in the context pipeline - a middleware (e.g. navigation or its hooks)
+     * or `handleRequest`. Runs inside the pipeline call, so the cleanups (e.g. closing the page) only fire after the
+     * error handler has seen the context.
+     */
+    async #handleRequestFailure(rawError, crawlingContext, requestSource, request) {
+        // Roll back *before* any error handler runs - error handlers write to real storage precisely
+        // because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.
+        currentStorageTransaction()?.rollback();
+        const skipped = rawError instanceof ContextPipelineInitializationError &&
+            rawError.cause instanceof ContextPipelineInterruptedError;
+        const err = this.unwrapError(rawError);
+        const context = crawlingContext;
+        const retryCountBefore = request.retryCount;
+        const resumeRequestTimeout = crawlingContext[suspendTimeoutKey]?.();
         try {
-            request.state = RequestState.REQUEST_HANDLER;
-            await this.runRequestHandler(crawlingContext);
-            // Commit *before* marking the request as handled, so a commit failure fails the request and
-            // it is retried. This also closes the transaction, so everything below passes through.
-            await transaction?.commit();
-            await this.timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-            isRequestLocked = false; // markRequestAsHandled succeeded and unlocked the request
-            this.statistics.recordRequestSuccess(statisticsId, request.retryCount);
-            // reclaim session if request finishes successfully
+            // ContextPipelineInterruptedError means the request was intentionally skipped
+            // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
+            if (skipped) {
+                this.statistics.discardRequestRecord(request.id || request.uniqueKey);
+                await this.timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+                return;
+            }
+            request.state = RequestState.ERROR_HANDLER;
+            await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context, request, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
             request.state = RequestState.DONE;
-            crawlingContext.session.markGood();
         }
-        catch (rawError) {
-            // Roll back *before* any error handler runs - error handlers write to real storage precisely
-            // because the transaction is already closed. A no-op when the commit above succeeded.
-            transaction?.rollback();
-            const err = this.unwrapError(rawError);
-            try {
-                request.state = RequestState.ERROR_HANDLER;
-                await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, crawlingContext, request, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-                if (!(err instanceof CriticalError)) {
-                    isRequestLocked = false; // requestFunctionErrorHandler calls either markRequestAsHandled or reclaimRequest
+        catch (secondaryError) {
+            const unwrappedSecondaryError = this.unwrapError(secondaryError);
+            const timedOut = unwrappedSecondaryError instanceof TimeoutError;
+            request.state = RequestState.ERROR;
+            if (timedOut) {
+                // Not worth crashing the crawler over. The request's state is unknown, so it gets retried - and the
+                // retry is counted, so that an error handler that always times out does not retry it forever.
+                this.log.warning(unwrappedSecondaryError.message);
+                if (!skipped && request.retryCount === retryCountBefore) {
+                    request.retryCount++;
                 }
-                request.state = RequestState.DONE;
             }
-            catch (secondaryError) {
-                const unwrappedSecondaryError = this.unwrapError(secondaryError);
-                if (!unwrappedSecondaryError.triggeredFromUserHandler &&
-                    // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
-                    !(unwrappedSecondaryError instanceof CriticalError)) {
-                    const apifySpecific = process.env.APIFY_IS_AT_HOME
-                        ? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
-                        : '';
-                    this.log.exception(unwrappedSecondaryError, 'An exception occurred during handling of failed request. ' +
-                        `This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`);
-                }
-                request.state = RequestState.ERROR;
-                throw unwrappedSecondaryError;
+            else if (!(unwrappedSecondaryError instanceof CriticalError)) {
+                // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
+                const apifySpecific = process.env.APIFY_IS_AT_HOME
+                    ? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
+                    : '';
+                this.log.exception(unwrappedSecondaryError, 'An exception occurred during handling of failed request. ' +
+                    `This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`);
+            }
+            // Safety net - the error handler neither marked the request as handled nor reclaimed it (e.g. after a
+            // CriticalError). Reclaiming a request that is no longer in progress is a harmless no-op on the storage backend.
+            try {
+                await addTimeoutToPromise(async () => requestSource.reclaimRequest(request), this.internalTimeoutMillis, `Reclaiming request ${request.url} (${request.id}) timed out.`);
             }
-            // decrease the session score if the request fails (but the error handler did not throw);
-            // skip when the error is a SessionError, which already retired the session
-            if (!this.errorAbsolvesSession(err)) {
-                crawlingContext.session.markBad();
+            catch {
+                // The request was never in progress, or could not be reclaimed. Either way it's fine.
+            }
+            if (!timedOut) {
+                throw unwrappedSecondaryError;
             }
         }
         finally {
-            // Safety net - return the request to the queue if nobody managed to mark it as handled
-            // or reclaim it before (e.g. after a CriticalError). Reclaiming a request that is no longer
-            // in progress is a harmless no-op on the storage backend.
-            if (isRequestLocked && requestSource instanceof RequestQueue) {
-                try {
-                    await requestSource.reclaimRequest(request);
-                }
-                catch {
-                    // The request was never in progress, or could not be reclaimed. Either way it's fine.
-                }
-            }
+            resumeRequestTimeout?.();
+        }
+        // decrease the session score if the request fails (but the error handler did not throw);
+        // skip when the error is a SessionError, which already retired the session
+        if (!skipped && !this.errorAbsolvesSession(err)) {
+            context.session?.markBad();
         }
     }
     /**
@@ -1641,10 +1632,12 @@ export class BasicCrawler {
     }
     /**
      * Unwraps errors thrown by the context pipeline to get the actual user error.
-     * RequestHandlerError and ContextPipelineCleanupError wrap the actual error.
+     * RequestHandlerError, ContextPipelineInitializationError and ContextPipelineCleanupError wrap the actual error.
      */
     unwrapError(error) {
-        if (error instanceof RequestHandlerError || error instanceof ContextPipelineCleanupError) {
+        if (error instanceof RequestHandlerError ||
+            error instanceof ContextPipelineInitializationError ||
+            error instanceof ContextPipelineCleanupError) {
             return this.unwrapError(error.cause);
         }
         return error;

package/internals/crawlers/context_pipeline.d.ts

@@ -1,7 +1,10 @@
 import type { Awaitable } from '@crawlee/types';
 /**
- * Registers a callback that runs after the final context consumer finishes. `error` is the consumer's failure, if any;
- * failures of downstream middlewares are not reported. Cleanups run in reverse registration order.
+ * Registers a callback that runs after the final context consumer finishes. Cleanups run in reverse registration
+ * order and receive the same `error` as the `onError` parameter of {@link ContextPipeline.call}: `undefined` on
+ * success, otherwise a {@link ContextPipelineInitializationError} (a middleware failed) or a
+ * {@link RequestHandlerError} (the consumer failed), with the original in `cause`. A cleanup that throws fails the
+ * whole call with a {@link ContextPipelineCleanupError}, which is critical.
  */
 export type CleanupRegistrar = (cleanup: (error?: unknown) => Awaitable<void>) => void;
 /**
@@ -60,12 +63,13 @@ export declare abstract class ContextPipeline<TContextBase, TCrawlingContext ext
      *
      * @param crawlingContext - The initial context to process through the pipeline
      * @param finalContextConsumer - The function that will receive the final enhanced context
-     * @param onInitializationError - Receives a middleware failure as-is; the consumer is then skipped. Runs before
-     *   the cleanups, so it still sees whatever the failed middleware's predecessors set up.
+     * @param onError - Receives a middleware failure wrapped in a {@link ContextPipelineInitializationError}, or a
+     *   consumer failure wrapped in a {@link RequestHandlerError}, with the original in `cause`. After a middleware
+     *   failure, the rest of the chain and the consumer are skipped. Runs before the cleanups, so it still sees
+     *   whatever the middlewares set up.
      *
-     * @throws {RequestHandlerError} When the final context consumer throws an exception
      * @throws {ContextPipelineCleanupError} When cleanup operations fail
-     * @throws {SessionError} Session errors are re-thrown as-is for special handling
+     * @throws Whatever `onError` throws
      */
-    abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>, onInitializationError: (error: unknown) => Awaitable<void>): Promise<void>;
+    abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>, onError: (error: unknown) => Awaitable<void>): Promise<void>;
 }

package/internals/crawlers/context_pipeline.js

@@ -1,5 +1,5 @@
-import { serviceLocator, SessionError } from '@crawlee/core';
-import { ContextPipelineCleanupError, RequestHandlerError } from '../errors.js';
+import { serviceLocator } from '@crawlee/core';
+import { ContextPipelineCleanupError, ContextPipelineInitializationError, RequestHandlerError } from '../errors.js';
 /**
  * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
  *
@@ -57,55 +57,47 @@ class ContextPipelineImpl extends ContextPipeline {
     /**
      * @inheritdoc
      */
-    async call(crawlingContext, finalContextConsumer, onInitializationError) {
+    async call(crawlingContext, finalContextConsumer, onError) {
         const middlewares = Array.from(this.middlewareChain()).reverse();
         const cleanupStack = [];
         const onCleanup = (cleanup) => {
             cleanupStack.push(cleanup);
         };
-        let consumerException;
+        let failure;
+        let consumerStarted = false;
         try {
             for (const middleware of middlewares) {
-                try {
-                    const contextExtension = await middleware(crawlingContext, onCleanup);
-                    const extensionNames = [
-                        ...Object.getOwnPropertyNames(contextExtension),
-                        ...Object.getOwnPropertySymbols(contextExtension),
-                    ];
-                    for (const key of extensionNames) {
-                        try {
-                            if (Object.getOwnPropertyDescriptor(crawlingContext, key)?.configurable !== false) {
-                                Object.defineProperty(crawlingContext, key, Object.getOwnPropertyDescriptor(contextExtension, key));
-                            }
-                        }
-                        catch (error) {
-                            serviceLocator
-                                .getLogger()
-                                .debug(`Context pipeline failed to define property ${key.toString()}:`, error);
+                const contextExtension = await middleware(crawlingContext, onCleanup);
+                const extensionNames = [
+                    ...Object.getOwnPropertyNames(contextExtension),
+                    ...Object.getOwnPropertySymbols(contextExtension),
+                ];
+                for (const key of extensionNames) {
+                    try {
+                        if (Object.getOwnPropertyDescriptor(crawlingContext, key)?.configurable !== false) {
+                            Object.defineProperty(crawlingContext, key, Object.getOwnPropertyDescriptor(contextExtension, key));
                         }
                     }
+                    catch (error) {
+                        serviceLocator
+                            .getLogger()
+                            .debug(`Context pipeline failed to define property ${key.toString()}:`, error);
+                    }
                 }
-                catch (exception) {
-                    await onInitializationError(exception);
-                    return;
-                }
-            }
-            try {
-                await finalContextConsumer(crawlingContext);
-            }
-            catch (exception) {
-                if (exception instanceof SessionError) {
-                    consumerException = exception;
-                    throw exception; // Session errors are re-thrown as-is
-                }
-                consumerException = exception;
-                throw new RequestHandlerError(exception);
             }
+            consumerStarted = true;
+            await finalContextConsumer(crawlingContext);
+        }
+        catch (exception) {
+            failure = consumerStarted
+                ? new RequestHandlerError(exception)
+                : new ContextPipelineInitializationError(exception);
+            await onError(failure);
         }
         finally {
             try {
                 for (const cleanup of cleanupStack.reverse()) {
-                    await cleanup(consumerException);
+                    await cleanup(failure);
                 }
             }
             catch (exception) {

package/internals/errors.d.ts

@@ -44,6 +44,11 @@ export declare class ContextPipelineInterruptedError extends Error {
 export declare class ContextPipelineCleanupError extends CriticalError {
     constructor(error: unknown, options?: ErrorOptions);
 }
+/** Wraps a failure of a context pipeline middleware, telling it apart from a {@link RequestHandlerError}. */
+export declare class ContextPipelineInitializationError extends Error {
+    constructor(error: unknown, options?: ErrorOptions);
+}
+/** Wraps a failure of the context pipeline's final consumer (the request handler), telling it apart from a middleware failure. */
 export declare class RequestHandlerError extends Error {
     constructor(error: unknown, options?: ErrorOptions);
 }

package/internals/errors.js

@@ -54,6 +54,13 @@ export class ContextPipelineCleanupError extends CriticalError {
         super(undefined, { cause: error, ...options });
     }
 }
+/** Wraps a failure of a context pipeline middleware, telling it apart from a {@link RequestHandlerError}. */
+export class ContextPipelineInitializationError extends Error {
+    constructor(error, options) {
+        super(undefined, { cause: error, ...options });
+    }
+}
+/** Wraps a failure of the context pipeline's final consumer (the request handler), telling it apart from a middleware failure. */
 export class RequestHandlerError extends Error {
     constructor(error, options) {
         super(undefined, { cause: error, ...options });

package/internals/request-timeout.d.ts

@@ -16,10 +16,16 @@ export declare const extendTimeoutKey: unique symbol;
  * bail, so a request whose timeout already elapsed does not carry on after the crawler moved past it.
  */
 export declare const timeoutExpiredKey: unique symbol;
+/**
+ * Stops the internal request timeout's clock until the returned callback is called. Error handling has a timeout
+ * of its own, and must not be cut short (and handled again) by the request timeout.
+ */
+export declare const suspendTimeoutKey: unique symbol;
 /** The slots the internal request timeout hangs on the in-flight crawling context. */
 export interface RequestTimeoutContext {
     [extendTimeoutKey]?: (extraMillis: number) => void;
     [timeoutExpiredKey]?: () => boolean;
+    [suspendTimeoutKey]?: () => () => void;
     [navigationDeadlineKey]?: number;
 }
 /**

package/internals/request-timeout.js

@@ -17,6 +17,11 @@ export const extendTimeoutKey = Symbol('extendTimeout');
  * bail, so a request whose timeout already elapsed does not carry on after the crawler moved past it.
  */
 export const timeoutExpiredKey = Symbol('timeoutExpired');
+/**
+ * Stops the internal request timeout's clock until the returned callback is called. Error handling has a timeout
+ * of its own, and must not be cut short (and handled again) by the request timeout.
+ */
+export const suspendTimeoutKey = Symbol('suspendTimeout');
 /**
  * Milliseconds left in the shared navigation window for `ctx`, lazily starting the window on first use.
  * @internal
@@ -46,21 +51,41 @@ export async function raceWithTimeout(context, work, { timeoutMillis, requestId
     let deadline = Date.now() + timeoutMillis;
     let settled = false;
     let firedByTimeout = false;
+    let suspendedAt;
     const timeout = new Promise((_, reject) => {
         const fire = () => {
             settled = true;
             firedByTimeout = true;
             reject(new TimeoutError(`Request timed out after ${timeoutMillis / 1e3} seconds (${requestId}).`));
         };
+        const schedule = () => {
+            clearTimeout(timer);
+            timer = setTimeout(fire, Math.max(deadline - Date.now(), 0));
+        };
         timer = setTimeout(fire, timeoutMillis);
         context[timeoutExpiredKey] = () => firedByTimeout;
         context[extendTimeoutKey] = (extraMillis) => {
             if (settled) {
                 return;
             }
-            clearTimeout(timer);
             deadline += extraMillis;
-            timer = setTimeout(fire, Math.max(deadline - Date.now(), 0));
+            if (suspendedAt === undefined) {
+                schedule();
+            }
+        };
+        context[suspendTimeoutKey] = () => {
+            if (settled || suspendedAt !== undefined) {
+                return () => { };
+            }
+            clearTimeout(timer);
+            suspendedAt = Date.now();
+            return () => {
+                deadline += Date.now() - suspendedAt;
+                suspendedAt = undefined;
+                if (!settled) {
+                    schedule();
+                }
+            };
         };
     });
     try {

package/internals/session_pool/session.d.ts

@@ -149,7 +149,8 @@ export declare class Session implements ISession {
      */
     setCookie(rawCookie: string, url: string): Promise<void>;
     /**
-     * Checks if session is not usable. if it is not retires the session.
+     * Makes an unusable session's state terminal. Unlike `retire()`, this leaves the error score alone so that
+     * `isBlocked()` keeps telling a burned identity from one that merely ran out of uses.
      */
     private maybeSelfRetire;
 }

package/internals/session_pool/session.js

@@ -140,6 +140,8 @@ export class Session {
      * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
      */
     markGood() {
+        if (this.#retired)
+            return;
         this.#usageCount += 1;
         if (this.#errorScore > 0) {
             this.#errorScore -= this.#errorScoreDecrement;
@@ -176,7 +178,7 @@ export class Session {
      * For transient external failures (such as `5XX` responses), use `markBad()` instead.
      */
     retire() {
-        if (this.#retired)
+        if (this.#retired && this.isBlocked())
             return;
         this.#errorScore += this.#maxErrorScore;
         this.#usageCount += 1;
@@ -212,11 +214,12 @@ export class Session {
         }
     }
     /**
-     * Checks if session is not usable. if it is not retires the session.
+     * Makes an unusable session's state terminal. Unlike `retire()`, this leaves the error score alone so that
+     * `isBlocked()` keeps telling a burned identity from one that merely ran out of uses.
      */
     maybeSelfRetire() {
         if (!this.isUsable()) {
-            this.retire();
+            this.#retired = true;
         }
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.207",
+    "version": "4.0.0-beta.209",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/core": "4.0.0-beta.207",
-        "@crawlee/http-client": "4.0.0-beta.207",
-        "@crawlee/types": "4.0.0-beta.207",
-        "@crawlee/utils": "4.0.0-beta.207",
+        "@crawlee/core": "4.0.0-beta.209",
+        "@crawlee/http-client": "4.0.0-beta.209",
+        "@crawlee/types": "4.0.0-beta.209",
+        "@crawlee/utils": "4.0.0-beta.209",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "csv-stringify": "^6.5.2",
@@ -57,7 +57,7 @@
         "zod": "^4.5.4"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "4.0.0-beta.207"
+        "@crawlee/impit-client": "4.0.0-beta.209"
     },
     "lerna": {
         "command": {
@@ -66,5 +66,5 @@
             }
         }
     },
-    "gitHead": "8d4e54dfe0343e33f40a90d12abfb1840340c520"
+    "gitHead": "ccf82c59e230033a34bddcbecb1f4a8e803eaa11"
 }