@crawlee/basic 4.0.0-beta.184 → 4.0.0-beta.185

package/internals/basic-crawler.d.ts

@@ -743,7 +743,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     private isTaskReadyFunction;
     /**
      * Unwraps errors thrown by the context pipeline to get the actual user error.
-     * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
+     * RequestHandlerError and ContextPipelineCleanupError wrap the actual error.
      */
     private unwrapError;
     /**

package/internals/basic-crawler.js

@@ -13,7 +13,7 @@ import { ConcurrencySystem } from './autoscaling/concurrency_system.js';
 import { mergeCookies } from './cookie_utils.js';
 import { ContextPipeline } from './crawlers/context_pipeline.js';
 import { CrawlingRequest, RequestState } from './crawling_request.js';
-import { ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
+import { ContextPipelineCleanupError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
 import { Statistics } from './crawlers/statistics.js';
 import { applyRequestTransform, buildEnqueueStrategyPatterns, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue_links/index.js';
 import { parseRetryAfterHeader } from './http.js';
@@ -54,7 +54,6 @@ class LazyDefaultHttpClient extends BaseHttpClient {
  * @ignore
  */
 const SAFE_MIGRATION_WAIT_MILLIS = 20000;
-const deferredCleanupKey = Symbol('deferredCleanup');
 // The request timeout plumbing (the window helper, the context symbols, and the race) lives in its own module.
 export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
 const urlPatternSchema = z.union([
@@ -463,6 +462,28 @@ export class BasicCrawler {
                     // matching `recordRequestStart`.
                     this.statistics.recordRequestStart(request.id || request.uniqueKey);
                     const crawlingContext = { request };
+                    // Runs inside the pipeline call, so the deferred cleanups registered by earlier middlewares
+                    // (e.g. closing the page) only fire after the error handler has seen the partial context.
+                    const handleInitializationError = async (rawError) => {
+                        // Roll back *before* the error handler runs, for the same reason `handleRequest` does.
+                        currentStorageTransaction()?.rollback();
+                        // ContextPipelineInterruptedError means the request was intentionally skipped
+                        // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
+                        if (rawError instanceof ContextPipelineInterruptedError) {
+                            this.statistics.discardRequestRecord(request.id || request.uniqueKey);
+                            await this.timeoutAndRetry(async () => this.requestManager?.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+                            return;
+                        }
+                        // An error during pipeline initialization (e.g., navigation timeout, session/proxy error,
+                        // i.e. not in user's requestHandler) goes through the normal error flow.
+                        const error = this.unwrapError(rawError);
+                        await this.requestFunctionErrorHandler(error, crawlingContext, request, this.requestManager);
+                        // SessionError already retired the session in `requestFunctionErrorHandler`;
+                        // skip `markBad` to avoid double-counting usage/error score.
+                        if (!this.errorAbsolvesSession(error)) {
+                            crawlingContext.session?.markBad();
+                        }
+                    };
                     try {
                         // The transaction spans the whole pipeline call, covering the navigation hooks
                         // and `extendContext` too; `handleRequest` drives its outcome explicitly.
@@ -472,27 +493,15 @@ export class BasicCrawler {
                         // catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
                         await this.withRequestTimeout(crawlingContext, this.basicContextPipeline
                             .chain(this.contextPipeline)
-                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request))));
+                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request), handleInitializationError)));
                     }
                     catch (error) {
-                        // ContextPipelineInterruptedError means the request was intentionally skipped
-                        // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
-                        if (error instanceof ContextPipelineInterruptedError) {
-                            this.statistics.discardRequestRecord(request.id || request.uniqueKey);
-                            await this.timeoutAndRetry(async () => this.requestManager?.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-                            return;
-                        }
-                        // If the error happened during pipeline initialization (e.g., navigation timeout, session/proxy error,
-                        // i.e. not in user's requestHandler), handle it through the normal error flow. A bare `TimeoutError`
-                        // here is the internal timeout above firing - anything else thrown inside the pipeline arrives wrapped.
-                        const isPipelineError = error instanceof ContextPipelineInitializationError ||
-                            error instanceof SessionError ||
-                            error instanceof TimeoutError;
-                        if (isPipelineError) {
+                        // A bare `TimeoutError` is the internal timeout above firing; a `SessionError` can only be a
+                        // secondary error from `handleRequest` - both take the normal error flow. Anything else thrown
+                        // inside the pipeline arrives wrapped.
+                        if (error instanceof SessionError || error instanceof TimeoutError) {
                             const unwrappedError = this.unwrapError(error);
                             await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
-                            // SessionError already retired the session in `requestFunctionErrorHandler`;
-                            // skip `markBad` to avoid double-counting usage/error score.
                             if (!this.errorAbsolvesSession(unwrappedError)) {
                                 crawlingContext.session?.markBad();
                             }
@@ -500,11 +509,6 @@ export class BasicCrawler {
                         }
                         throw this.unwrapError(error);
                     }
-                    finally {
-                        // Run request-scoped deferred cleanups only after the whole request lifecycle - including the user's error handler - has finished.
-                        const deferredCleanup = crawlingContext[deferredCleanupKey] ?? [];
-                        await Promise.all(deferredCleanup.map((fn) => fn().catch((cleanupError) => this.log.debug('Error in deferred cleanup', { error: cleanupError }))));
-                    }
                 },
                 isTaskReadyFunction: async () => {
                     if (isMaxPagesExceeded()) {
@@ -594,10 +598,10 @@ export class BasicCrawler {
      */
     buildBasicContextPipeline() {
         return ContextPipeline.create()
-            .compose({ action: this.checkRobotsTxt.bind(this) })
-            .compose({ action: (context) => this.createBaseContext(context) })
-            .compose({ action: this.resolveSession.bind(this) })
-            .compose({ action: this.createContextHelpers.bind(this) });
+            .compose(this.checkRobotsTxt.bind(this))
+            .compose(this.createBaseContext.bind(this))
+            .compose(this.resolveSession.bind(this))
+            .compose(this.createContextHelpers.bind(this));
     }
     async checkRobotsTxt({ request }) {
         if (!(await this.isAllowedBasedOnRobotsTxtFile(request.url))) {
@@ -619,8 +623,7 @@ export class BasicCrawler {
     buildContextPipeline() {
         return ContextPipeline.create();
     }
-    createBaseContext(context) {
-        const deferredCleanup = [];
+    createBaseContext(context, onCleanup) {
         return {
             id: cryptoRandomObjectId(10),
             log: this.log,
@@ -628,7 +631,14 @@ export class BasicCrawler {
             useState: this.useState.bind(this),
             getKeyValueStore: async (identifier) => KeyValueStore.open(identifier),
             registerDeferredCleanup: (cleanup) => {
-                deferredCleanup.push(cleanup);
+                onCleanup(async () => {
+                    try {
+                        await cleanup();
+                    }
+                    catch (error) {
+                        this.log.debug('Error in deferred cleanup', { error });
+                    }
+                });
             },
             afterStorageCommit: (callback) => {
                 const transaction = currentStorageTransaction();
@@ -651,7 +661,6 @@ export class BasicCrawler {
                     context[navigationDeadlineKey] += extraMillis;
                 }
             },
-            [deferredCleanupKey]: deferredCleanup,
         };
     }
     /**
@@ -673,7 +682,7 @@ export class BasicCrawler {
         const session = await this.timeoutAndRetry(async () => {
             const existingSession = await this.sessionPool.getSession(request.sessionId);
             if (!existingSession) {
-                throw new ContextPipelineInitializationError(new MissingSessionError(request.sessionId));
+                throw new MissingSessionError(request.sessionId);
             }
             return existingSession;
         }, this.internalTimeoutMillis, `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
@@ -705,26 +714,24 @@ export class BasicCrawler {
         let contextPipeline;
         if (extendContext !== undefined) {
             contextPipeline = ContextPipeline.create()
-                .compose({ action: async (context) => await extendContext(context) })
+                .compose(async (context) => await extendContext(context))
                 .chain(subclassPipeline);
         }
         else {
             contextPipeline = subclassPipeline;
         }
-        contextPipeline = contextPipeline.compose({
-            action: async (context) => {
-                const { request } = context;
-                if (request && !this.requestMatchesEnqueueStrategy(request)) {
-                    // eslint-disable-next-line dot-notation
-                    const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
-                    this.log.debug(message);
-                    request.noRetry = true;
-                    request.state = RequestState.SKIPPED;
-                    await this.#handleSkippedRequest({ request, reason: 'redirect' });
-                    throw new ContextPipelineInterruptedError(message);
-                }
-                return context;
-            },
+        contextPipeline = contextPipeline.compose(async (context) => {
+            const { request } = context;
+            if (request && !this.requestMatchesEnqueueStrategy(request)) {
+                // eslint-disable-next-line dot-notation
+                const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
+                this.log.debug(message);
+                request.noRetry = true;
+                request.state = RequestState.SKIPPED;
+                await this.#handleSkippedRequest({ request, reason: 'redirect' });
+                throw new ContextPipelineInterruptedError(message);
+            }
+            return context;
         });
         return contextPipeline;
     }
@@ -1642,12 +1649,10 @@ export class BasicCrawler {
     }
     /**
      * Unwraps errors thrown by the context pipeline to get the actual user error.
-     * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
+     * RequestHandlerError and ContextPipelineCleanupError wrap the actual error.
      */
     unwrapError(error) {
-        if (error instanceof RequestHandlerError ||
-            error instanceof ContextPipelineInitializationError ||
-            error instanceof ContextPipelineCleanupError) {
+        if (error instanceof RequestHandlerError || error instanceof ContextPipelineCleanupError) {
             return this.unwrapError(error.cause);
         }
         return error;

package/internals/crawlers/context_pipeline.d.ts

@@ -1,16 +1,17 @@
 import type { Awaitable } from '@crawlee/types';
 /**
- * Represents a middleware step in the context pipeline.
+ * Registers a callback that runs after the final context consumer finishes. `error` is the consumer's failure, if any;
+ * failures of downstream middlewares are not reported. Cleanups run in reverse registration order.
+ */
+export type CleanupRegistrar = (cleanup: (error?: unknown) => Awaitable<void>) => void;
+/**
+ * A middleware step in the context pipeline: receives the context built so far and returns an extension that gets
+ * merged into it. Anything that needs tearing down is registered via `onCleanup`.
  *
  * @template TCrawlingContext - The input context type for this middleware
  * @template TCrawlingContextExtension - The enhanced output context type
  */
-export interface ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> {
-    /** The main middleware function that enhances the context */
-    action: (context: TCrawlingContext) => Awaitable<TCrawlingContextExtension>;
-    /** Optional cleanup function called after the consumer finishes or fails */
-    cleanup?: (context: TCrawlingContext & TCrawlingContextExtension, error?: unknown) => Awaitable<void>;
-}
+export type ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> = (context: TCrawlingContext, onCleanup: CleanupRegistrar) => Awaitable<TCrawlingContextExtension>;
 /**
  * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
  *
@@ -59,12 +60,12 @@ export declare abstract class ContextPipeline<TContextBase, TCrawlingContext ext
      *
      * @param crawlingContext - The initial context to process through the pipeline
      * @param finalContextConsumer - The function that will receive the final enhanced context
+     * @param onInitializationError - Receives a middleware failure as-is; the consumer is then skipped. Runs before
+     *   the cleanups, so it still sees whatever the failed middleware's predecessors set up.
      *
-     * @throws {ContextPipelineInitializationError} When a middleware fails during initialization
-     * @throws {ContextPipelineInterruptedError} When the pipeline is intentionally interrupted during initialization
      * @throws {RequestHandlerError} When the final context consumer throws an exception
      * @throws {ContextPipelineCleanupError} When cleanup operations fail
      * @throws {SessionError} Session errors are re-thrown as-is for special handling
      */
-    abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>): Promise<void>;
+    abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>, onInitializationError: (error: unknown) => Awaitable<void>): Promise<void>;
 }

package/internals/crawlers/context_pipeline.js

@@ -1,5 +1,5 @@
 import { serviceLocator, SessionError } from '@crawlee/core';
-import { ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, RequestHandlerError, } from '../errors.js';
+import { ContextPipelineCleanupError, RequestHandlerError } from '../errors.js';
 /**
  * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
  *
@@ -18,7 +18,7 @@ export class ContextPipeline {
      * @returns A new ContextPipeline instance with no transformations
      */
     static create() {
-        return new ContextPipelineImpl({ action: async (context) => context });
+        return new ContextPipelineImpl(async (context) => context);
     }
 }
 /**
@@ -57,14 +57,17 @@ class ContextPipelineImpl extends ContextPipeline {
     /**
      * @inheritdoc
      */
-    async call(crawlingContext, finalContextConsumer) {
+    async call(crawlingContext, finalContextConsumer, onInitializationError) {
         const middlewares = Array.from(this.middlewareChain()).reverse();
         const cleanupStack = [];
+        const onCleanup = (cleanup) => {
+            cleanupStack.push(cleanup);
+        };
         let consumerException;
         try {
-            for (const { action, cleanup } of middlewares) {
+            for (const middleware of middlewares) {
                 try {
-                    const contextExtension = await action(crawlingContext);
+                    const contextExtension = await middleware(crawlingContext, onCleanup);
                     const extensionNames = [
                         ...Object.getOwnPropertyNames(contextExtension),
                         ...Object.getOwnPropertySymbols(contextExtension),
@@ -81,18 +84,10 @@ class ContextPipelineImpl extends ContextPipeline {
                                 .debug(`Context pipeline failed to define property ${key.toString()}:`, error);
                         }
                     }
-                    if (cleanup) {
-                        cleanupStack.push(cleanup);
-                    }
                 }
                 catch (exception) {
-                    if (exception instanceof SessionError) {
-                        throw exception; // Session errors are re-thrown as-is
-                    }
-                    if (exception instanceof ContextPipelineInterruptedError) {
-                        throw exception;
-                    }
-                    throw new ContextPipelineInitializationError(exception);
+                    await onInitializationError(exception);
+                    return;
                 }
             }
             try {
@@ -110,7 +105,7 @@ class ContextPipelineImpl extends ContextPipeline {
         finally {
             try {
                 for (const cleanup of cleanupStack.reverse()) {
-                    await cleanup(crawlingContext, consumerException);
+                    await cleanup(consumerException);
                 }
             }
             catch (exception) {

package/internals/errors.d.ts

@@ -41,9 +41,6 @@ export declare class MissingSessionError extends Error {
 export declare class ContextPipelineInterruptedError extends Error {
     constructor(message?: string);
 }
-export declare class ContextPipelineInitializationError extends Error {
-    constructor(error: unknown, options?: ErrorOptions);
-}
 export declare class ContextPipelineCleanupError extends CriticalError {
     constructor(error: unknown, options?: ErrorOptions);
 }

package/internals/errors.js

@@ -49,11 +49,6 @@ export class ContextPipelineInterruptedError extends Error {
         super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`);
     }
 }
-export class ContextPipelineInitializationError extends Error {
-    constructor(error, options) {
-        super(undefined, { cause: error, ...options });
-    }
-}
 export class ContextPipelineCleanupError extends CriticalError {
     constructor(error, options) {
         super(undefined, { cause: error, ...options });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.184",
+    "version": "4.0.0-beta.185",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/core": "4.0.0-beta.184",
-        "@crawlee/http-client": "4.0.0-beta.184",
-        "@crawlee/types": "4.0.0-beta.184",
-        "@crawlee/utils": "4.0.0-beta.184",
+        "@crawlee/core": "4.0.0-beta.185",
+        "@crawlee/http-client": "4.0.0-beta.185",
+        "@crawlee/types": "4.0.0-beta.185",
+        "@crawlee/utils": "4.0.0-beta.185",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "csv-stringify": "^6.5.2",
@@ -57,7 +57,7 @@
         "zod": "^4.5.4"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "4.0.0-beta.184"
+        "@crawlee/impit-client": "4.0.0-beta.185"
     },
     "lerna": {
         "command": {
@@ -66,5 +66,5 @@
             }
         }
     },
-    "gitHead": "19ebeeb9bb90e38496cbe90565c8ebb02e102193"
+    "gitHead": "642c670956a5a60384d5ed5e67ee809927cc419b"
 }