@crawlee/basic 4.0.0-beta.183 → 4.0.0-beta.185

package/index.d.ts

@@ -3,6 +3,7 @@ export * from './internals/autoscaling/index.js';
 export * from './internals/basic-crawler.js';
 export * from './internals/cookie_utils.js';
 export * from './internals/crawlers/index.js';
+export * from './internals/crawling_request.js';
 export * from './internals/enqueue_links/index.js';
 export * from './internals/errors.js';
 export * from './internals/http.js';

package/index.js

@@ -3,6 +3,7 @@ export * from './internals/autoscaling/index.js';
 export * from './internals/basic-crawler.js';
 export * from './internals/cookie_utils.js';
 export * from './internals/crawlers/index.js';
+export * from './internals/crawling_request.js';
 export * from './internals/enqueue_links/index.js';
 export * from './internals/errors.js';
 export * from './internals/http.js';

package/internals/basic-crawler.d.ts

@@ -12,6 +12,7 @@ import { ConcurrencySystem } from './autoscaling/concurrency_system.js';
 import type { FinalStatistics } from './autoscaling/system_status.js';
 import { ContextPipeline } from './crawlers/context_pipeline.js';
 import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler_commons.js';
+import { CrawlingRequest } from './crawling_request.js';
 import type { IStatistics, StatisticState } from './crawlers/statistics.js';
 import type { EnqueueUrlsOptions, SkippedRequestCallback } from './enqueue_links/index.js';
 import type { GetUserDataFromRequest, RouterHandler, RouterRoutes } from './router.js';
@@ -54,7 +55,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * To make this work, we should **always**
      * let our function throw exceptions rather than catch them.
      * The exceptions are logged to the request using the
-     * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
+     * {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
      */
     requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
     /**
@@ -441,7 +442,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * This is used e.g. in the {@link AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.
      */
     get basicContextPipeline(): ContextPipeline<{
-        request: Request;
+        request: CrawlingRequest;
     }, CrawlingContext>;
     get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
     /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
@@ -534,6 +535,11 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      */
     protected buildContextPipeline(): ContextPipeline<CrawlingContext, CrawlingContext>;
     private createBaseContext;
+    /**
+     * Fetches the next request and rebuilds it as a {@link CrawlingRequest}. A copy rather than the manager's
+     * instance: managers key on `uniqueKey`/`id`, and it is this copy that flows back into `reclaimRequest` /
+     * `markRequestAsHandled`, so nothing observes the difference.
+     */
     private resolveRequest;
     private resolveSession;
     private createContextHelpers;
@@ -737,7 +743,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     private isTaskReadyFunction;
     /**
      * Unwraps errors thrown by the context pipeline to get the actual user error.
-     * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
+     * RequestHandlerError and ContextPipelineCleanupError wrap the actual error.
      */
     private unwrapError;
     /**
@@ -780,7 +786,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     private requestMatchesEnqueueStrategy;
 }
 export interface CreateContextOptions {
-    request: Request;
+    request: CrawlingRequest;
     session: ISession;
     proxyInfo?: ProxyInfo;
 }

package/internals/basic-crawler.js

@@ -1,6 +1,6 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { bindMethodsToServiceLocator, Configuration, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestManagerTandem, RequestQueue, RequestState, ServiceLocator, serviceLocator, SessionError, validators, withDirectStorageAccess, } from '@crawlee/core';
+import { bindMethodsToServiceLocator, Configuration, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestManagerTandem, RequestQueue, ServiceLocator, serviceLocator, SessionError, validators, withDirectStorageAccess, } from '@crawlee/core';
 import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
 import { isAsyncIterable, isIterable, parseArgument, ROTATE_PROXY_ERRORS, schemas } from '@crawlee/utils/internal';
 import { RobotsTxtFile } from '@crawlee/utils';
@@ -12,7 +12,8 @@ import { cryptoRandomObjectId } from '@apify/utilities';
 import { ConcurrencySystem } from './autoscaling/concurrency_system.js';
 import { mergeCookies } from './cookie_utils.js';
 import { ContextPipeline } from './crawlers/context_pipeline.js';
-import { ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
+import { CrawlingRequest, RequestState } from './crawling_request.js';
+import { ContextPipelineCleanupError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
 import { Statistics } from './crawlers/statistics.js';
 import { applyRequestTransform, buildEnqueueStrategyPatterns, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue_links/index.js';
 import { parseRetryAfterHeader } from './http.js';
@@ -53,7 +54,6 @@ class LazyDefaultHttpClient extends BaseHttpClient {
  * @ignore
  */
 const SAFE_MIGRATION_WAIT_MILLIS = 20000;
-const deferredCleanupKey = Symbol('deferredCleanup');
 // The request timeout plumbing (the window helper, the context symbols, and the race) lives in its own module.
 export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
 const urlPatternSchema = z.union([
@@ -462,6 +462,28 @@ export class BasicCrawler {
                     // matching `recordRequestStart`.
                     this.statistics.recordRequestStart(request.id || request.uniqueKey);
                     const crawlingContext = { request };
+                    // Runs inside the pipeline call, so the deferred cleanups registered by earlier middlewares
+                    // (e.g. closing the page) only fire after the error handler has seen the partial context.
+                    const handleInitializationError = async (rawError) => {
+                        // Roll back *before* the error handler runs, for the same reason `handleRequest` does.
+                        currentStorageTransaction()?.rollback();
+                        // ContextPipelineInterruptedError means the request was intentionally skipped
+                        // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
+                        if (rawError instanceof ContextPipelineInterruptedError) {
+                            this.statistics.discardRequestRecord(request.id || request.uniqueKey);
+                            await this.timeoutAndRetry(async () => this.requestManager?.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+                            return;
+                        }
+                        // An error during pipeline initialization (e.g., navigation timeout, session/proxy error,
+                        // i.e. not in user's requestHandler) goes through the normal error flow.
+                        const error = this.unwrapError(rawError);
+                        await this.requestFunctionErrorHandler(error, crawlingContext, request, this.requestManager);
+                        // SessionError already retired the session in `requestFunctionErrorHandler`;
+                        // skip `markBad` to avoid double-counting usage/error score.
+                        if (!this.errorAbsolvesSession(error)) {
+                            crawlingContext.session?.markBad();
+                        }
+                    };
                     try {
                         // The transaction spans the whole pipeline call, covering the navigation hooks
                         // and `extendContext` too; `handleRequest` drives its outcome explicitly.
@@ -471,27 +493,15 @@ export class BasicCrawler {
                         // catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
                         await this.withRequestTimeout(crawlingContext, this.basicContextPipeline
                             .chain(this.contextPipeline)
-                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request))));
+                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request), handleInitializationError)));
                     }
                     catch (error) {
-                        // ContextPipelineInterruptedError means the request was intentionally skipped
-                        // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
-                        if (error instanceof ContextPipelineInterruptedError) {
-                            this.statistics.discardRequestRecord(request.id || request.uniqueKey);
-                            await this.timeoutAndRetry(async () => this.requestManager?.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-                            return;
-                        }
-                        // If the error happened during pipeline initialization (e.g., navigation timeout, session/proxy error,
-                        // i.e. not in user's requestHandler), handle it through the normal error flow. A bare `TimeoutError`
-                        // here is the internal timeout above firing - anything else thrown inside the pipeline arrives wrapped.
-                        const isPipelineError = error instanceof ContextPipelineInitializationError ||
-                            error instanceof SessionError ||
-                            error instanceof TimeoutError;
-                        if (isPipelineError) {
+                        // A bare `TimeoutError` is the internal timeout above firing; a `SessionError` can only be a
+                        // secondary error from `handleRequest` - both take the normal error flow. Anything else thrown
+                        // inside the pipeline arrives wrapped.
+                        if (error instanceof SessionError || error instanceof TimeoutError) {
                             const unwrappedError = this.unwrapError(error);
                             await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
-                            // SessionError already retired the session in `requestFunctionErrorHandler`;
-                            // skip `markBad` to avoid double-counting usage/error score.
                             if (!this.errorAbsolvesSession(unwrappedError)) {
                                 crawlingContext.session?.markBad();
                             }
@@ -499,11 +509,6 @@ export class BasicCrawler {
                         }
                         throw this.unwrapError(error);
                     }
-                    finally {
-                        // Run request-scoped deferred cleanups only after the whole request lifecycle - including the user's error handler - has finished.
-                        const deferredCleanup = crawlingContext[deferredCleanupKey] ?? [];
-                        await Promise.all(deferredCleanup.map((fn) => fn().catch((cleanupError) => this.log.debug('Error in deferred cleanup', { error: cleanupError }))));
-                    }
                 },
                 isTaskReadyFunction: async () => {
                     if (isMaxPagesExceeded()) {
@@ -593,10 +598,10 @@ export class BasicCrawler {
      */
     buildBasicContextPipeline() {
         return ContextPipeline.create()
-            .compose({ action: this.checkRobotsTxt.bind(this) })
-            .compose({ action: (context) => this.createBaseContext(context) })
-            .compose({ action: this.resolveSession.bind(this) })
-            .compose({ action: this.createContextHelpers.bind(this) });
+            .compose(this.checkRobotsTxt.bind(this))
+            .compose(this.createBaseContext.bind(this))
+            .compose(this.resolveSession.bind(this))
+            .compose(this.createContextHelpers.bind(this));
     }
     async checkRobotsTxt({ request }) {
         if (!(await this.isAllowedBasedOnRobotsTxtFile(request.url))) {
@@ -618,8 +623,7 @@ export class BasicCrawler {
     buildContextPipeline() {
         return ContextPipeline.create();
     }
-    createBaseContext(context) {
-        const deferredCleanup = [];
+    createBaseContext(context, onCleanup) {
         return {
             id: cryptoRandomObjectId(10),
             log: this.log,
@@ -627,7 +631,14 @@ export class BasicCrawler {
             useState: this.useState.bind(this),
             getKeyValueStore: async (identifier) => KeyValueStore.open(identifier),
             registerDeferredCleanup: (cleanup) => {
-                deferredCleanup.push(cleanup);
+                onCleanup(async () => {
+                    try {
+                        await cleanup();
+                    }
+                    catch (error) {
+                        this.log.debug('Error in deferred cleanup', { error });
+                    }
+                });
             },
             afterStorageCommit: (callback) => {
                 const transaction = currentStorageTransaction();
@@ -650,22 +661,28 @@ export class BasicCrawler {
                     context[navigationDeadlineKey] += extraMillis;
                 }
             },
-            [deferredCleanupKey]: deferredCleanup,
         };
     }
+    /**
+     * Fetches the next request and rebuilds it as a {@link CrawlingRequest}. A copy rather than the manager's
+     * instance: managers key on `uniqueKey`/`id`, and it is this copy that flows back into `reclaimRequest` /
+     * `markRequestAsHandled`, so nothing observes the difference.
+     */
     async resolveRequest() {
-        const request = await this.timeoutAndRetry(this.fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-        // Reset loadedUrl so an old one is not carried over to retries.
-        if (request) {
-            request.loadedUrl = undefined;
+        const fetched = await this.timeoutAndRetry(this.fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+        if (!fetched) {
+            return null;
         }
+        const request = CrawlingRequest.fromSchema(fetched);
+        // Reset loadedUrl so an old one is not carried over to retries.
+        request.loadedUrl = undefined;
         return request;
     }
     async resolveSession({ request }) {
         const session = await this.timeoutAndRetry(async () => {
             const existingSession = await this.sessionPool.getSession(request.sessionId);
             if (!existingSession) {
-                throw new ContextPipelineInitializationError(new MissingSessionError(request.sessionId));
+                throw new MissingSessionError(request.sessionId);
             }
             return existingSession;
         }, this.internalTimeoutMillis, `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
@@ -697,26 +714,24 @@ export class BasicCrawler {
         let contextPipeline;
         if (extendContext !== undefined) {
             contextPipeline = ContextPipeline.create()
-                .compose({ action: async (context) => await extendContext(context) })
+                .compose(async (context) => await extendContext(context))
                 .chain(subclassPipeline);
         }
         else {
             contextPipeline = subclassPipeline;
         }
-        contextPipeline = contextPipeline.compose({
-            action: async (context) => {
-                const { request } = context;
-                if (request && !this.requestMatchesEnqueueStrategy(request)) {
-                    // eslint-disable-next-line dot-notation
-                    const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
-                    this.log.debug(message);
-                    request.noRetry = true;
-                    request.state = RequestState.SKIPPED;
-                    await this.#handleSkippedRequest({ request, reason: 'redirect' });
-                    throw new ContextPipelineInterruptedError(message);
-                }
-                return context;
-            },
+        contextPipeline = contextPipeline.compose(async (context) => {
+            const { request } = context;
+            if (request && !this.requestMatchesEnqueueStrategy(request)) {
+                // eslint-disable-next-line dot-notation
+                const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
+                this.log.debug(message);
+                request.noRetry = true;
+                request.state = RequestState.SKIPPED;
+                await this.#handleSkippedRequest({ request, reason: 'redirect' });
+                throw new ContextPipelineInterruptedError(message);
+            }
+            return context;
         });
         return contextPipeline;
     }
@@ -1194,7 +1209,7 @@ export class BasicCrawler {
                 }
                 let [finalOptions] = filtered;
                 if (options.transformRequestFunction) {
-                    const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ source: keepSkippedSource(r), reason: r.skippedReason ?? 'transform' }));
+                    const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ source: keepSkippedSource(r), reason: 'transform' }));
                     if (transformed.length === 0) {
                         continue;
                     }
@@ -1590,6 +1605,11 @@ export class BasicCrawler {
             if (typeof request === 'string') {
                 yield { url: request, crawlDepth: newRequestDepth };
             }
+            else if (request instanceof Request) {
+                const crawlee = (request.userData.__crawlee ??= {});
+                crawlee.crawlDepth ??= newRequestDepth;
+                yield request;
+            }
             else {
                 request.crawlDepth ??= newRequestDepth;
                 yield request;
@@ -1629,12 +1649,10 @@ export class BasicCrawler {
     }
     /**
      * Unwraps errors thrown by the context pipeline to get the actual user error.
-     * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
+     * RequestHandlerError and ContextPipelineCleanupError wrap the actual error.
      */
     unwrapError(error) {
-        if (error instanceof RequestHandlerError ||
-            error instanceof ContextPipelineInitializationError ||
-            error instanceof ContextPipelineCleanupError) {
+        if (error instanceof RequestHandlerError || error instanceof ContextPipelineCleanupError) {
             return this.unwrapError(error.cause);
         }
         return error;
@@ -1809,8 +1827,7 @@ export class BasicCrawler {
             throw err;
         }
         const { url, loadedUrl } = request;
-        // eslint-disable-next-line dot-notation -- private access
-        const strategy = request['enqueueStrategy'];
+        const strategy = request.enqueueStrategy;
         // No strategy set, so we assume it matches, or it was added outside of enqueueLinks
         if (!strategy) {
             return true;

package/internals/crawlers/context_pipeline.d.ts

@@ -1,16 +1,17 @@
 import type { Awaitable } from '@crawlee/types';
 /**
- * Represents a middleware step in the context pipeline.
+ * Registers a callback that runs after the final context consumer finishes. `error` is the consumer's failure, if any;
+ * failures of downstream middlewares are not reported. Cleanups run in reverse registration order.
+ */
+export type CleanupRegistrar = (cleanup: (error?: unknown) => Awaitable<void>) => void;
+/**
+ * A middleware step in the context pipeline: receives the context built so far and returns an extension that gets
+ * merged into it. Anything that needs tearing down is registered via `onCleanup`.
  *
  * @template TCrawlingContext - The input context type for this middleware
  * @template TCrawlingContextExtension - The enhanced output context type
  */
-export interface ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> {
-    /** The main middleware function that enhances the context */
-    action: (context: TCrawlingContext) => Awaitable<TCrawlingContextExtension>;
-    /** Optional cleanup function called after the consumer finishes or fails */
-    cleanup?: (context: TCrawlingContext & TCrawlingContextExtension, error?: unknown) => Awaitable<void>;
-}
+export type ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> = (context: TCrawlingContext, onCleanup: CleanupRegistrar) => Awaitable<TCrawlingContextExtension>;
 /**
  * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
  *
@@ -59,12 +60,12 @@ export declare abstract class ContextPipeline<TContextBase, TCrawlingContext ext
      *
      * @param crawlingContext - The initial context to process through the pipeline
      * @param finalContextConsumer - The function that will receive the final enhanced context
+     * @param onInitializationError - Receives a middleware failure as-is; the consumer is then skipped. Runs before
+     *   the cleanups, so it still sees whatever the failed middleware's predecessors set up.
      *
-     * @throws {ContextPipelineInitializationError} When a middleware fails during initialization
-     * @throws {ContextPipelineInterruptedError} When the pipeline is intentionally interrupted during initialization
      * @throws {RequestHandlerError} When the final context consumer throws an exception
      * @throws {ContextPipelineCleanupError} When cleanup operations fail
      * @throws {SessionError} Session errors are re-thrown as-is for special handling
      */
-    abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>): Promise<void>;
+    abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>, onInitializationError: (error: unknown) => Awaitable<void>): Promise<void>;
 }

package/internals/crawlers/context_pipeline.js

@@ -1,5 +1,5 @@
 import { serviceLocator, SessionError } from '@crawlee/core';
-import { ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, RequestHandlerError, } from '../errors.js';
+import { ContextPipelineCleanupError, RequestHandlerError } from '../errors.js';
 /**
  * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
  *
@@ -18,7 +18,7 @@ export class ContextPipeline {
      * @returns A new ContextPipeline instance with no transformations
      */
     static create() {
-        return new ContextPipelineImpl({ action: async (context) => context });
+        return new ContextPipelineImpl(async (context) => context);
     }
 }
 /**
@@ -57,14 +57,17 @@ class ContextPipelineImpl extends ContextPipeline {
     /**
      * @inheritdoc
      */
-    async call(crawlingContext, finalContextConsumer) {
+    async call(crawlingContext, finalContextConsumer, onInitializationError) {
         const middlewares = Array.from(this.middlewareChain()).reverse();
         const cleanupStack = [];
+        const onCleanup = (cleanup) => {
+            cleanupStack.push(cleanup);
+        };
         let consumerException;
         try {
-            for (const { action, cleanup } of middlewares) {
+            for (const middleware of middlewares) {
                 try {
-                    const contextExtension = await action(crawlingContext);
+                    const contextExtension = await middleware(crawlingContext, onCleanup);
                     const extensionNames = [
                         ...Object.getOwnPropertyNames(contextExtension),
                         ...Object.getOwnPropertySymbols(contextExtension),
@@ -81,18 +84,10 @@ class ContextPipelineImpl extends ContextPipeline {
                                 .debug(`Context pipeline failed to define property ${key.toString()}:`, error);
                         }
                     }
-                    if (cleanup) {
-                        cleanupStack.push(cleanup);
-                    }
                 }
                 catch (exception) {
-                    if (exception instanceof SessionError) {
-                        throw exception; // Session errors are re-thrown as-is
-                    }
-                    if (exception instanceof ContextPipelineInterruptedError) {
-                        throw exception;
-                    }
-                    throw new ContextPipelineInitializationError(exception);
+                    await onInitializationError(exception);
+                    return;
                 }
             }
             try {
@@ -110,7 +105,7 @@ class ContextPipelineImpl extends ContextPipeline {
         finally {
             try {
                 for (const cleanup of cleanupStack.reverse()) {
-                    await cleanup(crawlingContext, consumerException);
+                    await cleanup(consumerException);
                 }
             }
             catch (exception) {

package/internals/crawlers/crawler_commons.d.ts

@@ -2,6 +2,7 @@ import type { Awaitable, Dictionary, HttpRequestOptions, ISession, ProxyInfo, Se
 import type { ReadonlyDeep } from 'type-fest';
 import type { AddRequestsBatchedResult, CrawleeLogger, Dataset, KeyValueStore, Request, RequestOptions, Source, StorageIdentifier } from '@crawlee/core';
 import type { EnqueueUrlsOptions } from '../enqueue_links/index.js';
+import type { CrawlingRequest } from '../crawling_request.js';
 /** @internal */
 export type IsAny<T> = 0 extends 1 & T ? true : false;
 /**
@@ -69,9 +70,9 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      */
     proxyInfo?: ProxyInfo;
     /**
-     * The original {@link Request} object.
+     * The request being processed.
      */
-    request: Request<UserData>;
+    request: CrawlingRequest<UserData>;
     /**
      * This function allows you to push data to a {@link Dataset} specified by name, or the one currently used by the crawler.
      *

package/internals/crawling_request.d.ts

@@ -0,0 +1,86 @@
+import type { EnqueueStrategyOption } from '@crawlee/core';
+import { Request } from '@crawlee/core';
+import type { Dictionary, RequestSchema } from '@crawlee/types';
+export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
+export declare enum RequestState {
+    UNPROCESSED = 0,
+    BEFORE_NAV = 1,
+    AFTER_NAV = 2,
+    REQUEST_HANDLER = 3,
+    DONE = 4,
+    ERROR_HANDLER = 5,
+    ERROR = 6,
+    SKIPPED = 7
+}
+export interface PushErrorMessageOptions {
+    /**
+     * Only push the error message without stack trace when true.
+     * @default false
+     */
+    omitStack?: boolean;
+}
+/**
+ * The crawler's per-request state, persisted inside `userData.__crawlee` so every storage carries it as plain user
+ * data. The settings are written by the `Request` constructor from `RequestOptions`; `state` only by a running crawler.
+ * @internal
+ */
+export interface CrawlingRequestData {
+    skipNavigation?: boolean;
+    crawlDepth?: number;
+    sessionId?: string;
+    maxRetries?: number;
+    enqueueStrategy?: EnqueueStrategyOption;
+    state?: RequestState;
+}
+/**
+ * A {@link Request} as seen by a crawler: the same stored record, with the crawler's own per-request state
+ * (`userData.__crawlee`) exposed as properties. This is what `request` is in every request handler.
+ *
+ * Requests fetched from a request manager are plain {@link Request}s; {@link BasicCrawler} rebuilds them
+ * as `CrawlingRequest`s before handing them to the pipeline.
+ * @category Sources
+ */
+export declare class CrawlingRequest<UserData extends Dictionary = Dictionary> extends Request<UserData> {
+    static fromSchema<UserData extends Dictionary = Dictionary>(schema: RequestSchema): CrawlingRequest<UserData>;
+    private get crawleeData();
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
+    get skipNavigation(): boolean;
+    set skipNavigation(value: boolean);
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     */
+    get crawlDepth(): number;
+    set crawlDepth(value: number);
+    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
+    get sessionId(): string | undefined;
+    set sessionId(value: string | undefined);
+    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
+    get maxRetries(): number | undefined;
+    set maxRetries(value: number | undefined);
+    /** Describes the request's current lifecycle state. */
+    get state(): RequestState;
+    set state(value: RequestState);
+    /** The strategy the request was enqueued under, if it came from `enqueueLinks`. @internal */
+    get enqueueStrategy(): EnqueueStrategyOption | undefined;
+    /**
+     * Stores information about an error that occurred during processing of this request.
+     *
+     * You should always use Error instances when throwing errors in JavaScript.
+     *
+     * Nevertheless, to improve the debugging experience when using third party libraries
+     * that may not always throw an Error instance, the function performs a type
+     * inspection of the passed argument and attempts to extract as much information
+     * as possible, since just throwing a bad type error makes any debugging rather difficult.
+     *
+     * @param errorOrMessage Error object or error message to be stored in the request.
+     * @param [options]
+     */
+    pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
+}

package/internals/crawling_request.js

@@ -0,0 +1,127 @@
+import util from 'node:util';
+import { Request } from '@crawlee/core';
+export var RequestState;
+(function (RequestState) {
+    RequestState[RequestState["UNPROCESSED"] = 0] = "UNPROCESSED";
+    RequestState[RequestState["BEFORE_NAV"] = 1] = "BEFORE_NAV";
+    RequestState[RequestState["AFTER_NAV"] = 2] = "AFTER_NAV";
+    RequestState[RequestState["REQUEST_HANDLER"] = 3] = "REQUEST_HANDLER";
+    RequestState[RequestState["DONE"] = 4] = "DONE";
+    RequestState[RequestState["ERROR_HANDLER"] = 5] = "ERROR_HANDLER";
+    RequestState[RequestState["ERROR"] = 6] = "ERROR";
+    RequestState[RequestState["SKIPPED"] = 7] = "SKIPPED";
+})(RequestState || (RequestState = {}));
+/**
+ * A {@link Request} as seen by a crawler: the same stored record, with the crawler's own per-request state
+ * (`userData.__crawlee`) exposed as properties. This is what `request` is in every request handler.
+ *
+ * Requests fetched from a request manager are plain {@link Request}s; {@link BasicCrawler} rebuilds them
+ * as `CrawlingRequest`s before handing them to the pipeline.
+ * @category Sources
+ */
+export class CrawlingRequest extends Request {
+    static fromSchema(schema) {
+        return super.fromSchema(schema);
+    }
+    get crawleeData() {
+        return (this.userData.__crawlee ??= {});
+    }
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
+    get skipNavigation() {
+        return this.crawleeData.skipNavigation ?? false;
+    }
+    set skipNavigation(value) {
+        this.crawleeData.skipNavigation = value;
+    }
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     */
+    get crawlDepth() {
+        return this.crawleeData.crawlDepth ?? 0;
+    }
+    set crawlDepth(value) {
+        this.crawleeData.crawlDepth = value;
+    }
+    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
+    get sessionId() {
+        return this.crawleeData.sessionId;
+    }
+    set sessionId(value) {
+        this.crawleeData.sessionId = value;
+    }
+    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
+    get maxRetries() {
+        return this.crawleeData.maxRetries;
+    }
+    set maxRetries(value) {
+        this.crawleeData.maxRetries = value;
+    }
+    /** Describes the request's current lifecycle state. */
+    get state() {
+        return this.crawleeData.state ?? RequestState.UNPROCESSED;
+    }
+    set state(value) {
+        this.crawleeData.state = value;
+    }
+    /** The strategy the request was enqueued under, if it came from `enqueueLinks`. @internal */
+    get enqueueStrategy() {
+        return this.crawleeData.enqueueStrategy;
+    }
+    /**
+     * Stores information about an error that occurred during processing of this request.
+     *
+     * You should always use Error instances when throwing errors in JavaScript.
+     *
+     * Nevertheless, to improve the debugging experience when using third party libraries
+     * that may not always throw an Error instance, the function performs a type
+     * inspection of the passed argument and attempts to extract as much information
+     * as possible, since just throwing a bad type error makes any debugging rather difficult.
+     *
+     * @param errorOrMessage Error object or error message to be stored in the request.
+     * @param [options]
+     */
+    pushErrorMessage(errorOrMessage, options = {}) {
+        const { omitStack } = options;
+        let message;
+        const type = typeof errorOrMessage;
+        if (type === 'object') {
+            if (!errorOrMessage) {
+                message = 'null';
+            }
+            else if (errorOrMessage instanceof Error) {
+                message = omitStack
+                    ? errorOrMessage.message
+                    : // .stack includes the message
+                        errorOrMessage.stack;
+            }
+            else if (Reflect.has(Object(errorOrMessage), 'message')) {
+                message = Reflect.get(Object(errorOrMessage), 'message');
+            }
+            else if (errorOrMessage.toString() !== '[object Object]') {
+                message = errorOrMessage.toString();
+            }
+            else {
+                try {
+                    message = util.inspect(errorOrMessage);
+                }
+                catch (err) {
+                    message = 'Unable to extract any message from the received object.';
+                }
+            }
+        }
+        else if (type === 'undefined') {
+            message = 'undefined';
+        }
+        else {
+            message = errorOrMessage.toString();
+        }
+        this.errorMessages.push(message);
+    }
+}

package/internals/enqueue_links/shared.d.ts

@@ -1,7 +1,8 @@
 import type { Awaitable, Dictionary } from '@crawlee/types';
 import { z } from 'zod';
-import type { EnqueueStrategyOption, RequestOptions, SkippedRequestReason, Source } from '@crawlee/core';
+import type { EnqueueStrategyOption, RequestOptions, Source } from '@crawlee/core';
 import { Request } from '@crawlee/core';
+import type { SkippedRequestReason } from '../crawling_request.js';
 export interface UrlPatternObject {
     glob?: string;
     regexp?: RegExp;

package/internals/enqueue_links/shared.js

@@ -27,10 +27,9 @@ export const urlPatternSchema = z.union([
  * @ignore
  */
 export function createSkippedRequestArgs(source, reason) {
-    const sourceReason = typeof source === 'string' ? undefined : source.skippedReason;
     let request;
     return {
-        reason: sourceReason ?? reason,
+        reason,
         get request() {
             request ??=
                 source instanceof Request

package/internals/errors.d.ts

@@ -41,9 +41,6 @@ export declare class MissingSessionError extends Error {
 export declare class ContextPipelineInterruptedError extends Error {
     constructor(message?: string);
 }
-export declare class ContextPipelineInitializationError extends Error {
-    constructor(error: unknown, options?: ErrorOptions);
-}
 export declare class ContextPipelineCleanupError extends CriticalError {
     constructor(error: unknown, options?: ErrorOptions);
 }

package/internals/errors.js

@@ -49,11 +49,6 @@ export class ContextPipelineInterruptedError extends Error {
         super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`);
     }
 }
-export class ContextPipelineInitializationError extends Error {
-    constructor(error, options) {
-        super(undefined, { cause: error, ...options });
-    }
-}
 export class ContextPipelineCleanupError extends CriticalError {
     constructor(error, options) {
         super(undefined, { cause: error, ...options });

package/internals/router.d.ts

@@ -1,4 +1,4 @@
-import type { Request } from '@crawlee/core';
+import type { CrawlingRequest } from './crawling_request.js';
 import type { Awaitable, Dictionary } from '@crawlee/types';
 import type { StandardSchemaV1 } from '@standard-schema/spec';
 import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext, TypedContextAddRequests, TypedContextEnqueueLinks } from './crawlers/crawler_commons.js';
@@ -14,7 +14,7 @@ export declare const defaultRoute: unique symbol;
  * request under a declared label requires the matching `userData` shape.
  */
 export type RouterHandlerContext<Context, UserData extends Dictionary, Routes extends Record<keyof Routes, Dictionary>> = Omit<Context, 'request' | 'addRequests' | 'enqueueLinks'> & {
-    request: LoadedRequest<Request<UserData>>;
+    request: LoadedRequest<CrawlingRequest<UserData>>;
     addRequests: TypedContextAddRequests<Routes>;
 } & (Context extends {
     enqueueLinks: infer EnqueueLinks;
@@ -68,7 +68,7 @@ export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = strin
 export interface RouterHandler<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends Router<Context, Routes> {
     (ctx: Context): Awaitable<void>;
 }
-export type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;
+export type GetUserDataFromRequest<T> = T extends CrawlingRequest<infer Y> ? Y : never;
 /**
  * Per-route overrides, passed as the last argument of {@link Router.addHandler|`addHandler`} and
  * {@link Router.addDefaultHandler|`addDefaultHandler`}.
@@ -85,7 +85,7 @@ export interface RouteOptions {
 }
 export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary>> = {
     [Label in keyof Routes]: (ctx: Omit<Context, 'request'> & {
-        request: Request<Routes[Label]>;
+        request: CrawlingRequest<Routes[Label]>;
     }) => Awaitable<void>;
 };
 /**

package/internals/throttling_request_manager.js

@@ -799,7 +799,7 @@ export class ThrottlingRequestManager {
     async #migrateToSubQueue(request, inner) {
         try {
             const subManager = await this.#selectManagerOrThrow(request.url);
-            await subManager.addRequest(new Request({ ...request, id: undefined }));
+            await subManager.addRequest(Request.fromSchema({ ...request, id: undefined }));
         }
         catch (error) {
             // Whatever the reason, the request is out of the wrapped manager and in nobody's hands, so it

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.183",
+    "version": "4.0.0-beta.185",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/core": "4.0.0-beta.183",
-        "@crawlee/http-client": "4.0.0-beta.183",
-        "@crawlee/types": "4.0.0-beta.183",
-        "@crawlee/utils": "4.0.0-beta.183",
+        "@crawlee/core": "4.0.0-beta.185",
+        "@crawlee/http-client": "4.0.0-beta.185",
+        "@crawlee/types": "4.0.0-beta.185",
+        "@crawlee/utils": "4.0.0-beta.185",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "csv-stringify": "^6.5.2",
@@ -57,7 +57,7 @@
         "zod": "^4.5.4"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "4.0.0-beta.183"
+        "@crawlee/impit-client": "4.0.0-beta.185"
     },
     "lerna": {
         "command": {
@@ -66,5 +66,5 @@
             }
         }
     },
-    "gitHead": "3832a8d3392bba9e3e7cc6d06b9be0be4ee1ab98"
+    "gitHead": "642c670956a5a60384d5ed5e67ee809927cc419b"
 }