@crawlee/basic 4.0.0-beta.183 → 4.0.0-beta.184

package/index.d.ts

@@ -3,6 +3,7 @@ export * from './internals/autoscaling/index.js';
 export * from './internals/basic-crawler.js';
 export * from './internals/cookie_utils.js';
 export * from './internals/crawlers/index.js';
+export * from './internals/crawling_request.js';
 export * from './internals/enqueue_links/index.js';
 export * from './internals/errors.js';
 export * from './internals/http.js';

package/index.js

@@ -3,6 +3,7 @@ export * from './internals/autoscaling/index.js';
 export * from './internals/basic-crawler.js';
 export * from './internals/cookie_utils.js';
 export * from './internals/crawlers/index.js';
+export * from './internals/crawling_request.js';
 export * from './internals/enqueue_links/index.js';
 export * from './internals/errors.js';
 export * from './internals/http.js';

package/internals/basic-crawler.d.ts

@@ -12,6 +12,7 @@ import { ConcurrencySystem } from './autoscaling/concurrency_system.js';
 import type { FinalStatistics } from './autoscaling/system_status.js';
 import { ContextPipeline } from './crawlers/context_pipeline.js';
 import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler_commons.js';
+import { CrawlingRequest } from './crawling_request.js';
 import type { IStatistics, StatisticState } from './crawlers/statistics.js';
 import type { EnqueueUrlsOptions, SkippedRequestCallback } from './enqueue_links/index.js';
 import type { GetUserDataFromRequest, RouterHandler, RouterRoutes } from './router.js';
@@ -54,7 +55,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * To make this work, we should **always**
      * let our function throw exceptions rather than catch them.
      * The exceptions are logged to the request using the
-     * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
+     * {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
      */
     requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
     /**
@@ -441,7 +442,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * This is used e.g. in the {@link AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.
      */
     get basicContextPipeline(): ContextPipeline<{
-        request: Request;
+        request: CrawlingRequest;
     }, CrawlingContext>;
     get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
     /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
@@ -534,6 +535,11 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      */
     protected buildContextPipeline(): ContextPipeline<CrawlingContext, CrawlingContext>;
     private createBaseContext;
+    /**
+     * Fetches the next request and rebuilds it as a {@link CrawlingRequest}. A copy rather than the manager's
+     * instance: managers key on `uniqueKey`/`id`, and it is this copy that flows back into `reclaimRequest` /
+     * `markRequestAsHandled`, so nothing observes the difference.
+     */
     private resolveRequest;
     private resolveSession;
     private createContextHelpers;
@@ -780,7 +786,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     private requestMatchesEnqueueStrategy;
 }
 export interface CreateContextOptions {
-    request: Request;
+    request: CrawlingRequest;
     session: ISession;
     proxyInfo?: ProxyInfo;
 }

package/internals/basic-crawler.js

@@ -1,6 +1,6 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { bindMethodsToServiceLocator, Configuration, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestManagerTandem, RequestQueue, RequestState, ServiceLocator, serviceLocator, SessionError, validators, withDirectStorageAccess, } from '@crawlee/core';
+import { bindMethodsToServiceLocator, Configuration, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestManagerTandem, RequestQueue, ServiceLocator, serviceLocator, SessionError, validators, withDirectStorageAccess, } from '@crawlee/core';
 import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
 import { isAsyncIterable, isIterable, parseArgument, ROTATE_PROXY_ERRORS, schemas } from '@crawlee/utils/internal';
 import { RobotsTxtFile } from '@crawlee/utils';
@@ -12,6 +12,7 @@ import { cryptoRandomObjectId } from '@apify/utilities';
 import { ConcurrencySystem } from './autoscaling/concurrency_system.js';
 import { mergeCookies } from './cookie_utils.js';
 import { ContextPipeline } from './crawlers/context_pipeline.js';
+import { CrawlingRequest, RequestState } from './crawling_request.js';
 import { ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
 import { Statistics } from './crawlers/statistics.js';
 import { applyRequestTransform, buildEnqueueStrategyPatterns, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue_links/index.js';
@@ -653,12 +654,19 @@ export class BasicCrawler {
             [deferredCleanupKey]: deferredCleanup,
         };
     }
+    /**
+     * Fetches the next request and rebuilds it as a {@link CrawlingRequest}. A copy rather than the manager's
+     * instance: managers key on `uniqueKey`/`id`, and it is this copy that flows back into `reclaimRequest` /
+     * `markRequestAsHandled`, so nothing observes the difference.
+     */
     async resolveRequest() {
-        const request = await this.timeoutAndRetry(this.fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-        // Reset loadedUrl so an old one is not carried over to retries.
-        if (request) {
-            request.loadedUrl = undefined;
+        const fetched = await this.timeoutAndRetry(this.fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+        if (!fetched) {
+            return null;
         }
+        const request = CrawlingRequest.fromSchema(fetched);
+        // Reset loadedUrl so an old one is not carried over to retries.
+        request.loadedUrl = undefined;
         return request;
     }
     async resolveSession({ request }) {
@@ -1194,7 +1202,7 @@ export class BasicCrawler {
                 }
                 let [finalOptions] = filtered;
                 if (options.transformRequestFunction) {
-                    const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ source: keepSkippedSource(r), reason: r.skippedReason ?? 'transform' }));
+                    const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ source: keepSkippedSource(r), reason: 'transform' }));
                     if (transformed.length === 0) {
                         continue;
                     }
@@ -1590,6 +1598,11 @@ export class BasicCrawler {
             if (typeof request === 'string') {
                 yield { url: request, crawlDepth: newRequestDepth };
             }
+            else if (request instanceof Request) {
+                const crawlee = (request.userData.__crawlee ??= {});
+                crawlee.crawlDepth ??= newRequestDepth;
+                yield request;
+            }
             else {
                 request.crawlDepth ??= newRequestDepth;
                 yield request;
@@ -1809,8 +1822,7 @@ export class BasicCrawler {
             throw err;
         }
         const { url, loadedUrl } = request;
-        // eslint-disable-next-line dot-notation -- private access
-        const strategy = request['enqueueStrategy'];
+        const strategy = request.enqueueStrategy;
         // No strategy set, so we assume it matches, or it was added outside of enqueueLinks
         if (!strategy) {
             return true;

package/internals/crawlers/crawler_commons.d.ts

@@ -2,6 +2,7 @@ import type { Awaitable, Dictionary, HttpRequestOptions, ISession, ProxyInfo, Se
 import type { ReadonlyDeep } from 'type-fest';
 import type { AddRequestsBatchedResult, CrawleeLogger, Dataset, KeyValueStore, Request, RequestOptions, Source, StorageIdentifier } from '@crawlee/core';
 import type { EnqueueUrlsOptions } from '../enqueue_links/index.js';
+import type { CrawlingRequest } from '../crawling_request.js';
 /** @internal */
 export type IsAny<T> = 0 extends 1 & T ? true : false;
 /**
@@ -69,9 +70,9 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      */
     proxyInfo?: ProxyInfo;
     /**
-     * The original {@link Request} object.
+     * The request being processed.
      */
-    request: Request<UserData>;
+    request: CrawlingRequest<UserData>;
     /**
      * This function allows you to push data to a {@link Dataset} specified by name, or the one currently used by the crawler.
      *

package/internals/crawling_request.d.ts

@@ -0,0 +1,86 @@
+import type { EnqueueStrategyOption } from '@crawlee/core';
+import { Request } from '@crawlee/core';
+import type { Dictionary, RequestSchema } from '@crawlee/types';
+export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
+export declare enum RequestState {
+    UNPROCESSED = 0,
+    BEFORE_NAV = 1,
+    AFTER_NAV = 2,
+    REQUEST_HANDLER = 3,
+    DONE = 4,
+    ERROR_HANDLER = 5,
+    ERROR = 6,
+    SKIPPED = 7
+}
+export interface PushErrorMessageOptions {
+    /**
+     * Only push the error message without stack trace when true.
+     * @default false
+     */
+    omitStack?: boolean;
+}
+/**
+ * The crawler's per-request state, persisted inside `userData.__crawlee` so every storage carries it as plain user
+ * data. The settings are written by the `Request` constructor from `RequestOptions`; `state` only by a running crawler.
+ * @internal
+ */
+export interface CrawlingRequestData {
+    skipNavigation?: boolean;
+    crawlDepth?: number;
+    sessionId?: string;
+    maxRetries?: number;
+    enqueueStrategy?: EnqueueStrategyOption;
+    state?: RequestState;
+}
+/**
+ * A {@link Request} as seen by a crawler: the same stored record, with the crawler's own per-request state
+ * (`userData.__crawlee`) exposed as properties. This is what `request` is in every request handler.
+ *
+ * Requests fetched from a request manager are plain {@link Request}s; {@link BasicCrawler} rebuilds them
+ * as `CrawlingRequest`s before handing them to the pipeline.
+ * @category Sources
+ */
+export declare class CrawlingRequest<UserData extends Dictionary = Dictionary> extends Request<UserData> {
+    static fromSchema<UserData extends Dictionary = Dictionary>(schema: RequestSchema): CrawlingRequest<UserData>;
+    private get crawleeData();
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
+    get skipNavigation(): boolean;
+    set skipNavigation(value: boolean);
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     */
+    get crawlDepth(): number;
+    set crawlDepth(value: number);
+    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
+    get sessionId(): string | undefined;
+    set sessionId(value: string | undefined);
+    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
+    get maxRetries(): number | undefined;
+    set maxRetries(value: number | undefined);
+    /** Describes the request's current lifecycle state. */
+    get state(): RequestState;
+    set state(value: RequestState);
+    /** The strategy the request was enqueued under, if it came from `enqueueLinks`. @internal */
+    get enqueueStrategy(): EnqueueStrategyOption | undefined;
+    /**
+     * Stores information about an error that occurred during processing of this request.
+     *
+     * You should always use Error instances when throwing errors in JavaScript.
+     *
+     * Nevertheless, to improve the debugging experience when using third party libraries
+     * that may not always throw an Error instance, the function performs a type
+     * inspection of the passed argument and attempts to extract as much information
+     * as possible, since just throwing a bad type error makes any debugging rather difficult.
+     *
+     * @param errorOrMessage Error object or error message to be stored in the request.
+     * @param [options]
+     */
+    pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
+}

package/internals/crawling_request.js

@@ -0,0 +1,127 @@
+import util from 'node:util';
+import { Request } from '@crawlee/core';
+export var RequestState;
+(function (RequestState) {
+    RequestState[RequestState["UNPROCESSED"] = 0] = "UNPROCESSED";
+    RequestState[RequestState["BEFORE_NAV"] = 1] = "BEFORE_NAV";
+    RequestState[RequestState["AFTER_NAV"] = 2] = "AFTER_NAV";
+    RequestState[RequestState["REQUEST_HANDLER"] = 3] = "REQUEST_HANDLER";
+    RequestState[RequestState["DONE"] = 4] = "DONE";
+    RequestState[RequestState["ERROR_HANDLER"] = 5] = "ERROR_HANDLER";
+    RequestState[RequestState["ERROR"] = 6] = "ERROR";
+    RequestState[RequestState["SKIPPED"] = 7] = "SKIPPED";
+})(RequestState || (RequestState = {}));
+/**
+ * A {@link Request} as seen by a crawler: the same stored record, with the crawler's own per-request state
+ * (`userData.__crawlee`) exposed as properties. This is what `request` is in every request handler.
+ *
+ * Requests fetched from a request manager are plain {@link Request}s; {@link BasicCrawler} rebuilds them
+ * as `CrawlingRequest`s before handing them to the pipeline.
+ * @category Sources
+ */
+export class CrawlingRequest extends Request {
+    static fromSchema(schema) {
+        return super.fromSchema(schema);
+    }
+    get crawleeData() {
+        return (this.userData.__crawlee ??= {});
+    }
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
+    get skipNavigation() {
+        return this.crawleeData.skipNavigation ?? false;
+    }
+    set skipNavigation(value) {
+        this.crawleeData.skipNavigation = value;
+    }
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     */
+    get crawlDepth() {
+        return this.crawleeData.crawlDepth ?? 0;
+    }
+    set crawlDepth(value) {
+        this.crawleeData.crawlDepth = value;
+    }
+    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
+    get sessionId() {
+        return this.crawleeData.sessionId;
+    }
+    set sessionId(value) {
+        this.crawleeData.sessionId = value;
+    }
+    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
+    get maxRetries() {
+        return this.crawleeData.maxRetries;
+    }
+    set maxRetries(value) {
+        this.crawleeData.maxRetries = value;
+    }
+    /** Describes the request's current lifecycle state. */
+    get state() {
+        return this.crawleeData.state ?? RequestState.UNPROCESSED;
+    }
+    set state(value) {
+        this.crawleeData.state = value;
+    }
+    /** The strategy the request was enqueued under, if it came from `enqueueLinks`. @internal */
+    get enqueueStrategy() {
+        return this.crawleeData.enqueueStrategy;
+    }
+    /**
+     * Stores information about an error that occurred during processing of this request.
+     *
+     * You should always use Error instances when throwing errors in JavaScript.
+     *
+     * Nevertheless, to improve the debugging experience when using third party libraries
+     * that may not always throw an Error instance, the function performs a type
+     * inspection of the passed argument and attempts to extract as much information
+     * as possible, since just throwing a bad type error makes any debugging rather difficult.
+     *
+     * @param errorOrMessage Error object or error message to be stored in the request.
+     * @param [options]
+     */
+    pushErrorMessage(errorOrMessage, options = {}) {
+        const { omitStack } = options;
+        let message;
+        const type = typeof errorOrMessage;
+        if (type === 'object') {
+            if (!errorOrMessage) {
+                message = 'null';
+            }
+            else if (errorOrMessage instanceof Error) {
+                message = omitStack
+                    ? errorOrMessage.message
+                    : // .stack includes the message
+                        errorOrMessage.stack;
+            }
+            else if (Reflect.has(Object(errorOrMessage), 'message')) {
+                message = Reflect.get(Object(errorOrMessage), 'message');
+            }
+            else if (errorOrMessage.toString() !== '[object Object]') {
+                message = errorOrMessage.toString();
+            }
+            else {
+                try {
+                    message = util.inspect(errorOrMessage);
+                }
+                catch (err) {
+                    message = 'Unable to extract any message from the received object.';
+                }
+            }
+        }
+        else if (type === 'undefined') {
+            message = 'undefined';
+        }
+        else {
+            message = errorOrMessage.toString();
+        }
+        this.errorMessages.push(message);
+    }
+}

package/internals/enqueue_links/shared.d.ts

@@ -1,7 +1,8 @@
 import type { Awaitable, Dictionary } from '@crawlee/types';
 import { z } from 'zod';
-import type { EnqueueStrategyOption, RequestOptions, SkippedRequestReason, Source } from '@crawlee/core';
+import type { EnqueueStrategyOption, RequestOptions, Source } from '@crawlee/core';
 import { Request } from '@crawlee/core';
+import type { SkippedRequestReason } from '../crawling_request.js';
 export interface UrlPatternObject {
     glob?: string;
     regexp?: RegExp;

package/internals/enqueue_links/shared.js

@@ -27,10 +27,9 @@ export const urlPatternSchema = z.union([
  * @ignore
  */
 export function createSkippedRequestArgs(source, reason) {
-    const sourceReason = typeof source === 'string' ? undefined : source.skippedReason;
     let request;
     return {
-        reason: sourceReason ?? reason,
+        reason,
         get request() {
             request ??=
                 source instanceof Request

package/internals/router.d.ts

@@ -1,4 +1,4 @@
-import type { Request } from '@crawlee/core';
+import type { CrawlingRequest } from './crawling_request.js';
 import type { Awaitable, Dictionary } from '@crawlee/types';
 import type { StandardSchemaV1 } from '@standard-schema/spec';
 import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext, TypedContextAddRequests, TypedContextEnqueueLinks } from './crawlers/crawler_commons.js';
@@ -14,7 +14,7 @@ export declare const defaultRoute: unique symbol;
  * request under a declared label requires the matching `userData` shape.
  */
 export type RouterHandlerContext<Context, UserData extends Dictionary, Routes extends Record<keyof Routes, Dictionary>> = Omit<Context, 'request' | 'addRequests' | 'enqueueLinks'> & {
-    request: LoadedRequest<Request<UserData>>;
+    request: LoadedRequest<CrawlingRequest<UserData>>;
     addRequests: TypedContextAddRequests<Routes>;
 } & (Context extends {
     enqueueLinks: infer EnqueueLinks;
@@ -68,7 +68,7 @@ export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = strin
 export interface RouterHandler<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends Router<Context, Routes> {
     (ctx: Context): Awaitable<void>;
 }
-export type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;
+export type GetUserDataFromRequest<T> = T extends CrawlingRequest<infer Y> ? Y : never;
 /**
  * Per-route overrides, passed as the last argument of {@link Router.addHandler|`addHandler`} and
  * {@link Router.addDefaultHandler|`addDefaultHandler`}.
@@ -85,7 +85,7 @@ export interface RouteOptions {
 }
 export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary>> = {
     [Label in keyof Routes]: (ctx: Omit<Context, 'request'> & {
-        request: Request<Routes[Label]>;
+        request: CrawlingRequest<Routes[Label]>;
     }) => Awaitable<void>;
 };
 /**

package/internals/throttling_request_manager.js

@@ -799,7 +799,7 @@ export class ThrottlingRequestManager {
     async #migrateToSubQueue(request, inner) {
         try {
             const subManager = await this.#selectManagerOrThrow(request.url);
-            await subManager.addRequest(new Request({ ...request, id: undefined }));
+            await subManager.addRequest(Request.fromSchema({ ...request, id: undefined }));
         }
         catch (error) {
             // Whatever the reason, the request is out of the wrapped manager and in nobody's hands, so it

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.183",
+    "version": "4.0.0-beta.184",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/core": "4.0.0-beta.183",
-        "@crawlee/http-client": "4.0.0-beta.183",
-        "@crawlee/types": "4.0.0-beta.183",
-        "@crawlee/utils": "4.0.0-beta.183",
+        "@crawlee/core": "4.0.0-beta.184",
+        "@crawlee/http-client": "4.0.0-beta.184",
+        "@crawlee/types": "4.0.0-beta.184",
+        "@crawlee/utils": "4.0.0-beta.184",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "csv-stringify": "^6.5.2",
@@ -57,7 +57,7 @@
         "zod": "^4.5.4"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "4.0.0-beta.183"
+        "@crawlee/impit-client": "4.0.0-beta.184"
     },
     "lerna": {
         "command": {
@@ -66,5 +66,5 @@
             }
         }
     },
-    "gitHead": "3832a8d3392bba9e3e7cc6d06b9be0be4ee1ab98"
+    "gitHead": "19ebeeb9bb90e38496cbe90565c8ebb02e102193"
 }