@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.130

package/request.d.ts

@@ -1,7 +1,7 @@
 import type { BinaryLike } from 'node:crypto';
-import type { Dictionary } from '@crawlee/types';
-import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
-import type { AllowedHttpMethods } from './typedefs.js';
+import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
+import type { EnqueueStrategyOption } from './enqueue_links/enqueue_links.js';
+import type { SkippedRequestReason } from './enqueue_links/shared.js';
 export declare enum RequestState {
     UNPROCESSED = 0,
     BEFORE_NAV = 1,
@@ -44,6 +44,7 @@ export declare enum RequestState {
  * @category Sources
  */
 declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
+    #private;
     /** Request ID */
     id?: string;
     /** URL of the web page to crawl. */
@@ -74,9 +75,12 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
     errorMessages: string[];
     /** Object with HTTP headers. Key is header name, value is the value. */
     headers?: Record<string, string>;
-    /** Private store for the custom user data assigned to the request. */
-    private _userData;
-    /** Custom user data assigned to the request. */
+    /**
+     * Custom user data assigned to the request.
+     *
+     * All data stored in `userData` must be JSON-serializable.
+     * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
+     */
     userData: UserData;
     /**
      * ISO datetime string that indicates the time when the request has been processed.
@@ -92,14 +96,35 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
      * @returns The native `fetch` API Request object.
      */
     intoFetchAPIRequest(): Request;
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
     get skipNavigation(): boolean;
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
     set skipNavigation(value: boolean);
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    get sessionRotationCount(): number;
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    set sessionRotationCount(value: number);
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     */
+    get crawlDepth(): number;
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     */
+    set crawlDepth(value: number);
+    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
+    get sessionId(): string | undefined;
+    set sessionId(value: string | undefined);
     /** shortcut for getting `request.userData.label` */
     get label(): string | undefined;
     /** shortcut for setting `request.userData.label` */
@@ -112,6 +137,14 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
     get state(): RequestState;
     /** Describes the request's current lifecycle state. */
     set state(value: RequestState);
+    /**
+     * Reason for skipping this request.
+     */
+    get skippedReason(): SkippedRequestReason | undefined;
+    /**
+     * Reason for skipping this request.
+     */
+    set skippedReason(value: SkippedRequestReason | undefined);
     private get enqueueStrategy();
     private set enqueueStrategy(value);
     /**
@@ -129,7 +162,7 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
      */
     pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
     /** @internal */
-    static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, }: ComputeUniqueKeyOptions): string;
+    static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, alwaysEnqueue, }: ComputeUniqueKeyOptions): string;
     /** @internal */
     static hashPayload(payload: BinaryLike): string;
 }
@@ -150,7 +183,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      * The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
      *
      * The `useExtendedUniqueKey` options determines whether method and payload are included in the `uniqueKey`,
-     * producing a `uniqueKey` in the following format: `METHOD(payloadHash):normalizedUrl`. This is useful
+     * producing a `uniqueKey` in the following format: `METHOD|payloadHash|normalizedUrl`. This is useful
      * when requests point to the same URL, but with different methods and payloads. For example: form submits.
      *
      * Pass an arbitrary non-empty text value to the `uniqueKey` property
@@ -174,6 +207,9 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
     /**
      * Custom user data assigned to the request. Use this to save any request related data to the
      * request's scope, keeping them accessible on retries, failures etc.
+     *
+     * All data stored in `userData` must be JSON-serializable.
+     * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
      */
     userData?: UserData;
     /**
@@ -195,17 +231,45 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      * @default false
      */
     useExtendedUniqueKey?: boolean;
+    /**
+     * If `true` then a random value is included in the `uniqueKey` computation, ensuring the request
+     * is always enqueued even if a request with the same URL (and method/payload) is already present
+     * in the queue. Cannot be used together with a custom `uniqueKey`.
+     * @default false
+     */
+    alwaysEnqueue?: boolean;
     /**
      * The `true` value indicates that the request will not be automatically retried on error.
      * @default false
      */
     noRetry?: boolean;
+    /**
+     * ID of a session from the crawler's `SessionPool` to use for this request.
+     * When set, the crawler will fetch this session from the pool instead of creating a new one.
+     */
+    sessionId?: string;
     /**
      * If set to `true` then the crawler processing this request evaluates
      * the `requestHandler` immediately without prior browser navigation.
+     *
+     * When enabled, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
      * @default false
      */
     skipNavigation?: boolean;
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     * @default 0
+     */
+    crawlDepth?: number;
+    /**
+     * Reason for skipping this request.
+     * This is used to provide more information about why the request was skipped.
+     * @internal
+     */
+    skippedReason?: SkippedRequestReason;
     /**
      * Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`.
      */
@@ -217,7 +281,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
     /** @internal */
     lockExpiresAt?: Date;
     /** @internal */
-    enqueueStrategy?: EnqueueLinksOptions['strategy'];
+    enqueueStrategy?: EnqueueStrategyOption;
 }
 export interface PushErrorMessageOptions {
     /**
@@ -232,6 +296,7 @@ interface ComputeUniqueKeyOptions {
     payload?: string | Buffer;
     keepUrlFragment?: boolean;
     useExtendedUniqueKey?: boolean;
+    alwaysEnqueue?: boolean;
 }
 export type Source = (Partial<RequestOptions> & {
     requestsFromUrl?: string;
@@ -243,4 +308,3 @@ export interface InternalSource {
     regex?: RegExp;
 }
 export { CrawleeRequest as Request };
-//# sourceMappingURL=request.d.ts.map

package/request.js

@@ -1,31 +1,13 @@
 import crypto from 'node:crypto';
 import util from 'node:util';
-import ow from 'ow';
-import { normalizeUrl } from '@apify/utilities';
-import { log as defaultLog } from './log.js';
+import { z } from 'zod';
+import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
+import { serviceLocator } from './service_locator.js';
 import { keys } from './typedefs.js';
-// new properties on the Request object breaks serialization
-const log = defaultLog.child({ prefix: 'Request' });
-const requestOptionalPredicates = {
-    id: ow.optional.string,
-    loadedUrl: ow.optional.string.url,
-    uniqueKey: ow.optional.string,
-    method: ow.optional.string,
-    payload: ow.optional.any(ow.string, ow.uint8Array),
-    noRetry: ow.optional.boolean,
-    retryCount: ow.optional.number,
-    sessionRotationCount: ow.optional.number,
-    maxRetries: ow.optional.number,
-    errorMessages: ow.optional.array.ofType(ow.string),
-    headers: ow.optional.object,
-    userData: ow.optional.object,
-    label: ow.optional.string,
-    handledAt: ow.optional.any(ow.string.date, ow.date),
-    keepUrlFragment: ow.optional.boolean,
-    useExtendedUniqueKey: ow.optional.boolean,
-    skipNavigation: ow.optional.boolean,
-    state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
-};
+import { parseArgument, schemas } from './validators.js';
+const dateString = z.string().refine((value) => !Number.isNaN(Date.parse(value)), {
+    message: 'Invalid input: expected a date string',
+});
 export var RequestState;
 (function (RequestState) {
     RequestState[RequestState["UNPROCESSED"] = 0] = "UNPROCESSED";
@@ -37,6 +19,34 @@ export var RequestState;
     RequestState[RequestState["ERROR"] = 6] = "ERROR";
     RequestState[RequestState["SKIPPED"] = 7] = "SKIPPED";
 })(RequestState || (RequestState = {}));
+const requestUrlSchema = z.object({ url: z.string() });
+// new properties on the Request object breaks serialization
+const requestOptionalSchemaShapes = {
+    id: z.string().optional(),
+    loadedUrl: z.url().optional(),
+    uniqueKey: z.string().optional(),
+    method: z.string().optional(),
+    payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
+    noRetry: z.boolean().optional(),
+    retryCount: schemas.anyNumber.optional(),
+    sessionId: z.string().optional(),
+    maxRetries: schemas.anyNumber.optional(),
+    errorMessages: schemas.arrayOf(z.string(), 'strings').optional(),
+    headers: z.looseObject({}).optional(),
+    userData: z.looseObject({}).optional(),
+    label: z.string().optional(),
+    handledAt: z.union([dateString, z.date()]).optional(),
+    keepUrlFragment: z.boolean().optional(),
+    useExtendedUniqueKey: z.boolean().optional(),
+    alwaysEnqueue: z.boolean().optional(),
+    skipNavigation: z.boolean().optional(),
+    crawlDepth: schemas.anyNumber
+        .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
+        .optional(),
+    state: z.enum(RequestState).optional(),
+};
+// Each schema is wrapped in a single-key object so validation errors carry the property name.
+const requestOptionalSchemas = Object.fromEntries(Object.entries(requestOptionalSchemaShapes).map(([key, schema]) => [key, z.object({ [key]: schema })]));
 /**
  * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
  * The `Request` object also stores information about errors that occurred during processing of the request.
@@ -100,8 +110,13 @@ class CrawleeRequest {
     /** Object with HTTP headers. Key is header name, value is the value. */
     headers;
     /** Private store for the custom user data assigned to the request. */
-    _userData = {};
-    /** Custom user data assigned to the request. */
+    #userData = {};
+    /**
+     * Custom user data assigned to the request.
+     *
+     * All data stored in `userData` must be JSON-serializable.
+     * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
+     */
     userData = {};
     /**
      * ISO datetime string that indicates the time when the request has been processed.
@@ -112,76 +127,89 @@ class CrawleeRequest {
      * `Request` parameters including the URL, HTTP method and headers, and others.
      */
     constructor(options) {
-        ow(options, 'RequestOptions', ow.object);
-        ow(options.url, 'RequestOptions.url', ow.string);
-        // 'ow' validation is slow, because it checks all predicates
+        // A bare URL is a common slip — point at the object form instead of a generic type error.
+        if (typeof options === 'string') {
+            throw new TypeError(`\`Request\` options must be an object, got the string '${options}'. ` +
+                'Did you mean `new Request({ url })`?');
+        }
+        parseArgument(options, schemas.anyObject, 'RequestOptions');
+        parseArgument(options, requestUrlSchema, 'RequestOptions');
+        // Full-shape validation is slow, because it checks all predicates
         // even if the validated object has only 1 property.
         // This custom validation loop iterates only over existing
         // properties and speeds up the validation cca 3-fold.
-        // See https://github.com/sindresorhus/ow/issues/193
         keys(options).forEach((prop) => {
             // skip url, because it is validated above
             if (prop === 'url') {
                 return;
             }
-            const predicate = requestOptionalPredicates[prop];
+            const schema = requestOptionalSchemas[prop];
             const value = options[prop];
-            if (predicate) {
-                ow(value, `RequestOptions.${prop}`, predicate);
+            if (schema) {
+                parseArgument({ [prop]: value }, schema, 'RequestOptions');
             }
         });
-        const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionRotationCount = 0, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, enqueueStrategy, } = options;
+        const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionId, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
         let { method = 'GET' } = options;
         method = method.toUpperCase();
         if (method === 'GET' && payload)
             throw new Error('Request with GET method cannot have a payload.');
+        if (uniqueKey && alwaysEnqueue) {
+            throw new Error('`alwaysEnqueue` cannot be used together with a custom `uniqueKey`.');
+        }
         this.id = id;
         this.url = url;
         this.loadedUrl = loadedUrl;
         this.uniqueKey =
             uniqueKey ||
-                CrawleeRequest.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
+                CrawleeRequest.computeUniqueKey({
+                    url,
+                    method,
+                    payload,
+                    keepUrlFragment,
+                    useExtendedUniqueKey,
+                    alwaysEnqueue,
+                });
         this.method = method;
         this.payload = payload;
         this.noRetry = noRetry;
         this.retryCount = retryCount;
-        this.sessionRotationCount = sessionRotationCount;
         this.errorMessages = [...errorMessages];
         this.headers = { ...headers };
         this.handledAt = handledAt instanceof Date ? handledAt.toISOString() : handledAt;
         if (label) {
             userData.label = label;
         }
+        // Read `__crawlee` explicitly - on a `userData` coming from another Request instance the
+        // bag is non-enumerable, so the spread alone would silently drop the internal state
+        // (e.g. `skipNavigation`) when a request is re-wrapped after a storage round trip.
+        this.#userData = { __crawlee: userData.__crawlee ?? {}, ...userData };
+        // `userData` must stay an enumerable own accessor — serialization in the storages relies on it
         Object.defineProperties(this, {
-            _userData: {
-                value: { __crawlee: {}, ...userData },
-                enumerable: false,
-                writable: true,
-            },
             userData: {
-                get: () => this._userData,
+                get: () => this.#userData,
                 set: (value) => {
                     Object.defineProperties(value, {
                         __crawlee: {
-                            value: this._userData.__crawlee,
+                            value: this.#userData.__crawlee,
                             enumerable: false,
                             writable: true,
                         },
                         toJSON: {
                             value: () => {
-                                if (Object.keys(this._userData.__crawlee).length > 0) {
+                                if (Object.keys(this.#userData.__crawlee).length > 0) {
                                     return {
-                                        ...this._userData,
-                                        __crawlee: this._userData.__crawlee,
+                                        ...this.#userData,
+                                        __crawlee: this.#userData.__crawlee,
                                     };
                                 }
-                                return this._userData;
+                                return this.#userData;
                             },
                             enumerable: false,
                             writable: true,
                         },
                     });
-                    this._userData = value;
+                    this.#userData = value;
                 },
                 enumerable: true,
             },
@@ -192,6 +220,10 @@ class CrawleeRequest {
             this.skipNavigation = skipNavigation;
         if (maxRetries != null)
             this.maxRetries = maxRetries;
+        if (crawlDepth != null)
+            this.userData.__crawlee.crawlDepth ??= crawlDepth;
+        if (sessionId)
+            this.sessionId = sessionId;
         // If it's already set, don't override it (for instance when fetching from storage)
         if (enqueueStrategy) {
             this.enqueueStrategy ??= enqueueStrategy;
@@ -208,11 +240,23 @@ class CrawleeRequest {
             body: this.payload,
         });
     }
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
     get skipNavigation() {
         return this.userData.__crawlee?.skipNavigation ?? false;
     }
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
     set skipNavigation(value) {
         if (!this.userData.__crawlee) {
             this.userData.__crawlee = { skipNavigation: value };
@@ -221,18 +265,28 @@ class CrawleeRequest {
             this.userData.__crawlee.skipNavigation = value;
         }
     }
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    get sessionRotationCount() {
-        return this.userData.__crawlee?.sessionRotationCount ?? 0;
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     */
+    get crawlDepth() {
+        return this.userData.__crawlee?.crawlDepth ?? 0;
     }
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    set sessionRotationCount(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { sessionRotationCount: value };
-        }
-        else {
-            this.userData.__crawlee.sessionRotationCount = value;
-        }
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     */
+    set crawlDepth(value) {
+        this.userData.__crawlee ??= {};
+        this.userData.__crawlee.crawlDepth = value;
+    }
+    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
+    get sessionId() {
+        return this.userData.__crawlee?.sessionId;
+    }
+    set sessionId(value) {
+        this.userData.__crawlee ??= {};
+        this.userData.__crawlee.sessionId = value;
     }
     /** shortcut for getting `request.userData.label` */
     get label() {
@@ -268,6 +322,23 @@ class CrawleeRequest {
             this.userData.__crawlee.state = value;
         }
     }
+    /**
+     * Reason for skipping this request.
+     */
+    get skippedReason() {
+        return this.userData.__crawlee?.skippedReason;
+    }
+    /**
+     * Reason for skipping this request.
+     */
+    set skippedReason(value) {
+        if (!this.userData.__crawlee) {
+            this.userData.__crawlee = { skippedReason: value };
+        }
+        else {
+            this.userData.__crawlee.skippedReason = value;
+        }
+    }
     get enqueueStrategy() {
         return this.userData.__crawlee?.enqueueStrategy;
     }
@@ -330,19 +401,28 @@ class CrawleeRequest {
         this.errorMessages.push(message);
     }
     /** @internal */
-    static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, }) {
+    static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, }) {
         const normalizedMethod = method.toUpperCase();
         const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
+        let uniqueKey;
         if (!useExtendedUniqueKey) {
             if (normalizedMethod !== 'GET' && payload) {
-                log.warningOnce(`We've encountered a ${normalizedMethod} Request with a payload. ` +
+                serviceLocator
+                    .getLogger()
+                    .warningOnce(`We've encountered a ${normalizedMethod} Request with a payload. ` +
                     'This is fine. Just letting you know that if your requests point to the same URL ' +
                     'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.');
             }
-            return normalizedUrl;
+            uniqueKey = normalizedUrl;
+        }
+        else {
+            const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
+            uniqueKey = `${normalizedMethod}|${payloadHash}|${normalizedUrl}`;
+        }
+        if (alwaysEnqueue) {
+            uniqueKey = `${cryptoRandomObjectId(17)}|${uniqueKey}`;
         }
-        const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
-        return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
+        return uniqueKey;
     }
     /** @internal */
     static hashPayload(payload) {
@@ -350,4 +430,3 @@ class CrawleeRequest {
     }
 }
 export { CrawleeRequest as Request };
-//# sourceMappingURL=request.js.map