@crawlee/core 4.0.0-beta.1 → 4.0.0-beta.100

package/recoverable_state.js

@@ -0,0 +1,143 @@
+import { KeyValueStore, serviceLocator } from '@crawlee/core';
+/**
+ * A class for managing persistent recoverable state using a plain JavaScript object.
+ *
+ * This class facilitates state persistence to a `KeyValueStore`, allowing data to be saved and retrieved
+ * across migrations or restarts. It manages the loading, saving, and resetting of state data,
+ * with optional persistence capabilities.
+ *
+ * The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
+ * The class automatically hooks into the event system to persist state when needed.
+ */
+export class RecoverableState {
+    defaultState;
+    state = null;
+    persistenceEnabled;
+    persistStateKey;
+    persistStateKvsName;
+    persistStateKvsId;
+    keyValueStore = null;
+    log;
+    serialize;
+    deserialize;
+    /**
+     * Initialize a new recoverable state object.
+     *
+     * @param options Configuration options for the recoverable state
+     */
+    constructor(options) {
+        this.defaultState = options.defaultState;
+        this.persistStateKey = options.persistStateKey;
+        this.persistenceEnabled = options.persistenceEnabled ?? false;
+        this.persistStateKvsName = options.persistStateKvsName;
+        this.persistStateKvsId = options.persistStateKvsId;
+        this.log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
+        this.serialize = options.serialize ?? JSON.stringify;
+        this.deserialize = options.deserialize ?? JSON.parse;
+        this.persistState = this.persistState.bind(this);
+    }
+    /**
+     * Initialize the recoverable state.
+     *
+     * This method must be called before using the recoverable state. It loads the saved state
+     * if persistence is enabled and registers the object to listen for PERSIST_STATE events.
+     *
+     * @returns The loaded state object
+     */
+    async initialize() {
+        if (this.state !== null && this.state !== undefined) {
+            return this.currentValue;
+        }
+        if (!this.persistenceEnabled) {
+            this.state = this.deserialize(this.serialize(this.defaultState));
+            return this.currentValue;
+        }
+        let kvsIdentifier = null;
+        if (this.persistStateKvsName) {
+            kvsIdentifier = { name: this.persistStateKvsName };
+        }
+        else if (this.persistStateKvsId) {
+            kvsIdentifier = { id: this.persistStateKvsId };
+        }
+        this.keyValueStore = await KeyValueStore.open(kvsIdentifier, {
+            configuration: serviceLocator.getConfiguration(),
+        });
+        await this.loadSavedState();
+        // Register for persist state events
+        const eventManager = serviceLocator.getEventManager();
+        eventManager.on("persistState" /* EventType.PERSIST_STATE */, this.persistState);
+        return this.currentValue;
+    }
+    /**
+     * Clean up resources used by the recoverable state.
+     *
+     * If persistence is enabled, this method deregisters the object from PERSIST_STATE events
+     * and persists the current state one last time.
+     */
+    async teardown() {
+        if (!this.persistenceEnabled || !this.persistState) {
+            return;
+        }
+        const eventManager = serviceLocator.getEventManager();
+        eventManager.off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
+        await this.persistState();
+    }
+    /**
+     * Get the current state.
+     */
+    get currentValue() {
+        if (this.state === null) {
+            throw new Error('Recoverable state has not yet been loaded');
+        }
+        return this.state;
+    }
+    /**
+     * Reset the state to the default values and clear any persisted state.
+     *
+     * Resets the current state to the default state and, if persistence is enabled,
+     * clears the persisted state from the KeyValueStore.
+     */
+    async reset() {
+        this.state = this.deserialize(this.serialize(this.defaultState));
+        if (this.persistenceEnabled) {
+            if (this.keyValueStore === null) {
+                throw new Error('Recoverable state has not yet been initialized');
+            }
+            await this.keyValueStore.setValue(this.persistStateKey, null);
+        }
+    }
+    /**
+     * Persist the current state to the KeyValueStore.
+     *
+     * This method is typically called in response to a PERSIST_STATE event, but can also be called
+     * directly when needed.
+     *
+     * @param eventData Optional data associated with a PERSIST_STATE event
+     */
+    async persistState(eventData) {
+        this.log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
+        if (this.keyValueStore === null || this.state === null) {
+            throw new Error('Recoverable state has not yet been initialized');
+        }
+        if (this.persistenceEnabled) {
+            await this.keyValueStore.setValue(this.persistStateKey, this.serialize(this.state), {
+                contentType: 'text/plain', // HACK - the result is expected to be JSON, but we do this to avoid the implicit JSON.parse in `KeyValueStore.getValue`
+            });
+        }
+    }
+    /**
+     * Load the saved state from the KeyValueStore
+     */
+    async loadSavedState() {
+        if (this.keyValueStore === null) {
+            throw new Error('Recoverable state has not yet been initialized');
+        }
+        const storedState = await this.keyValueStore.getValue(this.persistStateKey);
+        if (storedState === null || storedState === undefined) {
+            this.state = this.deserialize(this.serialize(this.defaultState));
+        }
+        else {
+            this.state = this.deserialize(storedState);
+        }
+    }
+}

package/request.d.ts

@@ -1,7 +1,7 @@
 import type { BinaryLike } from 'node:crypto';
-import type { Dictionary } from '@crawlee/types';
+import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
 import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
-import type { AllowedHttpMethods } from './typedefs.js';
+import type { SkippedRequestReason } from './enqueue_links/shared.js';
 export declare enum RequestState {
     UNPROCESSED = 0,
     BEFORE_NAV = 1,
@@ -43,7 +43,7 @@ export declare enum RequestState {
  * ```
  * @category Sources
  */
-export declare class Request<UserData extends Dictionary = Dictionary> {
+declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
     /** Request ID */
     id?: string;
     /** URL of the web page to crawl. */
@@ -76,7 +76,12 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
     headers?: Record<string, string>;
     /** Private store for the custom user data assigned to the request. */
     private _userData;
-    /** Custom user data assigned to the request. */
+    /**
+     * Custom user data assigned to the request.
+     *
+     * All data stored in `userData` must be JSON-serializable.
+     * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
+     */
     userData: UserData;
     /**
      * ISO datetime string that indicates the time when the request has been processed.
@@ -87,14 +92,40 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
      * `Request` parameters including the URL, HTTP method and headers, and others.
      */
     constructor(options: RequestOptions<UserData>);
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+    /**
+     * Converts the Crawlee Request object to a `fetch` API Request object.
+     * @returns The native `fetch` API Request object.
+     */
+    intoFetchAPIRequest(): Request;
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
     get skipNavigation(): boolean;
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
     set skipNavigation(value: boolean);
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    get sessionRotationCount(): number;
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    set sessionRotationCount(value: number);
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     */
+    get crawlDepth(): number;
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     */
+    set crawlDepth(value: number);
+    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
+    get sessionId(): string | undefined;
+    set sessionId(value: string | undefined);
     /** shortcut for getting `request.userData.label` */
     get label(): string | undefined;
     /** shortcut for setting `request.userData.label` */
@@ -107,6 +138,14 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
     get state(): RequestState;
     /** Describes the request's current lifecycle state. */
     set state(value: RequestState);
+    /**
+     * Reason for skipping this request.
+     */
+    get skippedReason(): SkippedRequestReason | undefined;
+    /**
+     * Reason for skipping this request.
+     */
+    set skippedReason(value: SkippedRequestReason | undefined);
     private get enqueueStrategy();
     private set enqueueStrategy(value);
     /**
@@ -123,10 +162,8 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
      * @param [options]
      */
     pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
-    protected _computeUniqueKey(options: ComputeUniqueKeyOptions): string;
-    protected _hashPayload(payload: BinaryLike): string;
     /** @internal */
-    static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, }: ComputeUniqueKeyOptions): string;
+    static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, alwaysEnqueue, }: ComputeUniqueKeyOptions): string;
     /** @internal */
     static hashPayload(payload: BinaryLike): string;
 }
@@ -147,7 +184,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      * The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
      *
      * The `useExtendedUniqueKey` options determines whether method and payload are included in the `uniqueKey`,
-     * producing a `uniqueKey` in the following format: `METHOD(payloadHash):normalizedUrl`. This is useful
+     * producing a `uniqueKey` in the following format: `METHOD|payloadHash|normalizedUrl`. This is useful
      * when requests point to the same URL, but with different methods and payloads. For example: form submits.
      *
      * Pass an arbitrary non-empty text value to the `uniqueKey` property
@@ -171,6 +208,9 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
     /**
      * Custom user data assigned to the request. Use this to save any request related data to the
      * request's scope, keeping them accessible on retries, failures etc.
+     *
+     * All data stored in `userData` must be JSON-serializable.
+     * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
      */
     userData?: UserData;
     /**
@@ -192,17 +232,45 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      * @default false
      */
     useExtendedUniqueKey?: boolean;
+    /**
+     * If `true` then a random value is included in the `uniqueKey` computation, ensuring the request
+     * is always enqueued even if a request with the same URL (and method/payload) is already present
+     * in the queue. Cannot be used together with a custom `uniqueKey`.
+     * @default false
+     */
+    alwaysEnqueue?: boolean;
     /**
      * The `true` value indicates that the request will not be automatically retried on error.
      * @default false
      */
     noRetry?: boolean;
+    /**
+     * ID of a session from the crawler's `SessionPool` to use for this request.
+     * When set, the crawler will fetch this session from the pool instead of creating a new one.
+     */
+    sessionId?: string;
     /**
      * If set to `true` then the crawler processing this request evaluates
      * the `requestHandler` immediately without prior browser navigation.
+     *
+     * When enabled, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
      * @default false
      */
     skipNavigation?: boolean;
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     * @default 0
+     */
+    crawlDepth?: number;
+    /**
+     * Reason for skipping this request.
+     * This is used to provide more information about why the request was skipped.
+     * @internal
+     */
+    skippedReason?: SkippedRequestReason;
     /**
      * Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`.
      */
@@ -229,15 +297,15 @@ interface ComputeUniqueKeyOptions {
     payload?: string | Buffer;
     keepUrlFragment?: boolean;
     useExtendedUniqueKey?: boolean;
+    alwaysEnqueue?: boolean;
 }
 export type Source = (Partial<RequestOptions> & {
     requestsFromUrl?: string;
     regex?: RegExp;
-}) | Request;
+}) | CrawleeRequest;
 /** @internal */
 export interface InternalSource {
     requestsFromUrl: string;
     regex?: RegExp;
 }
-export {};
-//# sourceMappingURL=request.d.ts.map
+export { CrawleeRequest as Request };

package/request.js

@@ -1,11 +1,10 @@
 import crypto from 'node:crypto';
 import util from 'node:util';
 import ow from 'ow';
-import { normalizeUrl } from '@apify/utilities';
-import { log as defaultLog } from './log.js';
+import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
+import { serviceLocator } from './service_locator.js';
 import { keys } from './typedefs.js';
 // new properties on the Request object breaks serialization
-const log = defaultLog.child({ prefix: 'Request' });
 const requestOptionalPredicates = {
     id: ow.optional.string,
     loadedUrl: ow.optional.string.url,
@@ -14,7 +13,7 @@ const requestOptionalPredicates = {
     payload: ow.optional.any(ow.string, ow.uint8Array),
     noRetry: ow.optional.boolean,
     retryCount: ow.optional.number,
-    sessionRotationCount: ow.optional.number,
+    sessionId: ow.optional.string,
     maxRetries: ow.optional.number,
     errorMessages: ow.optional.array.ofType(ow.string),
     headers: ow.optional.object,
@@ -23,7 +22,9 @@ const requestOptionalPredicates = {
     handledAt: ow.optional.any(ow.string.date, ow.date),
     keepUrlFragment: ow.optional.boolean,
     useExtendedUniqueKey: ow.optional.boolean,
+    alwaysEnqueue: ow.optional.boolean,
     skipNavigation: ow.optional.boolean,
+    crawlDepth: ow.optional.number.greaterThanOrEqual(0),
     state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
 };
 export var RequestState;
@@ -68,7 +69,7 @@ export var RequestState;
  * ```
  * @category Sources
  */
-export class Request {
+class CrawleeRequest {
     /** Request ID */
     id;
     /** URL of the web page to crawl. */
@@ -101,7 +102,12 @@ export class Request {
     headers;
     /** Private store for the custom user data assigned to the request. */
     _userData = {};
-    /** Custom user data assigned to the request. */
+    /**
+     * Custom user data assigned to the request.
+     *
+     * All data stored in `userData` must be JSON-serializable.
+     * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
+     */
     userData = {};
     /**
      * ISO datetime string that indicates the time when the request has been processed.
@@ -130,21 +136,31 @@ export class Request {
                 ow(value, `RequestOptions.${prop}`, predicate);
             }
         });
-        const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionRotationCount = 0, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, enqueueStrategy, } = options;
+        const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionId, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
         let { method = 'GET' } = options;
         method = method.toUpperCase();
         if (method === 'GET' && payload)
             throw new Error('Request with GET method cannot have a payload.');
+        if (uniqueKey && alwaysEnqueue) {
+            throw new Error('`alwaysEnqueue` cannot be used together with a custom `uniqueKey`.');
+        }
         this.id = id;
         this.url = url;
         this.loadedUrl = loadedUrl;
         this.uniqueKey =
-            uniqueKey || Request.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
+            uniqueKey ||
+                CrawleeRequest.computeUniqueKey({
+                    url,
+                    method,
+                    payload,
+                    keepUrlFragment,
+                    useExtendedUniqueKey,
+                    alwaysEnqueue,
+                });
         this.method = method;
         this.payload = payload;
         this.noRetry = noRetry;
         this.retryCount = retryCount;
-        this.sessionRotationCount = sessionRotationCount;
         this.errorMessages = [...errorMessages];
         this.headers = { ...headers };
         this.handledAt = handledAt instanceof Date ? handledAt.toISOString() : handledAt;
@@ -191,16 +207,43 @@ export class Request {
             this.skipNavigation = skipNavigation;
         if (maxRetries != null)
             this.maxRetries = maxRetries;
+        if (crawlDepth != null)
+            this.userData.__crawlee.crawlDepth ??= crawlDepth;
+        if (sessionId)
+            this.sessionId = sessionId;
         // If it's already set, don't override it (for instance when fetching from storage)
         if (enqueueStrategy) {
             this.enqueueStrategy ??= enqueueStrategy;
         }
     }
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+    /**
+     * Converts the Crawlee Request object to a `fetch` API Request object.
+     * @returns The native `fetch` API Request object.
+     */
+    intoFetchAPIRequest() {
+        return new Request(this.url, {
+            method: this.method,
+            headers: this.headers,
+            body: this.payload,
+        });
+    }
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
     get skipNavigation() {
         return this.userData.__crawlee?.skipNavigation ?? false;
     }
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
     set skipNavigation(value) {
         if (!this.userData.__crawlee) {
             this.userData.__crawlee = { skipNavigation: value };
@@ -209,18 +252,28 @@ export class Request {
             this.userData.__crawlee.skipNavigation = value;
         }
     }
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    get sessionRotationCount() {
-        return this.userData.__crawlee?.sessionRotationCount ?? 0;
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     */
+    get crawlDepth() {
+        return this.userData.__crawlee?.crawlDepth ?? 0;
     }
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    set sessionRotationCount(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { sessionRotationCount: value };
-        }
-        else {
-            this.userData.__crawlee.sessionRotationCount = value;
-        }
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     */
+    set crawlDepth(value) {
+        this.userData.__crawlee ??= {};
+        this.userData.__crawlee.crawlDepth = value;
+    }
+    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
+    get sessionId() {
+        return this.userData.__crawlee?.sessionId;
+    }
+    set sessionId(value) {
+        this.userData.__crawlee ??= {};
+        this.userData.__crawlee.sessionId = value;
     }
     /** shortcut for getting `request.userData.label` */
     get label() {
@@ -256,6 +309,23 @@ export class Request {
             this.userData.__crawlee.state = value;
         }
     }
+    /**
+     * Reason for skipping this request.
+     */
+    get skippedReason() {
+        return this.userData.__crawlee?.skippedReason;
+    }
+    /**
+     * Reason for skipping this request.
+     */
+    set skippedReason(value) {
+        if (!this.userData.__crawlee) {
+            this.userData.__crawlee = { skippedReason: value };
+        }
+        else {
+            this.userData.__crawlee.skippedReason = value;
+        }
+    }
     get enqueueStrategy() {
         return this.userData.__crawlee?.enqueueStrategy;
     }
@@ -317,32 +387,33 @@ export class Request {
         }
         this.errorMessages.push(message);
     }
-    // TODO: only for better BC, remove in v4
-    _computeUniqueKey(options) {
-        return Request.computeUniqueKey(options);
-    }
-    // TODO: only for better BC, remove in v4
-    _hashPayload(payload) {
-        return Request.hashPayload(payload);
-    }
     /** @internal */
-    static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, }) {
+    static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, }) {
         const normalizedMethod = method.toUpperCase();
         const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
+        let uniqueKey;
         if (!useExtendedUniqueKey) {
             if (normalizedMethod !== 'GET' && payload) {
-                log.warningOnce(`We've encountered a ${normalizedMethod} Request with a payload. ` +
+                serviceLocator
+                    .getLogger()
+                    .warningOnce(`We've encountered a ${normalizedMethod} Request with a payload. ` +
                     'This is fine. Just letting you know that if your requests point to the same URL ' +
                     'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.');
             }
-            return normalizedUrl;
+            uniqueKey = normalizedUrl;
+        }
+        else {
+            const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
+            uniqueKey = `${normalizedMethod}|${payloadHash}|${normalizedUrl}`;
+        }
+        if (alwaysEnqueue) {
+            uniqueKey = `${cryptoRandomObjectId(17)}|${uniqueKey}`;
         }
-        const payloadHash = payload ? Request.hashPayload(payload) : '';
-        return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
+        return uniqueKey;
     }
     /** @internal */
     static hashPayload(payload) {
         return crypto.createHash('sha256').update(payload).digest('base64').replace(/[+/=]/g, '').substring(0, 8);
     }
 }
-//# sourceMappingURL=request.js.map
+export { CrawleeRequest as Request };