@crawlee/core 4.0.0-beta.12 → 4.0.0-beta.121

package/recoverable_state.js

@@ -0,0 +1,212 @@
+import { addTimeoutToPromise, storage as timeoutStorage } from '@apify/timeout';
+import { EventType, KeyValueStore, serviceLocator, StateValidationError } from '@crawlee/core';
+const DEFAULT_PERSISTENCE_TIMEOUT_MILLIS = 60_000;
+/**
+ * A class for managing persistent recoverable state using a plain JavaScript object.
+ *
+ * This class facilitates state persistence to a `KeyValueStore`, allowing data to be saved and retrieved
+ * across migrations or restarts. It manages the loading, saving, and resetting of state data,
+ * with optional persistence capabilities.
+ *
+ * The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
+ * The class automatically hooks into the event system to persist state when needed.
+ */
+export class RecoverableState {
+    #defaultState;
+    #state = null;
+    #initialized = false;
+    #listening = false;
+    #persistenceEnabled;
+    #persistStateKey;
+    #persistenceTimeoutMillis;
+    #configuration;
+    #keyValueStore;
+    #log;
+    #serialize;
+    #deserialize;
+    #persistStateQuietly;
+    /**
+     * Initialize a new recoverable state object.
+     *
+     * @param options Configuration options for the recoverable state
+     */
+    constructor(options) {
+        const { defaultState } = options;
+        this.#defaultState =
+            typeof defaultState === 'function'
+                ? defaultState
+                : () => structuredClone(defaultState);
+        this.#persistStateKey = options.persistStateKey;
+        this.#persistenceEnabled = options.persistenceEnabled ?? false;
+        this.#persistenceTimeoutMillis = options.persistenceTimeoutMillis ?? DEFAULT_PERSISTENCE_TIMEOUT_MILLIS;
+        this.#configuration = options.configuration;
+        this.#keyValueStore = options.keyValueStore ?? null;
+        this.#log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
+        this.#serialize = this.#toConversion(options.serialize);
+        this.#deserialize = this.#toConversion(options.deserialize);
+        // The automatic persists, where a rejection has nowhere useful to go - the event manager does not catch
+        // listener errors, and throwing from teardown would bury the outcome of the work it cleans up after.
+        this.#persistStateQuietly = async (eventData) => this.persistState(eventData).catch((error) => this.#log.warning(`Failed to persist the state under key '${this.#persistStateKey}'.`, { error }));
+    }
+    /** Normalizes a conversion option into a function. Absent conversions pass the value through unchanged. */
+    #toConversion(conversion) {
+        if (conversion === undefined) {
+            return async (value) => value;
+        }
+        if (typeof conversion === 'function') {
+            return async (value) => conversion(value);
+        }
+        return async (value) => {
+            const result = await conversion['~standard'].validate(value);
+            if (result.issues) {
+                throw new StateValidationError(this.#persistStateKey, result.issues);
+            }
+            return result.value;
+        };
+    }
+    /**
+     * Initialize the recoverable state.
+     *
+     * If persistence is enabled, this method loads the saved state and registers the object to listen for
+     * PERSIST_STATE events. A state established beforehand by {@link RecoverableState.reset} survives if there
+     * is no record to restore.
+     *
+     * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
+     * listener is registered again and the record reloaded.
+     *
+     * @returns The loaded state object
+     */
+    async initialize() {
+        if (this.#initialized) {
+            return this.currentValue;
+        }
+        if (this.#persistenceEnabled) {
+            this.#keyValueStore ??= KeyValueStore.open(null, {
+                configuration: this.#configuration ?? serviceLocator.getConfiguration(),
+            });
+            await this.#resolveKeyValueStore();
+            serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.#persistStateQuietly);
+            this.#listening = true;
+        }
+        // Flipped before the record is loaded, so that a caller catching a `StateValidationError` is left with a
+        // fully wired object running on the default state rather than a half-initialized one.
+        this.#initialized = true;
+        this.#state ??= this.#defaultState();
+        await this.#loadSavedState();
+        return this.currentValue;
+    }
+    /**
+     * Clean up resources used by the recoverable state.
+     *
+     * If persistence is enabled, this method deregisters the object from PERSIST_STATE events
+     * and persists the current state one last time, warning rather than throwing if that write fails - cleanup
+     * runs when the work is already done, and failing it would bury whatever the caller was doing. The in-memory
+     * state is left alone, and {@link RecoverableState.initialize} can be called again to open a new
+     * persistence window.
+     */
+    async teardown() {
+        this.#initialized = false;
+        if (!this.#persistenceEnabled) {
+            return;
+        }
+        serviceLocator.getEventManager().off(EventType.PERSIST_STATE, this.#persistStateQuietly);
+        this.#listening = false;
+        await this.#persistStateQuietly();
+    }
+    /**
+     * Get the current state.
+     *
+     * Throws until the state has been established, by either {@link RecoverableState.initialize} or the
+     * synchronous {@link RecoverableState.reset} - the latter being how a caller that cannot await in its
+     * constructor gets a usable state right away.
+     */
+    get currentValue() {
+        if (this.#state === null) {
+            throw new Error('Recoverable state has not yet been loaded - call initialize() or reset() first');
+        }
+        return this.#state;
+    }
+    /**
+     * Reset the in-memory state to the default values, leaving any persisted record alone.
+     *
+     * Use {@link RecoverableState.resetStore} to clear the persisted record as well.
+     */
+    reset() {
+        this.#state = this.#defaultState();
+    }
+    /**
+     * Clear the persisted state record, leaving the in-memory state alone.
+     *
+     * This is a between-lifecycles operation - its point is to stop the next {@link RecoverableState.initialize}
+     * from restoring the record, so it throws while PERSIST_STATE events are still being handled, where the next
+     * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
+     * or {@link RecoverableState.teardown} before clearing the record.
+     *
+     * A no-op if persistence is disabled or no KeyValueStore is available yet.
+     */
+    async resetStore() {
+        if (this.#listening) {
+            throw new Error(`Cannot clear the state persisted under key '${this.#persistStateKey}' while it is still being persisted periodically - the next PERSIST_STATE event would write it straight back. Use reset() to reset the state itself, or teardown() before clearing the record.`);
+        }
+        if (!this.#persistenceEnabled) {
+            return;
+        }
+        const keyValueStore = await this.#resolveKeyValueStore();
+        if (keyValueStore === null) {
+            return;
+        }
+        await this.#withTimeout(async () => keyValueStore.setValue(this.#persistStateKey, null), 'Clearing the persisted state');
+    }
+    /**
+     * Persist the current state to the KeyValueStore.
+     *
+     * This method is typically called in response to a PERSIST_STATE event, but can also be called
+     * directly when needed. It is a no-op if persistence is disabled, if no KeyValueStore is available yet, or if
+     * there is no state to write. A failed write only rejects here - the periodic and teardown ones warn instead.
+     *
+     * @param eventData Optional data associated with a PERSIST_STATE event
+     */
+    async persistState(eventData) {
+        if (!this.#persistenceEnabled || this.#state === null) {
+            return;
+        }
+        const keyValueStore = await this.#resolveKeyValueStore();
+        if (keyValueStore === null) {
+            return;
+        }
+        this.#log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
+        const serializedState = await this.#serialize(this.currentValue);
+        await this.#withTimeout(async () => keyValueStore.setValue(this.#persistStateKey, serializedState), 'Persisting the state');
+    }
+    /** Awaits a store handed over as a pending `open()`, keeping the resolved instance for later calls. */
+    async #resolveKeyValueStore() {
+        if (this.#keyValueStore === null) {
+            return null;
+        }
+        this.#keyValueStore = await this.#keyValueStore;
+        return this.#keyValueStore;
+    }
+    /**
+     * Load the saved state from the KeyValueStore. Leaves the current state alone if there is no record to load.
+     */
+    async #loadSavedState() {
+        if (!this.#persistenceEnabled) {
+            return;
+        }
+        const keyValueStore = await this.#resolveKeyValueStore();
+        if (keyValueStore === null) {
+            return;
+        }
+        const storedState = await this.#withTimeout(async () => keyValueStore.getValue(this.#persistStateKey), 'Loading the persisted state');
+        if (storedState === null || storedState === undefined) {
+            return;
+        }
+        this.#state = await this.#deserialize(storedState);
+    }
+    async #withTimeout(operation, description) {
+        // `@apify/timeout` shares one `AbortController` across nested frames and `KeyValueStore` checks it on
+        // every operation, so a teardown-time persist running inside an already-expired request handler timeout
+        // would be aborted before it started. Hence a fresh timeout context.
+        return timeoutStorage.exit(async () => addTimeoutToPromise(operation, this.#persistenceTimeoutMillis, `${description} under key '${this.#persistStateKey}' timed out after ${this.#persistenceTimeoutMillis / 1000} seconds.`));
+    }
+}

package/request.d.ts

@@ -1,7 +1,7 @@
 import type { BinaryLike } from 'node:crypto';
-import type { Dictionary } from '@crawlee/types';
+import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
 import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
-import type { AllowedHttpMethods } from './typedefs.js';
+import type { SkippedRequestReason } from './enqueue_links/shared.js';
 export declare enum RequestState {
     UNPROCESSED = 0,
     BEFORE_NAV = 1,
@@ -43,7 +43,8 @@ export declare enum RequestState {
  * ```
  * @category Sources
  */
-export declare class Request<UserData extends Dictionary = Dictionary> {
+declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
+    #private;
     /** Request ID */
     id?: string;
     /** URL of the web page to crawl. */
@@ -74,9 +75,12 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
     errorMessages: string[];
     /** Object with HTTP headers. Key is header name, value is the value. */
     headers?: Record<string, string>;
-    /** Private store for the custom user data assigned to the request. */
-    private _userData;
-    /** Custom user data assigned to the request. */
+    /**
+     * Custom user data assigned to the request.
+     *
+     * All data stored in `userData` must be JSON-serializable.
+     * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
+     */
     userData: UserData;
     /**
      * ISO datetime string that indicates the time when the request has been processed.
@@ -87,14 +91,40 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
      * `Request` parameters including the URL, HTTP method and headers, and others.
      */
     constructor(options: RequestOptions<UserData>);
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+    /**
+     * Converts the Crawlee Request object to a `fetch` API Request object.
+     * @returns The native `fetch` API Request object.
+     */
+    intoFetchAPIRequest(): Request;
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
     get skipNavigation(): boolean;
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
     set skipNavigation(value: boolean);
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    get sessionRotationCount(): number;
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    set sessionRotationCount(value: number);
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     */
+    get crawlDepth(): number;
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     */
+    set crawlDepth(value: number);
+    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
+    get sessionId(): string | undefined;
+    set sessionId(value: string | undefined);
     /** shortcut for getting `request.userData.label` */
     get label(): string | undefined;
     /** shortcut for setting `request.userData.label` */
@@ -107,6 +137,14 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
     get state(): RequestState;
     /** Describes the request's current lifecycle state. */
     set state(value: RequestState);
+    /**
+     * Reason for skipping this request.
+     */
+    get skippedReason(): SkippedRequestReason | undefined;
+    /**
+     * Reason for skipping this request.
+     */
+    set skippedReason(value: SkippedRequestReason | undefined);
     private get enqueueStrategy();
     private set enqueueStrategy(value);
     /**
@@ -124,7 +162,7 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
      */
     pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
     /** @internal */
-    static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, }: ComputeUniqueKeyOptions): string;
+    static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, alwaysEnqueue, }: ComputeUniqueKeyOptions): string;
     /** @internal */
     static hashPayload(payload: BinaryLike): string;
 }
@@ -145,7 +183,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      * The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
      *
      * The `useExtendedUniqueKey` options determines whether method and payload are included in the `uniqueKey`,
-     * producing a `uniqueKey` in the following format: `METHOD(payloadHash):normalizedUrl`. This is useful
+     * producing a `uniqueKey` in the following format: `METHOD|payloadHash|normalizedUrl`. This is useful
      * when requests point to the same URL, but with different methods and payloads. For example: form submits.
      *
      * Pass an arbitrary non-empty text value to the `uniqueKey` property
@@ -169,6 +207,9 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
     /**
      * Custom user data assigned to the request. Use this to save any request related data to the
      * request's scope, keeping them accessible on retries, failures etc.
+     *
+     * All data stored in `userData` must be JSON-serializable.
+     * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
      */
     userData?: UserData;
     /**
@@ -190,17 +231,45 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      * @default false
      */
     useExtendedUniqueKey?: boolean;
+    /**
+     * If `true` then a random value is included in the `uniqueKey` computation, ensuring the request
+     * is always enqueued even if a request with the same URL (and method/payload) is already present
+     * in the queue. Cannot be used together with a custom `uniqueKey`.
+     * @default false
+     */
+    alwaysEnqueue?: boolean;
     /**
      * The `true` value indicates that the request will not be automatically retried on error.
      * @default false
      */
     noRetry?: boolean;
+    /**
+     * ID of a session from the crawler's `SessionPool` to use for this request.
+     * When set, the crawler will fetch this session from the pool instead of creating a new one.
+     */
+    sessionId?: string;
     /**
      * If set to `true` then the crawler processing this request evaluates
      * the `requestHandler` immediately without prior browser navigation.
+     *
+     * When enabled, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
      * @default false
      */
     skipNavigation?: boolean;
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     * @default 0
+     */
+    crawlDepth?: number;
+    /**
+     * Reason for skipping this request.
+     * This is used to provide more information about why the request was skipped.
+     * @internal
+     */
+    skippedReason?: SkippedRequestReason;
     /**
      * Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`.
      */
@@ -227,15 +296,15 @@ interface ComputeUniqueKeyOptions {
     payload?: string | Buffer;
     keepUrlFragment?: boolean;
     useExtendedUniqueKey?: boolean;
+    alwaysEnqueue?: boolean;
 }
 export type Source = (Partial<RequestOptions> & {
     requestsFromUrl?: string;
     regex?: RegExp;
-}) | Request;
+}) | CrawleeRequest;
 /** @internal */
 export interface InternalSource {
     requestsFromUrl: string;
     regex?: RegExp;
 }
-export {};
-//# sourceMappingURL=request.d.ts.map
+export { CrawleeRequest as Request };