@crawlee/core 4.0.0-beta.14 → 4.0.0-beta.140

package/recoverable_state.js

@@ -1,5 +1,25 @@
-import { Configuration, KeyValueStore } from '@crawlee/core';
-import log from '@apify/log';
+import { addTimeoutToPromise, storage as timeoutStorage } from '@apify/timeout';
+import { EventType, KeyValueStore, serviceLocator, StateValidationError } from '@crawlee/core';
+const DEFAULT_PERSISTENCE_TIMEOUT_MILLIS = 60_000;
+/**
+ * Applies a {@link SyncStateConversion}, throwing a {@link StateValidationError} for a schema that rejects
+ * the value.
+ *
+ * @internal
+ */
+export function convertStateSync(conversion, value, persistStateKey) {
+    if (typeof conversion === 'function') {
+        return conversion(value);
+    }
+    const result = conversion['~standard'].validate(value);
+    if ('then' in result) {
+        throw new Error(`The state conversion for '${persistStateKey}' validated asynchronously, which this caller cannot await.`);
+    }
+    if (result.issues) {
+        throw new StateValidationError(persistStateKey, result.issues);
+    }
+    return result.value;
+}
 /**
  * A class for managing persistent recoverable state using a plain JavaScript object.
  *
@@ -11,130 +31,201 @@ import log from '@apify/log';
  * The class automatically hooks into the event system to persist state when needed.
  */
 export class RecoverableState {
-    defaultState;
-    state = null;
-    persistenceEnabled;
-    persistStateKey;
-    persistStateKvsName;
-    persistStateKvsId;
-    keyValueStore = null;
-    log;
-    config;
-    serialize;
-    deserialize;
+    #defaultState;
+    #state = null;
+    #initialized = false;
+    #listening = false;
+    #persistenceEnabled;
+    #persistStateKey;
+    #persistenceTimeoutMillis;
+    #configuration;
+    #keyValueStore;
+    #log;
+    #serialize;
+    #deserialize;
+    #persistStateQuietly;
     /**
      * Initialize a new recoverable state object.
      *
      * @param options Configuration options for the recoverable state
      */
     constructor(options) {
-        this.defaultState = options.defaultState;
-        this.persistStateKey = options.persistStateKey;
-        this.persistenceEnabled = options.persistenceEnabled ?? false;
-        this.persistStateKvsName = options.persistStateKvsName;
-        this.persistStateKvsId = options.persistStateKvsId;
-        this.log = options.logger ?? log.child({ prefix: 'RecoverableState' });
-        this.config = options.config ?? Configuration.getGlobalConfig();
-        this.serialize = options.serialize ?? JSON.stringify;
-        this.deserialize = options.deserialize ?? JSON.parse;
-        this.persistState = this.persistState.bind(this);
+        const { defaultState } = options;
+        this.#defaultState =
+            typeof defaultState === 'function'
+                ? defaultState
+                : () => structuredClone(defaultState);
+        this.#persistStateKey = options.persistStateKey;
+        this.#persistenceEnabled = options.persistenceEnabled ?? false;
+        this.#persistenceTimeoutMillis = options.persistenceTimeoutMillis ?? DEFAULT_PERSISTENCE_TIMEOUT_MILLIS;
+        this.#configuration = options.configuration;
+        this.#keyValueStore = options.keyValueStore ?? null;
+        this.#log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
+        this.#serialize = this.#toConversion(options.serialize);
+        this.#deserialize = this.#toConversion(options.deserialize);
+        // The automatic persists, where a rejection has nowhere useful to go - the event manager does not catch
+        // listener errors, and throwing from teardown would bury the outcome of the work it cleans up after.
+        this.#persistStateQuietly = async (eventData) => this.persistState(eventData).catch((error) => this.#log.warning(`Failed to persist the state under key '${this.#persistStateKey}'.`, { error }));
+    }
+    /** Normalizes a conversion option into a function. Absent conversions pass the value through unchanged. */
+    #toConversion(conversion) {
+        if (conversion === undefined) {
+            return async (value) => value;
+        }
+        if (typeof conversion === 'function') {
+            return async (value) => conversion(value);
+        }
+        return async (value) => {
+            const result = await conversion['~standard'].validate(value);
+            if (result.issues) {
+                throw new StateValidationError(this.#persistStateKey, result.issues);
+            }
+            return result.value;
+        };
     }
     /**
      * Initialize the recoverable state.
      *
-     * This method must be called before using the recoverable state. It loads the saved state
-     * if persistence is enabled and registers the object to listen for PERSIST_STATE events.
+     * If persistence is enabled, this method loads the saved state and registers the object to listen for
+     * PERSIST_STATE events. A state established beforehand by {@link RecoverableState.reset} survives if there
+     * is no record to restore.
+     *
+     * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
+     * listener is registered again and the record reloaded.
      *
      * @returns The loaded state object
      */
     async initialize() {
-        if (this.state !== null && this.state !== undefined) {
+        if (this.#initialized) {
             return this.currentValue;
         }
-        if (!this.persistenceEnabled) {
-            this.state = this.deserialize(this.serialize(this.defaultState));
-            return this.currentValue;
+        if (this.#persistenceEnabled) {
+            this.#keyValueStore ??= KeyValueStore.open(null, {
+                configuration: this.#configuration ?? serviceLocator.getConfiguration(),
+            });
+            await this.#resolveKeyValueStore();
+            serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.#persistStateQuietly);
+            this.#listening = true;
         }
-        this.keyValueStore = await KeyValueStore.open(this.persistStateKvsName ?? this.persistStateKvsId, {
-            config: this.config,
-        });
-        await this.loadSavedState();
-        // Register for persist state events
-        const eventManager = this.config.getEventManager();
-        eventManager.on("persistState" /* EventType.PERSIST_STATE */, this.persistState);
+        // Flipped before the record is loaded, so that a caller catching a `StateValidationError` is left with a
+        // fully wired object running on the default state rather than a half-initialized one.
+        this.#initialized = true;
+        this.#state ??= this.#defaultState();
+        await this.#loadSavedState();
         return this.currentValue;
     }
     /**
      * Clean up resources used by the recoverable state.
      *
      * If persistence is enabled, this method deregisters the object from PERSIST_STATE events
-     * and persists the current state one last time.
+     * and persists the current state one last time, warning rather than throwing if that write fails - cleanup
+     * runs when the work is already done, and failing it would bury whatever the caller was doing. The in-memory
+     * state is left alone, and {@link RecoverableState.initialize} can be called again to open a new
+     * persistence window.
      */
     async teardown() {
-        if (!this.persistenceEnabled || !this.persistState) {
+        this.#initialized = false;
+        if (!this.#persistenceEnabled) {
             return;
         }
-        const eventManager = this.config.getEventManager();
-        eventManager.off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
-        await this.persistState();
+        serviceLocator.getEventManager().off(EventType.PERSIST_STATE, this.#persistStateQuietly);
+        this.#listening = false;
+        await this.#persistStateQuietly();
     }
     /**
      * Get the current state.
+     *
+     * Throws until the state has been established, by either {@link RecoverableState.initialize} or the
+     * synchronous {@link RecoverableState.reset} - the latter being how a caller that cannot await in its
+     * constructor gets a usable state right away.
      */
     get currentValue() {
-        if (this.state === null) {
-            throw new Error('Recoverable state has not yet been loaded');
+        if (this.#state === null) {
+            throw new Error('Recoverable state has not yet been loaded - call initialize() or reset() first');
         }
-        return this.state;
+        return this.#state;
     }
     /**
-     * Reset the state to the default values and clear any persisted state.
+     * Reset the in-memory state to the default values, leaving any persisted record alone.
      *
-     * Resets the current state to the default state and, if persistence is enabled,
-     * clears the persisted state from the KeyValueStore.
+     * Use {@link RecoverableState.resetStore} to clear the persisted record as well.
      */
-    async reset() {
-        this.state = this.deserialize(this.serialize(this.defaultState));
-        if (this.persistenceEnabled) {
-            if (this.keyValueStore === null) {
-                throw new Error('Recoverable state has not yet been initialized');
-            }
-            await this.keyValueStore.setValue(this.persistStateKey, null);
+    reset() {
+        this.#state = this.#defaultState();
+    }
+    /**
+     * Clear the persisted state record, leaving the in-memory state alone.
+     *
+     * This is a between-lifecycles operation - its point is to stop the next {@link RecoverableState.initialize}
+     * from restoring the record, so it throws while PERSIST_STATE events are still being handled, where the next
+     * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
+     * or {@link RecoverableState.teardown} before clearing the record.
+     *
+     * A no-op if persistence is disabled or no KeyValueStore is available yet.
+     */
+    async resetStore() {
+        if (this.#listening) {
+            throw new Error(`Cannot clear the state persisted under key '${this.#persistStateKey}' while it is still being persisted periodically - the next PERSIST_STATE event would write it straight back. Use reset() to reset the state itself, or teardown() before clearing the record.`);
+        }
+        if (!this.#persistenceEnabled) {
+            return;
         }
+        const keyValueStore = await this.#resolveKeyValueStore();
+        if (keyValueStore === null) {
+            return;
+        }
+        await this.#withTimeout(async () => keyValueStore.setValue(this.#persistStateKey, null), 'Clearing the persisted state');
     }
     /**
      * Persist the current state to the KeyValueStore.
      *
      * This method is typically called in response to a PERSIST_STATE event, but can also be called
-     * directly when needed.
+     * directly when needed. It is a no-op if persistence is disabled, if no KeyValueStore is available yet, or if
+     * there is no state to write. A failed write only rejects here - the periodic and teardown ones warn instead.
      *
      * @param eventData Optional data associated with a PERSIST_STATE event
      */
     async persistState(eventData) {
-        this.log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
-        if (this.keyValueStore === null || this.state === null) {
-            throw new Error('Recoverable state has not yet been initialized');
+        if (!this.#persistenceEnabled || this.#state === null) {
+            return;
         }
-        if (this.persistenceEnabled) {
-            await this.keyValueStore.setValue(this.persistStateKey, this.serialize(this.state), {
-                contentType: 'text/plain', // HACK - the result is expected to be JSON, but we do this to avoid the implicit JSON.parse in `KeyValueStore.getValue`
-            });
+        const keyValueStore = await this.#resolveKeyValueStore();
+        if (keyValueStore === null) {
+            return;
+        }
+        this.#log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
+        const serializedState = await this.#serialize(this.currentValue);
+        await this.#withTimeout(async () => keyValueStore.setValue(this.#persistStateKey, serializedState), 'Persisting the state');
+    }
+    /** Awaits a store handed over as a pending `open()`, keeping the resolved instance for later calls. */
+    async #resolveKeyValueStore() {
+        if (this.#keyValueStore === null) {
+            return null;
         }
+        this.#keyValueStore = await this.#keyValueStore;
+        return this.#keyValueStore;
     }
     /**
-     * Load the saved state from the KeyValueStore
+     * Load the saved state from the KeyValueStore. Leaves the current state alone if there is no record to load.
      */
-    async loadSavedState() {
-        if (this.keyValueStore === null) {
-            throw new Error('Recoverable state has not yet been initialized');
+    async #loadSavedState() {
+        if (!this.#persistenceEnabled) {
+            return;
         }
-        const storedState = await this.keyValueStore.getValue(this.persistStateKey);
-        if (storedState === null || storedState === undefined) {
-            this.state = this.deserialize(this.serialize(this.defaultState));
+        const keyValueStore = await this.#resolveKeyValueStore();
+        if (keyValueStore === null) {
+            return;
         }
-        else {
-            this.state = this.deserialize(storedState);
+        const storedState = await this.#withTimeout(async () => keyValueStore.getValue(this.#persistStateKey), 'Loading the persisted state');
+        if (storedState === null || storedState === undefined) {
+            return;
         }
+        this.#state = await this.#deserialize(storedState);
+    }
+    async #withTimeout(operation, description) {
+        // `@apify/timeout` shares one `AbortController` across nested frames and `KeyValueStore` checks it on
+        // every operation, so a teardown-time persist running inside an already-expired request handler timeout
+        // would be aborted before it started. Hence a fresh timeout context.
+        return timeoutStorage.exit(async () => addTimeoutToPromise(operation, this.#persistenceTimeoutMillis, `${description} under key '${this.#persistStateKey}' timed out after ${this.#persistenceTimeoutMillis / 1000} seconds.`));
     }
 }
-//# sourceMappingURL=recoverable_state.js.map

package/request.d.ts

@@ -1,8 +1,7 @@
 import type { BinaryLike } from 'node:crypto';
-import type { Dictionary } from '@crawlee/types';
-import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
+import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
+import type { EnqueueStrategyOption } from './enqueue_links/enqueue_links.js';
 import type { SkippedRequestReason } from './enqueue_links/shared.js';
-import type { AllowedHttpMethods } from './typedefs.js';
 export declare enum RequestState {
     UNPROCESSED = 0,
     BEFORE_NAV = 1,
@@ -45,6 +44,7 @@ export declare enum RequestState {
  * @category Sources
  */
 declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
+    #private;
     /** Request ID */
     id?: string;
     /** URL of the web page to crawl. */
@@ -75,9 +75,12 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
     errorMessages: string[];
     /** Object with HTTP headers. Key is header name, value is the value. */
     headers?: Record<string, string>;
-    /** Private store for the custom user data assigned to the request. */
-    private _userData;
-    /** Custom user data assigned to the request. */
+    /**
+     * Custom user data assigned to the request.
+     *
+     * All data stored in `userData` must be JSON-serializable.
+     * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
+     */
     userData: UserData;
     /**
      * ISO datetime string that indicates the time when the request has been processed.
@@ -93,9 +96,21 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
      * @returns The native `fetch` API Request object.
      */
     intoFetchAPIRequest(): Request;
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
     get skipNavigation(): boolean;
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
     set skipNavigation(value: boolean);
     /**
      * Depth of the request in the current crawl tree.
@@ -107,10 +122,9 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
      * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
      */
     set crawlDepth(value: number);
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    get sessionRotationCount(): number;
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    set sessionRotationCount(value: number);
+    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
+    get sessionId(): string | undefined;
+    set sessionId(value: string | undefined);
     /** shortcut for getting `request.userData.label` */
     get label(): string | undefined;
     /** shortcut for setting `request.userData.label` */
@@ -123,6 +137,14 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
     get state(): RequestState;
     /** Describes the request's current lifecycle state. */
     set state(value: RequestState);
+    /**
+     * Reason for skipping this request.
+     */
+    get skippedReason(): SkippedRequestReason | undefined;
+    /**
+     * Reason for skipping this request.
+     */
+    set skippedReason(value: SkippedRequestReason | undefined);
     private get enqueueStrategy();
     private set enqueueStrategy(value);
     /**
@@ -140,7 +162,7 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
      */
     pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
     /** @internal */
-    static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, }: ComputeUniqueKeyOptions): string;
+    static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, alwaysEnqueue, }: ComputeUniqueKeyOptions): string;
     /** @internal */
     static hashPayload(payload: BinaryLike): string;
 }
@@ -161,7 +183,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      * The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
      *
      * The `useExtendedUniqueKey` options determines whether method and payload are included in the `uniqueKey`,
-     * producing a `uniqueKey` in the following format: `METHOD(payloadHash):normalizedUrl`. This is useful
+     * producing a `uniqueKey` in the following format: `METHOD|payloadHash|normalizedUrl`. This is useful
      * when requests point to the same URL, but with different methods and payloads. For example: form submits.
      *
      * Pass an arbitrary non-empty text value to the `uniqueKey` property
@@ -185,6 +207,9 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
     /**
      * Custom user data assigned to the request. Use this to save any request related data to the
      * request's scope, keeping them accessible on retries, failures etc.
+     *
+     * All data stored in `userData` must be JSON-serializable.
+     * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
      */
     userData?: UserData;
     /**
@@ -206,14 +231,30 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      * @default false
      */
     useExtendedUniqueKey?: boolean;
+    /**
+     * If `true` then a random value is included in the `uniqueKey` computation, ensuring the request
+     * is always enqueued even if a request with the same URL (and method/payload) is already present
+     * in the queue. Cannot be used together with a custom `uniqueKey`.
+     * @default false
+     */
+    alwaysEnqueue?: boolean;
     /**
      * The `true` value indicates that the request will not be automatically retried on error.
      * @default false
      */
     noRetry?: boolean;
+    /**
+     * ID of a session from the crawler's `SessionPool` to use for this request.
+     * When set, the crawler will fetch this session from the pool instead of creating a new one.
+     */
+    sessionId?: string;
     /**
      * If set to `true` then the crawler processing this request evaluates
      * the `requestHandler` immediately without prior browser navigation.
+     *
+     * When enabled, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
      * @default false
      */
     skipNavigation?: boolean;
@@ -240,7 +281,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
     /** @internal */
     lockExpiresAt?: Date;
     /** @internal */
-    enqueueStrategy?: EnqueueLinksOptions['strategy'];
+    enqueueStrategy?: EnqueueStrategyOption;
 }
 export interface PushErrorMessageOptions {
     /**
@@ -255,6 +296,7 @@ interface ComputeUniqueKeyOptions {
     payload?: string | Buffer;
     keepUrlFragment?: boolean;
     useExtendedUniqueKey?: boolean;
+    alwaysEnqueue?: boolean;
 }
 export type Source = (Partial<RequestOptions> & {
     requestsFromUrl?: string;
@@ -266,4 +308,3 @@ export interface InternalSource {
     regex?: RegExp;
 }
 export { CrawleeRequest as Request };
-//# sourceMappingURL=request.d.ts.map