@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.0

package/recoverable_state.d.ts

@@ -1,4 +1,31 @@
 import type { Configuration, CrawleeLogger } from '@crawlee/core';
+import { KeyValueStore } from '@crawlee/core';
+import type { Awaitable } from '@crawlee/types';
+import type { StandardSchemaV1 } from '@standard-schema/spec';
+/**
+ * One direction of the conversion between the state model and its persisted form - either a plain function, or a
+ * [Standard Schema](https://standardschema.dev) whose validated output is the result.
+ *
+ * A schema that fails to validate makes {@link RecoverableState} throw a {@link StateValidationError}. Zod
+ * codecs work directly, as their validation *is* the decode direction; use `(state) => codec.encode(state)` for the
+ * other one.
+ */
+export type StateConversion<TFrom, TTo> = ((value: TFrom) => Awaitable<TTo>) | StandardSchemaV1<TFrom, TTo>;
+/**
+ * A {@link StateConversion} for a caller that cannot await one - {@link Statistics}, whose `toJSON()` is
+ * synchronous, being the reason this exists.
+ *
+ * Only the function arm can be narrowed here: a Standard Schema is free to validate asynchronously, so a schema
+ * that does is rejected when it runs rather than when it is passed.
+ */
+export type SyncStateConversion<TFrom, TTo> = ((value: TFrom) => TTo) | StandardSchemaV1<TFrom, TTo>;
+/**
+ * Applies a {@link SyncStateConversion}, throwing a {@link StateValidationError} for a schema that rejects
+ * the value.
+ *
+ * @internal
+ */
+export declare function convertStateSync<TFrom, TTo>(conversion: SyncStateConversion<TFrom, TTo>, value: TFrom, persistStateKey: string): TTo;
 export interface RecoverableStatePersistenceOptions {
     /**
      * The key under which the state is stored in the KeyValueStore
@@ -9,44 +36,45 @@ export interface RecoverableStatePersistenceOptions {
      */
     persistenceEnabled?: boolean;
     /**
-     * The name of the KeyValueStore to use for persistence.
-     * If neither a name nor an id are supplied, the default store will be used.
+     * The KeyValueStore to persist into, defaulting to the default store. Accepts a pending
+     * {@link KeyValueStore.open} so that callers do not have to be async to point at a specific store.
      */
-    persistStateKvsName?: string;
+    keyValueStore?: KeyValueStore | PromiseLike<KeyValueStore>;
     /**
-     * The identifier of the KeyValueStore to use for persistence.
-     * If neither a name nor an id are supplied, the default store will be used.
+     * Time limit for a single load or save of the state, in milliseconds.
+     * @default 60_000
      */
-    persistStateKvsId?: string;
+    persistenceTimeoutMillis?: number;
 }
 /**
  * Options for configuring the RecoverableState
  */
-export interface RecoverableStateOptions<TStateModel = Record<string, unknown>> extends RecoverableStatePersistenceOptions {
+export interface RecoverableStateOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> extends RecoverableStatePersistenceOptions {
     /**
-     * The default state used if no persisted state is found.
-     * A deep copy is made each time the state is used.
+     * The state used when no persisted state is found, and the state {@link RecoverableState.reset} restores.
+     *
+     * A plain value is deep-copied with `structuredClone` each time it is used, so pass a factory for a state
+     * that `structuredClone` cannot rebuild - one holding class instances, say, or one derived from a schema.
      */
-    defaultState: TStateModel;
+    defaultState: TStateModel | (() => TStateModel);
     /**
      * A logger instance for logging operations related to state persistence
      */
     logger?: CrawleeLogger;
     /**
-     * Configuration instance to use
+     * Configuration instance to use when opening the KeyValueStore
      */
     configuration?: Configuration;
     /**
-     * Optional function to transform the state to a JSON string before persistence.
-     * If not provided, JSON.stringify will be used.
+     * Optional conversion of the state to a plain JSON-serializable value before it is persisted.
+     * If not provided, the state is persisted as is.
      */
-    serialize?: (state: TStateModel) => string;
+    serialize?: StateConversion<TStateModel, TPersistedState>;
     /**
-     * Optional function to transform a JSON-serialized object back to the state model.
-     * If not provided, JSON.parse is used.
-     * It is advisable to perform validation in this function and to throw an exception if it fails.
+     * Optional conversion of a persisted value back to the state model, and the place to validate a record before
+     * trusting it. If not provided, the persisted value is used as is.
      */
-    deserialize?: (serializedState: string) => TStateModel;
+    deserialize?: StateConversion<TPersistedState, TStateModel>;
 }
 /**
  * A class for managing persistent recoverable state using a plain JavaScript object.
@@ -58,28 +86,23 @@ export interface RecoverableStateOptions<TStateModel = Record<string, unknown>>
  * The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
  * The class automatically hooks into the event system to persist state when needed.
  */
-export declare class RecoverableState<TStateModel = Record<string, unknown>> {
-    private readonly defaultState;
-    private state;
-    private readonly persistenceEnabled;
-    private readonly persistStateKey;
-    private readonly persistStateKvsName?;
-    private readonly persistStateKvsId?;
-    private keyValueStore;
-    private readonly log;
-    private readonly serialize;
-    private readonly deserialize;
+export declare class RecoverableState<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> {
+    #private;
     /**
      * Initialize a new recoverable state object.
      *
      * @param options Configuration options for the recoverable state
      */
-    constructor(options: RecoverableStateOptions<TStateModel>);
+    constructor(options: RecoverableStateOptions<TStateModel, TPersistedState>);
     /**
      * Initialize the recoverable state.
      *
-     * This method must be called before using the recoverable state. It loads the saved state
-     * if persistence is enabled and registers the object to listen for PERSIST_STATE events.
+     * If persistence is enabled, this method loads the saved state and registers the object to listen for
+     * PERSIST_STATE events. A state established beforehand by {@link RecoverableState.reset} survives if there
+     * is no record to restore.
+     *
+     * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
+     * listener is registered again and the record reloaded.
      *
      * @returns The loaded state object
      */
@@ -88,33 +111,45 @@ export declare class RecoverableState<TStateModel = Record<string, unknown>> {
      * Clean up resources used by the recoverable state.
      *
      * If persistence is enabled, this method deregisters the object from PERSIST_STATE events
-     * and persists the current state one last time.
+     * and persists the current state one last time, warning rather than throwing if that write fails - cleanup
+     * runs when the work is already done, and failing it would bury whatever the caller was doing. The in-memory
+     * state is left alone, and {@link RecoverableState.initialize} can be called again to open a new
+     * persistence window.
      */
     teardown(): Promise<void>;
     /**
      * Get the current state.
+     *
+     * Throws until the state has been established, by either {@link RecoverableState.initialize} or the
+     * synchronous {@link RecoverableState.reset} - the latter being how a caller that cannot await in its
+     * constructor gets a usable state right away.
      */
     get currentValue(): TStateModel;
     /**
-     * Reset the state to the default values and clear any persisted state.
+     * Reset the in-memory state to the default values, leaving any persisted record alone.
      *
-     * Resets the current state to the default state and, if persistence is enabled,
-     * clears the persisted state from the KeyValueStore.
+     * Use {@link RecoverableState.resetStore} to clear the persisted record as well.
      */
-    reset(): Promise<void>;
+    reset(): void;
+    /**
+     * Clear the persisted state record, leaving the in-memory state alone.
+     *
+     * This is a between-lifecycles operation - its point is to stop the next {@link RecoverableState.initialize}
+     * from restoring the record, so it throws while PERSIST_STATE events are still being handled, where the next
+     * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
+     * or {@link RecoverableState.teardown} before clearing the record.
+     *
+     * A no-op if persistence is disabled or no KeyValueStore is available yet.
+     */
+    resetStore(): Promise<void>;
     /**
      * Persist the current state to the KeyValueStore.
      *
      * This method is typically called in response to a PERSIST_STATE event, but can also be called
-     * directly when needed.
+     * directly when needed. It is a no-op if persistence is disabled, if no KeyValueStore is available yet, or if
+     * there is no state to write. A failed write only rejects here - the periodic and teardown ones warn instead.
      *
      * @param eventData Optional data associated with a PERSIST_STATE event
      */
-    persistState(eventData?: {
-        isMigrating: boolean;
-    }): Promise<void>;
-    /**
-     * Load the saved state from the KeyValueStore
-     */
-    private loadSavedState;
+    persistState(eventData?: Record<string, unknown>): Promise<void>;
 }

package/recoverable_state.js

@@ -1,4 +1,25 @@
-import { KeyValueStore, serviceLocator } from '@crawlee/core';
+import { addTimeoutToPromise, storage as timeoutStorage } from '@apify/timeout';
+import { EventType, KeyValueStore, serviceLocator, StateValidationError } from '@crawlee/core';
+const DEFAULT_PERSISTENCE_TIMEOUT_MILLIS = 60_000;
+/**
+ * Applies a {@link SyncStateConversion}, throwing a {@link StateValidationError} for a schema that rejects
+ * the value.
+ *
+ * @internal
+ */
+export function convertStateSync(conversion, value, persistStateKey) {
+    if (typeof conversion === 'function') {
+        return conversion(value);
+    }
+    const result = conversion['~standard'].validate(value);
+    if ('then' in result) {
+        throw new Error(`The state conversion for '${persistStateKey}' validated asynchronously, which this caller cannot await.`);
+    }
+    if (result.issues) {
+        throw new StateValidationError(persistStateKey, result.issues);
+    }
+    return result.value;
+}
 /**
  * A class for managing persistent recoverable state using a plain JavaScript object.
  *
@@ -10,134 +31,201 @@ import { KeyValueStore, serviceLocator } from '@crawlee/core';
  * The class automatically hooks into the event system to persist state when needed.
  */
 export class RecoverableState {
-    defaultState;
-    state = null;
-    persistenceEnabled;
-    persistStateKey;
-    persistStateKvsName;
-    persistStateKvsId;
-    keyValueStore = null;
-    log;
-    serialize;
-    deserialize;
+    #defaultState;
+    #state = null;
+    #initialized = false;
+    #listening = false;
+    #persistenceEnabled;
+    #persistStateKey;
+    #persistenceTimeoutMillis;
+    #configuration;
+    #keyValueStore;
+    #log;
+    #serialize;
+    #deserialize;
+    #persistStateQuietly;
     /**
      * Initialize a new recoverable state object.
      *
      * @param options Configuration options for the recoverable state
      */
     constructor(options) {
-        this.defaultState = options.defaultState;
-        this.persistStateKey = options.persistStateKey;
-        this.persistenceEnabled = options.persistenceEnabled ?? false;
-        this.persistStateKvsName = options.persistStateKvsName;
-        this.persistStateKvsId = options.persistStateKvsId;
-        this.log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
-        this.serialize = options.serialize ?? JSON.stringify;
-        this.deserialize = options.deserialize ?? JSON.parse;
-        this.persistState = this.persistState.bind(this);
+        const { defaultState } = options;
+        this.#defaultState =
+            typeof defaultState === 'function'
+                ? defaultState
+                : () => structuredClone(defaultState);
+        this.#persistStateKey = options.persistStateKey;
+        this.#persistenceEnabled = options.persistenceEnabled ?? false;
+        this.#persistenceTimeoutMillis = options.persistenceTimeoutMillis ?? DEFAULT_PERSISTENCE_TIMEOUT_MILLIS;
+        this.#configuration = options.configuration;
+        this.#keyValueStore = options.keyValueStore ?? null;
+        this.#log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
+        this.#serialize = this.#toConversion(options.serialize);
+        this.#deserialize = this.#toConversion(options.deserialize);
+        // The automatic persists, where a rejection has nowhere useful to go - the event manager does not catch
+        // listener errors, and throwing from teardown would bury the outcome of the work it cleans up after.
+        this.#persistStateQuietly = async (eventData) => this.persistState(eventData).catch((error) => this.#log.warning(`Failed to persist the state under key '${this.#persistStateKey}'.`, { error }));
+    }
+    /** Normalizes a conversion option into a function. Absent conversions pass the value through unchanged. */
+    #toConversion(conversion) {
+        if (conversion === undefined) {
+            return async (value) => value;
+        }
+        if (typeof conversion === 'function') {
+            return async (value) => conversion(value);
+        }
+        return async (value) => {
+            const result = await conversion['~standard'].validate(value);
+            if (result.issues) {
+                throw new StateValidationError(this.#persistStateKey, result.issues);
+            }
+            return result.value;
+        };
     }
     /**
      * Initialize the recoverable state.
      *
-     * This method must be called before using the recoverable state. It loads the saved state
-     * if persistence is enabled and registers the object to listen for PERSIST_STATE events.
+     * If persistence is enabled, this method loads the saved state and registers the object to listen for
+     * PERSIST_STATE events. A state established beforehand by {@link RecoverableState.reset} survives if there
+     * is no record to restore.
+     *
+     * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
+     * listener is registered again and the record reloaded.
      *
      * @returns The loaded state object
      */
     async initialize() {
-        if (this.state !== null && this.state !== undefined) {
+        if (this.#initialized) {
             return this.currentValue;
         }
-        if (!this.persistenceEnabled) {
-            this.state = this.deserialize(this.serialize(this.defaultState));
-            return this.currentValue;
-        }
-        let kvsIdentifier = null;
-        if (this.persistStateKvsName) {
-            kvsIdentifier = { name: this.persistStateKvsName };
-        }
-        else if (this.persistStateKvsId) {
-            kvsIdentifier = { id: this.persistStateKvsId };
+        if (this.#persistenceEnabled) {
+            this.#keyValueStore ??= KeyValueStore.open(null, {
+                configuration: this.#configuration ?? serviceLocator.getConfiguration(),
+            });
+            await this.#resolveKeyValueStore();
+            serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.#persistStateQuietly);
+            this.#listening = true;
         }
-        this.keyValueStore = await KeyValueStore.open(kvsIdentifier, {
-            configuration: serviceLocator.getConfiguration(),
-        });
-        await this.loadSavedState();
-        // Register for persist state events
-        const eventManager = serviceLocator.getEventManager();
-        eventManager.on("persistState" /* EventType.PERSIST_STATE */, this.persistState);
+        // Flipped before the record is loaded, so that a caller catching a `StateValidationError` is left with a
+        // fully wired object running on the default state rather than a half-initialized one.
+        this.#initialized = true;
+        this.#state ??= this.#defaultState();
+        await this.#loadSavedState();
         return this.currentValue;
     }
     /**
      * Clean up resources used by the recoverable state.
      *
      * If persistence is enabled, this method deregisters the object from PERSIST_STATE events
-     * and persists the current state one last time.
+     * and persists the current state one last time, warning rather than throwing if that write fails - cleanup
+     * runs when the work is already done, and failing it would bury whatever the caller was doing. The in-memory
+     * state is left alone, and {@link RecoverableState.initialize} can be called again to open a new
+     * persistence window.
      */
     async teardown() {
-        if (!this.persistenceEnabled || !this.persistState) {
+        this.#initialized = false;
+        if (!this.#persistenceEnabled) {
             return;
         }
-        const eventManager = serviceLocator.getEventManager();
-        eventManager.off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
-        await this.persistState();
+        serviceLocator.getEventManager().off(EventType.PERSIST_STATE, this.#persistStateQuietly);
+        this.#listening = false;
+        await this.#persistStateQuietly();
     }
     /**
      * Get the current state.
+     *
+     * Throws until the state has been established, by either {@link RecoverableState.initialize} or the
+     * synchronous {@link RecoverableState.reset} - the latter being how a caller that cannot await in its
+     * constructor gets a usable state right away.
      */
     get currentValue() {
-        if (this.state === null) {
-            throw new Error('Recoverable state has not yet been loaded');
+        if (this.#state === null) {
+            throw new Error('Recoverable state has not yet been loaded - call initialize() or reset() first');
         }
-        return this.state;
+        return this.#state;
     }
     /**
-     * Reset the state to the default values and clear any persisted state.
+     * Reset the in-memory state to the default values, leaving any persisted record alone.
      *
-     * Resets the current state to the default state and, if persistence is enabled,
-     * clears the persisted state from the KeyValueStore.
+     * Use {@link RecoverableState.resetStore} to clear the persisted record as well.
      */
-    async reset() {
-        this.state = this.deserialize(this.serialize(this.defaultState));
-        if (this.persistenceEnabled) {
-            if (this.keyValueStore === null) {
-                throw new Error('Recoverable state has not yet been initialized');
-            }
-            await this.keyValueStore.setValue(this.persistStateKey, null);
+    reset() {
+        this.#state = this.#defaultState();
+    }
+    /**
+     * Clear the persisted state record, leaving the in-memory state alone.
+     *
+     * This is a between-lifecycles operation - its point is to stop the next {@link RecoverableState.initialize}
+     * from restoring the record, so it throws while PERSIST_STATE events are still being handled, where the next
+     * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
+     * or {@link RecoverableState.teardown} before clearing the record.
+     *
+     * A no-op if persistence is disabled or no KeyValueStore is available yet.
+     */
+    async resetStore() {
+        if (this.#listening) {
+            throw new Error(`Cannot clear the state persisted under key '${this.#persistStateKey}' while it is still being persisted periodically - the next PERSIST_STATE event would write it straight back. Use reset() to reset the state itself, or teardown() before clearing the record.`);
+        }
+        if (!this.#persistenceEnabled) {
+            return;
+        }
+        const keyValueStore = await this.#resolveKeyValueStore();
+        if (keyValueStore === null) {
+            return;
         }
+        await this.#withTimeout(async () => keyValueStore.setValue(this.#persistStateKey, null), 'Clearing the persisted state');
     }
     /**
      * Persist the current state to the KeyValueStore.
      *
      * This method is typically called in response to a PERSIST_STATE event, but can also be called
-     * directly when needed.
+     * directly when needed. It is a no-op if persistence is disabled, if no KeyValueStore is available yet, or if
+     * there is no state to write. A failed write only rejects here - the periodic and teardown ones warn instead.
      *
      * @param eventData Optional data associated with a PERSIST_STATE event
      */
     async persistState(eventData) {
-        this.log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
-        if (this.keyValueStore === null || this.state === null) {
-            throw new Error('Recoverable state has not yet been initialized');
+        if (!this.#persistenceEnabled || this.#state === null) {
+            return;
         }
-        if (this.persistenceEnabled) {
-            await this.keyValueStore.setValue(this.persistStateKey, this.serialize(this.state), {
-                contentType: 'text/plain', // HACK - the result is expected to be JSON, but we do this to avoid the implicit JSON.parse in `KeyValueStore.getValue`
-            });
+        const keyValueStore = await this.#resolveKeyValueStore();
+        if (keyValueStore === null) {
+            return;
+        }
+        this.#log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
+        const serializedState = await this.#serialize(this.currentValue);
+        await this.#withTimeout(async () => keyValueStore.setValue(this.#persistStateKey, serializedState), 'Persisting the state');
+    }
+    /** Awaits a store handed over as a pending `open()`, keeping the resolved instance for later calls. */
+    async #resolveKeyValueStore() {
+        if (this.#keyValueStore === null) {
+            return null;
         }
+        this.#keyValueStore = await this.#keyValueStore;
+        return this.#keyValueStore;
     }
     /**
-     * Load the saved state from the KeyValueStore
+     * Load the saved state from the KeyValueStore. Leaves the current state alone if there is no record to load.
      */
-    async loadSavedState() {
-        if (this.keyValueStore === null) {
-            throw new Error('Recoverable state has not yet been initialized');
+    async #loadSavedState() {
+        if (!this.#persistenceEnabled) {
+            return;
         }
-        const storedState = await this.keyValueStore.getValue(this.persistStateKey);
-        if (storedState === null || storedState === undefined) {
-            this.state = this.deserialize(this.serialize(this.defaultState));
+        const keyValueStore = await this.#resolveKeyValueStore();
+        if (keyValueStore === null) {
+            return;
         }
-        else {
-            this.state = this.deserialize(storedState);
+        const storedState = await this.#withTimeout(async () => keyValueStore.getValue(this.#persistStateKey), 'Loading the persisted state');
+        if (storedState === null || storedState === undefined) {
+            return;
         }
+        this.#state = await this.#deserialize(storedState);
+    }
+    async #withTimeout(operation, description) {
+        // `@apify/timeout` shares one `AbortController` across nested frames and `KeyValueStore` checks it on
+        // every operation, so a teardown-time persist running inside an already-expired request handler timeout
+        // would be aborted before it started. Hence a fresh timeout context.
+        return timeoutStorage.exit(async () => addTimeoutToPromise(operation, this.#persistenceTimeoutMillis, `${description} under key '${this.#persistStateKey}' timed out after ${this.#persistenceTimeoutMillis / 1000} seconds.`));
     }
 }

package/request.d.ts

@@ -1,6 +1,6 @@
 import type { BinaryLike } from 'node:crypto';
 import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
-import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
+import type { EnqueueStrategyOption } from './enqueue_links/enqueue_links.js';
 import type { SkippedRequestReason } from './enqueue_links/shared.js';
 export declare enum RequestState {
     UNPROCESSED = 0,
@@ -44,6 +44,7 @@ export declare enum RequestState {
  * @category Sources
  */
 declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
+    #private;
     /** Request ID */
     id?: string;
     /** URL of the web page to crawl. */
@@ -74,8 +75,6 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
     errorMessages: string[];
     /** Object with HTTP headers. Key is header name, value is the value. */
     headers?: Record<string, string>;
-    /** Private store for the custom user data assigned to the request. */
-    private _userData;
     /**
      * Custom user data assigned to the request.
      *
@@ -282,7 +281,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
     /** @internal */
     lockExpiresAt?: Date;
     /** @internal */
-    enqueueStrategy?: EnqueueLinksOptions['strategy'];
+    enqueueStrategy?: EnqueueStrategyOption;
 }
 export interface PushErrorMessageOptions {
     /**