@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

package/recoverable_state.js

@@ -1,4 +1,29 @@
-import { KeyValueStore, serviceLocator } from '@crawlee/core';
+import { Readable } from 'node:stream';
+import { addTimeoutToPromise, storage as timeoutStorage } from '@apify/timeout';
+import { StateValidationError } from './errors.js';
+import { EventType } from './events/event_manager.js';
+import { serviceLocator } from './service_locator.js';
+import { KeyValueStore } from './storages/key_value_store.js';
+const DEFAULT_PERSISTENCE_TIMEOUT_MILLIS = 60_000;
+/**
+ * Applies a {@link SyncStateConversion}, throwing a {@link StateValidationError} for a schema that rejects
+ * the value.
+ *
+ * @internal
+ */
+export function convertStateSync(conversion, value, persistStateKey) {
+    if (typeof conversion === 'function') {
+        return conversion(value);
+    }
+    const result = conversion['~standard'].validate(value);
+    if ('then' in result) {
+        throw new Error(`The state conversion for '${persistStateKey}' validated asynchronously, which this caller cannot await.`);
+    }
+    if (result.issues) {
+        throw new StateValidationError(persistStateKey, result.issues);
+    }
+    return result.value;
+}
 /**
  * A class for managing persistent recoverable state using a plain JavaScript object.
  *
@@ -10,134 +35,234 @@ import { KeyValueStore, serviceLocator } from '@crawlee/core';
  * The class automatically hooks into the event system to persist state when needed.
  */
 export class RecoverableState {
-    defaultState;
-    state = null;
-    persistenceEnabled;
-    persistStateKey;
-    persistStateKvsName;
-    persistStateKvsId;
-    keyValueStore = null;
-    log;
-    serialize;
-    deserialize;
+    #defaultState;
+    #state = null;
+    #initialized = false;
+    #recordLoaded = false;
+    #listening = false;
+    #persistenceEnabled;
+    #persistStateKey;
+    #persistenceTimeoutMillis;
+    #configuration;
+    #keyValueStore;
+    #log;
+    #serialize;
+    #deserialize;
+    #contentType;
+    #persistStateQuietly;
     /**
      * Initialize a new recoverable state object.
      *
      * @param options Configuration options for the recoverable state
      */
     constructor(options) {
-        this.defaultState = options.defaultState;
-        this.persistStateKey = options.persistStateKey;
-        this.persistenceEnabled = options.persistenceEnabled ?? false;
-        this.persistStateKvsName = options.persistStateKvsName;
-        this.persistStateKvsId = options.persistStateKvsId;
-        this.log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
-        this.serialize = options.serialize ?? JSON.stringify;
-        this.deserialize = options.deserialize ?? JSON.parse;
-        this.persistState = this.persistState.bind(this);
+        const { defaultState } = options;
+        this.#defaultState =
+            typeof defaultState === 'function'
+                ? defaultState
+                : () => structuredClone(defaultState);
+        this.#persistStateKey = options.persistStateKey;
+        this.#persistenceEnabled = options.persistenceEnabled ?? false;
+        this.#persistenceTimeoutMillis = options.persistenceTimeoutMillis ?? DEFAULT_PERSISTENCE_TIMEOUT_MILLIS;
+        this.#configuration = options.configuration;
+        this.#keyValueStore = options.keyValueStore ?? null;
+        this.#log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
+        this.#contentType = options.contentType;
+        if (this.#contentType !== undefined && (options.serialize === undefined || options.deserialize === undefined)) {
+            throw new Error(`A 'contentType' for the state persisted under key '${this.#persistStateKey}' requires both 'serialize' and 'deserialize' - the record is no longer JSON the default codec can handle.`);
+        }
+        this.#serialize = this.#toConversion(options.serialize);
+        this.#deserialize = this.#toConversion(options.deserialize);
+        // A pending open() may reject before anything here awaits it. Observe the rejection so it surfaces on
+        // first use rather than as an unhandled rejection - most immediately when persistence is disabled and
+        // nothing ever awaits the store.
+        if (this.#keyValueStore !== null) {
+            Promise.resolve(this.#keyValueStore).then(undefined, () => { });
+        }
+        // The automatic persists, where a rejection has nowhere useful to go - the event manager does not catch
+        // listener errors, and throwing from teardown would bury the outcome of the work it cleans up after.
+        this.#persistStateQuietly = async (eventData) => this.persistState(eventData).catch((error) => this.#log.warning(`Failed to persist the state under key '${this.#persistStateKey}'.`, { error }));
+    }
+    /** Normalizes a conversion option into a function. Absent conversions pass the value through unchanged. */
+    #toConversion(conversion) {
+        if (conversion === undefined) {
+            return async (value) => value;
+        }
+        if (typeof conversion === 'function') {
+            return async (value) => conversion(value);
+        }
+        return async (value) => {
+            const result = await conversion['~standard'].validate(value);
+            if (result.issues) {
+                throw new StateValidationError(this.#persistStateKey, result.issues);
+            }
+            return result.value;
+        };
     }
     /**
      * Initialize the recoverable state.
      *
-     * This method must be called before using the recoverable state. It loads the saved state
-     * if persistence is enabled and registers the object to listen for PERSIST_STATE events.
+     * If persistence is enabled, this method loads the saved state and registers the object to listen for
+     * PERSIST_STATE events. A state established beforehand by {@link RecoverableState.reset} survives if there
+     * is no record to restore.
+     *
+     * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
+     * listener is registered again. The record is not reloaded: the in-memory state is what the teardown wrote, and
+     * reloading it would only drop whatever the deserialization leaves out.
      *
      * @returns The loaded state object
      */
     async initialize() {
-        if (this.state !== null && this.state !== undefined) {
-            return this.currentValue;
-        }
-        if (!this.persistenceEnabled) {
-            this.state = this.deserialize(this.serialize(this.defaultState));
+        if (this.#initialized) {
             return this.currentValue;
         }
-        let kvsIdentifier = null;
-        if (this.persistStateKvsName) {
-            kvsIdentifier = { name: this.persistStateKvsName };
+        if (this.#persistenceEnabled) {
+            await this.#openKeyValueStore();
+            serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.#persistStateQuietly);
+            this.#listening = true;
         }
-        else if (this.persistStateKvsId) {
-            kvsIdentifier = { id: this.persistStateKvsId };
+        // Flipped before the record is loaded, so that a caller catching a `StateValidationError` is left with a
+        // fully wired object running on the default state rather than a half-initialized one.
+        this.#initialized = true;
+        this.#state ??= this.#defaultState();
+        if (!this.#recordLoaded) {
+            this.#recordLoaded = true;
+            await this.#loadSavedState();
         }
-        this.keyValueStore = await KeyValueStore.open(kvsIdentifier, {
-            configuration: serviceLocator.getConfiguration(),
-        });
-        await this.loadSavedState();
-        // Register for persist state events
-        const eventManager = serviceLocator.getEventManager();
-        eventManager.on("persistState" /* EventType.PERSIST_STATE */, this.persistState);
         return this.currentValue;
     }
     /**
      * Clean up resources used by the recoverable state.
      *
      * If persistence is enabled, this method deregisters the object from PERSIST_STATE events
-     * and persists the current state one last time.
+     * and persists the current state one last time, warning rather than throwing if that write fails - cleanup
+     * runs when the work is already done, and failing it would bury whatever the caller was doing. The in-memory
+     * state is left alone, and {@link RecoverableState.initialize} can be called again to open a new
+     * persistence window.
      */
     async teardown() {
-        if (!this.persistenceEnabled || !this.persistState) {
+        this.#initialized = false;
+        if (!this.#persistenceEnabled) {
             return;
         }
-        const eventManager = serviceLocator.getEventManager();
-        eventManager.off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
-        await this.persistState();
+        serviceLocator.getEventManager().off(EventType.PERSIST_STATE, this.#persistStateQuietly);
+        this.#listening = false;
+        await this.#persistStateQuietly();
     }
     /**
      * Get the current state.
+     *
+     * Throws until the state has been established, by either {@link RecoverableState.initialize} or the
+     * synchronous {@link RecoverableState.reset} - the latter being how a caller that cannot await in its
+     * constructor gets a usable state right away.
      */
     get currentValue() {
-        if (this.state === null) {
-            throw new Error('Recoverable state has not yet been loaded');
+        if (this.#state === null) {
+            throw new Error('Recoverable state has not yet been loaded - call initialize() or reset() first');
         }
-        return this.state;
+        return this.#state;
     }
     /**
-     * Reset the state to the default values and clear any persisted state.
+     * Reset the in-memory state to the default values, leaving any persisted record alone.
      *
-     * Resets the current state to the default state and, if persistence is enabled,
-     * clears the persisted state from the KeyValueStore.
+     * Use {@link RecoverableState.resetStore} to clear the persisted record as well.
      */
-    async reset() {
-        this.state = this.deserialize(this.serialize(this.defaultState));
-        if (this.persistenceEnabled) {
-            if (this.keyValueStore === null) {
-                throw new Error('Recoverable state has not yet been initialized');
-            }
-            await this.keyValueStore.setValue(this.persistStateKey, null);
+    reset() {
+        this.#state = this.#defaultState();
+    }
+    /**
+     * Clear the persisted state record, leaving the in-memory state alone.
+     *
+     * This is a between-lifecycles operation - its point is to stop the next {@link RecoverableState.initialize}
+     * from restoring the record, so it throws while PERSIST_STATE events are still being handled, where the next
+     * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
+     * or {@link RecoverableState.teardown} before clearing the record.
+     *
+     * A no-op if persistence is disabled.
+     */
+    async resetStore() {
+        if (this.#listening) {
+            throw new Error(`Cannot clear the state persisted under key '${this.#persistStateKey}' while it is still being persisted periodically - the next PERSIST_STATE event would write it straight back. Use reset() to reset the state itself, or teardown() before clearing the record.`);
+        }
+        if (!this.#persistenceEnabled) {
+            return;
         }
+        const keyValueStore = await this.#openKeyValueStore();
+        await this.#withTimeout(async () => keyValueStore.setValue(this.#persistStateKey, null), 'Clearing the persisted state');
     }
     /**
      * Persist the current state to the KeyValueStore.
      *
      * This method is typically called in response to a PERSIST_STATE event, but can also be called
-     * directly when needed.
+     * directly when needed. It is a no-op if persistence is disabled, if no KeyValueStore is available yet, or if
+     * there is no state to write. A failed write only rejects here - the periodic and teardown ones warn instead.
      *
      * @param eventData Optional data associated with a PERSIST_STATE event
      */
     async persistState(eventData) {
-        this.log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
-        if (this.keyValueStore === null || this.state === null) {
-            throw new Error('Recoverable state has not yet been initialized');
+        if (!this.#persistenceEnabled || this.#state === null) {
+            return;
         }
-        if (this.persistenceEnabled) {
-            await this.keyValueStore.setValue(this.persistStateKey, this.serialize(this.state), {
-                contentType: 'text/plain', // HACK - the result is expected to be JSON, but we do this to avoid the implicit JSON.parse in `KeyValueStore.getValue`
-            });
+        const keyValueStore = await this.#resolveKeyValueStore();
+        if (keyValueStore === null) {
+            return;
+        }
+        this.#log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
+        const serializedState = await this.#serialize(this.currentValue);
+        await this.#withTimeout(async () => this.#contentType === undefined
+            ? keyValueStore.setValue(this.#persistStateKey, serializedState)
+            : keyValueStore.setValue(this.#persistStateKey, serializedState, {
+                contentType: this.#contentType,
+            }), 'Persisting the state');
+    }
+    /** Falls back to the default store when none was supplied - persistence has to write somewhere. */
+    async #openKeyValueStore() {
+        this.#keyValueStore ??= KeyValueStore.open(null, {
+            configuration: this.#configuration ?? serviceLocator.getConfiguration(),
+        });
+        return (await this.#resolveKeyValueStore());
+    }
+    /** Awaits a store handed over as a pending `open()`, keeping the resolved instance for later calls. */
+    async #resolveKeyValueStore() {
+        if (this.#keyValueStore === null) {
+            return null;
         }
+        this.#keyValueStore = await this.#keyValueStore;
+        return this.#keyValueStore;
     }
     /**
-     * Load the saved state from the KeyValueStore
+     * Load the saved state from the KeyValueStore. Leaves the current state alone if there is no record to load.
      */
-    async loadSavedState() {
-        if (this.keyValueStore === null) {
-            throw new Error('Recoverable state has not yet been initialized');
+    async #loadSavedState() {
+        if (!this.#persistenceEnabled) {
+            return;
         }
-        const storedState = await this.keyValueStore.getValue(this.persistStateKey);
-        if (storedState === null || storedState === undefined) {
-            this.state = this.deserialize(this.serialize(this.defaultState));
+        const keyValueStore = await this.#resolveKeyValueStore();
+        if (keyValueStore === null) {
+            return;
         }
-        else {
-            this.state = this.deserialize(storedState);
+        // With a content type, the record is whatever `serialize` produced, so the codec's parse is skipped and
+        // `deserialize` gets the bytes as a stream - the shape a streaming parser wants.
+        // TODO: read the record as a stream instead of buffering it and wrapping (https://github.com/apify/crawlee/issues/2929).
+        const storedState = await this.#withTimeout(async () => {
+            if (this.#contentType === undefined) {
+                return keyValueStore.getValue(this.#persistStateKey);
+            }
+            const record = await keyValueStore.getRecord(this.#persistStateKey);
+            if (record === null) {
+                return null;
+            }
+            return Readable.from(Buffer.isBuffer(record.value) ? record.value : Buffer.from(record.value));
+        }, 'Loading the persisted state');
+        if (storedState === null || storedState === undefined) {
+            return;
         }
+        this.#state = await this.#deserialize(storedState);
+    }
+    async #withTimeout(operation, description) {
+        // `@apify/timeout` shares one `AbortController` across nested frames and `KeyValueStore` checks it on
+        // every operation, so a teardown-time persist running inside an already-expired request handler timeout
+        // would be aborted before it started. Hence a fresh timeout context.
+        return timeoutStorage.exit(async () => addTimeoutToPromise(operation, this.#persistenceTimeoutMillis, `${description} under key '${this.#persistStateKey}' timed out after ${this.#persistenceTimeoutMillis / 1000} seconds.`));
     }
 }

package/request.d.ts

@@ -1,20 +1,12 @@
 import type { BinaryLike } from 'node:crypto';
-import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
-import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
-import type { SkippedRequestReason } from './enqueue_links/shared.js';
-export declare enum RequestState {
-    UNPROCESSED = 0,
-    BEFORE_NAV = 1,
-    AFTER_NAV = 2,
-    REQUEST_HANDLER = 3,
-    DONE = 4,
-    ERROR_HANDLER = 5,
-    ERROR = 6,
-    SKIPPED = 7
-}
+import type { AllowedHttpMethods, Dictionary, RequestSchema } from '@crawlee/types';
+import type { EnqueueStrategy } from '@crawlee/utils';
+/** The `strategy` option accepted by {@link ExtractLinksOptions} and {@link EnqueueUrlsOptions}. */
+export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
 /**
- * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
- * The `Request` object also stores information about errors that occurred during processing of the request.
+ * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata, together
+ * with the processing state a request queue keeps about it (`retryCount`, `errorMessages`, `handledAt`, ...).
+ * Inside a crawler, requests are {@link CrawlingRequest}s, which add the crawler's own per-request settings.
  *
  * Each `Request` instance has the `uniqueKey` property, which can be either specified
  * manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
@@ -30,21 +22,16 @@ export declare enum RequestState {
  * const request = new Request({
  *     url: 'http://www.example.com',
  *     headers: { Accept: 'application/json' },
+ *     userData: { foo: 'bar' },
  * });
  *
- * ...
- *
- * request.userData.foo = 'bar';
- * request.pushErrorMessage(new Error('Request failed!'));
- *
- * ...
- *
- * const foo = request.userData.foo;
+ * await requestQueue.addRequest(request);
  * ```
  * @category Sources
  */
 declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
-    /** Request ID */
+    #private;
+    /** Storage-assigned request ID. Only present on requests that went through a {@link RequestQueue}. */
     id?: string;
     /** URL of the web page to crawl. */
     url: string;
@@ -74,8 +61,6 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
     errorMessages: string[];
     /** Object with HTTP headers. Key is header name, value is the value. */
     headers?: Record<string, string>;
-    /** Private store for the custom user data assigned to the request. */
-    private _userData;
     /**
      * Custom user data assigned to the request.
      *
@@ -90,78 +75,25 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
     handledAt?: string;
     /**
      * `Request` parameters including the URL, HTTP method and headers, and others.
+     *
+     * Processing state (`retryCount`, `errorMessages`, `handledAt`, `loadedUrl`, `id`) is not an option;
+     * requests coming back from a storage are rebuilt with {@link Request.fromSchema}.
      */
     constructor(options: RequestOptions<UserData>);
+    /**
+     * Rebuilds a request from its stored form, including the processing state a
+     * {@link RequestOptions} object cannot carry. Subclasses get instances of themselves.
+     */
+    static fromSchema<UserData extends Dictionary = Dictionary>(schema: RequestSchema): CrawleeRequest<UserData>;
     /**
      * Converts the Crawlee Request object to a `fetch` API Request object.
      * @returns The native `fetch` API Request object.
      */
     intoFetchAPIRequest(): Request;
-    /**
-     * Tells the crawler processing this request to skip the navigation and process the request directly.
-     *
-     * When this is set to `true`, the crawling context will not contain the results of the navigation
-     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
-     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
-     */
-    get skipNavigation(): boolean;
-    /**
-     * Tells the crawler processing this request to skip the navigation and process the request directly.
-     *
-     * When this is set to `true`, the crawling context will not contain the results of the navigation
-     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
-     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
-     */
-    set skipNavigation(value: boolean);
-    /**
-     * Depth of the request in the current crawl tree.
-     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
-     */
-    get crawlDepth(): number;
-    /**
-     * Depth of the request in the current crawl tree.
-     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
-     */
-    set crawlDepth(value: number);
-    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
-    get sessionId(): string | undefined;
-    set sessionId(value: string | undefined);
     /** shortcut for getting `request.userData.label` */
     get label(): string | undefined;
     /** shortcut for setting `request.userData.label` */
     set label(value: string | undefined);
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    get maxRetries(): number | undefined;
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    set maxRetries(value: number | undefined);
-    /** Describes the request's current lifecycle state. */
-    get state(): RequestState;
-    /** Describes the request's current lifecycle state. */
-    set state(value: RequestState);
-    /**
-     * Reason for skipping this request.
-     */
-    get skippedReason(): SkippedRequestReason | undefined;
-    /**
-     * Reason for skipping this request.
-     */
-    set skippedReason(value: SkippedRequestReason | undefined);
-    private get enqueueStrategy();
-    private set enqueueStrategy(value);
-    /**
-     * Stores information about an error that occurred during processing of this request.
-     *
-     * You should always use Error instances when throwing errors in JavaScript.
-     *
-     * Nevertheless, to improve the debugging experience when using third party libraries
-     * that may not always throw an Error instance, the function performs a type
-     * inspection of the passed argument and attempts to extract as much information
-     * as possible, since just throwing a bad type error makes any debugging rather difficult.
-     *
-     * @param errorOrMessage Error object or error message to be stored in the request.
-     * @param [options]
-     */
-    pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
     /** @internal */
     static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, alwaysEnqueue, }: ComputeUniqueKeyOptions): string;
     /** @internal */
@@ -265,31 +197,12 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      * @default 0
      */
     crawlDepth?: number;
-    /**
-     * Reason for skipping this request.
-     * This is used to provide more information about why the request was skipped.
-     * @internal
-     */
-    skippedReason?: SkippedRequestReason;
     /**
      * Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`.
      */
     maxRetries?: number;
     /** @internal */
-    id?: string;
-    /** @internal */
-    handledAt?: string;
-    /** @internal */
-    lockExpiresAt?: Date;
-    /** @internal */
-    enqueueStrategy?: EnqueueLinksOptions['strategy'];
-}
-export interface PushErrorMessageOptions {
-    /**
-     * Only push the error message without stack trace when true.
-     * @default false
-     */
-    omitStack?: boolean;
+    enqueueStrategy?: EnqueueStrategyOption;
 }
 interface ComputeUniqueKeyOptions {
     url: string;