@crawlee/core 4.0.0-beta.2 → 4.0.0-beta.200

package/recoverable_state.d.ts

@@ -0,0 +1,177 @@
+import type { Configuration } from './configuration.js';
+import type { CrawleeLogger } from './log.js';
+import { KeyValueStore } from './storages/key_value_store.js';
+import type { Awaitable } from '@crawlee/types';
+import type { StandardSchemaV1 } from '@standard-schema/spec';
+/**
+ * One direction of the conversion between the state model and its persisted form - either a plain function, or a
+ * [Standard Schema](https://standardschema.dev) whose validated output is the result.
+ *
+ * A schema that fails to validate makes {@link RecoverableState} throw a {@link StateValidationError}. Zod
+ * codecs work directly, as their validation *is* the decode direction; use `(state) => codec.encode(state)` for the
+ * other one.
+ */
+export type StateConversion<TFrom, TTo> = ((value: TFrom) => Awaitable<TTo>) | StandardSchemaV1<TFrom, TTo>;
+/**
+ * A {@link StateConversion} for a caller that cannot await one - {@link Statistics}, whose `toJSON()` is
+ * synchronous, being the reason this exists.
+ *
+ * Only the function arm can be narrowed here: a Standard Schema is free to validate asynchronously, so a schema
+ * that does is rejected when it runs rather than when it is passed.
+ */
+export type SyncStateConversion<TFrom, TTo> = ((value: TFrom) => TTo) | StandardSchemaV1<TFrom, TTo>;
+/**
+ * Applies a {@link SyncStateConversion}, throwing a {@link StateValidationError} for a schema that rejects
+ * the value.
+ *
+ * @internal
+ */
+export declare function convertStateSync<TFrom, TTo>(conversion: SyncStateConversion<TFrom, TTo>, value: TFrom, persistStateKey: string): TTo;
+export interface RecoverableStatePersistenceOptions {
+    /**
+     * The key under which the state is stored in the KeyValueStore
+     */
+    persistStateKey: string;
+    /**
+     * Flag to enable or disable state persistence
+     */
+    persistenceEnabled?: boolean;
+    /**
+     * The KeyValueStore to persist into, defaulting to the default store. Accepts a pending
+     * {@link KeyValueStore.open} so that callers do not have to be async to point at a specific store.
+     */
+    keyValueStore?: KeyValueStore | PromiseLike<KeyValueStore>;
+    /**
+     * Time limit for a single load or save of the state, in milliseconds.
+     * @default 60_000
+     */
+    persistenceTimeoutMillis?: number;
+}
+/**
+ * The fields of {@link RecoverableStateOptions}, without the constraint tying `contentType` to the conversions.
+ */
+export interface RecoverableStateBaseOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> extends RecoverableStatePersistenceOptions {
+    /**
+     * The state used when no persisted state is found, and the state {@link RecoverableState.reset} restores.
+     *
+     * A plain value is deep-copied with `structuredClone` each time it is used, so pass a factory for a state
+     * that `structuredClone` cannot rebuild - one holding class instances, say, or one derived from a schema.
+     */
+    defaultState: TStateModel | (() => TStateModel);
+    /**
+     * A logger instance for logging operations related to state persistence
+     */
+    logger?: CrawleeLogger;
+    /**
+     * Configuration instance to use when opening the KeyValueStore
+     */
+    configuration?: Configuration;
+    /**
+     * Optional conversion of the state to a plain JSON-serializable value before it is persisted.
+     * If not provided, the state is persisted as is.
+     *
+     * With {@link RecoverableStateBaseOptions.contentType} set, it has to produce what
+     * {@link KeyValueStore.setValue} accepts alongside an explicit content type - a `string`, a `Buffer` or a
+     * stream.
+     */
+    serialize?: StateConversion<TStateModel, TPersistedState>;
+    /**
+     * Optional conversion of a persisted value back to the state model, and the place to validate a record before
+     * trusting it. If not provided, the persisted value is used as is.
+     *
+     * With {@link RecoverableStateBaseOptions.contentType} set, it receives a `Readable` of the record bytes
+     * instead of a parsed value.
+     */
+    deserialize?: StateConversion<TPersistedState, TStateModel>;
+    /**
+     * Content type of the persisted record. Setting it hands the record encoding over to
+     * {@link RecoverableStateBaseOptions.serialize} and {@link RecoverableStateBaseOptions.deserialize}, both of
+     * which are then required - the default JSON codec is bypassed in both directions. Meant for a state too large
+     * for `JSON.stringify`, which `serialize` can then stream out instead.
+     */
+    contentType?: string;
+}
+/**
+ * Options for configuring the RecoverableState
+ */
+export type RecoverableStateOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> = RecoverableStateBaseOptions<TStateModel, TPersistedState> & ({
+    contentType?: undefined;
+} | Required<Pick<RecoverableStateBaseOptions<TStateModel, TPersistedState>, 'serialize' | 'deserialize' | 'contentType'>>);
+/**
+ * A class for managing persistent recoverable state using a plain JavaScript object.
+ *
+ * This class facilitates state persistence to a `KeyValueStore`, allowing data to be saved and retrieved
+ * across migrations or restarts. It manages the loading, saving, and resetting of state data,
+ * with optional persistence capabilities.
+ *
+ * The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
+ * The class automatically hooks into the event system to persist state when needed.
+ */
+export declare class RecoverableState<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> {
+    #private;
+    /**
+     * Initialize a new recoverable state object.
+     *
+     * @param options Configuration options for the recoverable state
+     */
+    constructor(options: RecoverableStateOptions<TStateModel, TPersistedState>);
+    /**
+     * Initialize the recoverable state.
+     *
+     * If persistence is enabled, this method loads the saved state and registers the object to listen for
+     * PERSIST_STATE events. A state established beforehand by {@link RecoverableState.reset} survives if there
+     * is no record to restore.
+     *
+     * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
+     * listener is registered again. The record is not reloaded: the in-memory state is what the teardown wrote, and
+     * reloading it would only drop whatever the deserialization leaves out.
+     *
+     * @returns The loaded state object
+     */
+    initialize(): Promise<TStateModel>;
+    /**
+     * Clean up resources used by the recoverable state.
+     *
+     * If persistence is enabled, this method deregisters the object from PERSIST_STATE events
+     * and persists the current state one last time, warning rather than throwing if that write fails - cleanup
+     * runs when the work is already done, and failing it would bury whatever the caller was doing. The in-memory
+     * state is left alone, and {@link RecoverableState.initialize} can be called again to open a new
+     * persistence window.
+     */
+    teardown(): Promise<void>;
+    /**
+     * Get the current state.
+     *
+     * Throws until the state has been established, by either {@link RecoverableState.initialize} or the
+     * synchronous {@link RecoverableState.reset} - the latter being how a caller that cannot await in its
+     * constructor gets a usable state right away.
+     */
+    get currentValue(): TStateModel;
+    /**
+     * Reset the in-memory state to the default values, leaving any persisted record alone.
+     *
+     * Use {@link RecoverableState.resetStore} to clear the persisted record as well.
+     */
+    reset(): void;
+    /**
+     * Clear the persisted state record, leaving the in-memory state alone.
+     *
+     * This is a between-lifecycles operation - its point is to stop the next {@link RecoverableState.initialize}
+     * from restoring the record, so it throws while PERSIST_STATE events are still being handled, where the next
+     * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
+     * or {@link RecoverableState.teardown} before clearing the record.
+     *
+     * A no-op if persistence is disabled.
+     */
+    resetStore(): Promise<void>;
+    /**
+     * Persist the current state to the KeyValueStore.
+     *
+     * This method is typically called in response to a PERSIST_STATE event, but can also be called
+     * directly when needed. It is a no-op if persistence is disabled, if no KeyValueStore is available yet, or if
+     * there is no state to write. A failed write only rejects here - the periodic and teardown ones warn instead.
+     *
+     * @param eventData Optional data associated with a PERSIST_STATE event
+     */
+    persistState(eventData?: Record<string, unknown>): Promise<void>;
+}

package/recoverable_state.js

@@ -0,0 +1,268 @@
+import { Readable } from 'node:stream';
+import { addTimeoutToPromise, storage as timeoutStorage } from '@apify/timeout';
+import { StateValidationError } from './errors.js';
+import { EventType } from './events/event_manager.js';
+import { serviceLocator } from './service_locator.js';
+import { KeyValueStore } from './storages/key_value_store.js';
+const DEFAULT_PERSISTENCE_TIMEOUT_MILLIS = 60_000;
+/**
+ * Applies a {@link SyncStateConversion}, throwing a {@link StateValidationError} for a schema that rejects
+ * the value.
+ *
+ * @internal
+ */
+export function convertStateSync(conversion, value, persistStateKey) {
+    if (typeof conversion === 'function') {
+        return conversion(value);
+    }
+    const result = conversion['~standard'].validate(value);
+    if ('then' in result) {
+        throw new Error(`The state conversion for '${persistStateKey}' validated asynchronously, which this caller cannot await.`);
+    }
+    if (result.issues) {
+        throw new StateValidationError(persistStateKey, result.issues);
+    }
+    return result.value;
+}
+/**
+ * A class for managing persistent recoverable state using a plain JavaScript object.
+ *
+ * This class facilitates state persistence to a `KeyValueStore`, allowing data to be saved and retrieved
+ * across migrations or restarts. It manages the loading, saving, and resetting of state data,
+ * with optional persistence capabilities.
+ *
+ * The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
+ * The class automatically hooks into the event system to persist state when needed.
+ */
+export class RecoverableState {
+    #defaultState;
+    #state = null;
+    #initialized = false;
+    #recordLoaded = false;
+    #listening = false;
+    #persistenceEnabled;
+    #persistStateKey;
+    #persistenceTimeoutMillis;
+    #configuration;
+    #keyValueStore;
+    #log;
+    #serialize;
+    #deserialize;
+    #contentType;
+    #persistStateQuietly;
+    /**
+     * Initialize a new recoverable state object.
+     *
+     * @param options Configuration options for the recoverable state
+     */
+    constructor(options) {
+        const { defaultState } = options;
+        this.#defaultState =
+            typeof defaultState === 'function'
+                ? defaultState
+                : () => structuredClone(defaultState);
+        this.#persistStateKey = options.persistStateKey;
+        this.#persistenceEnabled = options.persistenceEnabled ?? false;
+        this.#persistenceTimeoutMillis = options.persistenceTimeoutMillis ?? DEFAULT_PERSISTENCE_TIMEOUT_MILLIS;
+        this.#configuration = options.configuration;
+        this.#keyValueStore = options.keyValueStore ?? null;
+        this.#log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
+        this.#contentType = options.contentType;
+        if (this.#contentType !== undefined && (options.serialize === undefined || options.deserialize === undefined)) {
+            throw new Error(`A 'contentType' for the state persisted under key '${this.#persistStateKey}' requires both 'serialize' and 'deserialize' - the record is no longer JSON the default codec can handle.`);
+        }
+        this.#serialize = this.#toConversion(options.serialize);
+        this.#deserialize = this.#toConversion(options.deserialize);
+        // A pending open() may reject before anything here awaits it. Observe the rejection so it surfaces on
+        // first use rather than as an unhandled rejection - most immediately when persistence is disabled and
+        // nothing ever awaits the store.
+        if (this.#keyValueStore !== null) {
+            Promise.resolve(this.#keyValueStore).then(undefined, () => { });
+        }
+        // The automatic persists, where a rejection has nowhere useful to go - the event manager does not catch
+        // listener errors, and throwing from teardown would bury the outcome of the work it cleans up after.
+        this.#persistStateQuietly = async (eventData) => this.persistState(eventData).catch((error) => this.#log.warning(`Failed to persist the state under key '${this.#persistStateKey}'.`, { error }));
+    }
+    /** Normalizes a conversion option into a function. Absent conversions pass the value through unchanged. */
+    #toConversion(conversion) {
+        if (conversion === undefined) {
+            return async (value) => value;
+        }
+        if (typeof conversion === 'function') {
+            return async (value) => conversion(value);
+        }
+        return async (value) => {
+            const result = await conversion['~standard'].validate(value);
+            if (result.issues) {
+                throw new StateValidationError(this.#persistStateKey, result.issues);
+            }
+            return result.value;
+        };
+    }
+    /**
+     * Initialize the recoverable state.
+     *
+     * If persistence is enabled, this method loads the saved state and registers the object to listen for
+     * PERSIST_STATE events. A state established beforehand by {@link RecoverableState.reset} survives if there
+     * is no record to restore.
+     *
+     * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
+     * listener is registered again. The record is not reloaded: the in-memory state is what the teardown wrote, and
+     * reloading it would only drop whatever the deserialization leaves out.
+     *
+     * @returns The loaded state object
+     */
+    async initialize() {
+        if (this.#initialized) {
+            return this.currentValue;
+        }
+        if (this.#persistenceEnabled) {
+            await this.#openKeyValueStore();
+            serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.#persistStateQuietly);
+            this.#listening = true;
+        }
+        // Flipped before the record is loaded, so that a caller catching a `StateValidationError` is left with a
+        // fully wired object running on the default state rather than a half-initialized one.
+        this.#initialized = true;
+        this.#state ??= this.#defaultState();
+        if (!this.#recordLoaded) {
+            this.#recordLoaded = true;
+            await this.#loadSavedState();
+        }
+        return this.currentValue;
+    }
+    /**
+     * Clean up resources used by the recoverable state.
+     *
+     * If persistence is enabled, this method deregisters the object from PERSIST_STATE events
+     * and persists the current state one last time, warning rather than throwing if that write fails - cleanup
+     * runs when the work is already done, and failing it would bury whatever the caller was doing. The in-memory
+     * state is left alone, and {@link RecoverableState.initialize} can be called again to open a new
+     * persistence window.
+     */
+    async teardown() {
+        this.#initialized = false;
+        if (!this.#persistenceEnabled) {
+            return;
+        }
+        serviceLocator.getEventManager().off(EventType.PERSIST_STATE, this.#persistStateQuietly);
+        this.#listening = false;
+        await this.#persistStateQuietly();
+    }
+    /**
+     * Get the current state.
+     *
+     * Throws until the state has been established, by either {@link RecoverableState.initialize} or the
+     * synchronous {@link RecoverableState.reset} - the latter being how a caller that cannot await in its
+     * constructor gets a usable state right away.
+     */
+    get currentValue() {
+        if (this.#state === null) {
+            throw new Error('Recoverable state has not yet been loaded - call initialize() or reset() first');
+        }
+        return this.#state;
+    }
+    /**
+     * Reset the in-memory state to the default values, leaving any persisted record alone.
+     *
+     * Use {@link RecoverableState.resetStore} to clear the persisted record as well.
+     */
+    reset() {
+        this.#state = this.#defaultState();
+    }
+    /**
+     * Clear the persisted state record, leaving the in-memory state alone.
+     *
+     * This is a between-lifecycles operation - its point is to stop the next {@link RecoverableState.initialize}
+     * from restoring the record, so it throws while PERSIST_STATE events are still being handled, where the next
+     * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
+     * or {@link RecoverableState.teardown} before clearing the record.
+     *
+     * A no-op if persistence is disabled.
+     */
+    async resetStore() {
+        if (this.#listening) {
+            throw new Error(`Cannot clear the state persisted under key '${this.#persistStateKey}' while it is still being persisted periodically - the next PERSIST_STATE event would write it straight back. Use reset() to reset the state itself, or teardown() before clearing the record.`);
+        }
+        if (!this.#persistenceEnabled) {
+            return;
+        }
+        const keyValueStore = await this.#openKeyValueStore();
+        await this.#withTimeout(async () => keyValueStore.setValue(this.#persistStateKey, null), 'Clearing the persisted state');
+    }
+    /**
+     * Persist the current state to the KeyValueStore.
+     *
+     * This method is typically called in response to a PERSIST_STATE event, but can also be called
+     * directly when needed. It is a no-op if persistence is disabled, if no KeyValueStore is available yet, or if
+     * there is no state to write. A failed write only rejects here - the periodic and teardown ones warn instead.
+     *
+     * @param eventData Optional data associated with a PERSIST_STATE event
+     */
+    async persistState(eventData) {
+        if (!this.#persistenceEnabled || this.#state === null) {
+            return;
+        }
+        const keyValueStore = await this.#resolveKeyValueStore();
+        if (keyValueStore === null) {
+            return;
+        }
+        this.#log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
+        const serializedState = await this.#serialize(this.currentValue);
+        await this.#withTimeout(async () => this.#contentType === undefined
+            ? keyValueStore.setValue(this.#persistStateKey, serializedState)
+            : keyValueStore.setValue(this.#persistStateKey, serializedState, {
+                contentType: this.#contentType,
+            }), 'Persisting the state');
+    }
+    /** Falls back to the default store when none was supplied - persistence has to write somewhere. */
+    async #openKeyValueStore() {
+        this.#keyValueStore ??= KeyValueStore.open(null, {
+            configuration: this.#configuration ?? serviceLocator.getConfiguration(),
+        });
+        return (await this.#resolveKeyValueStore());
+    }
+    /** Awaits a store handed over as a pending `open()`, keeping the resolved instance for later calls. */
+    async #resolveKeyValueStore() {
+        if (this.#keyValueStore === null) {
+            return null;
+        }
+        this.#keyValueStore = await this.#keyValueStore;
+        return this.#keyValueStore;
+    }
+    /**
+     * Load the saved state from the KeyValueStore. Leaves the current state alone if there is no record to load.
+     */
+    async #loadSavedState() {
+        if (!this.#persistenceEnabled) {
+            return;
+        }
+        const keyValueStore = await this.#resolveKeyValueStore();
+        if (keyValueStore === null) {
+            return;
+        }
+        // With a content type, the record is whatever `serialize` produced, so the codec's parse is skipped and
+        // `deserialize` gets the bytes as a stream - the shape a streaming parser wants.
+        // TODO: read the record as a stream instead of buffering it and wrapping (https://github.com/apify/crawlee/issues/2929).
+        const storedState = await this.#withTimeout(async () => {
+            if (this.#contentType === undefined) {
+                return keyValueStore.getValue(this.#persistStateKey);
+            }
+            const record = await keyValueStore.getRecord(this.#persistStateKey);
+            if (record === null) {
+                return null;
+            }
+            return Readable.from(Buffer.isBuffer(record.value) ? record.value : Buffer.from(record.value));
+        }, 'Loading the persisted state');
+        if (storedState === null || storedState === undefined) {
+            return;
+        }
+        this.#state = await this.#deserialize(storedState);
+    }
+    async #withTimeout(operation, description) {
+        // `@apify/timeout` shares one `AbortController` across nested frames and `KeyValueStore` checks it on
+        // every operation, so a teardown-time persist running inside an already-expired request handler timeout
+        // would be aborted before it started. Hence a fresh timeout context.
+        return timeoutStorage.exit(async () => addTimeoutToPromise(operation, this.#persistenceTimeoutMillis, `${description} under key '${this.#persistStateKey}' timed out after ${this.#persistenceTimeoutMillis / 1000} seconds.`));
+    }
+}

package/request.d.ts

@@ -1,20 +1,12 @@
 import type { BinaryLike } from 'node:crypto';
-import type { Dictionary } from '@crawlee/types';
-import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
-import type { AllowedHttpMethods } from './typedefs.js';
-export declare enum RequestState {
-    UNPROCESSED = 0,
-    BEFORE_NAV = 1,
-    AFTER_NAV = 2,
-    REQUEST_HANDLER = 3,
-    DONE = 4,
-    ERROR_HANDLER = 5,
-    ERROR = 6,
-    SKIPPED = 7
-}
+import type { AllowedHttpMethods, Dictionary, RequestSchema } from '@crawlee/types';
+import type { EnqueueStrategy } from '@crawlee/utils';
+/** The `strategy` option accepted by {@link ExtractLinksOptions} and {@link EnqueueUrlsOptions}. */
+export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
 /**
- * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
- * The `Request` object also stores information about errors that occurred during processing of the request.
+ * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata, together
+ * with the processing state a request queue keeps about it (`retryCount`, `errorMessages`, `handledAt`, ...).
+ * Inside a crawler, requests are {@link CrawlingRequest}s, which add the crawler's own per-request settings.
  *
  * Each `Request` instance has the `uniqueKey` property, which can be either specified
  * manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
@@ -30,21 +22,16 @@ export declare enum RequestState {
  * const request = new Request({
  *     url: 'http://www.example.com',
  *     headers: { Accept: 'application/json' },
+ *     userData: { foo: 'bar' },
  * });
  *
- * ...
- *
- * request.userData.foo = 'bar';
- * request.pushErrorMessage(new Error('Request failed!'));
- *
- * ...
- *
- * const foo = request.userData.foo;
+ * await requestQueue.addRequest(request);
  * ```
  * @category Sources
  */
-export declare class Request<UserData extends Dictionary = Dictionary> {
-    /** Request ID */
+declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
+    #private;
+    /** Storage-assigned request ID. Only present on requests that went through a {@link RequestQueue}. */
     id?: string;
     /** URL of the web page to crawl. */
     url: string;
@@ -74,9 +61,12 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
     errorMessages: string[];
     /** Object with HTTP headers. Key is header name, value is the value. */
     headers?: Record<string, string>;
-    /** Private store for the custom user data assigned to the request. */
-    private _userData;
-    /** Custom user data assigned to the request. */
+    /**
+     * Custom user data assigned to the request.
+     *
+     * All data stored in `userData` must be JSON-serializable.
+     * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
+     */
     userData: UserData;
     /**
      * ISO datetime string that indicates the time when the request has been processed.
@@ -85,48 +75,27 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
     handledAt?: string;
     /**
      * `Request` parameters including the URL, HTTP method and headers, and others.
+     *
+     * Processing state (`retryCount`, `errorMessages`, `handledAt`, `loadedUrl`, `id`) is not an option;
+     * requests coming back from a storage are rebuilt with {@link Request.fromSchema}.
      */
     constructor(options: RequestOptions<UserData>);
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
-    get skipNavigation(): boolean;
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
-    set skipNavigation(value: boolean);
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    get sessionRotationCount(): number;
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    set sessionRotationCount(value: number);
+    /**
+     * Rebuilds a request from its stored form, including the processing state a
+     * {@link RequestOptions} object cannot carry. Subclasses get instances of themselves.
+     */
+    static fromSchema<UserData extends Dictionary = Dictionary>(schema: RequestSchema): CrawleeRequest<UserData>;
+    /**
+     * Converts the Crawlee Request object to a `fetch` API Request object.
+     * @returns The native `fetch` API Request object.
+     */
+    intoFetchAPIRequest(): Request;
     /** shortcut for getting `request.userData.label` */
     get label(): string | undefined;
     /** shortcut for setting `request.userData.label` */
     set label(value: string | undefined);
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    get maxRetries(): number | undefined;
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    set maxRetries(value: number | undefined);
-    /** Describes the request's current lifecycle state. */
-    get state(): RequestState;
-    /** Describes the request's current lifecycle state. */
-    set state(value: RequestState);
-    private get enqueueStrategy();
-    private set enqueueStrategy(value);
-    /**
-     * Stores information about an error that occurred during processing of this request.
-     *
-     * You should always use Error instances when throwing errors in JavaScript.
-     *
-     * Nevertheless, to improve the debugging experience when using third party libraries
-     * that may not always throw an Error instance, the function performs a type
-     * inspection of the passed argument and attempts to extract as much information
-     * as possible, since just throwing a bad type error makes any debugging rather difficult.
-     *
-     * @param errorOrMessage Error object or error message to be stored in the request.
-     * @param [options]
-     */
-    pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
-    protected _computeUniqueKey(options: ComputeUniqueKeyOptions): string;
-    protected _hashPayload(payload: BinaryLike): string;
     /** @internal */
-    static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, }: ComputeUniqueKeyOptions): string;
+    static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, alwaysEnqueue, }: ComputeUniqueKeyOptions): string;
     /** @internal */
     static hashPayload(payload: BinaryLike): string;
 }
@@ -147,7 +116,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      * The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
      *
      * The `useExtendedUniqueKey` options determines whether method and payload are included in the `uniqueKey`,
-     * producing a `uniqueKey` in the following format: `METHOD(payloadHash):normalizedUrl`. This is useful
+     * producing a `uniqueKey` in the following format: `METHOD|payloadHash|normalizedUrl`. This is useful
      * when requests point to the same URL, but with different methods and payloads. For example: form submits.
      *
      * Pass an arbitrary non-empty text value to the `uniqueKey` property
@@ -171,6 +140,9 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
     /**
      * Custom user data assigned to the request. Use this to save any request related data to the
      * request's scope, keeping them accessible on retries, failures etc.
+     *
+     * All data stored in `userData` must be JSON-serializable.
+     * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
      */
     userData?: UserData;
     /**
@@ -192,36 +164,45 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      * @default false
      */
     useExtendedUniqueKey?: boolean;
+    /**
+     * If `true` then a random value is included in the `uniqueKey` computation, ensuring the request
+     * is always enqueued even if a request with the same URL (and method/payload) is already present
+     * in the queue. Cannot be used together with a custom `uniqueKey`.
+     * @default false
+     */
+    alwaysEnqueue?: boolean;
     /**
      * The `true` value indicates that the request will not be automatically retried on error.
      * @default false
      */
     noRetry?: boolean;
+    /**
+     * ID of a session from the crawler's `SessionPool` to use for this request.
+     * When set, the crawler will fetch this session from the pool instead of creating a new one.
+     */
+    sessionId?: string;
     /**
      * If set to `true` then the crawler processing this request evaluates
      * the `requestHandler` immediately without prior browser navigation.
+     *
+     * When enabled, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
      * @default false
      */
     skipNavigation?: boolean;
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     * @default 0
+     */
+    crawlDepth?: number;
     /**
      * Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`.
      */
     maxRetries?: number;
     /** @internal */
-    id?: string;
-    /** @internal */
-    handledAt?: string;
-    /** @internal */
-    lockExpiresAt?: Date;
-    /** @internal */
-    enqueueStrategy?: EnqueueLinksOptions['strategy'];
-}
-export interface PushErrorMessageOptions {
-    /**
-     * Only push the error message without stack trace when true.
-     * @default false
-     */
-    omitStack?: boolean;
+    enqueueStrategy?: EnqueueStrategyOption;
 }
 interface ComputeUniqueKeyOptions {
     url: string;
@@ -229,15 +210,15 @@ interface ComputeUniqueKeyOptions {
     payload?: string | Buffer;
     keepUrlFragment?: boolean;
     useExtendedUniqueKey?: boolean;
+    alwaysEnqueue?: boolean;
 }
 export type Source = (Partial<RequestOptions> & {
     requestsFromUrl?: string;
     regex?: RegExp;
-}) | Request;
+}) | CrawleeRequest;
 /** @internal */
 export interface InternalSource {
     requestsFromUrl: string;
     regex?: RegExp;
 }
-export {};
-//# sourceMappingURL=request.d.ts.map
+export { CrawleeRequest as Request };