@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

package/proxy_configuration.d.ts

@@ -1,28 +1,28 @@
 import type { ProxyInfo } from '@crawlee/types';
-import type { Request } from './request.js';
 export interface ProxyConfigurationFunction {
-    (options?: {
-        request?: Request;
-    }): string | null | Promise<string | null>;
+    (): string | null | Promise<string | null>;
 }
-type UrlList = (string | null)[];
 export interface ProxyConfigurationOptions {
     /**
      * An array of custom proxy URLs to be rotated.
      * Custom proxies are not compatible with Apify Proxy and an attempt to use both
      * configuration options will cause an error to be thrown on initialize.
      */
-    proxyUrls?: UrlList;
+    proxyUrls?: (string | null)[];
     /**
-     * Custom function that allows you to generate the new proxy URL dynamically. It gets an optional parameter with the `Request` object when applicable.
+     * Custom function that allows you to generate the new proxy URL dynamically.
      * Can return either stringified proxy URL or `null` if the proxy should not be used. Can be asynchronous.
      *
      * This function is used to generate the URL when {@link ProxyConfiguration.newUrl} or {@link ProxyConfiguration.newProxyInfo} is called.
      */
     newUrlFunction?: ProxyConfigurationFunction;
-}
-interface NewUrlOptions {
-    request?: Request;
+    /**
+     * When truthy, the constructor throws unless one of `proxyUrls` / `newUrlFunction` was given. Falsy by
+     * default, so a bare `ProxyConfiguration` can be constructed. Set by the Apify SDK, which builds the options
+     * object itself; declared here only so that it stays type-checkable.
+     * @internal
+     */
+    validateRequired?: boolean;
 }
 /**
  * Minimal contract that any object passed to a crawler as its `proxyConfiguration`
@@ -36,10 +36,14 @@ interface NewUrlOptions {
  */
 export interface IProxyConfiguration {
     /**
-     * Creates a new {@link ProxyInfo} object describing the proxy to use for the given
-     * request. Returns `undefined` when no proxy should be used.
+     * Creates a new {@link ProxyInfo} object describing the proxy to use.
+     * Returns `undefined` when no proxy should be used.
+     *
+     * @param proxyInfo A previously created `ProxyInfo`, e.g. one restored with a persisted session. Implementations
+     *  should return an equivalent `ProxyInfo` that is usable in the current environment, or the argument itself when
+     *  there is nothing to refresh.
      */
-    newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
+    newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined>;
 }
 /**
  * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
@@ -70,10 +74,8 @@ export interface IProxyConfiguration {
  * @category Scaling
  */
 export declare class ProxyConfiguration implements IProxyConfiguration {
+    #private;
     readonly isManInTheMiddle = false;
-    private nextCustomUrlIndex;
-    private proxyUrls?;
-    private newUrlFunction?;
     /**
      * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
      * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
@@ -102,22 +104,15 @@ export declare class ProxyConfiguration implements IProxyConfiguration {
      * Use it if you want to work with a rich representation of a proxy URL.
      * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
      *
+     * @param proxyInfo A previously created `ProxyInfo`, returned unchanged.
      * @return Represents information about used proxy and its configuration.
      */
-    newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
+    newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined>;
     /**
      * Returns a new proxy URL based on provided configuration options.
      *
      * @return A string with a proxy URL, including authentication credentials and port number.
      *  For example, `http://bob:password123@proxy.example.com:8000`
      */
-    newUrl(options?: NewUrlOptions): Promise<string | undefined>;
-    private handleProxyUrlsList;
-    /**
-     * Calls the custom newUrlFunction and checks format of its return value
-     */
-    private callNewUrlFunction;
-    private throwCannotCombineCustomMethods;
-    private throwNoOptionsProvided;
+    newUrl(): Promise<string | undefined>;
 }
-export {};

package/proxy_configuration.js

@@ -1,4 +1,12 @@
-import ow from 'ow';
+import { z } from 'zod';
+import { parseArgument, schemas } from './validators.js';
+const proxyConfigurationOptionsSchema = z.strictObject({
+    proxyUrls: z
+        .array(z.union([z.url(), z.null()]))
+        .nonempty()
+        .optional(),
+    newUrlFunction: schemas.anyFunction.optional(),
+});
 /**
  * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
  * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
@@ -29,9 +37,9 @@ import ow from 'ow';
  */
 export class ProxyConfiguration {
     isManInTheMiddle = false;
-    nextCustomUrlIndex = 0;
-    proxyUrls;
-    newUrlFunction;
+    #nextCustomUrlIndex = 0;
+    #proxyUrls;
+    #newUrlFunction;
     /**
      * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
      * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
@@ -53,22 +61,21 @@ export class ProxyConfiguration {
      * ```
      */
     constructor(options = {}) {
+        // `validateRequired` is destructured off before the strict-object parse on purpose: the Apify SDK passes it
+        // through a computed key (`['validateRequired' as string]: false`), and leaving it in `rest` would make
+        // `Actor.createProxyConfiguration()` fail the `z.strictObject` check with a `ZodError`.
         const { validateRequired, ...rest } = options;
         if ('tieredProxyUrls' in rest) {
             throw new Error('The `tieredProxyUrls` option has been removed in Crawlee v4. ' +
                 'See the v4 upgrading guide for the recommended migration to named sessions.');
         }
-        ow(rest, ow.object.exactShape({
-            proxyUrls: ow.optional.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null)),
-            newUrlFunction: ow.optional.function,
-        }));
-        const { proxyUrls, newUrlFunction } = options;
+        const { proxyUrls, newUrlFunction } = parseArgument(rest, proxyConfigurationOptionsSchema);
         if (proxyUrls && newUrlFunction)
-            this.throwCannotCombineCustomMethods();
+            this.#throwCannotCombineCustomMethods();
         if (!proxyUrls && !newUrlFunction && validateRequired)
-            this.throwNoOptionsProvided();
-        this.proxyUrls = proxyUrls;
-        this.newUrlFunction = newUrlFunction;
+            this.#throwNoOptionsProvided();
+        this.#proxyUrls = proxyUrls;
+        this.#newUrlFunction = newUrlFunction;
     }
     /**
      * This function creates a new {@link ProxyInfo} info object.
@@ -77,10 +84,13 @@ export class ProxyConfiguration {
      * Use it if you want to work with a rich representation of a proxy URL.
      * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
      *
+     * @param proxyInfo A previously created `ProxyInfo`, returned unchanged.
      * @return Represents information about used proxy and its configuration.
      */
-    async newProxyInfo(options) {
-        const url = await this.newUrl(options);
+    async newProxyInfo(proxyInfo) {
+        if (proxyInfo)
+            return proxyInfo;
+        const url = await this.newUrl();
         if (!url)
             return undefined;
         const { username, password, port, hostname } = new URL(url);
@@ -98,20 +108,20 @@ export class ProxyConfiguration {
      * @return A string with a proxy URL, including authentication credentials and port number.
      *  For example, `http://bob:password123@proxy.example.com:8000`
      */
-    async newUrl(options) {
-        if (this.newUrlFunction) {
-            return (await this.callNewUrlFunction({ request: options?.request })) ?? undefined;
+    async newUrl() {
+        if (this.#newUrlFunction) {
+            return (await this.#callNewUrlFunction()) ?? undefined;
         }
-        return this.handleProxyUrlsList() ?? undefined;
+        return this.#handleProxyUrlsList() ?? undefined;
     }
-    handleProxyUrlsList() {
-        return this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
+    #handleProxyUrlsList() {
+        return this.#proxyUrls[this.#nextCustomUrlIndex++ % this.#proxyUrls.length];
     }
     /**
      * Calls the custom newUrlFunction and checks format of its return value
      */
-    async callNewUrlFunction(options) {
-        const proxyUrl = await this.newUrlFunction(options);
+    async #callNewUrlFunction() {
+        const proxyUrl = await this.#newUrlFunction();
         try {
             if (proxyUrl) {
                 new URL(proxyUrl); // eslint-disable-line no-new
@@ -122,10 +132,10 @@ export class ProxyConfiguration {
             throw new Error(`The provided newUrlFunction did not return a valid URL.\nCause: ${err.message}`);
         }
     }
-    throwCannotCombineCustomMethods() {
+    #throwCannotCombineCustomMethods() {
         throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".');
     }
-    throwNoOptionsProvided() {
+    #throwNoOptionsProvided() {
         throw new Error('One of "options.proxyUrls" or "options.newUrlFunction" needs to be provided.');
     }
 }

package/recoverable_state.d.ts

@@ -1,4 +1,32 @@
-import type { Configuration, CrawleeLogger } from '@crawlee/core';
+import type { Configuration } from './configuration.js';
+import type { CrawleeLogger } from './log.js';
+import { KeyValueStore } from './storages/key_value_store.js';
+import type { Awaitable } from '@crawlee/types';
+import type { StandardSchemaV1 } from '@standard-schema/spec';
+/**
+ * One direction of the conversion between the state model and its persisted form - either a plain function, or a
+ * [Standard Schema](https://standardschema.dev) whose validated output is the result.
+ *
+ * A schema that fails to validate makes {@link RecoverableState} throw a {@link StateValidationError}. Zod
+ * codecs work directly, as their validation *is* the decode direction; use `(state) => codec.encode(state)` for the
+ * other one.
+ */
+export type StateConversion<TFrom, TTo> = ((value: TFrom) => Awaitable<TTo>) | StandardSchemaV1<TFrom, TTo>;
+/**
+ * A {@link StateConversion} for a caller that cannot await one - {@link Statistics}, whose `toJSON()` is
+ * synchronous, being the reason this exists.
+ *
+ * Only the function arm can be narrowed here: a Standard Schema is free to validate asynchronously, so a schema
+ * that does is rejected when it runs rather than when it is passed.
+ */
+export type SyncStateConversion<TFrom, TTo> = ((value: TFrom) => TTo) | StandardSchemaV1<TFrom, TTo>;
+/**
+ * Applies a {@link SyncStateConversion}, throwing a {@link StateValidationError} for a schema that rejects
+ * the value.
+ *
+ * @internal
+ */
+export declare function convertStateSync<TFrom, TTo>(conversion: SyncStateConversion<TFrom, TTo>, value: TFrom, persistStateKey: string): TTo;
 export interface RecoverableStatePersistenceOptions {
     /**
      * The key under which the state is stored in the KeyValueStore
@@ -9,45 +37,66 @@ export interface RecoverableStatePersistenceOptions {
      */
     persistenceEnabled?: boolean;
     /**
-     * The name of the KeyValueStore to use for persistence.
-     * If neither a name nor an id are supplied, the default store will be used.
+     * The KeyValueStore to persist into, defaulting to the default store. Accepts a pending
+     * {@link KeyValueStore.open} so that callers do not have to be async to point at a specific store.
      */
-    persistStateKvsName?: string;
+    keyValueStore?: KeyValueStore | PromiseLike<KeyValueStore>;
     /**
-     * The identifier of the KeyValueStore to use for persistence.
-     * If neither a name nor an id are supplied, the default store will be used.
+     * Time limit for a single load or save of the state, in milliseconds.
+     * @default 60_000
      */
-    persistStateKvsId?: string;
+    persistenceTimeoutMillis?: number;
 }
 /**
- * Options for configuring the RecoverableState
+ * The fields of {@link RecoverableStateOptions}, without the constraint tying `contentType` to the conversions.
  */
-export interface RecoverableStateOptions<TStateModel = Record<string, unknown>> extends RecoverableStatePersistenceOptions {
+export interface RecoverableStateBaseOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> extends RecoverableStatePersistenceOptions {
     /**
-     * The default state used if no persisted state is found.
-     * A deep copy is made each time the state is used.
+     * The state used when no persisted state is found, and the state {@link RecoverableState.reset} restores.
+     *
+     * A plain value is deep-copied with `structuredClone` each time it is used, so pass a factory for a state
+     * that `structuredClone` cannot rebuild - one holding class instances, say, or one derived from a schema.
      */
-    defaultState: TStateModel;
+    defaultState: TStateModel | (() => TStateModel);
     /**
      * A logger instance for logging operations related to state persistence
      */
     logger?: CrawleeLogger;
     /**
-     * Configuration instance to use
+     * Configuration instance to use when opening the KeyValueStore
      */
     configuration?: Configuration;
     /**
-     * Optional function to transform the state to a JSON string before persistence.
-     * If not provided, JSON.stringify will be used.
+     * Optional conversion of the state to a plain JSON-serializable value before it is persisted.
+     * If not provided, the state is persisted as is.
+     *
+     * With {@link RecoverableStateBaseOptions.contentType} set, it has to produce what
+     * {@link KeyValueStore.setValue} accepts alongside an explicit content type - a `string`, a `Buffer` or a
+     * stream.
+     */
+    serialize?: StateConversion<TStateModel, TPersistedState>;
+    /**
+     * Optional conversion of a persisted value back to the state model, and the place to validate a record before
+     * trusting it. If not provided, the persisted value is used as is.
+     *
+     * With {@link RecoverableStateBaseOptions.contentType} set, it receives a `Readable` of the record bytes
+     * instead of a parsed value.
      */
-    serialize?: (state: TStateModel) => string;
+    deserialize?: StateConversion<TPersistedState, TStateModel>;
     /**
-     * Optional function to transform a JSON-serialized object back to the state model.
-     * If not provided, JSON.parse is used.
-     * It is advisable to perform validation in this function and to throw an exception if it fails.
+     * Content type of the persisted record. Setting it hands the record encoding over to
+     * {@link RecoverableStateBaseOptions.serialize} and {@link RecoverableStateBaseOptions.deserialize}, both of
+     * which are then required - the default JSON codec is bypassed in both directions. Meant for a state too large
+     * for `JSON.stringify`, which `serialize` can then stream out instead.
      */
-    deserialize?: (serializedState: string) => TStateModel;
+    contentType?: string;
 }
+/**
+ * Options for configuring the RecoverableState
+ */
+export type RecoverableStateOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> = RecoverableStateBaseOptions<TStateModel, TPersistedState> & ({
+    contentType?: undefined;
+} | Required<Pick<RecoverableStateBaseOptions<TStateModel, TPersistedState>, 'serialize' | 'deserialize' | 'contentType'>>);
 /**
  * A class for managing persistent recoverable state using a plain JavaScript object.
  *
@@ -58,28 +107,24 @@ export interface RecoverableStateOptions<TStateModel = Record<string, unknown>>
  * The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
  * The class automatically hooks into the event system to persist state when needed.
  */
-export declare class RecoverableState<TStateModel = Record<string, unknown>> {
-    private readonly defaultState;
-    private state;
-    private readonly persistenceEnabled;
-    private readonly persistStateKey;
-    private readonly persistStateKvsName?;
-    private readonly persistStateKvsId?;
-    private keyValueStore;
-    private readonly log;
-    private readonly serialize;
-    private readonly deserialize;
+export declare class RecoverableState<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> {
+    #private;
     /**
      * Initialize a new recoverable state object.
      *
      * @param options Configuration options for the recoverable state
      */
-    constructor(options: RecoverableStateOptions<TStateModel>);
+    constructor(options: RecoverableStateOptions<TStateModel, TPersistedState>);
     /**
      * Initialize the recoverable state.
      *
-     * This method must be called before using the recoverable state. It loads the saved state
-     * if persistence is enabled and registers the object to listen for PERSIST_STATE events.
+     * If persistence is enabled, this method loads the saved state and registers the object to listen for
+     * PERSIST_STATE events. A state established beforehand by {@link RecoverableState.reset} survives if there
+     * is no record to restore.
+     *
+     * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
+     * listener is registered again. The record is not reloaded: the in-memory state is what the teardown wrote, and
+     * reloading it would only drop whatever the deserialization leaves out.
      *
      * @returns The loaded state object
      */
@@ -88,33 +133,45 @@ export declare class RecoverableState<TStateModel = Record<string, unknown>> {
      * Clean up resources used by the recoverable state.
      *
      * If persistence is enabled, this method deregisters the object from PERSIST_STATE events
-     * and persists the current state one last time.
+     * and persists the current state one last time, warning rather than throwing if that write fails - cleanup
+     * runs when the work is already done, and failing it would bury whatever the caller was doing. The in-memory
+     * state is left alone, and {@link RecoverableState.initialize} can be called again to open a new
+     * persistence window.
      */
     teardown(): Promise<void>;
     /**
      * Get the current state.
+     *
+     * Throws until the state has been established, by either {@link RecoverableState.initialize} or the
+     * synchronous {@link RecoverableState.reset} - the latter being how a caller that cannot await in its
+     * constructor gets a usable state right away.
      */
     get currentValue(): TStateModel;
     /**
-     * Reset the state to the default values and clear any persisted state.
+     * Reset the in-memory state to the default values, leaving any persisted record alone.
      *
-     * Resets the current state to the default state and, if persistence is enabled,
-     * clears the persisted state from the KeyValueStore.
+     * Use {@link RecoverableState.resetStore} to clear the persisted record as well.
      */
-    reset(): Promise<void>;
+    reset(): void;
+    /**
+     * Clear the persisted state record, leaving the in-memory state alone.
+     *
+     * This is a between-lifecycles operation - its point is to stop the next {@link RecoverableState.initialize}
+     * from restoring the record, so it throws while PERSIST_STATE events are still being handled, where the next
+     * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
+     * or {@link RecoverableState.teardown} before clearing the record.
+     *
+     * A no-op if persistence is disabled.
+     */
+    resetStore(): Promise<void>;
     /**
      * Persist the current state to the KeyValueStore.
      *
      * This method is typically called in response to a PERSIST_STATE event, but can also be called
-     * directly when needed.
+     * directly when needed. It is a no-op if persistence is disabled, if no KeyValueStore is available yet, or if
+     * there is no state to write. A failed write only rejects here - the periodic and teardown ones warn instead.
      *
      * @param eventData Optional data associated with a PERSIST_STATE event
      */
-    persistState(eventData?: {
-        isMigrating: boolean;
-    }): Promise<void>;
-    /**
-     * Load the saved state from the KeyValueStore
-     */
-    private loadSavedState;
+    persistState(eventData?: Record<string, unknown>): Promise<void>;
 }