@crawlee/core 4.0.0-beta.14 → 4.0.0-beta.140

package/proxy_configuration.js

@@ -1,54 +1,12 @@
-import ow from 'ow';
-import log from '@apify/log';
-/**
- * Internal class for tracking the proxy tier history for a specific domain.
- *
- * Predicts the best proxy tier for the next request based on the error history for different proxy tiers.
- */
-class ProxyTierTracker {
-    histogram;
-    currentTier;
-    constructor(tieredProxyUrls) {
-        this.histogram = tieredProxyUrls.map(() => 0);
-        this.currentTier = 0;
-    }
-    /**
-     * Processes a single step of the algorithm and updates the current tier prediction based on the error history.
-     */
-    processStep() {
-        this.histogram.forEach((x, i) => {
-            if (this.currentTier === i)
-                return;
-            if (x > 0)
-                this.histogram[i]--;
-        });
-        const left = this.currentTier > 0 ? this.histogram[this.currentTier - 1] : Infinity;
-        const right = this.currentTier < this.histogram.length - 1 ? this.histogram[this.currentTier + 1] : Infinity;
-        if (this.histogram[this.currentTier] > Math.min(left, right)) {
-            this.currentTier = left <= right ? this.currentTier - 1 : this.currentTier + 1;
-        }
-        else if (this.histogram[this.currentTier] === left) {
-            this.currentTier--;
-        }
-    }
-    /**
-     * Increases the error score for the given proxy tier. This raises the chance of picking a different proxy tier for the subsequent requests.
-     *
-     * The error score is increased by 10 for the given tier. This means that this tier will be disadvantaged for the next 10 requests (every new request prediction decreases the error score by 1).
-     * @param tier The proxy tier to mark as problematic.
-     */
-    addError(tier) {
-        this.histogram[tier] += 10;
-    }
-    /**
-     * Returns the best proxy tier for the next request based on the error history for different proxy tiers.
-     * @returns The proxy tier prediction
-     */
-    predictTier() {
-        this.processStep();
-        return this.currentTier;
-    }
-}
+import { z } from 'zod';
+import { parseArgument, schemas } from './validators.js';
+const proxyConfigurationOptionsSchema = z.strictObject({
+    proxyUrls: z
+        .array(z.union([z.url(), z.null()]))
+        .nonempty()
+        .optional(),
+    newUrlFunction: schemas.anyFunction.optional(),
+});
 /**
  * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
  * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
@@ -79,13 +37,9 @@ class ProxyTierTracker {
  */
 export class ProxyConfiguration {
     isManInTheMiddle = false;
-    nextCustomUrlIndex = 0;
-    proxyUrls;
-    tieredProxyUrls;
-    usedProxyUrls = new Map();
-    newUrlFunction;
-    log = log.child({ prefix: 'ProxyConfiguration' });
-    domainTiers = new Map();
+    #nextCustomUrlIndex = 0;
+    #proxyUrls;
+    #newUrlFunction;
     /**
      * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
      * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
@@ -108,19 +62,17 @@ export class ProxyConfiguration {
      */
     constructor(options = {}) {
         const { validateRequired, ...rest } = options;
-        ow(rest, ow.object.exactShape({
-            proxyUrls: ow.optional.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null)),
-            newUrlFunction: ow.optional.function,
-            tieredProxyUrls: ow.optional.array.nonEmpty.ofType(ow.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null))),
-        }));
-        const { proxyUrls, newUrlFunction, tieredProxyUrls } = options;
-        if ([proxyUrls, newUrlFunction, tieredProxyUrls].filter((x) => x).length > 1)
-            this._throwCannotCombineCustomMethods();
+        if ('tieredProxyUrls' in rest) {
+            throw new Error('The `tieredProxyUrls` option has been removed in Crawlee v4. ' +
+                'See the v4 upgrading guide for the recommended migration to named sessions.');
+        }
+        const { proxyUrls, newUrlFunction } = parseArgument(rest, proxyConfigurationOptionsSchema);
+        if (proxyUrls && newUrlFunction)
+            this.throwCannotCombineCustomMethods();
         if (!proxyUrls && !newUrlFunction && validateRequired)
-            this._throwNoOptionsProvided();
-        this.proxyUrls = proxyUrls;
-        this.newUrlFunction = newUrlFunction;
-        this.tieredProxyUrls = tieredProxyUrls;
+            this.throwNoOptionsProvided();
+        this.#proxyUrls = proxyUrls;
+        this.#newUrlFunction = newUrlFunction;
     }
     /**
      * This function creates a new {@link ProxyInfo} info object.
@@ -132,16 +84,7 @@ export class ProxyConfiguration {
      * @return Represents information about used proxy and its configuration.
      */
     async newProxyInfo(options) {
-        let url;
-        let tier;
-        if (this.tieredProxyUrls) {
-            const { proxyUrl, proxyTier } = this._handleTieredUrl(options);
-            url = proxyUrl ?? undefined;
-            tier = proxyTier;
-        }
-        else {
-            url = await this.newUrl(options);
-        }
+        const url = await this.newUrl(options);
         if (!url)
             return undefined;
         const { username, password, port, hostname } = new URL(url);
@@ -151,59 +94,8 @@ export class ProxyConfiguration {
             password: decodeURIComponent(password),
             hostname,
             port: port,
-            proxyTier: tier,
-        };
-    }
-    /**
-     * Given a request / proxy tier, this function returns a new proxy URL based on the provided configuration options.
-     * @param options Options for the tiered proxy rotation
-     * @returns An object with the proxy URL and the proxy tier used.
-     */
-    _handleTieredUrl(options) {
-        if (!this.tieredProxyUrls)
-            throw new Error('Tiered proxy URLs are not set');
-        if (!options || (!options?.request && options?.proxyTier === undefined)) {
-            const allProxyUrls = this.tieredProxyUrls.flat();
-            return {
-                proxyUrl: allProxyUrls[this.nextCustomUrlIndex++ % allProxyUrls.length],
-            };
-        }
-        let tierPrediction = options.proxyTier;
-        if (typeof tierPrediction !== 'number') {
-            tierPrediction = this.predictProxyTier(options.request);
-        }
-        const proxyTier = this.tieredProxyUrls[tierPrediction];
-        return {
-            proxyUrl: proxyTier[this.nextCustomUrlIndex++ % proxyTier.length],
-            proxyTier: tierPrediction,
         };
     }
-    /**
-     * Given a `Request` object, this function returns the tier of the proxy that should be used for the request.
-     *
-     * This returns `null` if `tieredProxyUrls` option is not set.
-     */
-    predictProxyTier(request) {
-        if (!this.tieredProxyUrls)
-            return null;
-        const domain = new URL(request.url).hostname;
-        if (!this.domainTiers.has(domain)) {
-            this.domainTiers.set(domain, new ProxyTierTracker(this.tieredProxyUrls));
-        }
-        request.userData.__crawlee ??= {};
-        const tracker = this.domainTiers.get(domain);
-        if (typeof request.userData.__crawlee.lastProxyTier === 'number') {
-            tracker.addError(request.userData.__crawlee.lastProxyTier);
-        }
-        const tierPrediction = tracker.predictTier();
-        if (typeof request.userData.__crawlee.lastProxyTier === 'number' &&
-            request.userData.__crawlee.lastProxyTier !== tierPrediction) {
-            log.debug(`Changing proxy tier for domain "${domain}" from ${request.userData.__crawlee.lastProxyTier} to ${tierPrediction}.`);
-        }
-        request.userData.__crawlee.lastProxyTier = tierPrediction;
-        request.userData.__crawlee.forefront = true;
-        return tierPrediction;
-    }
     /**
      * Returns a new proxy URL based on provided configuration options.
      *
@@ -211,22 +103,19 @@ export class ProxyConfiguration {
      *  For example, `http://bob:password123@proxy.example.com:8000`
      */
     async newUrl(options) {
-        if (this.newUrlFunction) {
-            return (await this._callNewUrlFunction({ request: options?.request })) ?? undefined;
-        }
-        if (this.tieredProxyUrls) {
-            return this._handleTieredUrl(options).proxyUrl ?? undefined;
+        if (this.#newUrlFunction) {
+            return (await this.callNewUrlFunction({ request: options?.request })) ?? undefined;
         }
-        return this._handleProxyUrlsList() ?? undefined;
+        return this.handleProxyUrlsList() ?? undefined;
     }
-    _handleProxyUrlsList() {
-        return this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
+    handleProxyUrlsList() {
+        return this.#proxyUrls[this.#nextCustomUrlIndex++ % this.#proxyUrls.length];
     }
     /**
      * Calls the custom newUrlFunction and checks format of its return value
      */
-    async _callNewUrlFunction(options) {
-        const proxyUrl = await this.newUrlFunction(options);
+    async callNewUrlFunction(options) {
+        const proxyUrl = await this.#newUrlFunction(options);
         try {
             if (proxyUrl) {
                 new URL(proxyUrl); // eslint-disable-line no-new
@@ -237,11 +126,10 @@ export class ProxyConfiguration {
             throw new Error(`The provided newUrlFunction did not return a valid URL.\nCause: ${err.message}`);
         }
     }
-    _throwCannotCombineCustomMethods() {
+    throwCannotCombineCustomMethods() {
         throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".');
     }
-    _throwNoOptionsProvided() {
+    throwNoOptionsProvided() {
         throw new Error('One of "options.proxyUrls" or "options.newUrlFunction" needs to be provided.');
     }
 }
-//# sourceMappingURL=proxy_configuration.js.map

package/recoverable_state.d.ts

@@ -1,5 +1,31 @@
-import { Configuration } from '@crawlee/core';
-import type { Log } from '@apify/log';
+import type { Configuration, CrawleeLogger } from '@crawlee/core';
+import { KeyValueStore } from '@crawlee/core';
+import type { Awaitable } from '@crawlee/types';
+import type { StandardSchemaV1 } from '@standard-schema/spec';
+/**
+ * One direction of the conversion between the state model and its persisted form - either a plain function, or a
+ * [Standard Schema](https://standardschema.dev) whose validated output is the result.
+ *
+ * A schema that fails to validate makes {@link RecoverableState} throw a {@link StateValidationError}. Zod
+ * codecs work directly, as their validation *is* the decode direction; use `(state) => codec.encode(state)` for the
+ * other one.
+ */
+export type StateConversion<TFrom, TTo> = ((value: TFrom) => Awaitable<TTo>) | StandardSchemaV1<TFrom, TTo>;
+/**
+ * A {@link StateConversion} for a caller that cannot await one - {@link Statistics}, whose `toJSON()` is
+ * synchronous, being the reason this exists.
+ *
+ * Only the function arm can be narrowed here: a Standard Schema is free to validate asynchronously, so a schema
+ * that does is rejected when it runs rather than when it is passed.
+ */
+export type SyncStateConversion<TFrom, TTo> = ((value: TFrom) => TTo) | StandardSchemaV1<TFrom, TTo>;
+/**
+ * Applies a {@link SyncStateConversion}, throwing a {@link StateValidationError} for a schema that rejects
+ * the value.
+ *
+ * @internal
+ */
+export declare function convertStateSync<TFrom, TTo>(conversion: SyncStateConversion<TFrom, TTo>, value: TFrom, persistStateKey: string): TTo;
 export interface RecoverableStatePersistenceOptions {
     /**
      * The key under which the state is stored in the KeyValueStore
@@ -10,44 +36,45 @@ export interface RecoverableStatePersistenceOptions {
      */
     persistenceEnabled?: boolean;
     /**
-     * The name of the KeyValueStore to use for persistence.
-     * If neither a name nor an id are supplied, the default store will be used.
+     * The KeyValueStore to persist into, defaulting to the default store. Accepts a pending
+     * {@link KeyValueStore.open} so that callers do not have to be async to point at a specific store.
      */
-    persistStateKvsName?: string;
+    keyValueStore?: KeyValueStore | PromiseLike<KeyValueStore>;
     /**
-     * The identifier of the KeyValueStore to use for persistence.
-     * If neither a name nor an id are supplied, the default store will be used.
+     * Time limit for a single load or save of the state, in milliseconds.
+     * @default 60_000
      */
-    persistStateKvsId?: string;
+    persistenceTimeoutMillis?: number;
 }
 /**
  * Options for configuring the RecoverableState
  */
-export interface RecoverableStateOptions<TStateModel = Record<string, unknown>> extends RecoverableStatePersistenceOptions {
+export interface RecoverableStateOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> extends RecoverableStatePersistenceOptions {
     /**
-     * The default state used if no persisted state is found.
-     * A deep copy is made each time the state is used.
+     * The state used when no persisted state is found, and the state {@link RecoverableState.reset} restores.
+     *
+     * A plain value is deep-copied with `structuredClone` each time it is used, so pass a factory for a state
+     * that `structuredClone` cannot rebuild - one holding class instances, say, or one derived from a schema.
      */
-    defaultState: TStateModel;
+    defaultState: TStateModel | (() => TStateModel);
     /**
      * A logger instance for logging operations related to state persistence
      */
-    logger?: Log;
+    logger?: CrawleeLogger;
     /**
-     * Configuration instance to use
+     * Configuration instance to use when opening the KeyValueStore
      */
-    config?: Configuration;
+    configuration?: Configuration;
     /**
-     * Optional function to transform the state to a JSON string before persistence.
-     * If not provided, JSON.stringify will be used.
+     * Optional conversion of the state to a plain JSON-serializable value before it is persisted.
+     * If not provided, the state is persisted as is.
      */
-    serialize?: (state: TStateModel) => string;
+    serialize?: StateConversion<TStateModel, TPersistedState>;
     /**
-     * Optional function to transform a JSON-serialized object back to the state model.
-     * If not provided, JSON.parse is used.
-     * It is advisable to perform validation in this function and to throw an exception if it fails.
+     * Optional conversion of a persisted value back to the state model, and the place to validate a record before
+     * trusting it. If not provided, the persisted value is used as is.
      */
-    deserialize?: (serializedState: string) => TStateModel;
+    deserialize?: StateConversion<TPersistedState, TStateModel>;
 }
 /**
  * A class for managing persistent recoverable state using a plain JavaScript object.
@@ -59,29 +86,23 @@ export interface RecoverableStateOptions<TStateModel = Record<string, unknown>>
  * The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
  * The class automatically hooks into the event system to persist state when needed.
  */
-export declare class RecoverableState<TStateModel = Record<string, unknown>> {
-    private readonly defaultState;
-    private state;
-    private readonly persistenceEnabled;
-    private readonly persistStateKey;
-    private readonly persistStateKvsName?;
-    private readonly persistStateKvsId?;
-    private keyValueStore;
-    private readonly log;
-    private readonly config;
-    private readonly serialize;
-    private readonly deserialize;
+export declare class RecoverableState<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> {
+    #private;
     /**
      * Initialize a new recoverable state object.
      *
      * @param options Configuration options for the recoverable state
      */
-    constructor(options: RecoverableStateOptions<TStateModel>);
+    constructor(options: RecoverableStateOptions<TStateModel, TPersistedState>);
     /**
      * Initialize the recoverable state.
      *
-     * This method must be called before using the recoverable state. It loads the saved state
-     * if persistence is enabled and registers the object to listen for PERSIST_STATE events.
+     * If persistence is enabled, this method loads the saved state and registers the object to listen for
+     * PERSIST_STATE events. A state established beforehand by {@link RecoverableState.reset} survives if there
+     * is no record to restore.
+     *
+     * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
+     * listener is registered again and the record reloaded.
      *
      * @returns The loaded state object
      */
@@ -90,34 +111,45 @@ export declare class RecoverableState<TStateModel = Record<string, unknown>> {
      * Clean up resources used by the recoverable state.
      *
      * If persistence is enabled, this method deregisters the object from PERSIST_STATE events
-     * and persists the current state one last time.
+     * and persists the current state one last time, warning rather than throwing if that write fails - cleanup
+     * runs when the work is already done, and failing it would bury whatever the caller was doing. The in-memory
+     * state is left alone, and {@link RecoverableState.initialize} can be called again to open a new
+     * persistence window.
      */
     teardown(): Promise<void>;
     /**
      * Get the current state.
+     *
+     * Throws until the state has been established, by either {@link RecoverableState.initialize} or the
+     * synchronous {@link RecoverableState.reset} - the latter being how a caller that cannot await in its
+     * constructor gets a usable state right away.
      */
     get currentValue(): TStateModel;
     /**
-     * Reset the state to the default values and clear any persisted state.
+     * Reset the in-memory state to the default values, leaving any persisted record alone.
      *
-     * Resets the current state to the default state and, if persistence is enabled,
-     * clears the persisted state from the KeyValueStore.
+     * Use {@link RecoverableState.resetStore} to clear the persisted record as well.
      */
-    reset(): Promise<void>;
+    reset(): void;
+    /**
+     * Clear the persisted state record, leaving the in-memory state alone.
+     *
+     * This is a between-lifecycles operation - its point is to stop the next {@link RecoverableState.initialize}
+     * from restoring the record, so it throws while PERSIST_STATE events are still being handled, where the next
+     * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
+     * or {@link RecoverableState.teardown} before clearing the record.
+     *
+     * A no-op if persistence is disabled or no KeyValueStore is available yet.
+     */
+    resetStore(): Promise<void>;
     /**
      * Persist the current state to the KeyValueStore.
      *
      * This method is typically called in response to a PERSIST_STATE event, but can also be called
-     * directly when needed.
+     * directly when needed. It is a no-op if persistence is disabled, if no KeyValueStore is available yet, or if
+     * there is no state to write. A failed write only rejects here - the periodic and teardown ones warn instead.
      *
      * @param eventData Optional data associated with a PERSIST_STATE event
      */
-    persistState(eventData?: {
-        isMigrating: boolean;
-    }): Promise<void>;
-    /**
-     * Load the saved state from the KeyValueStore
-     */
-    private loadSavedState;
+    persistState(eventData?: Record<string, unknown>): Promise<void>;
 }
-//# sourceMappingURL=recoverable_state.d.ts.map