@crawlee/core 4.0.0-beta.161 → 4.0.0-beta.163

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.161",
+    "version": "4.0.0-beta.163",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,10 +52,10 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.161",
-        "@crawlee/http-client": "4.0.0-beta.161",
-        "@crawlee/types": "4.0.0-beta.161",
-        "@crawlee/utils": "4.0.0-beta.161",
+        "@crawlee/fs-storage": "4.0.0-beta.163",
+        "@crawlee/http-client": "4.0.0-beta.163",
+        "@crawlee/types": "4.0.0-beta.163",
+        "@crawlee/utils": "4.0.0-beta.163",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "dc353e294d5f4d2ee468325ef11dd7af44443be2"
+    "gitHead": "3bcb1a803a1ba3082734c4b1aee72f7b3b51f427"
 }

package/recoverable_state.d.ts

@@ -102,7 +102,8 @@ export declare class RecoverableState<TStateModel = Record<string, unknown>, TPe
      * is no record to restore.
      *
      * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
-     * listener is registered again and the record reloaded.
+     * listener is registered again. The record is not reloaded: the in-memory state is what the teardown wrote, and
+     * reloading it would only drop whatever the deserialization leaves out.
      *
      * @returns The loaded state object
      */
@@ -139,7 +140,7 @@ export declare class RecoverableState<TStateModel = Record<string, unknown>, TPe
      * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
      * or {@link RecoverableState.teardown} before clearing the record.
      *
-     * A no-op if persistence is disabled or no KeyValueStore is available yet.
+     * A no-op if persistence is disabled.
      */
     resetStore(): Promise<void>;
     /**

package/recoverable_state.js

@@ -34,6 +34,7 @@ export class RecoverableState {
     #defaultState;
     #state = null;
     #initialized = false;
+    #recordLoaded = false;
     #listening = false;
     #persistenceEnabled;
     #persistStateKey;
@@ -63,6 +64,12 @@ export class RecoverableState {
         this.#log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
         this.#serialize = this.#toConversion(options.serialize);
         this.#deserialize = this.#toConversion(options.deserialize);
+        // A pending open() may reject before anything here awaits it. Observe the rejection so it surfaces on
+        // first use rather than as an unhandled rejection - most immediately when persistence is disabled and
+        // nothing ever awaits the store.
+        if (this.#keyValueStore !== null) {
+            Promise.resolve(this.#keyValueStore).then(undefined, () => { });
+        }
         // The automatic persists, where a rejection has nowhere useful to go - the event manager does not catch
         // listener errors, and throwing from teardown would bury the outcome of the work it cleans up after.
         this.#persistStateQuietly = async (eventData) => this.persistState(eventData).catch((error) => this.#log.warning(`Failed to persist the state under key '${this.#persistStateKey}'.`, { error }));
@@ -91,7 +98,8 @@ export class RecoverableState {
      * is no record to restore.
      *
      * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
-     * listener is registered again and the record reloaded.
+     * listener is registered again. The record is not reloaded: the in-memory state is what the teardown wrote, and
+     * reloading it would only drop whatever the deserialization leaves out.
      *
      * @returns The loaded state object
      */
@@ -100,10 +108,7 @@ export class RecoverableState {
             return this.currentValue;
         }
         if (this.#persistenceEnabled) {
-            this.#keyValueStore ??= KeyValueStore.open(null, {
-                configuration: this.#configuration ?? serviceLocator.getConfiguration(),
-            });
-            await this.#resolveKeyValueStore();
+            await this.#openKeyValueStore();
             serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.#persistStateQuietly);
             this.#listening = true;
         }
@@ -111,7 +116,10 @@ export class RecoverableState {
         // fully wired object running on the default state rather than a half-initialized one.
         this.#initialized = true;
         this.#state ??= this.#defaultState();
-        await this.#loadSavedState();
+        if (!this.#recordLoaded) {
+            this.#recordLoaded = true;
+            await this.#loadSavedState();
+        }
         return this.currentValue;
     }
     /**
@@ -161,7 +169,7 @@ export class RecoverableState {
      * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
      * or {@link RecoverableState.teardown} before clearing the record.
      *
-     * A no-op if persistence is disabled or no KeyValueStore is available yet.
+     * A no-op if persistence is disabled.
      */
     async resetStore() {
         if (this.#listening) {
@@ -170,10 +178,7 @@ export class RecoverableState {
         if (!this.#persistenceEnabled) {
             return;
         }
-        const keyValueStore = await this.#resolveKeyValueStore();
-        if (keyValueStore === null) {
-            return;
-        }
+        const keyValueStore = await this.#openKeyValueStore();
         await this.#withTimeout(async () => keyValueStore.setValue(this.#persistStateKey, null), 'Clearing the persisted state');
     }
     /**
@@ -197,6 +202,13 @@ export class RecoverableState {
         const serializedState = await this.#serialize(this.currentValue);
         await this.#withTimeout(async () => keyValueStore.setValue(this.#persistStateKey, serializedState), 'Persisting the state');
     }
+    /** Falls back to the default store when none was supplied - persistence has to write somewhere. */
+    async #openKeyValueStore() {
+        this.#keyValueStore ??= KeyValueStore.open(null, {
+            configuration: this.#configuration ?? serviceLocator.getConfiguration(),
+        });
+        return (await this.#resolveKeyValueStore());
+    }
     /** Awaits a store handed over as a pending `open()`, keeping the resolved instance for later calls. */
     async #resolveKeyValueStore() {
         if (this.#keyValueStore === null) {

package/session_pool/session.d.ts

@@ -119,7 +119,7 @@ export declare class Session implements ISession {
     markGood(): void;
     /**
      * Gets session state for persistence in KeyValueStore.
-     * @returns Represents session internal state.
+     * @internal
      */
     getState(): SessionState;
     /**

package/session_pool/session.js

@@ -148,7 +148,7 @@ export class Session {
     }
     /**
      * Gets session state for persistence in KeyValueStore.
-     * @returns Represents session internal state.
+     * @internal
      */
     getState() {
         return {

package/session_pool/session_pool.d.ts

@@ -1,10 +1,21 @@
-import type { ISessionPool } from '@crawlee/types';
+import type { ISessionPool, SessionState } from '@crawlee/types';
+import { z } from 'zod';
 import type { PersistenceOptions } from '../crawlers/statistics.js';
 import type { CrawleeLogger } from '../log.js';
 import type { SessionOptions } from './session.js';
 import { Session } from './session.js';
 declare const SESSION_REUSE_STRATEGIES: readonly ['random', 'round-robin', 'use-until-failure'];
 export type SessionReuseStrategy = (typeof SESSION_REUSE_STRATEGIES)[number];
+declare const persistedSessionPoolState: z.ZodObject<{
+    usableSessionsCount: z.ZodNumber;
+    retiredSessionsCount: z.ZodNumber;
+    sessions: z.ZodArray<z.ZodCustom<SessionState, SessionState>>;
+}, z.core.$strip>;
+/**
+ * The shape of the persisted {@link SessionPool} record, as returned by {@link SessionPool.getState}.
+ * @internal
+ */
+export type SessionPoolPersistedState = z.infer<typeof persistedSessionPoolState>;
 /**
  * Factory user-function which creates customized {@link Session} instances.
  */
@@ -152,34 +163,36 @@ export declare class SessionPool implements ISessionPool {
      */
     getSession(sessionId?: string): Promise<Session | undefined>;
     /**
-     * @param options - Override the persistence options provided in the constructor
+     * Discards all sessions in the pool, both usable and retired, resetting it to a blank state.
+     *
+     * The persisted record is left alone - use {@link SessionPool.resetStore} to clear that as well.
+     */
+    reset(): void;
+    /**
+     * Clears the persisted pool record, leaving the in-memory sessions alone.
+     *
+     * Throws while the state is still being persisted periodically - the next PERSIST_STATE event would write
+     * the record straight back. Call {@link SessionPool.teardown} first, or use {@link SessionPool.reset}
+     * to discard the sessions themselves. A no-op if persistence is disabled.
      */
-    resetStore(options?: PersistenceOptions): Promise<void>;
+    resetStore(): Promise<void>;
     /**
      * Returns an object representing the internal state of the `SessionPool` instance.
-     * Note that the object's fields can change in future releases.
+     * @internal
      */
-    getState(): Promise<{
-        usableSessionsCount: number;
-        retiredSessionsCount: number;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        sessions: import("@crawlee/types").SessionState[];
-    }>;
+    getState(): Promise<SessionPoolPersistedState>;
     /**
      * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
      * The state is persisted automatically in regular intervals.
-     * @param options - Override the persistence options provided in the constructor
      */
-    persistState(options?: PersistenceOptions): Promise<void>;
+    persistState(): Promise<void>;
     [Symbol.asyncDispose](): Promise<void>;
     /**
-     * Removes listener from `persistState` event.
+     * Stops the periodic state persistence and persists the state one last time.
      * This function should be called after you are done with using the `SessionPool` instance.
-     * @param options - Set `persistState` to false when the final state was already persisted by the event manager.
+     * Using the pool again afterwards initializes it anew.
      */
-    teardown({ persistState }?: {
-        persistState?: boolean;
-    }): Promise<void>;
+    teardown(): Promise<void>;
     /**
      * Removes retired `Session` instances from `SessionPool`.
      */
@@ -225,10 +238,5 @@ export declare class SessionPool implements ISessionPool {
      * Returns `undefined` when no session should be reused and a new one should be created instead.
      */
     private pickSession;
-    /**
-     * Potentially loads `SessionPool`.
-     * If the state was persisted it loads the `SessionPool` from the persisted state.
-     */
-    private maybeLoadSessionPool;
 }
 export {};

package/session_pool/session_pool.js

@@ -1,6 +1,6 @@
 import { AsyncQueue } from '@sapphire/async-queue';
 import { z } from 'zod';
-import { EventType } from '../events/event_manager.js';
+import { RecoverableState } from '../recoverable_state.js';
 import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from '../storages/key_value_store.js';
 import { parseArgument, schemas, validators } from '../validators.js';
@@ -24,6 +24,43 @@ const sessionPoolOptionsSchema = z.strictObject({
 const createSessionOptionsSchema = z.strictObject({
     sessionOptions: schemas.anyObject.default(() => ({})),
 });
+// Only what the recreation reads is pinned down - the rest of a session's state is the session's own business.
+const persistedSessionState = z.custom((value) => typeof value === 'object' &&
+    value !== null &&
+    typeof value.createdAt === 'string' &&
+    typeof value.expiresAt === 'string', 'Expected a persisted session with `createdAt` and `expiresAt` timestamps');
+const persistedSessionPoolState = z.object({
+    usableSessionsCount: z.number(),
+    retiredSessionsCount: z.number(),
+    sessions: z.array(persistedSessionState),
+});
+const sessionPoolInternalState = z.object({
+    sessions: z.array(z.instanceof(Session)),
+});
+/**
+ * The conversion between the pool's sessions and its persisted record. Decoding recreates the sessions through
+ * the pool's factory and keeps only the usable ones, which is why the codec is built per pool.
+ */
+function buildSessionPoolStateCodec(pool) {
+    return z.codec(persistedSessionPoolState, sessionPoolInternalState, {
+        decode: async (record) => {
+            const sessions = [];
+            for (const sessionState of record.sessions) {
+                const session = await pool.recreateSession({
+                    ...sessionState,
+                    createdAt: new Date(sessionState.createdAt),
+                    expiresAt: new Date(sessionState.expiresAt),
+                });
+                if (session.isUsable()) {
+                    sessions.push(session);
+                }
+            }
+            pool.log.debug(`${sessions.length} active sessions loaded from KeyValueStore`);
+            return { sessions };
+        },
+        encode: ({ sessions }) => pool.toRecord(sessions),
+    });
+}
 /**
  * Handles the rotation, creation and persistence of user-like sessions.
  * Creates a pool of {@link Session} instances, that are randomly rotated.
@@ -81,18 +118,20 @@ export class SessionPool {
     static #nextId = 0;
     id;
     #log;
-    #sessions = [];
     #maxPoolSize;
     #createSessionFunction;
-    #keyValueStore;
     #sessionMap = new Map();
     #sessionOptions;
     #persistStateKeyValueStoreId;
     #persistStateKey;
-    #listener;
-    #events;
     #persistenceOptions;
     #sessionReuseStrategy;
+    /**
+     * The sessions live in a {@link RecoverableState} that handles the loading, recreation and periodic
+     * persistence of the record under `persistStateKey`.
+     */
+    #state;
+    #stateCodec;
     #initPromise;
     #queue = new AsyncQueue();
     #roundRobinIndex = 0;
@@ -100,7 +139,6 @@ export class SessionPool {
         const { id, maxPoolSize, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions, log, persistenceOptions, sessionReuseStrategy, } = parseArgument(options, sessionPoolOptionsSchema);
         this.id = id != null ? String(id) : String(SessionPool.#nextId++);
         this.#sessionReuseStrategy = sessionReuseStrategy;
-        this.#events = serviceLocator.getEventManager();
         this.#log = log.child({ prefix: 'SessionPool' });
         this.#persistenceOptions = persistenceOptions;
         // Pool Configuration
@@ -116,20 +154,38 @@ export class SessionPool {
         // Session keyValueStore
         this.#persistStateKeyValueStoreId = persistStateKeyValueStoreId;
         this.#persistStateKey = persistStateKey ?? `${PERSIST_STATE_KEY}_${this.id}`;
+        this.#stateCodec = buildSessionPoolStateCodec({
+            recreateSession: (sessionState) => this.invokeCreateSessionFunction(sessionState),
+            toRecord: (sessions) => this.#buildPersistedState(sessions),
+            log: this.#log,
+        });
+        this.#state = new RecoverableState({
+            defaultState: () => ({ sessions: [] }),
+            persistStateKey: this.#persistStateKey,
+            persistenceEnabled: !!this.#persistenceOptions.enable,
+            logger: this.#log,
+            keyValueStore: this.#persistenceOptions.enable && persistStateKeyValueStoreId
+                ? KeyValueStore.open({ id: persistStateKeyValueStoreId }, { configuration: serviceLocator.getConfiguration() })
+                : undefined,
+            // The codec validates in the decode direction, so it is a Standard Schema as-is; encoding needs a call.
+            deserialize: this.#stateCodec,
+            serialize: (state) => this.#stateCodec.encode(state),
+        });
+        this.#state.reset();
     }
     /**
      * Gets count of usable sessions in the pool.
      */
     async usableSessionsCount() {
         await this.ensureInitialized();
-        return this.#sessions.filter((session) => session.isUsable()).length;
+        return this.#state.currentValue.sessions.filter((session) => session.isUsable()).length;
     }
     /**
      * Gets count of retired sessions in the pool.
      */
     async retiredSessionsCount() {
         await this.ensureInitialized();
-        return this.#sessions.filter((session) => !session.isUsable()).length;
+        return this.#state.currentValue.sessions.filter((session) => !session.isUsable()).length;
     }
     /**
      * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
@@ -142,19 +198,10 @@ export class SessionPool {
         return this.#initPromise;
     }
     async setupPool() {
-        if (!this.#persistenceOptions.enable) {
-            return;
-        }
-        this.#keyValueStore = await KeyValueStore.open(this.#persistStateKeyValueStoreId ? { id: this.#persistStateKeyValueStoreId } : null, {
-            configuration: serviceLocator.getConfiguration(),
-        });
-        if (!this.#persistStateKeyValueStoreId) {
-            this.#log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.#keyValueStore.id}`);
+        await this.#state.initialize();
+        for (const session of this.#state.currentValue.sessions) {
+            this.#sessionMap.set(session.id, session);
         }
-        // in case of migration happened and SessionPool state should be restored from the keyValueStore.
-        await this.maybeLoadSessionPool();
-        this.#listener = this.persistState.bind(this);
-        this.#events.on(EventType.PERSIST_STATE, this.#listener);
     }
     /**
      * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
@@ -221,34 +268,39 @@ export class SessionPool {
         }
     }
     /**
-     * @param options - Override the persistence options provided in the constructor
+     * Discards all sessions in the pool, both usable and retired, resetting it to a blank state.
+     *
+     * The persisted record is left alone - use {@link SessionPool.resetStore} to clear that as well.
      */
-    async resetStore(options) {
-        if (!this.#persistenceOptions.enable && !options?.enable) {
-            return;
-        }
-        await this.ensureInitialized();
-        await this.#keyValueStore?.setValue(this.#persistStateKey, null);
+    reset() {
+        this.#state.reset();
+        this.#sessionMap.clear();
+        this.#roundRobinIndex = 0;
+    }
+    /**
+     * Clears the persisted pool record, leaving the in-memory sessions alone.
+     *
+     * Throws while the state is still being persisted periodically - the next PERSIST_STATE event would write
+     * the record straight back. Call {@link SessionPool.teardown} first, or use {@link SessionPool.reset}
+     * to discard the sessions themselves. A no-op if persistence is disabled.
+     */
+    async resetStore() {
+        await this.#state.resetStore();
     }
     /**
      * Returns an object representing the internal state of the `SessionPool` instance.
-     * Note that the object's fields can change in future releases.
+     * @internal
      */
     async getState() {
         await this.ensureInitialized();
-        return {
-            usableSessionsCount: await this.usableSessionsCount(),
-            retiredSessionsCount: await this.retiredSessionsCount(),
-            sessions: this.#sessions.map((session) => session.getState()),
-        };
+        return this.#buildPersistedState(this.#state.currentValue.sessions);
     }
     /**
      * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
      * The state is persisted automatically in regular intervals.
-     * @param options - Override the persistence options provided in the constructor
      */
-    async persistState(options) {
-        if (!this.#persistenceOptions.enable && !options?.enable) {
+    async persistState() {
+        if (!this.#persistenceOptions.enable) {
             return;
         }
         await this.ensureInitialized();
@@ -256,34 +308,32 @@ export class SessionPool {
             persistStateKeyValueStoreId: this.#persistStateKeyValueStoreId,
             persistStateKey: this.#persistStateKey,
         });
-        await this.#keyValueStore
-            ?.setValue(this.#persistStateKey, await this.getState())
+        await this.#state
+            .persistState()
             .catch((error) => this.#log.warning(`Failed to persist the session pool stats to ${this.#persistStateKey}`, { error }));
     }
     async [Symbol.asyncDispose]() {
-        await this.teardown({ persistState: true });
+        await this.teardown();
     }
     /**
-     * Removes listener from `persistState` event.
+     * Stops the periodic state persistence and persists the state one last time.
      * This function should be called after you are done with using the `SessionPool` instance.
-     * @param options - Set `persistState` to false when the final state was already persisted by the event manager.
+     * Using the pool again afterwards initializes it anew.
      */
-    async teardown({ persistState = true } = {}) {
+    async teardown() {
         if (!this.#initPromise)
             return;
         await this.ensureInitialized();
-        if (this.#listener) {
-            this.#events.off(EventType.PERSIST_STATE, this.#listener);
-        }
-        if (persistState) {
-            await this.persistState();
-        }
+        await this.#state.teardown();
+        // The next use initializes the pool again, restoring the record and resuming the periodic persistence -
+        // a crawler that is run twice gets a persisting pool both times.
+        this.#initPromise = undefined;
     }
     /**
      * Removes retired `Session` instances from `SessionPool`.
      */
     removeRetiredSessions() {
-        this.#sessions = this.#sessions.filter((storedSession) => {
+        this.#state.currentValue.sessions = this.#state.currentValue.sessions.filter((storedSession) => {
             if (storedSession.isUsable())
                 return true;
             this.#sessionMap.delete(storedSession.id);
@@ -296,14 +346,14 @@ export class SessionPool {
      * @param newSession `Session` instance to be added.
      */
     registerSession(newSession) {
-        this.#sessions.push(newSession);
+        this.#state.currentValue.sessions.push(newSession);
         this.#sessionMap.set(newSession.id, newSession);
     }
     /**
      * Gets random index.
      */
     getRandomIndex() {
-        return Math.floor(Math.random() * this.#sessions.length);
+        return Math.floor(Math.random() * this.#state.currentValue.sessions.length);
     }
     /**
      * Creates new session without any extra behavior.
@@ -347,7 +397,7 @@ export class SessionPool {
      * Decides whether there is enough space for creating new session.
      */
     hasSpaceForSession() {
-        return this.#sessions.length < this.#maxPoolSize;
+        return this.#state.currentValue.sessions.length < this.#maxPoolSize;
     }
     /**
      * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
@@ -357,40 +407,25 @@ export class SessionPool {
         if (this.#sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession())
             return undefined;
         if (this.#sessionReuseStrategy === 'use-until-failure') {
-            return this.#sessions.find((session) => session.isUsable());
+            return this.#state.currentValue.sessions.find((session) => session.isUsable());
         }
         let picked;
         if (this.#sessionReuseStrategy === 'round-robin') {
-            const index = this.#roundRobinIndex % this.#sessions.length;
+            const index = this.#roundRobinIndex % this.#state.currentValue.sessions.length;
             this.#roundRobinIndex = index + 1;
-            picked = this.#sessions[index];
+            picked = this.#state.currentValue.sessions[index];
         }
         else {
-            picked = this.#sessions[this.getRandomIndex()];
+            picked = this.#state.currentValue.sessions[this.getRandomIndex()];
         }
         return picked.isUsable() ? picked : undefined;
     }
-    /**
-     * Potentially loads `SessionPool`.
-     * If the state was persisted it loads the `SessionPool` from the persisted state.
-     */
-    async maybeLoadSessionPool() {
-        const loadedSessionPool = await this.#keyValueStore?.getValue(this.#persistStateKey);
-        if (!loadedSessionPool)
-            return;
-        // Invalidate old sessions and load active sessions only
-        this.#log.debug('Recreating state from KeyValueStore', {
-            persistStateKeyValueStoreId: this.#persistStateKeyValueStoreId,
-            persistStateKey: this.#persistStateKey,
-        });
-        for (const sessionObject of loadedSessionPool.sessions) {
-            sessionObject.createdAt = new Date(sessionObject.createdAt);
-            sessionObject.expiresAt = new Date(sessionObject.expiresAt);
-            const recreatedSession = await this.invokeCreateSessionFunction(sessionObject);
-            if (recreatedSession.isUsable()) {
-                this.registerSession(recreatedSession);
-            }
-        }
-        this.#log.debug(`${this.#sessions.length} active sessions loaded from KeyValueStore`);
+    /** The persisted form of the pool, which is also what {@link SessionPool.getState} returns. */
+    #buildPersistedState(sessions) {
+        return {
+            usableSessionsCount: sessions.filter((session) => session.isUsable()).length,
+            retiredSessionsCount: sessions.filter((session) => !session.isUsable()).length,
+            sessions: sessions.map((session) => session.getState()),
+        };
     }
 }

package/storages/request_list.d.ts

@@ -127,7 +127,8 @@ export interface RequestListOptions {
      * ```
      *
      * Note that the preferred (and simpler) way to persist the state of crawling of the `RequestList`
-     * is to use the `stateKeyPrefix` parameter instead.
+     * is to use the `stateKeyPrefix` parameter instead. When both are set, a record persisted under
+     * {@link RequestListOptions.persistStateKey} takes precedence over this option.
      */
     state?: RequestListState;
     /**
@@ -228,12 +229,7 @@ export declare class RequestList implements IRequestLoader {
      * Set of `uniqueKey`s of requests that were returned by fetchNextRequest().
      * @internal
      */
-    inProgress: Set<string>;
-    /**
-     * Starts as true because until we handle the first request, the list is effectively persisted by doing nothing.
-     * @internal
-     */
-    isStatePersisted: boolean;
+    get inProgress(): Set<string>;
     /**
      * Starts as false because we don't know yet and sources might change in the meantime (eg. download from live list).
      * @internal
@@ -279,15 +275,6 @@ export declare class RequestList implements IRequestLoader {
      * but there's no reason to persist it again afterwards, because RequestList is immutable.
      */
     private persistRequests;
-    /**
-     * Restores RequestList state from a state object.
-     */
-    private restoreState;
-    /**
-     * Attempts to load state and requests using the `RequestList` configuration
-     * and returns a tuple of [state, requests] where each may be null if not loaded.
-     */
-    private loadStateAndPersistedRequests;
     /**
      * Returns an object representing the internal state of the `RequestList` instance.
      * Note that the object's fields can change in future releases.

package/storages/request_list.js

@@ -1,6 +1,6 @@
 import { downloadListOfUrls } from '@crawlee/utils';
 import { z } from 'zod';
-import { EventType } from '../events/event_manager.js';
+import { convertStateSync, RecoverableState } from '../recoverable_state.js';
 import { Request } from '../request.js';
 import { createDeserialize, serializeArray } from '../serialization.js';
 import { serviceLocator } from '../service_locator.js';
@@ -12,24 +12,87 @@ export const STATE_PERSISTENCE_KEY = 'REQUEST_LIST_STATE';
 /** @internal */
 export const REQUESTS_PERSISTENCE_KEY = 'REQUEST_LIST_REQUESTS';
 const CONTENT_TYPE_BINARY = 'application/octet-stream';
+const persistedRequestListState = z.object({
+    nextIndex: z.number().int().nonnegative(),
+    nextUniqueKey: z.string().nullable(),
+    inProgress: z.array(z.string()),
+});
+const requestListInternalState = z.object({
+    /** Position of the next request to be processed. */
+    nextIndex: z.number().int().nonnegative(),
+    /** `uniqueKey`s of the requests being processed at the moment. */
+    inProgress: z.set(z.string()),
+});
 const requestListOptionsSchema = z.strictObject({
     sources: schemas.anyArray.optional(), // check only for array and not subtypes to avoid iteration over the whole thing
     sourcesFunction: schemas.anyFunction.optional(),
     persistStateKey: z.string().optional(),
     persistRequestsKey: z.string().optional(),
-    state: z
-        .strictObject({
-        nextIndex: schemas.anyNumber,
-        nextUniqueKey: z.string(),
-        inProgress: schemas.anyObject, // persisted as an array of unique keys
-    })
-        .optional(),
+    state: persistedRequestListState.optional(),
     keepDuplicateUrls: z.boolean().default(false),
     proxyConfiguration: validators.proxyConfiguration.optional(),
     httpClient: schemas.httpClient.optional(),
 });
 const listNameSchema = z.string().nullish();
 const openOptionsSchema = z.looseObject({});
+/**
+ * The conversion between the crawling position and its persisted record. Decoding also checks the record against
+ * the loaded requests, which is why the codec is built per list.
+ */
+function buildRequestListStateCodec(list) {
+    return z.codec(persistedRequestListState, requestListInternalState, {
+        decode: (record, ctx) => {
+            const { requests } = list;
+            const { nextIndex, nextUniqueKey } = record;
+            const inconsistent = (message) => {
+                ctx.issues.push({ code: 'custom', message, input: record });
+                return z.NEVER;
+            };
+            if (nextIndex > requests.length) {
+                return inconsistent('The state object is not consistent with RequestList, too few requests loaded.');
+            }
+            if (nextIndex < requests.length && requests[nextIndex].uniqueKey !== nextUniqueKey) {
+                return inconsistent('The state object is not consistent with RequestList the order of URLs seems to have changed.');
+            }
+            const inProgress = new Set(record.inProgress);
+            const deleteFromInProgress = [];
+            for (const uniqueKey of inProgress) {
+                const index = list.indexOf(uniqueKey);
+                if (index === undefined) {
+                    return inconsistent('The state object is not consistent with RequestList. Unknown uniqueKey is present in the state.');
+                }
+                if (index >= nextIndex) {
+                    deleteFromInProgress.push(uniqueKey);
+                }
+            }
+            // WORKAROUND:
+            // It happened to some users that state object contained something like:
+            // {
+            //   "nextIndex": 11308,
+            //   "nextUniqueKey": "https://www.anychart.com",
+            //   "inProgress": {
+            //      "https://www.ams360.com": true,
+            //      ...
+            //        "https://www.anychart.com": true,
+            // }
+            // Which then caused error "The request is not being processed (uniqueKey: https://www.anychart.com)"
+            // As a workaround, we just remove all inProgress requests whose index >= nextIndex,
+            // since they will be crawled again.
+            if (deleteFromInProgress.length) {
+                list.log.warning("RequestList's in-progress field is not consistent, skipping invalid in-progress entries", { deleteFromInProgress });
+                for (const uniqueKey of deleteFromInProgress) {
+                    inProgress.delete(uniqueKey);
+                }
+            }
+            return { nextIndex, inProgress };
+        },
+        encode: ({ nextIndex, inProgress }) => ({
+            nextIndex,
+            nextUniqueKey: nextIndex < list.requests.length ? list.requests[nextIndex].uniqueKey : null,
+            inProgress: [...inProgress],
+        }),
+    });
+}
 /**
  * Represents a static list of URLs to crawl.
  * The URLs can be provided either in code or parsed from a text file hosted on the web.
@@ -99,26 +162,27 @@ export class RequestList {
      * @internal
      */
     requests = [];
-    /** Index to the next item in requests array to fetch. All previous requests are either handled or in progress. */
-    #nextIndex = 0;
     /** Dictionary, key is Request.uniqueKey, value is corresponding index in the requests array. */
     #uniqueKeyToIndex = {};
+    /**
+     * The crawling position, kept in a {@link RecoverableState} that handles the loading, validation and
+     * periodic persistence of the record under `persistStateKey`.
+     */
+    #state;
+    #stateCodec;
     /**
      * Set of `uniqueKey`s of requests that were returned by fetchNextRequest().
      * @internal
      */
-    inProgress = new Set();
+    get inProgress() {
+        return this.#state.currentValue.inProgress;
+    }
     /**
      * `uniqueKey`s of requests that were in progress when the state was last persisted and thus need to be
      * re-crawled after a restart. They are served before advancing through the rest of the sources.
      * @internal
      */
     #requestsToRetry = [];
-    /**
-     * Starts as true because until we handle the first request, the list is effectively persisted by doing nothing.
-     * @internal
-     */
-    isStatePersisted = true;
     /**
      * Starts as false because we don't know yet and sources might change in the meantime (eg. download from live list).
      * @internal
@@ -128,7 +192,6 @@ export class RequestList {
     #isInitialized = false;
     #persistStateKey;
     #persistRequestsKey;
-    #initialState;
     #store;
     #keepDuplicateUrls;
     #sources;
@@ -147,8 +210,29 @@ export class RequestList {
         }
         this.#persistStateKey = persistStateKey ? `CRAWLEE_${persistStateKey}` : persistStateKey;
         this.#persistRequestsKey = persistRequestsKey ? `CRAWLEE_${persistRequestsKey}` : persistRequestsKey;
-        this.#initialState = state;
         this.#httpClient = httpClient;
+        this.#stateCodec = buildRequestListStateCodec({
+            requests: this.requests,
+            indexOf: (uniqueKey) => {
+                const index = this.#uniqueKeyToIndex[uniqueKey];
+                return typeof index === 'number' ? index : undefined;
+            },
+            log: this.#log,
+        });
+        const stateKey = this.#persistStateKey ?? STATE_PERSISTENCE_KEY;
+        this.#state = new RecoverableState({
+            // The `state` option is where the list starts when there is no persisted record to restore. The factory
+            // runs on `initialize()`, once the requests it is validated against are loaded.
+            defaultState: () => state
+                ? convertStateSync(this.#stateCodec, state, stateKey)
+                : { nextIndex: 0, inProgress: new Set() },
+            persistStateKey: stateKey,
+            persistenceEnabled: !!this.#persistStateKey,
+            logger: this.#log,
+            // The codec validates in the decode direction, so it is a Standard Schema as-is; encoding needs a call.
+            deserialize: this.#stateCodec,
+            serialize: (internalState) => this.#stateCodec.encode(internalState),
+        });
         // If this option is set then all requests will get a pre-generated unique ID and duplicate URLs will be kept in the list.
         this.#keepDuplicateUrls = keepDuplicateUrls;
         // Will be empty after initialization to save memory.
@@ -156,7 +240,6 @@ export class RequestList {
         this.#sourcesFunction = sourcesFunction;
         // The proxy configuration used for `requestsFromUrl` requests.
         this.#proxyConfiguration = proxyConfiguration;
-        this.persistState = this.persistState.bind(this);
     }
     /**
      * Loads all remote sources of URLs and potentially starts periodic state persistence.
@@ -168,7 +251,12 @@ export class RequestList {
         }
         this.#isLoading = true;
         await purgeDefaultStorages({ onlyPurgeOnce: true });
-        const [state, persistedRequests] = await this.loadStateAndPersistedRequests();
+        let persistedRequests;
+        if (this.#persistRequestsKey) {
+            persistedRequests = await this.getPersistedState(this.#persistRequestsKey);
+            if (persistedRequests)
+                this.#log.debug('Loaded requests from key value store using the persistRequestsKey.');
+        }
         // Add persisted requests / new sources in a memory efficient way because with very
         // large lists, we were running out of memory.
         if (persistedRequests) {
@@ -177,13 +265,14 @@ export class RequestList {
         else {
             await this.addRequestsFromSources();
         }
-        this.restoreState(state);
+        // Restores the crawling position - the requests have to be loaded first, as the record is validated
+        // against them. Also starts the periodic state persistence when `persistStateKey` is set.
+        await this.#state.initialize();
+        // All in-progress requests were interrupted and need to be re-crawled.
+        this.#requestsToRetry = [...this.inProgress];
         this.#isInitialized = true;
         if (this.#persistRequestsKey && !this.areRequestsPersisted)
             await this.persistRequests();
-        if (this.#persistStateKey) {
-            serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.persistState);
-        }
         return this;
     }
     /**
@@ -255,12 +344,8 @@ export class RequestList {
         if (!this.#persistStateKey) {
             throw new Error('Cannot persist state. options.persistStateKey is not set.');
         }
-        if (this.isStatePersisted)
-            return;
         try {
-            this.#store ??= await KeyValueStore.open();
-            await this.#store.setValue(this.#persistStateKey, this.getState());
-            this.isStatePersisted = true;
+            await this.#state.persistState();
         }
         catch (e) {
             const err = e;
@@ -273,10 +358,7 @@ export class RequestList {
      * leaking the listener (and the requests it retains) on the shared event manager.
      */
     async teardown() {
-        serviceLocator.getEventManager().off(EventType.PERSIST_STATE, this.persistState);
-        if (this.#persistStateKey) {
-            await this.persistState();
-        }
+        await this.#state.teardown();
     }
     /**
      * Unlike persistState(), this is used only internally, since the sources
@@ -289,101 +371,20 @@ export class RequestList {
         await this.#store.setValue(this.#persistRequestsKey, serializedRequests, { contentType: CONTENT_TYPE_BINARY });
         this.areRequestsPersisted = true;
     }
-    /**
-     * Restores RequestList state from a state object.
-     */
-    restoreState(state) {
-        // If there's no state it means we've not persisted any (yet).
-        if (!state)
-            return;
-        // Restore previous state.
-        if (typeof state.nextIndex !== 'number' || state.nextIndex < 0) {
-            throw new Error('The state object is invalid: nextIndex must be a non-negative number.');
-        }
-        if (state.nextIndex > this.requests.length) {
-            throw new Error('The state object is not consistent with RequestList, too few requests loaded.');
-        }
-        if (state.nextIndex < this.requests.length &&
-            this.requests[state.nextIndex].uniqueKey !== state.nextUniqueKey) {
-            throw new Error('The state object is not consistent with RequestList the order of URLs seems to have changed.');
-        }
-        const deleteFromInProgress = [];
-        state.inProgress.forEach((uniqueKey) => {
-            const index = this.#uniqueKeyToIndex[uniqueKey];
-            if (typeof index !== 'number') {
-                throw new Error('The state object is not consistent with RequestList. Unknown uniqueKey is present in the state.');
-            }
-            if (index >= state.nextIndex) {
-                deleteFromInProgress.push(uniqueKey);
-            }
-        });
-        this.#nextIndex = state.nextIndex;
-        this.inProgress = new Set(state.inProgress);
-        // WORKAROUND:
-        // It happened to some users that state object contained something like:
-        // {
-        //   "nextIndex": 11308,
-        //   "nextUniqueKey": "https://www.anychart.com",
-        //   "inProgress": {
-        //      "https://www.ams360.com": true,
-        //      ...
-        //        "https://www.anychart.com": true,
-        // }
-        // Which then caused error "The request is not being processed (uniqueKey: https://www.anychart.com)"
-        // As a workaround, we just remove all inProgress requests whose index >= nextIndex,
-        // since they will be crawled again.
-        if (deleteFromInProgress.length) {
-            this.#log.warning("RequestList's in-progress field is not consistent, skipping invalid in-progress entries", {
-                deleteFromInProgress,
-            });
-            for (const uniqueKey of deleteFromInProgress) {
-                this.inProgress.delete(uniqueKey);
-            }
-        }
-        // All in-progress requests were interrupted and need to be re-crawled.
-        this.#requestsToRetry = [...this.inProgress];
-    }
-    /**
-     * Attempts to load state and requests using the `RequestList` configuration
-     * and returns a tuple of [state, requests] where each may be null if not loaded.
-     */
-    async loadStateAndPersistedRequests() {
-        let state;
-        let persistedRequests;
-        if (this.#initialState) {
-            state = this.#initialState;
-            this.#log.debug('Loaded state from options.state argument.');
-        }
-        else if (this.#persistStateKey) {
-            state = await this.getPersistedState(this.#persistStateKey);
-            if (state)
-                this.#log.debug('Loaded state from key value store using the persistStateKey.');
-        }
-        if (this.#persistRequestsKey) {
-            persistedRequests = await this.getPersistedState(this.#persistRequestsKey);
-            if (persistedRequests)
-                this.#log.debug('Loaded requests from key value store using the persistRequestsKey.');
-        }
-        return [state, persistedRequests];
-    }
     /**
      * Returns an object representing the internal state of the `RequestList` instance.
      * Note that the object's fields can change in future releases.
      */
     getState() {
         this.ensureIsInitialized();
-        return {
-            nextIndex: this.#nextIndex,
-            nextUniqueKey: this.#nextIndex < this.requests.length ? this.requests[this.#nextIndex].uniqueKey : null,
-            inProgress: [...this.inProgress],
-        };
+        return this.#stateCodec.encode(this.#state.currentValue);
     }
     /**
      * @inheritDoc
      */
     async checkReadiness() {
         this.ensureIsInitialized();
-        if (this.#requestsToRetry.length > 0 || this.#nextIndex < this.requests.length) {
+        if (this.#requestsToRetry.length > 0 || this.#state.currentValue.nextIndex < this.requests.length) {
             return { status: 'ready' };
         }
         // `#requestsToRetry` is a subset of `inProgress`, so nothing in progress means nothing left to re-serve.
@@ -401,12 +402,12 @@ export class RequestList {
             return this.ensureRequest(this.requests[index], index);
         }
         // Otherwise return next request.
-        if (this.#nextIndex < this.requests.length) {
-            const index = this.#nextIndex;
+        const state = this.#state.currentValue;
+        if (state.nextIndex < this.requests.length) {
+            const index = state.nextIndex;
             const request = this.requests[index];
-            this.inProgress.add(request.uniqueKey);
-            this.#nextIndex++;
-            this.isStatePersisted = false;
+            state.inProgress.add(request.uniqueKey);
+            state.nextIndex++;
             return this.ensureRequest(request, index);
         }
         return null;
@@ -438,7 +439,6 @@ export class RequestList {
         this.ensureInProgress(uniqueKey);
         this.ensureIsInitialized();
         this.inProgress.delete(uniqueKey);
-        this.isStatePersisted = false;
     }
     /**
      * Adds all fetched requests from a URL from a remote resource.
@@ -561,7 +561,7 @@ export class RequestList {
      */
     async getPendingCount() {
         this.ensureIsInitialized();
-        return this.requests.length - (this.#nextIndex - this.inProgress.size);
+        return this.requests.length - (this.#state.currentValue.nextIndex - this.inProgress.size);
     }
     /**
      * Combines this list with a request manager (a {@link RequestQueue} by default) into a
@@ -579,7 +579,7 @@ export class RequestList {
      */
     async getHandledCount() {
         this.ensureIsInitialized();
-        return this.#nextIndex - this.inProgress.size;
+        return this.#state.currentValue.nextIndex - this.inProgress.size;
     }
     /**
      * Opens a request list and returns a promise resolving to an instance