@crawlee/core 4.0.0-beta.17 → 4.0.0-beta.170

package/session_pool/session_pool.js

@@ -1,31 +1,74 @@
-import { EventEmitter } from 'node:events';
 import { AsyncQueue } from '@sapphire/async-queue';
-import ow from 'ow';
-import { Configuration } from '../configuration.js';
-import { log as defaultLog } from '../log.js';
+import { z } from 'zod';
+import { RecoverableState } from '../recoverable_state.js';
+import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from '../storages/key_value_store.js';
-import { BLOCKED_STATUS_CODES, MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
+import { parseArgument, schemas, validators } from '../validators.js';
+import { MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
+import { createDefaultSessionFingerprint } from './fingerprint.js';
 import { Session } from './session.js';
+const SESSION_REUSE_STRATEGIES = ['random', 'round-robin', 'use-until-failure'];
+// `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
+// copy), so class instances like loggers keep their prototype.
+const sessionPoolOptionsSchema = z.strictObject({
+    id: z.union([schemas.anyNumber, z.string()]).optional(),
+    maxPoolSize: schemas.anyNumber.default(MAX_POOL_SIZE),
+    persistStateKeyValueStoreId: z.string().optional(),
+    persistStateKey: z.string().optional(),
+    createSessionFunction: schemas.anyFunction.optional(),
+    sessionOptions: schemas.anyObject.default(() => ({})),
+    log: validators.logger.default(() => serviceLocator.getLogger()),
+    persistenceOptions: schemas.anyObject.default(() => ({ enable: true })),
+    sessionReuseStrategy: z.enum(SESSION_REUSE_STRATEGIES).default('random'),
+});
+const createSessionOptionsSchema = z.strictObject({
+    sessionOptions: schemas.anyObject.default(() => ({})),
+});
+// Only what the recreation reads is pinned down - the rest of a session's state is the session's own business.
+const persistedSessionState = z.custom((value) => typeof value === 'object' &&
+    value !== null &&
+    typeof value.createdAt === 'string' &&
+    typeof value.expiresAt === 'string', 'Expected a persisted session with `createdAt` and `expiresAt` timestamps');
+const persistedSessionPoolState = z.object({
+    usableSessionsCount: z.number(),
+    retiredSessionsCount: z.number(),
+    sessions: z.array(persistedSessionState),
+});
+const sessionPoolInternalState = z.object({
+    sessions: z.array(z.instanceof(Session)),
+});
+/**
+ * The conversion between the pool's sessions and its persisted record. Decoding recreates the sessions through
+ * the pool's factory and keeps only the usable ones, which is why the codec is built per pool.
+ */
+function buildSessionPoolStateCodec(pool) {
+    return z.codec(persistedSessionPoolState, sessionPoolInternalState, {
+        decode: async (record) => {
+            const sessions = [];
+            for (const sessionState of record.sessions) {
+                const session = await pool.recreateSession({
+                    ...sessionState,
+                    createdAt: new Date(sessionState.createdAt),
+                    expiresAt: new Date(sessionState.expiresAt),
+                });
+                if (session.isUsable()) {
+                    sessions.push(session);
+                }
+            }
+            pool.log.debug(`${sessions.length} active sessions loaded from KeyValueStore`);
+            return { sessions };
+        },
+        encode: ({ sessions }) => pool.toRecord(sessions),
+    });
+}
 /**
  * Handles the rotation, creation and persistence of user-like sessions.
  * Creates a pool of {@link Session} instances, that are randomly rotated.
  * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
  * Learn more in the {@doclink guides/session-management | Session management guide}.
  *
- * You can create one by calling the {@link SessionPool.open} function.
- *
- * Session pool is already integrated into crawlers, and it can significantly improve your scraper
- * performance with just 2 lines of code.
- *
- * **Example usage:**
- *
- * ```javascript
- * const crawler = new CheerioCrawler({
- *     useSessionPool: true,
- *     persistCookiesPerSession: true,
- *     // ...
- * })
- * ```
+ * Session pool is already integrated into crawlers and is always active.
+ * All public methods are lazy-initialized — the pool initializes itself on first use.
  *
  * You can configure the pool with many options. See the {@link SessionPoolOptions}.
  * Session pool is by default persisted in default {@link KeyValueStore}.
@@ -35,7 +78,7 @@ import { Session } from './session.js';
  * **Advanced usage:**
  *
  * ```javascript
- * const sessionPool = await SessionPool.open({
+ * const sessionPool = new SessionPool({
  *     maxPoolSize: 25,
  *     sessionOptions:{
  *          maxAgeSecs: 10,
@@ -71,94 +114,94 @@ import { Session } from './session.js';
  *
  * @category Scaling
  */
-export class SessionPool extends EventEmitter {
-    config;
-    log;
-    maxPoolSize;
-    createSessionFunction;
-    keyValueStore;
-    sessions = [];
-    sessionMap = new Map();
-    sessionOptions;
-    persistStateKeyValueStoreId;
-    persistStateKey;
-    _listener;
-    events;
-    blockedStatusCodes;
-    persistenceOptions;
-    isInitialized = false;
-    queue = new AsyncQueue();
+export class SessionPool {
+    static #nextId = 0;
+    id;
+    #log;
+    #maxPoolSize;
+    #createSessionFunction;
+    #sessionMap = new Map();
+    #sessionOptions;
+    #persistStateKeyValueStoreId;
+    #persistStateKey;
+    #persistenceOptions;
+    #sessionReuseStrategy;
     /**
-     * @internal
+     * The sessions live in a {@link RecoverableState} that handles the loading, recreation and periodic
+     * persistence of the record under `persistStateKey`.
      */
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
-        super();
-        this.config = config;
-        ow(options, ow.object.exactShape({
-            maxPoolSize: ow.optional.number,
-            persistStateKeyValueStoreId: ow.optional.string,
-            persistStateKey: ow.optional.string,
-            createSessionFunction: ow.optional.function,
-            sessionOptions: ow.optional.object,
-            blockedStatusCodes: ow.optional.array.ofType(ow.number),
-            log: ow.optional.object,
-            persistenceOptions: ow.optional.object,
-        }));
-        const { maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey = PERSIST_STATE_KEY, createSessionFunction, sessionOptions = {}, blockedStatusCodes = BLOCKED_STATUS_CODES, log = defaultLog, persistenceOptions = {
-            enable: true,
-        }, } = options;
-        this.config = config;
-        this.blockedStatusCodes = blockedStatusCodes;
-        this.events = config.getEventManager();
-        this.log = log.child({ prefix: 'SessionPool' });
-        this.persistenceOptions = persistenceOptions;
+    #state;
+    #stateCodec;
+    #initPromise;
+    #queue = new AsyncQueue();
+    #roundRobinIndex = 0;
+    constructor(options = {}) {
+        const { id, maxPoolSize, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions, log, persistenceOptions, sessionReuseStrategy, } = parseArgument(options, sessionPoolOptionsSchema);
+        this.id = id != null ? String(id) : String(SessionPool.#nextId++);
+        this.#sessionReuseStrategy = sessionReuseStrategy;
+        this.#log = log.child({ prefix: 'SessionPool' });
+        this.#persistenceOptions = persistenceOptions;
         // Pool Configuration
-        this.maxPoolSize = maxPoolSize;
-        this.createSessionFunction = createSessionFunction || this._defaultCreateSessionFunction;
-        // Session configuration
-        this.sessionOptions = {
+        this.#maxPoolSize = maxPoolSize;
+        this.#createSessionFunction = createSessionFunction || this.defaultCreateSessionFunction;
+        // Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside
+        // `invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
+        // having to know about it.
+        this.#sessionOptions = {
             ...sessionOptions,
-            // the log needs to propagate to createSessionFunction as in "new Session({ ...sessionPool.sessionOptions })"
-            // and can't go inside _defaultCreateSessionFunction
-            log: this.log,
+            log: this.#log,
         };
         // Session keyValueStore
-        this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
-        this.persistStateKey = persistStateKey;
+        this.#persistStateKeyValueStoreId = persistStateKeyValueStoreId;
+        this.#persistStateKey = persistStateKey ?? `${PERSIST_STATE_KEY}_${this.id}`;
+        this.#stateCodec = buildSessionPoolStateCodec({
+            recreateSession: (sessionState) => this.invokeCreateSessionFunction(sessionState),
+            toRecord: (sessions) => this.#buildPersistedState(sessions),
+            log: this.#log,
+        });
+        this.#state = new RecoverableState({
+            defaultState: () => ({ sessions: [] }),
+            persistStateKey: this.#persistStateKey,
+            persistenceEnabled: !!this.#persistenceOptions.enable,
+            logger: this.#log,
+            keyValueStore: this.#persistenceOptions.enable && persistStateKeyValueStoreId
+                ? KeyValueStore.open({ id: persistStateKeyValueStoreId }, { configuration: serviceLocator.getConfiguration() })
+                : undefined,
+            // The codec validates in the decode direction, so it is a Standard Schema as-is; encoding needs a call.
+            deserialize: this.#stateCodec,
+            serialize: (state) => this.#stateCodec.encode(state),
+        });
+        this.#state.reset();
     }
     /**
      * Gets count of usable sessions in the pool.
      */
-    get usableSessionsCount() {
-        return this.sessions.filter((session) => session.isUsable()).length;
+    async usableSessionsCount() {
+        await this.ensureInitialized();
+        return this.#state.currentValue.sessions.filter((session) => session.isUsable()).length;
     }
     /**
      * Gets count of retired sessions in the pool.
      */
-    get retiredSessionsCount() {
-        return this.sessions.filter((session) => !session.isUsable()).length;
+    async retiredSessionsCount() {
+        await this.ensureInitialized();
+        return this.#state.currentValue.sessions.filter((session) => !session.isUsable()).length;
     }
     /**
      * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
-     * It is called automatically by the {@link SessionPool.open} function.
+     * Called automatically on first use of any public method.
      */
-    async initialize() {
-        if (this.isInitialized) {
-            return;
+    async ensureInitialized() {
+        if (!this.#initPromise) {
+            this.#initPromise = this.setupPool();
         }
-        this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId, { config: this.config });
-        if (!this.persistenceOptions.enable) {
-            this.isInitialized = true;
-            return;
-        }
-        if (!this.persistStateKeyValueStoreId) {
-            this.log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
+        return this.#initPromise;
+    }
+    async setupPool() {
+        await this.#state.initialize();
+        for (const session of this.#state.currentValue.sessions) {
+            this.#sessionMap.set(session.id, session);
         }
-        // in case of migration happened and SessionPool state should be restored from the keyValueStore.
-        await this._maybeLoadSessionPool();
-        this._listener = this.persistState.bind(this);
-        this.events.on("persistState" /* EventType.PERSIST_STATE */, this._listener);
-        this.isInitialized = true;
     }
     /**
      * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
@@ -167,20 +210,20 @@ export class SessionPool extends EventEmitter {
      * @param [options] The configuration options for the session being added to the session pool.
      */
     async addSession(options = {}) {
-        this._throwIfNotInitialized();
+        await this.ensureInitialized();
         const { id } = options;
         if (id) {
-            const sessionExists = this.sessionMap.has(id);
+            const sessionExists = this.#sessionMap.has(id);
             if (sessionExists) {
                 throw new Error(`Cannot add session with id '${id}' as it already exists in the pool`);
             }
         }
-        if (!this._hasSpaceForSession()) {
-            this._removeRetiredSessions();
+        if (!this.hasSpaceForSession()) {
+            this.removeRetiredSessions();
         }
-        const newSession = options instanceof Session ? options : await this.createSessionFunction(this, { sessionOptions: options });
-        this.log.debug(`Adding new Session - ${newSession.id}`);
-        this._addSession(newSession);
+        const newSession = options instanceof Session ? options : await this.invokeCreateSessionFunction(options);
+        this.#log.debug(`Adding new Session - ${newSession.id}`);
+        this.registerSession(newSession);
     }
     /**
      * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
@@ -189,9 +232,9 @@ export class SessionPool extends EventEmitter {
      * @param [options] The configuration options for the session being added to the session pool.
      */
     async newSession(sessionOptions) {
-        this._throwIfNotInitialized();
-        const newSession = await this.createSessionFunction(this, { sessionOptions });
-        this._addSession(newSession);
+        await this.ensureInitialized();
+        const newSession = await this.invokeCreateSessionFunction(sessionOptions);
+        this.registerSession(newSession);
         return newSession;
     }
     /**
@@ -202,96 +245,99 @@ export class SessionPool extends EventEmitter {
      * @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
      */
     async getSession(sessionId) {
-        await this.queue.wait();
+        await this.ensureInitialized();
+        await this.#queue.wait();
         try {
-            this._throwIfNotInitialized();
             if (sessionId) {
-                const session = this.sessionMap.get(sessionId);
-                if (session && session.isUsable())
+                const session = this.#sessionMap.get(sessionId);
+                if (session?.isUsable())
                     return session;
                 return undefined;
             }
-            if (this._hasSpaceForSession()) {
-                return await this._createSession();
-            }
-            const pickedSession = this._pickSession();
-            if (pickedSession.isUsable()) {
+            const pickedSession = this.pickSession();
+            if (pickedSession)
                 return pickedSession;
+            if (this.hasSpaceForSession()) {
+                return await this.createSession();
             }
-            this._removeRetiredSessions();
-            return await this._createSession();
+            this.removeRetiredSessions();
+            return await this.createSession();
         }
         finally {
-            this.queue.shift();
+            this.#queue.shift();
         }
     }
     /**
-     * @param options - Override the persistence options provided in the constructor
+     * Discards all sessions in the pool, both usable and retired, resetting it to a blank state.
+     *
+     * The persisted record is left alone - use {@link SessionPool.resetStore} to clear that as well.
      */
-    async resetStore(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
-            return;
-        }
-        await this.keyValueStore?.setValue(this.persistStateKey, null);
+    reset() {
+        this.#state.reset();
+        this.#sessionMap.clear();
+        this.#roundRobinIndex = 0;
+    }
+    /**
+     * Clears the persisted pool record, leaving the in-memory sessions alone.
+     *
+     * Throws while the state is still being persisted periodically - the next PERSIST_STATE event would write
+     * the record straight back. Call {@link SessionPool.teardown} first, or use {@link SessionPool.reset}
+     * to discard the sessions themselves. A no-op if persistence is disabled.
+     */
+    async resetStore() {
+        await this.#state.resetStore();
     }
     /**
      * Returns an object representing the internal state of the `SessionPool` instance.
-     * Note that the object's fields can change in future releases.
+     * @internal
      */
-    getState() {
-        return {
-            usableSessionsCount: this.usableSessionsCount,
-            retiredSessionsCount: this.retiredSessionsCount,
-            sessions: this.sessions.map((session) => session.getState()),
-        };
+    async getState() {
+        await this.ensureInitialized();
+        return this.#buildPersistedState(this.#state.currentValue.sessions);
     }
     /**
      * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
      * The state is persisted automatically in regular intervals.
-     * @param options - Override the persistence options provided in the constructor
      */
-    async persistState(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
+    async persistState() {
+        if (!this.#persistenceOptions.enable) {
             return;
         }
-        this.log.debug('Persisting state', {
-            persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
-            persistStateKey: this.persistStateKey,
+        await this.ensureInitialized();
+        this.#log.debug('Persisting state', {
+            persistStateKeyValueStoreId: this.#persistStateKeyValueStoreId,
+            persistStateKey: this.#persistStateKey,
         });
-        // use half the interval of `persistState` to avoid race conditions
-        const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
-        const timeoutSecs = persistStateIntervalMillis / 2_000;
-        await this.keyValueStore
-            .setValue(this.persistStateKey, this.getState(), {
-            timeoutSecs,
-            doNotRetryTimeouts: true,
-        })
-            .catch((error) => this.log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
+        await this.#state
+            .persistState()
+            .catch((error) => this.#log.warning(`Failed to persist the session pool stats to ${this.#persistStateKey}`, { error }));
+    }
+    async [Symbol.asyncDispose]() {
+        await this.teardown();
     }
     /**
-     * Removes listener from `persistState` event.
+     * Stops the periodic state persistence and persists the state one last time.
      * This function should be called after you are done with using the `SessionPool` instance.
+     * Using the pool again afterwards initializes it anew.
      */
     async teardown() {
-        this.events.off("persistState" /* EventType.PERSIST_STATE */, this._listener);
-        await this.persistState();
-    }
-    /**
-     * SessionPool should not work before initialization.
-     */
-    _throwIfNotInitialized() {
-        if (!this.isInitialized)
-            throw new Error('SessionPool is not initialized.');
+        if (!this.#initPromise)
+            return;
+        await this.ensureInitialized();
+        await this.#state.teardown();
+        // The next use initializes the pool again, restoring the record and resuming the periodic persistence -
+        // a crawler that is run twice gets a persisting pool both times.
+        this.#initPromise = undefined;
     }
     /**
      * Removes retired `Session` instances from `SessionPool`.
      */
-    _removeRetiredSessions() {
-        this.sessions = this.sessions.filter((storedSession) => {
+    removeRetiredSessions() {
+        this.#state.currentValue.sessions = this.#state.currentValue.sessions.filter((storedSession) => {
             if (storedSession.isUsable())
                 return true;
-            this.sessionMap.delete(storedSession.id);
-            this.log.debug(`Removed Session - ${storedSession.id}`);
+            this.#sessionMap.delete(storedSession.id);
+            this.#log.debug(`Removed Session - ${storedSession.id}`);
             return false;
         });
     }
@@ -299,89 +345,87 @@ export class SessionPool extends EventEmitter {
      * Adds `Session` instance to `SessionPool`.
      * @param newSession `Session` instance to be added.
      */
-    _addSession(newSession) {
-        this.sessions.push(newSession);
-        this.sessionMap.set(newSession.id, newSession);
+    registerSession(newSession) {
+        this.#state.currentValue.sessions.push(newSession);
+        this.#sessionMap.set(newSession.id, newSession);
     }
     /**
      * Gets random index.
      */
-    _getRandomIndex() {
-        return Math.floor(Math.random() * this.sessions.length);
+    getRandomIndex() {
+        return Math.floor(Math.random() * this.#state.currentValue.sessions.length);
     }
     /**
      * Creates new session without any extra behavior.
-     * @param sessionPool
      * @param [options]
      * @param [options.sessionOptions] The configuration options for the session being created.
      * @returns New session.
      */
-    async _defaultCreateSessionFunction(sessionPool, options = {}) {
-        ow(options, ow.object.exactShape({ sessionOptions: ow.optional.object }));
-        const { sessionOptions = {} } = options;
-        return new Session({
-            ...this.sessionOptions,
-            ...sessionOptions,
-            sessionPool,
-        });
+    async defaultCreateSessionFunction(options = {}) {
+        const { sessionOptions } = parseArgument(options, createSessionOptionsSchema);
+        return new Session(sessionOptions);
+    }
+    /**
+     * Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
+     * the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
+     *
+     * A default {@link SessionFingerprint} is generated up front (host OS as
+     * `platform`, a random valid `browser`/`device` for that platform). Pool-wide
+     * and per-call options override it, and a persisted fingerprint coming
+     * through `maybeLoadSessionPool` naturally wins because it arrives in
+     * `perCallOptions`.
+     */
+    async invokeCreateSessionFunction(perCallOptions) {
+        const sessionOptions = {
+            fingerprint: createDefaultSessionFingerprint(),
+            ...this.#sessionOptions,
+            ...perCallOptions,
+        };
+        return this.#createSessionFunction({ sessionOptions });
     }
     /**
      * Creates new session and adds it to the pool.
      * @returns Newly created `Session` instance.
      */
-    async _createSession() {
-        const newSession = await this.createSessionFunction(this);
-        this._addSession(newSession);
-        this.log.debug(`Created new Session - ${newSession.id}`);
+    async createSession() {
+        const newSession = await this.invokeCreateSessionFunction();
+        this.registerSession(newSession);
+        this.#log.debug(`Created new Session - ${newSession.id}`);
         return newSession;
     }
     /**
      * Decides whether there is enough space for creating new session.
      */
-    _hasSpaceForSession() {
-        return this.sessions.length < this.maxPoolSize;
+    hasSpaceForSession() {
+        return this.#state.currentValue.sessions.length < this.#maxPoolSize;
     }
     /**
-     * Picks random session from the `SessionPool`.
-     * @returns Picked `Session`.
+     * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
+     * Returns `undefined` when no session should be reused and a new one should be created instead.
      */
-    _pickSession() {
-        return this.sessions[this._getRandomIndex()]; // Or maybe we should let the developer to customize the picking algorithm
-    }
-    /**
-     * Potentially loads `SessionPool`.
-     * If the state was persisted it loads the `SessionPool` from the persisted state.
-     */
-    async _maybeLoadSessionPool() {
-        const loadedSessionPool = await this.keyValueStore.getValue(this.persistStateKey);
-        if (!loadedSessionPool)
-            return;
-        // Invalidate old sessions and load active sessions only
-        this.log.debug('Recreating state from KeyValueStore', {
-            persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
-            persistStateKey: this.persistStateKey,
-        });
-        for (const sessionObject of loadedSessionPool.sessions) {
-            sessionObject.sessionPool = this;
-            sessionObject.createdAt = new Date(sessionObject.createdAt);
-            sessionObject.expiresAt = new Date(sessionObject.expiresAt);
-            const recreatedSession = await this.createSessionFunction(this, { sessionOptions: sessionObject });
-            if (recreatedSession.isUsable()) {
-                this._addSession(recreatedSession);
-            }
+    pickSession() {
+        if (this.#sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession())
+            return undefined;
+        if (this.#sessionReuseStrategy === 'use-until-failure') {
+            return this.#state.currentValue.sessions.find((session) => session.isUsable());
+        }
+        let picked;
+        if (this.#sessionReuseStrategy === 'round-robin') {
+            const index = this.#roundRobinIndex % this.#state.currentValue.sessions.length;
+            this.#roundRobinIndex = index + 1;
+            picked = this.#state.currentValue.sessions[index];
         }
-        this.log.debug(`${this.usableSessionsCount} active sessions loaded from KeyValueStore`);
+        else {
+            picked = this.#state.currentValue.sessions[this.getRandomIndex()];
+        }
+        return picked.isUsable() ? picked : undefined;
     }
-    /**
-     * Opens a SessionPool and returns a promise resolving to an instance
-     * of the {@link SessionPool} class that is already initialized.
-     *
-     * For more details and code examples, see the {@link SessionPool} class.
-     */
-    static async open(options, config) {
-        const sessionPool = new SessionPool(options, config);
-        await sessionPool.initialize();
-        return sessionPool;
+    /** The persisted form of the pool, which is also what {@link SessionPool.getState} returns. */
+    #buildPersistedState(sessions) {
+        return {
+            usableSessionsCount: sessions.filter((session) => session.isUsable()).length,
+            retiredSessionsCount: sessions.filter((session) => !session.isUsable()).length,
+            sessions: sessions.map((session) => session.getState()),
+        };
     }
 }
-//# sourceMappingURL=session_pool.js.map

package/storages/batched_adds.d.ts

@@ -0,0 +1,37 @@
+import type { ProcessedRequest } from '@crawlee/types';
+import type { Source } from '../request.js';
+import type { AddRequestsBatchedResult } from './request_queue.js';
+export interface DrainRequestBatchesOptions<TItem extends Source> {
+    /**
+     * The requests to add, already normalized by the caller. Consumed lazily: an unbounded or expensive
+     * iterable is only pulled from as far as the batching (and any `maxNewRequests` budget) requires.
+     */
+    items: AsyncGenerator<TItem>;
+    batchSize: number;
+    waitBetweenBatchesMillis: number;
+    waitForAllRequestsToBeAdded: boolean;
+    maxNewRequests?: number;
+    /**
+     * Adds a single chunk and reports what it processed.
+     *
+     * @param isInitial Whether this is the first chunk, which is added before this function returns. Later
+     *  chunks land in the background, which is why some callers cache only the first.
+     */
+    processChunk: (chunk: TItem[], isInitial: boolean) => Promise<ProcessedRequest[]>;
+    /**
+     * Called with the promise covering every chunk after the first, so the caller can keep its own
+     * `isFinished` honest while batches are still landing.
+     */
+    trackBackgroundBatches?: (batches: Promise<unknown>) => void;
+}
+/**
+ * Drives the chunk-by-chunk half of `addRequestsBatched`: the first chunk is added before returning and the
+ * rest continue in the background, paced by `waitBetweenBatchesMillis`.
+ *
+ * Callers differ only in how a chunk is added and how the input is normalized, so that is all
+ * {@link DrainRequestBatchesOptions} asks for - the budget arithmetic, the lazy chunking, the
+ * over-limit reporting and the transaction handling are identical for everyone and live here. In
+ * particular, every caller has to keep its background chunks out of a transaction they will outlive,
+ * so that is read from the ambient transaction rather than asked of the caller.
+ */
+export declare function drainRequestBatches<TItem extends Source>(options: DrainRequestBatchesOptions<TItem>): Promise<AddRequestsBatchedResult>;