@crawlee/core 4.0.0-beta.17 → 4.0.0-beta.170

package/session_pool/session_pool.d.ts

@@ -1,24 +1,39 @@
-import { EventEmitter } from 'node:events';
-import type { Log } from '@apify/log';
-import { Configuration } from '../configuration.js';
+import type { ISessionPool, SessionState } from '@crawlee/types';
+import { z } from 'zod';
 import type { PersistenceOptions } from '../crawlers/statistics.js';
-import type { EventManager } from '../events/event_manager.js';
-import { KeyValueStore } from '../storages/key_value_store.js';
+import type { CrawleeLogger } from '../log.js';
 import type { SessionOptions } from './session.js';
 import { Session } from './session.js';
+declare const SESSION_REUSE_STRATEGIES: readonly ['random', 'round-robin', 'use-until-failure'];
+export type SessionReuseStrategy = (typeof SESSION_REUSE_STRATEGIES)[number];
+declare const persistedSessionPoolState: z.ZodObject<{
+    usableSessionsCount: z.ZodNumber;
+    retiredSessionsCount: z.ZodNumber;
+    sessions: z.ZodArray<z.ZodCustom<SessionState, SessionState>>;
+}, z.core.$strip>;
+/**
+ * The shape of the persisted {@link SessionPool} record, as returned by {@link SessionPool.getState}.
+ * @internal
+ */
+export type SessionPoolPersistedState = z.infer<typeof persistedSessionPoolState>;
 /**
  * Factory user-function which creates customized {@link Session} instances.
  */
 export interface CreateSession {
     /**
-     * @param sessionPool Pool requesting the new session.
-     * @param options
+     * @param options.sessionOptions Per-call session options already merged with the pool-wide defaults.
      */
-    (sessionPool: SessionPool, options?: {
+    (options?: {
         sessionOptions?: SessionOptions;
     }): Session | Promise<Session>;
 }
 export interface SessionPoolOptions {
+    /**
+     * Unique identifier for this session pool instance. Used to generate a unique
+     * persistence key when `persistStateKey` is not provided.
+     * If not provided, an auto-incrementing ID is used.
+     */
+    id?: string | number;
     /**
      * Maximum size of the pool. Indicates how many sessions are rotated.
      * @default 1000
@@ -29,24 +44,26 @@ export interface SessionPoolOptions {
     /** Name or Id of `KeyValueStore` where is the `SessionPool` state stored. */
     persistStateKeyValueStoreId?: string;
     /**
-     * Session pool persists it's state under this key in Key value store.
-     * @default SESSION_POOL_STATE
+     * Session pool persists its state under this key in Key value store.
+     * @default CRAWLEE_SESSION_POOL_STATE_{id}
      */
     persistStateKey?: string;
     /**
      * Custom function that should return a `Session` instance, or a promise resolving to such instance.
-     * Any error thrown from this function will terminate the process.
-     * Function receives `SessionPool` instance as a parameter
+     * Any error thrown from this function will terminate the process. Receives `{ sessionOptions }`
+     * already merged from the pool-wide defaults and the per-call overrides.
      */
     createSessionFunction?: CreateSession;
     /**
-     * Specifies which response status codes are considered as blocked.
-     * Session connected to such request will be marked as retired.
-     * @default [401, 403, 429]
+     * Strategy for picking sessions from the pool.
+     * - `'random'` (default): fills the pool up to `maxPoolSize`, then picks a random usable session
+     * - `'round-robin'`: fills the pool up to `maxPoolSize`, then reuses sessions cycling through them in order
+     * - `'use-until-failure'`: always reuses the same session until it is retired, then moves to the next one
+     * @default 'random'
      */
-    blockedStatusCodes?: number[];
+    sessionReuseStrategy?: SessionReuseStrategy;
     /** @internal */
-    log?: Log;
+    log?: CrawleeLogger;
     /**
      * Control how and when to persist the state of the session pool.
      */
@@ -58,20 +75,8 @@ export interface SessionPoolOptions {
  * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
  * Learn more in the {@doclink guides/session-management | Session management guide}.
  *
- * You can create one by calling the {@link SessionPool.open} function.
- *
- * Session pool is already integrated into crawlers, and it can significantly improve your scraper
- * performance with just 2 lines of code.
- *
- * **Example usage:**
- *
- * ```javascript
- * const crawler = new CheerioCrawler({
- *     useSessionPool: true,
- *     persistCookiesPerSession: true,
- *     // ...
- * })
- * ```
+ * Session pool is already integrated into crawlers and is always active.
+ * All public methods are lazy-initialized — the pool initializes itself on first use.
  *
  * You can configure the pool with many options. See the {@link SessionPoolOptions}.
  * Session pool is by default persisted in default {@link KeyValueStore}.
@@ -81,7 +86,7 @@ export interface SessionPoolOptions {
  * **Advanced usage:**
  *
  * ```javascript
- * const sessionPool = await SessionPool.open({
+ * const sessionPool = new SessionPool({
  *     maxPoolSize: 25,
  *     sessionOptions:{
  *          maxAgeSecs: 10,
@@ -117,40 +122,24 @@ export interface SessionPoolOptions {
  *
  * @category Scaling
  */
-export declare class SessionPool extends EventEmitter {
-    readonly config: Configuration;
-    protected log: Log;
-    protected maxPoolSize: number;
-    protected createSessionFunction: CreateSession;
-    protected keyValueStore: KeyValueStore;
-    protected sessions: Session[];
-    protected sessionMap: Map<string, Session>;
-    protected sessionOptions: SessionOptions;
-    protected persistStateKeyValueStoreId?: string;
-    protected persistStateKey: string;
-    protected _listener: () => Promise<void>;
-    protected events: EventManager;
-    protected readonly blockedStatusCodes: number[];
-    protected persistenceOptions: PersistenceOptions;
-    protected isInitialized: boolean;
-    private queue;
-    /**
-     * @internal
-     */
-    constructor(options?: SessionPoolOptions, config?: Configuration);
+export declare class SessionPool implements ISessionPool {
+    #private;
+    readonly id: string;
+    constructor(options?: SessionPoolOptions);
     /**
      * Gets count of usable sessions in the pool.
      */
-    get usableSessionsCount(): number;
+    usableSessionsCount(): Promise<number>;
     /**
      * Gets count of retired sessions in the pool.
      */
-    get retiredSessionsCount(): number;
+    retiredSessionsCount(): Promise<number>;
     /**
      * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
-     * It is called automatically by the {@link SessionPool.open} function.
+     * Called automatically on first use of any public method.
      */
-    initialize(): Promise<void>;
+    private ensureInitialized;
+    private setupPool;
     /**
      * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
      * but this allows you to add more sessions once the max pool size is reached.
@@ -170,89 +159,84 @@ export declare class SessionPool extends EventEmitter {
      * If there is space for new session, it creates and returns new session.
      * If the session pool is full, it picks a session from the pool,
      * If the picked session is usable it is returned, otherwise it creates and returns a new one.
+     * @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
      */
-    getSession(): Promise<Session>;
+    getSession(sessionId?: string): Promise<Session | undefined>;
     /**
-     * Gets session based on the provided session id or `undefined.
+     * Discards all sessions in the pool, both usable and retired, resetting it to a blank state.
+     *
+     * The persisted record is left alone - use {@link SessionPool.resetStore} to clear that as well.
      */
-    getSession(sessionId: string): Promise<Session>;
+    reset(): void;
     /**
-     * @param options - Override the persistence options provided in the constructor
+     * Clears the persisted pool record, leaving the in-memory sessions alone.
+     *
+     * Throws while the state is still being persisted periodically - the next PERSIST_STATE event would write
+     * the record straight back. Call {@link SessionPool.teardown} first, or use {@link SessionPool.reset}
+     * to discard the sessions themselves. A no-op if persistence is disabled.
      */
-    resetStore(options?: PersistenceOptions): Promise<void>;
+    resetStore(): Promise<void>;
     /**
      * Returns an object representing the internal state of the `SessionPool` instance.
-     * Note that the object's fields can change in future releases.
+     * @internal
      */
-    getState(): {
-        usableSessionsCount: number;
-        retiredSessionsCount: number;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        sessions: import("./session.js").SessionState[];
-    };
+    getState(): Promise<SessionPoolPersistedState>;
     /**
      * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
      * The state is persisted automatically in regular intervals.
-     * @param options - Override the persistence options provided in the constructor
      */
-    persistState(options?: PersistenceOptions): Promise<void>;
+    persistState(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
     /**
-     * Removes listener from `persistState` event.
+     * Stops the periodic state persistence and persists the state one last time.
      * This function should be called after you are done with using the `SessionPool` instance.
+     * Using the pool again afterwards initializes it anew.
      */
     teardown(): Promise<void>;
-    /**
-     * SessionPool should not work before initialization.
-     */
-    protected _throwIfNotInitialized(): void;
     /**
      * Removes retired `Session` instances from `SessionPool`.
      */
-    protected _removeRetiredSessions(): void;
+    private removeRetiredSessions;
     /**
      * Adds `Session` instance to `SessionPool`.
      * @param newSession `Session` instance to be added.
      */
-    protected _addSession(newSession: Session): void;
+    private registerSession;
     /**
      * Gets random index.
      */
-    protected _getRandomIndex(): number;
+    private getRandomIndex;
     /**
      * Creates new session without any extra behavior.
-     * @param sessionPool
      * @param [options]
      * @param [options.sessionOptions] The configuration options for the session being created.
      * @returns New session.
      */
-    protected _defaultCreateSessionFunction(sessionPool: SessionPool, options?: {
-        sessionOptions?: SessionOptions;
-    }): Promise<Session>;
+    private defaultCreateSessionFunction;
+    /**
+     * Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
+     * the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
+     *
+     * A default {@link SessionFingerprint} is generated up front (host OS as
+     * `platform`, a random valid `browser`/`device` for that platform). Pool-wide
+     * and per-call options override it, and a persisted fingerprint coming
+     * through `maybeLoadSessionPool` naturally wins because it arrives in
+     * `perCallOptions`.
+     */
+    private invokeCreateSessionFunction;
     /**
      * Creates new session and adds it to the pool.
      * @returns Newly created `Session` instance.
      */
-    protected _createSession(): Promise<Session>;
+    private createSession;
     /**
      * Decides whether there is enough space for creating new session.
      */
-    protected _hasSpaceForSession(): boolean;
-    /**
-     * Picks random session from the `SessionPool`.
-     * @returns Picked `Session`.
-     */
-    protected _pickSession(): Session;
+    private hasSpaceForSession;
     /**
-     * Potentially loads `SessionPool`.
-     * If the state was persisted it loads the `SessionPool` from the persisted state.
-     */
-    protected _maybeLoadSessionPool(): Promise<void>;
-    /**
-     * Opens a SessionPool and returns a promise resolving to an instance
-     * of the {@link SessionPool} class that is already initialized.
-     *
-     * For more details and code examples, see the {@link SessionPool} class.
+     * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
+     * Returns `undefined` when no session should be reused and a new one should be created instead.
      */
-    static open(options?: SessionPoolOptions, config?: Configuration): Promise<SessionPool>;
+    private pickSession;
 }
-//# sourceMappingURL=session_pool.d.ts.map
+export {};