@crawlee/core 4.0.0-beta.11 → 4.0.0-beta.111

package/session_pool/session_pool.js

@@ -1,31 +1,19 @@
-import { EventEmitter } from 'node:events';
 import { AsyncQueue } from '@sapphire/async-queue';
 import ow from 'ow';
-import { Configuration } from '../configuration.js';
-import { log as defaultLog } from '../log.js';
+import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from '../storages/key_value_store.js';
-import { BLOCKED_STATUS_CODES, MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
+import { MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
+import { createDefaultSessionFingerprint } from './fingerprint.js';
 import { Session } from './session.js';
+const SESSION_REUSE_STRATEGIES = ['random', 'round-robin', 'use-until-failure'];
 /**
  * Handles the rotation, creation and persistence of user-like sessions.
  * Creates a pool of {@link Session} instances, that are randomly rotated.
  * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
  * Learn more in the {@doclink guides/session-management | Session management guide}.
  *
- * You can create one by calling the {@link SessionPool.open} function.
- *
- * Session pool is already integrated into crawlers, and it can significantly improve your scraper
- * performance with just 2 lines of code.
- *
- * **Example usage:**
- *
- * ```javascript
- * const crawler = new CheerioCrawler({
- *     useSessionPool: true,
- *     persistCookiesPerSession: true,
- *     // ...
- * })
- * ```
+ * Session pool is already integrated into crawlers and is always active.
+ * All public methods are lazy-initialized — the pool initializes itself on first use.
  *
  * You can configure the pool with many options. See the {@link SessionPoolOptions}.
  * Session pool is by default persisted in default {@link KeyValueStore}.
@@ -35,7 +23,7 @@ import { Session } from './session.js';
  * **Advanced usage:**
  *
  * ```javascript
- * const sessionPool = await SessionPool.open({
+ * const sessionPool = new SessionPool({
  *     maxPoolSize: 25,
  *     sessionOptions:{
  *          maxAgeSecs: 10,
@@ -71,94 +59,98 @@ import { Session } from './session.js';
  *
  * @category Scaling
  */
-export class SessionPool extends EventEmitter {
-    config;
-    log;
+export class SessionPool {
+    static #nextId = 0;
+    id;
+    #log;
+    #sessions = [];
+    // kept as TS-private: session_pool tests read/override the members below directly
     maxPoolSize;
     createSessionFunction;
     keyValueStore;
-    sessions = [];
     sessionMap = new Map();
     sessionOptions;
     persistStateKeyValueStoreId;
     persistStateKey;
-    _listener;
-    events;
-    blockedStatusCodes;
-    persistenceOptions;
-    isInitialized = false;
-    queue = new AsyncQueue();
-    /**
-     * @internal
-     */
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
-        super();
-        this.config = config;
+    #listener;
+    #events;
+    #persistenceOptions;
+    #sessionReuseStrategy;
+    #initPromise;
+    #queue = new AsyncQueue();
+    #roundRobinIndex = 0;
+    constructor(options = {}) {
         ow(options, ow.object.exactShape({
+            id: ow.optional.any(ow.number, ow.string),
             maxPoolSize: ow.optional.number,
             persistStateKeyValueStoreId: ow.optional.string,
             persistStateKey: ow.optional.string,
             createSessionFunction: ow.optional.function,
             sessionOptions: ow.optional.object,
-            blockedStatusCodes: ow.optional.array.ofType(ow.number),
             log: ow.optional.object,
             persistenceOptions: ow.optional.object,
+            sessionReuseStrategy: ow.optional.string.oneOf([...SESSION_REUSE_STRATEGIES]),
         }));
-        const { maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey = PERSIST_STATE_KEY, createSessionFunction, sessionOptions = {}, blockedStatusCodes = BLOCKED_STATUS_CODES, log = defaultLog, persistenceOptions = {
+        const { id, maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions = {}, log = serviceLocator.getLogger(), persistenceOptions = {
             enable: true,
-        }, } = options;
-        this.config = config;
-        this.blockedStatusCodes = blockedStatusCodes;
-        this.events = config.getEventManager();
-        this.log = log.child({ prefix: 'SessionPool' });
-        this.persistenceOptions = persistenceOptions;
+        }, sessionReuseStrategy = 'random', } = options;
+        this.id = id != null ? String(id) : String(SessionPool.#nextId++);
+        this.#sessionReuseStrategy = sessionReuseStrategy;
+        this.#events = serviceLocator.getEventManager();
+        this.#log = log.child({ prefix: 'SessionPool' });
+        this.#persistenceOptions = persistenceOptions;
         // Pool Configuration
         this.maxPoolSize = maxPoolSize;
-        this.createSessionFunction = createSessionFunction || this._defaultCreateSessionFunction;
-        // Session configuration
+        this.createSessionFunction = createSessionFunction || this.defaultCreateSessionFunction;
+        // Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside
+        // `invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
+        // having to know about it.
         this.sessionOptions = {
             ...sessionOptions,
-            // the log needs to propagate to createSessionFunction as in "new Session({ ...sessionPool.sessionOptions })"
-            // and can't go inside _defaultCreateSessionFunction
-            log: this.log,
+            log: this.#log,
         };
         // Session keyValueStore
         this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
-        this.persistStateKey = persistStateKey;
+        this.persistStateKey = persistStateKey ?? `${PERSIST_STATE_KEY}_${this.id}`;
     }
     /**
      * Gets count of usable sessions in the pool.
      */
-    get usableSessionsCount() {
-        return this.sessions.filter((session) => session.isUsable()).length;
+    async usableSessionsCount() {
+        await this.ensureInitialized();
+        return this.#sessions.filter((session) => session.isUsable()).length;
     }
     /**
      * Gets count of retired sessions in the pool.
      */
-    get retiredSessionsCount() {
-        return this.sessions.filter((session) => !session.isUsable()).length;
+    async retiredSessionsCount() {
+        await this.ensureInitialized();
+        return this.#sessions.filter((session) => !session.isUsable()).length;
     }
     /**
      * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
-     * It is called automatically by the {@link SessionPool.open} function.
+     * Called automatically on first use of any public method.
      */
-    async initialize() {
-        if (this.isInitialized) {
-            return;
+    async ensureInitialized() {
+        if (!this.#initPromise) {
+            this.#initPromise = this.setupPool();
         }
-        this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId, { config: this.config });
-        if (!this.persistenceOptions.enable) {
-            this.isInitialized = true;
+        return this.#initPromise;
+    }
+    async setupPool() {
+        if (!this.#persistenceOptions.enable) {
             return;
         }
+        this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId ? { id: this.persistStateKeyValueStoreId } : null, {
+            configuration: serviceLocator.getConfiguration(),
+        });
         if (!this.persistStateKeyValueStoreId) {
-            this.log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
+            this.#log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
         }
         // in case of migration happened and SessionPool state should be restored from the keyValueStore.
-        await this._maybeLoadSessionPool();
-        this._listener = this.persistState.bind(this);
-        this.events.on("persistState" /* EventType.PERSIST_STATE */, this._listener);
-        this.isInitialized = true;
+        await this.maybeLoadSessionPool();
+        this.#listener = this.persistState.bind(this);
+        this.#events.on("persistState" /* EventType.PERSIST_STATE */, this.#listener);
     }
     /**
      * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
@@ -167,7 +159,7 @@ export class SessionPool extends EventEmitter {
      * @param [options] The configuration options for the session being added to the session pool.
      */
     async addSession(options = {}) {
-        this._throwIfNotInitialized();
+        await this.ensureInitialized();
         const { id } = options;
         if (id) {
             const sessionExists = this.sessionMap.has(id);
@@ -175,12 +167,24 @@ export class SessionPool extends EventEmitter {
                 throw new Error(`Cannot add session with id '${id}' as it already exists in the pool`);
             }
         }
-        if (!this._hasSpaceForSession()) {
-            this._removeRetiredSessions();
+        if (!this.hasSpaceForSession()) {
+            this.removeRetiredSessions();
         }
-        const newSession = options instanceof Session ? options : await this.createSessionFunction(this, { sessionOptions: options });
-        this.log.debug(`Adding new Session - ${newSession.id}`);
-        this._addSession(newSession);
+        const newSession = options instanceof Session ? options : await this.invokeCreateSessionFunction(options);
+        this.#log.debug(`Adding new Session - ${newSession.id}`);
+        this.registerSession(newSession);
+    }
+    /**
+     * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
+     * but this allows you to add more sessions once the max pool size is reached.
+     * This also allows you to add session with overridden session options (e.g. with specific session id).
+     * @param [options] The configuration options for the session being added to the session pool.
+     */
+    async newSession(sessionOptions) {
+        await this.ensureInitialized();
+        const newSession = await this.invokeCreateSessionFunction(sessionOptions);
+        this.registerSession(newSession);
+        return newSession;
     }
     /**
      * Gets session.
@@ -190,47 +194,48 @@ export class SessionPool extends EventEmitter {
      * @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
      */
     async getSession(sessionId) {
-        await this.queue.wait();
+        await this.ensureInitialized();
+        await this.#queue.wait();
         try {
-            this._throwIfNotInitialized();
             if (sessionId) {
                 const session = this.sessionMap.get(sessionId);
-                if (session && session.isUsable())
+                if (session?.isUsable())
                     return session;
                 return undefined;
             }
-            if (this._hasSpaceForSession()) {
-                return await this._createSession();
-            }
-            const pickedSession = this._pickSession();
-            if (pickedSession.isUsable()) {
+            const pickedSession = this.pickSession();
+            if (pickedSession)
                 return pickedSession;
+            if (this.hasSpaceForSession()) {
+                return await this.createSession();
             }
-            this._removeRetiredSessions();
-            return await this._createSession();
+            this.removeRetiredSessions();
+            return await this.createSession();
         }
         finally {
-            this.queue.shift();
+            this.#queue.shift();
         }
     }
     /**
      * @param options - Override the persistence options provided in the constructor
      */
     async resetStore(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
+        if (!this.#persistenceOptions.enable && !options?.enable) {
             return;
         }
+        await this.ensureInitialized();
         await this.keyValueStore?.setValue(this.persistStateKey, null);
     }
     /**
      * Returns an object representing the internal state of the `SessionPool` instance.
      * Note that the object's fields can change in future releases.
      */
-    getState() {
+    async getState() {
+        await this.ensureInitialized();
         return {
-            usableSessionsCount: this.usableSessionsCount,
-            retiredSessionsCount: this.retiredSessionsCount,
-            sessions: this.sessions.map((session) => session.getState()),
+            usableSessionsCount: await this.usableSessionsCount(),
+            retiredSessionsCount: await this.retiredSessionsCount(),
+            sessions: this.#sessions.map((session) => session.getState()),
         };
     }
     /**
@@ -239,47 +244,40 @@ export class SessionPool extends EventEmitter {
      * @param options - Override the persistence options provided in the constructor
      */
     async persistState(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
+        if (!this.#persistenceOptions.enable && !options?.enable) {
             return;
         }
-        this.log.debug('Persisting state', {
+        await this.ensureInitialized();
+        this.#log.debug('Persisting state', {
             persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
             persistStateKey: this.persistStateKey,
         });
-        // use half the interval of `persistState` to avoid race conditions
-        const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
-        const timeoutSecs = persistStateIntervalMillis / 2_000;
         await this.keyValueStore
-            .setValue(this.persistStateKey, this.getState(), {
-            timeoutSecs,
-            doNotRetryTimeouts: true,
-        })
-            .catch((error) => this.log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
+            ?.setValue(this.persistStateKey, await this.getState())
+            .catch((error) => this.#log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
     }
     /**
      * Removes listener from `persistState` event.
      * This function should be called after you are done with using the `SessionPool` instance.
      */
     async teardown() {
-        this.events.off("persistState" /* EventType.PERSIST_STATE */, this._listener);
+        if (!this.#initPromise)
+            return;
+        await this.ensureInitialized();
+        if (this.#listener) {
+            this.#events.off("persistState" /* EventType.PERSIST_STATE */, this.#listener);
+        }
         await this.persistState();
     }
-    /**
-     * SessionPool should not work before initialization.
-     */
-    _throwIfNotInitialized() {
-        if (!this.isInitialized)
-            throw new Error('SessionPool is not initialized.');
-    }
     /**
      * Removes retired `Session` instances from `SessionPool`.
      */
-    _removeRetiredSessions() {
-        this.sessions = this.sessions.filter((storedSession) => {
+    removeRetiredSessions() {
+        this.#sessions = this.#sessions.filter((storedSession) => {
             if (storedSession.isUsable())
                 return true;
             this.sessionMap.delete(storedSession.id);
-            this.log.debug(`Removed Session - ${storedSession.id}`);
+            this.#log.debug(`Removed Session - ${storedSession.id}`);
             return false;
         });
     }
@@ -287,89 +285,103 @@ export class SessionPool extends EventEmitter {
      * Adds `Session` instance to `SessionPool`.
      * @param newSession `Session` instance to be added.
      */
-    _addSession(newSession) {
-        this.sessions.push(newSession);
+    registerSession(newSession) {
+        this.#sessions.push(newSession);
         this.sessionMap.set(newSession.id, newSession);
     }
     /**
      * Gets random index.
      */
-    _getRandomIndex() {
-        return Math.floor(Math.random() * this.sessions.length);
+    getRandomIndex() {
+        return Math.floor(Math.random() * this.#sessions.length);
     }
     /**
      * Creates new session without any extra behavior.
-     * @param sessionPool
      * @param [options]
      * @param [options.sessionOptions] The configuration options for the session being created.
      * @returns New session.
      */
-    _defaultCreateSessionFunction(sessionPool, options = {}) {
+    async defaultCreateSessionFunction(options = {}) {
         ow(options, ow.object.exactShape({ sessionOptions: ow.optional.object }));
         const { sessionOptions = {} } = options;
-        return new Session({
+        return new Session(sessionOptions);
+    }
+    /**
+     * Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
+     * the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
+     *
+     * A default {@link SessionFingerprint} is generated up front (host OS as
+     * `platform`, a random valid `browser`/`device` for that platform). Pool-wide
+     * and per-call options override it, and a persisted fingerprint coming
+     * through `maybeLoadSessionPool` naturally wins because it arrives in
+     * `perCallOptions`.
+     */
+    async invokeCreateSessionFunction(perCallOptions) {
+        const sessionOptions = {
+            fingerprint: createDefaultSessionFingerprint(),
             ...this.sessionOptions,
-            ...sessionOptions,
-            sessionPool,
-        });
+            ...perCallOptions,
+        };
+        return this.createSessionFunction({ sessionOptions });
     }
     /**
      * Creates new session and adds it to the pool.
      * @returns Newly created `Session` instance.
      */
-    async _createSession() {
-        const newSession = await this.createSessionFunction(this);
-        this._addSession(newSession);
-        this.log.debug(`Created new Session - ${newSession.id}`);
+    async createSession() {
+        const newSession = await this.invokeCreateSessionFunction();
+        this.registerSession(newSession);
+        this.#log.debug(`Created new Session - ${newSession.id}`);
         return newSession;
     }
     /**
      * Decides whether there is enough space for creating new session.
      */
-    _hasSpaceForSession() {
-        return this.sessions.length < this.maxPoolSize;
+    hasSpaceForSession() {
+        return this.#sessions.length < this.maxPoolSize;
     }
     /**
-     * Picks random session from the `SessionPool`.
-     * @returns Picked `Session`.
+     * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
+     * Returns `undefined` when no session should be reused and a new one should be created instead.
      */
-    _pickSession() {
-        return this.sessions[this._getRandomIndex()]; // Or maybe we should let the developer to customize the picking algorithm
+    pickSession() {
+        if (this.#sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession())
+            return undefined;
+        if (this.#sessionReuseStrategy === 'use-until-failure') {
+            return this.#sessions.find((session) => session.isUsable());
+        }
+        let picked;
+        if (this.#sessionReuseStrategy === 'round-robin') {
+            const index = this.#roundRobinIndex % this.#sessions.length;
+            this.#roundRobinIndex = index + 1;
+            picked = this.#sessions[index];
+        }
+        else {
+            picked = this.#sessions[this.getRandomIndex()];
+        }
+        return picked.isUsable() ? picked : undefined;
     }
     /**
      * Potentially loads `SessionPool`.
      * If the state was persisted it loads the `SessionPool` from the persisted state.
      */
-    async _maybeLoadSessionPool() {
-        const loadedSessionPool = await this.keyValueStore.getValue(this.persistStateKey);
+    async maybeLoadSessionPool() {
+        const loadedSessionPool = await this.keyValueStore?.getValue(this.persistStateKey);
         if (!loadedSessionPool)
             return;
         // Invalidate old sessions and load active sessions only
-        this.log.debug('Recreating state from KeyValueStore', {
+        this.#log.debug('Recreating state from KeyValueStore', {
             persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
             persistStateKey: this.persistStateKey,
         });
         for (const sessionObject of loadedSessionPool.sessions) {
-            sessionObject.sessionPool = this;
             sessionObject.createdAt = new Date(sessionObject.createdAt);
             sessionObject.expiresAt = new Date(sessionObject.expiresAt);
-            const recreatedSession = await this.createSessionFunction(this, { sessionOptions: sessionObject });
+            const recreatedSession = await this.invokeCreateSessionFunction(sessionObject);
             if (recreatedSession.isUsable()) {
-                this._addSession(recreatedSession);
+                this.registerSession(recreatedSession);
             }
         }
-        this.log.debug(`${this.usableSessionsCount} active sessions loaded from KeyValueStore`);
-    }
-    /**
-     * Opens a SessionPool and returns a promise resolving to an instance
-     * of the {@link SessionPool} class that is already initialized.
-     *
-     * For more details and code examples, see the {@link SessionPool} class.
-     */
-    static async open(options, config) {
-        const sessionPool = new SessionPool(options, config);
-        await sessionPool.initialize();
-        return sessionPool;
+        this.#log.debug(`${this.#sessions.length} active sessions loaded from KeyValueStore`);
     }
 }
-//# sourceMappingURL=session_pool.js.map