@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.80

package/session_pool/session_pool.js

@@ -1,31 +1,19 @@
-import { EventEmitter } from 'node:events';
 import { AsyncQueue } from '@sapphire/async-queue';
 import ow from 'ow';
-import { Configuration } from '../configuration.js';
-import { log as defaultLog } from '../log.js';
+import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from '../storages/key_value_store.js';
-import { BLOCKED_STATUS_CODES, MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
+import { MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
+import { createDefaultSessionFingerprint } from './fingerprint.js';
 import { Session } from './session.js';
+const SESSION_REUSE_STRATEGIES = ['random', 'round-robin', 'use-until-failure'];
 /**
  * Handles the rotation, creation and persistence of user-like sessions.
  * Creates a pool of {@link Session} instances, that are randomly rotated.
  * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
  * Learn more in the {@doclink guides/session-management | Session management guide}.
  *
- * You can create one by calling the {@link SessionPool.open} function.
- *
- * Session pool is already integrated into crawlers, and it can significantly improve your scraper
- * performance with just 2 lines of code.
- *
- * **Example usage:**
- *
- * ```javascript
- * const crawler = new CheerioCrawler({
- *     useSessionPool: true,
- *     persistCookiesPerSession: true,
- *     // ...
- * })
- * ```
+ * Session pool is already integrated into crawlers and is always active.
+ * All public methods are lazy-initialized — the pool initializes itself on first use.
  *
  * You can configure the pool with many options. See the {@link SessionPoolOptions}.
  * Session pool is by default persisted in default {@link KeyValueStore}.
@@ -35,7 +23,7 @@ import { Session } from './session.js';
  * **Advanced usage:**
  *
  * ```javascript
- * const sessionPool = await SessionPool.open({
+ * const sessionPool = new SessionPool({
  *     maxPoolSize: 25,
  *     sessionOptions:{
  *          maxAgeSecs: 10,
@@ -71,8 +59,9 @@ import { Session } from './session.js';
  *
  * @category Scaling
  */
-export class SessionPool extends EventEmitter {
-    config;
+export class SessionPool {
+    static nextId = 0;
+    id;
     log;
     maxPoolSize;
     createSessionFunction;
@@ -84,73 +73,76 @@ export class SessionPool extends EventEmitter {
     persistStateKey;
     _listener;
     events;
-    blockedStatusCodes;
     persistenceOptions;
-    isInitialized = false;
+    sessionReuseStrategy;
+    initPromise;
     queue = new AsyncQueue();
-    /**
-     * @internal
-     */
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
-        super();
-        this.config = config;
+    roundRobinIndex = 0;
+    constructor(options = {}) {
         ow(options, ow.object.exactShape({
+            id: ow.optional.any(ow.number, ow.string),
             maxPoolSize: ow.optional.number,
             persistStateKeyValueStoreId: ow.optional.string,
             persistStateKey: ow.optional.string,
             createSessionFunction: ow.optional.function,
             sessionOptions: ow.optional.object,
-            blockedStatusCodes: ow.optional.array.ofType(ow.number),
             log: ow.optional.object,
             persistenceOptions: ow.optional.object,
+            sessionReuseStrategy: ow.optional.string.oneOf([...SESSION_REUSE_STRATEGIES]),
         }));
-        const { maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey = PERSIST_STATE_KEY, createSessionFunction, sessionOptions = {}, blockedStatusCodes = BLOCKED_STATUS_CODES, log = defaultLog, persistenceOptions = {
+        const { id, maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions = {}, log = serviceLocator.getLogger(), persistenceOptions = {
             enable: true,
-        }, } = options;
-        this.config = config;
-        this.blockedStatusCodes = blockedStatusCodes;
-        this.events = config.getEventManager();
+        }, sessionReuseStrategy = 'random', } = options;
+        this.id = id != null ? String(id) : String(SessionPool.nextId++);
+        this.sessionReuseStrategy = sessionReuseStrategy;
+        this.events = serviceLocator.getEventManager();
         this.log = log.child({ prefix: 'SessionPool' });
         this.persistenceOptions = persistenceOptions;
         // Pool Configuration
         this.maxPoolSize = maxPoolSize;
         this.createSessionFunction = createSessionFunction || this._defaultCreateSessionFunction;
-        // Session configuration
+        // Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside
+        // `_invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
+        // having to know about it.
         this.sessionOptions = {
             ...sessionOptions,
-            // the log needs to propagate to createSessionFunction as in "new Session({ ...sessionPool.sessionOptions })"
-            // and can't go inside _defaultCreateSessionFunction
             log: this.log,
         };
         // Session keyValueStore
         this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
-        this.persistStateKey = persistStateKey;
+        this.persistStateKey = persistStateKey ?? `${PERSIST_STATE_KEY}_${this.id}`;
     }
     /**
      * Gets count of usable sessions in the pool.
      */
-    get usableSessionsCount() {
+    async usableSessionsCount() {
+        await this.ensureInitialized();
         return this.sessions.filter((session) => session.isUsable()).length;
     }
     /**
      * Gets count of retired sessions in the pool.
      */
-    get retiredSessionsCount() {
+    async retiredSessionsCount() {
+        await this.ensureInitialized();
         return this.sessions.filter((session) => !session.isUsable()).length;
     }
     /**
      * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
-     * It is called automatically by the {@link SessionPool.open} function.
+     * Called automatically on first use of any public method.
      */
-    async initialize() {
-        if (this.isInitialized) {
-            return;
+    async ensureInitialized() {
+        if (!this.initPromise) {
+            this.initPromise = this.setupPool();
         }
-        this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId, { config: this.config });
+        return this.initPromise;
+    }
+    async setupPool() {
         if (!this.persistenceOptions.enable) {
-            this.isInitialized = true;
             return;
         }
+        this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId ? { id: this.persistStateKeyValueStoreId } : null, {
+            config: serviceLocator.getConfiguration(),
+        });
         if (!this.persistStateKeyValueStoreId) {
             this.log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
         }
@@ -158,7 +150,6 @@ export class SessionPool extends EventEmitter {
         await this._maybeLoadSessionPool();
         this._listener = this.persistState.bind(this);
         this.events.on("persistState" /* EventType.PERSIST_STATE */, this._listener);
-        this.isInitialized = true;
     }
     /**
      * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
@@ -167,7 +158,7 @@ export class SessionPool extends EventEmitter {
      * @param [options] The configuration options for the session being added to the session pool.
      */
     async addSession(options = {}) {
-        this._throwIfNotInitialized();
+        await this.ensureInitialized();
         const { id } = options;
         if (id) {
             const sessionExists = this.sessionMap.has(id);
@@ -178,10 +169,22 @@ export class SessionPool extends EventEmitter {
         if (!this._hasSpaceForSession()) {
             this._removeRetiredSessions();
         }
-        const newSession = options instanceof Session ? options : await this.createSessionFunction(this, { sessionOptions: options });
+        const newSession = options instanceof Session ? options : await this._invokeCreateSessionFunction(options);
         this.log.debug(`Adding new Session - ${newSession.id}`);
         this._addSession(newSession);
     }
+    /**
+     * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
+     * but this allows you to add more sessions once the max pool size is reached.
+     * This also allows you to add session with overridden session options (e.g. with specific session id).
+     * @param [options] The configuration options for the session being added to the session pool.
+     */
+    async newSession(sessionOptions) {
+        await this.ensureInitialized();
+        const newSession = await this._invokeCreateSessionFunction(sessionOptions);
+        this._addSession(newSession);
+        return newSession;
+    }
     /**
      * Gets session.
      * If there is space for new session, it creates and returns new session.
@@ -190,21 +193,20 @@ export class SessionPool extends EventEmitter {
      * @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
      */
     async getSession(sessionId) {
+        await this.ensureInitialized();
         await this.queue.wait();
         try {
-            this._throwIfNotInitialized();
             if (sessionId) {
                 const session = this.sessionMap.get(sessionId);
-                if (session && session.isUsable())
+                if (session?.isUsable())
                     return session;
                 return undefined;
             }
-            if (this._hasSpaceForSession()) {
-                return await this._createSession();
-            }
             const pickedSession = this._pickSession();
-            if (pickedSession.isUsable()) {
+            if (pickedSession)
                 return pickedSession;
+            if (this._hasSpaceForSession()) {
+                return await this._createSession();
             }
             this._removeRetiredSessions();
             return await this._createSession();
@@ -220,16 +222,18 @@ export class SessionPool extends EventEmitter {
         if (!this.persistenceOptions.enable && !options?.enable) {
             return;
         }
+        await this.ensureInitialized();
         await this.keyValueStore?.setValue(this.persistStateKey, null);
     }
     /**
      * Returns an object representing the internal state of the `SessionPool` instance.
      * Note that the object's fields can change in future releases.
      */
-    getState() {
+    async getState() {
+        await this.ensureInitialized();
         return {
-            usableSessionsCount: this.usableSessionsCount,
-            retiredSessionsCount: this.retiredSessionsCount,
+            usableSessionsCount: await this.usableSessionsCount(),
+            retiredSessionsCount: await this.retiredSessionsCount(),
             sessions: this.sessions.map((session) => session.getState()),
         };
     }
@@ -242,18 +246,13 @@ export class SessionPool extends EventEmitter {
         if (!this.persistenceOptions.enable && !options?.enable) {
             return;
         }
+        await this.ensureInitialized();
         this.log.debug('Persisting state', {
             persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
             persistStateKey: this.persistStateKey,
         });
-        // use half the interval of `persistState` to avoid race conditions
-        const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
-        const timeoutSecs = persistStateIntervalMillis / 2_000;
         await this.keyValueStore
-            .setValue(this.persistStateKey, this.getState(), {
-            timeoutSecs,
-            doNotRetryTimeouts: true,
-        })
+            ?.setValue(this.persistStateKey, await this.getState())
             .catch((error) => this.log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
     }
     /**
@@ -261,16 +260,14 @@ export class SessionPool extends EventEmitter {
      * This function should be called after you are done with using the `SessionPool` instance.
      */
     async teardown() {
-        this.events.off("persistState" /* EventType.PERSIST_STATE */, this._listener);
+        if (!this.initPromise)
+            return;
+        await this.ensureInitialized();
+        if (this._listener) {
+            this.events.off("persistState" /* EventType.PERSIST_STATE */, this._listener);
+        }
         await this.persistState();
     }
-    /**
-     * SessionPool should not work before initialization.
-     */
-    _throwIfNotInitialized() {
-        if (!this.isInitialized)
-            throw new Error('SessionPool is not initialized.');
-    }
     /**
      * Removes retired `Session` instances from `SessionPool`.
      */
@@ -299,26 +296,39 @@ export class SessionPool extends EventEmitter {
     }
     /**
      * Creates new session without any extra behavior.
-     * @param sessionPool
      * @param [options]
      * @param [options.sessionOptions] The configuration options for the session being created.
      * @returns New session.
      */
-    _defaultCreateSessionFunction(sessionPool, options = {}) {
+    async _defaultCreateSessionFunction(options = {}) {
         ow(options, ow.object.exactShape({ sessionOptions: ow.optional.object }));
         const { sessionOptions = {} } = options;
-        return new Session({
+        return new Session(sessionOptions);
+    }
+    /**
+     * Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
+     * the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
+     *
+     * A default {@link SessionFingerprint} is generated up front (host OS as
+     * `platform`, a random valid `browser`/`device` for that platform). Pool-wide
+     * and per-call options override it, and a persisted fingerprint coming
+     * through `_maybeLoadSessionPool` naturally wins because it arrives in
+     * `perCallOptions`.
+     */
+    async _invokeCreateSessionFunction(perCallOptions) {
+        const sessionOptions = {
+            fingerprint: createDefaultSessionFingerprint(),
             ...this.sessionOptions,
-            ...sessionOptions,
-            sessionPool,
-        });
+            ...perCallOptions,
+        };
+        return this.createSessionFunction({ sessionOptions });
     }
     /**
      * Creates new session and adds it to the pool.
      * @returns Newly created `Session` instance.
      */
     async _createSession() {
-        const newSession = await this.createSessionFunction(this);
+        const newSession = await this._invokeCreateSessionFunction();
         this._addSession(newSession);
         this.log.debug(`Created new Session - ${newSession.id}`);
         return newSession;
@@ -330,18 +340,32 @@ export class SessionPool extends EventEmitter {
         return this.sessions.length < this.maxPoolSize;
     }
     /**
-     * Picks random session from the `SessionPool`.
-     * @returns Picked `Session`.
+     * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
+     * Returns `undefined` when no session should be reused and a new one should be created instead.
      */
     _pickSession() {
-        return this.sessions[this._getRandomIndex()]; // Or maybe we should let the developer to customize the picking algorithm
+        if (this.sessionReuseStrategy !== 'use-until-failure' && this._hasSpaceForSession())
+            return undefined;
+        if (this.sessionReuseStrategy === 'use-until-failure') {
+            return this.sessions.find((session) => session.isUsable());
+        }
+        let picked;
+        if (this.sessionReuseStrategy === 'round-robin') {
+            const index = this.roundRobinIndex % this.sessions.length;
+            this.roundRobinIndex = index + 1;
+            picked = this.sessions[index];
+        }
+        else {
+            picked = this.sessions[this._getRandomIndex()];
+        }
+        return picked.isUsable() ? picked : undefined;
     }
     /**
      * Potentially loads `SessionPool`.
      * If the state was persisted it loads the `SessionPool` from the persisted state.
      */
     async _maybeLoadSessionPool() {
-        const loadedSessionPool = await this.keyValueStore.getValue(this.persistStateKey);
+        const loadedSessionPool = await this.keyValueStore?.getValue(this.persistStateKey);
         if (!loadedSessionPool)
             return;
         // Invalidate old sessions and load active sessions only
@@ -350,26 +374,13 @@ export class SessionPool extends EventEmitter {
             persistStateKey: this.persistStateKey,
         });
         for (const sessionObject of loadedSessionPool.sessions) {
-            sessionObject.sessionPool = this;
             sessionObject.createdAt = new Date(sessionObject.createdAt);
             sessionObject.expiresAt = new Date(sessionObject.expiresAt);
-            const recreatedSession = await this.createSessionFunction(this, { sessionOptions: sessionObject });
+            const recreatedSession = await this._invokeCreateSessionFunction(sessionObject);
             if (recreatedSession.isUsable()) {
                 this._addSession(recreatedSession);
             }
         }
-        this.log.debug(`${this.usableSessionsCount} active sessions loaded from KeyValueStore`);
-    }
-    /**
-     * Opens a SessionPool and returns a promise resolving to an instance
-     * of the {@link SessionPool} class that is already initialized.
-     *
-     * For more details and code examples, see the {@link SessionPool} class.
-     */
-    static async open(options, config) {
-        const sessionPool = new SessionPool(options, config);
-        await sessionPool.initialize();
-        return sessionPool;
+        this.log.debug(`${this.sessions.length} active sessions loaded from KeyValueStore`);
     }
 }
-//# sourceMappingURL=session_pool.js.map

package/storages/access_checking.d.ts

@@ -10,4 +10,3 @@ export declare const checkStorageAccess: () => void | undefined;
  * @param callback The code that should be invoked with the `checkFunction` setting
  */
 export declare const withCheckedStorageAccess: <T>(checkFunction: () => void, callback: () => Awaitable<T>) => Promise<T>;
-//# sourceMappingURL=access_checking.d.ts.map

package/storages/access_checking.js

@@ -1,9 +1,13 @@
 import { AsyncLocalStorage } from 'node:async_hooks';
+import { tryCancel } from '@apify/timeout';
 const storage = new AsyncLocalStorage();
 /**
  * Invoke a storage access checker function defined using {@link withCheckedStorageAccess} higher up in the call stack.
  */
-export const checkStorageAccess = () => storage.getStore()?.checkFunction();
+export const checkStorageAccess = () => {
+    tryCancel();
+    return storage.getStore()?.checkFunction();
+};
 /**
  * Define a storage access checker function that should be used by calls to {@link checkStorageAccess} in the callbacks.
  *
@@ -11,4 +15,3 @@ export const checkStorageAccess = () => storage.getStore()?.checkFunction();
  * @param callback The code that should be invoked with the `checkFunction` setting
  */
 export const withCheckedStorageAccess = async (checkFunction, callback) => storage.run({ checkFunction }, callback);
-//# sourceMappingURL=access_checking.js.map