@crawlee/core 4.0.0-beta.11 → 4.0.0-beta.111

package/session_pool/session.js

@@ -1,10 +1,8 @@
-import { EventEmitter } from 'node:events';
 import ow from 'ow';
 import { CookieJar } from 'tough-cookie';
 import { cryptoRandomObjectId } from '@apify/utilities';
-import { browserPoolCookieToToughCookie, getCookiesFromResponse, getDefaultCookieExpirationDate, toughCookieToBrowserPoolCookie, } from '../cookie_utils.js';
-import { log as defaultLog } from '../log.js';
-import { EVENT_SESSION_RETIRED } from './events.js';
+import { getDefaultCookieExpirationDate } from '../cookie_utils.js';
+import { serviceLocator } from '../service_locator.js';
 /**
  * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
  * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
@@ -13,50 +11,67 @@ import { EVENT_SESSION_RETIRED } from './events.js';
  */
 export class Session {
     id;
-    maxAgeSecs;
     userData;
-    _maxErrorScore;
-    _errorScoreDecrement;
-    _createdAt;
-    _expiresAt;
-    _usageCount;
-    _maxUsageCount;
-    sessionPool;
-    _errorScore;
-    _cookieJar;
-    log;
+    #maxErrorScore;
+    #errorScoreDecrement;
+    #createdAt;
+    #expiresAt;
+    #usageCount;
+    #maxUsageCount;
+    #errorScore;
+    #retired = false;
+    #proxyInfo;
+    #cookieJar;
+    #fingerprint;
+    #log;
     get errorScore() {
-        return this._errorScore;
+        return this.#errorScore;
     }
     get usageCount() {
-        return this._usageCount;
+        return this.#usageCount;
     }
     get maxErrorScore() {
-        return this._maxErrorScore;
+        return this.#maxErrorScore;
     }
     get errorScoreDecrement() {
-        return this._errorScoreDecrement;
+        return this.#errorScoreDecrement;
     }
     get expiresAt() {
-        return this._expiresAt;
+        return this.#expiresAt;
     }
     get createdAt() {
-        return this._createdAt;
+        return this.#createdAt;
     }
     get maxUsageCount() {
-        return this._maxUsageCount;
+        return this.#maxUsageCount;
     }
     get cookieJar() {
-        return this._cookieJar;
+        return this.#cookieJar;
+    }
+    get proxyInfo() {
+        return this.#proxyInfo;
+    }
+    get fingerprint() {
+        return this.#fingerprint;
+    }
+    set fingerprint(fingerprint) {
+        this.#fingerprint = fingerprint;
+    }
+    /**
+     * `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
+     * a retired session is never picked by the pool and cannot be revived via `markGood()`.
+     */
+    get retired() {
+        return this.#retired;
     }
     /**
      * Session configuration.
      */
-    constructor(options) {
+    constructor(options = {}) {
         ow(options, ow.object.exactShape({
-            sessionPool: ow.object.instanceOf(EventEmitter),
             id: ow.optional.string,
             cookieJar: ow.optional.object,
+            proxyInfo: ow.optional.object,
             maxAgeSecs: ow.optional.number,
             userData: ow.optional.object,
             maxErrorScore: ow.optional.number,
@@ -66,24 +81,27 @@ export class Session {
             usageCount: ow.optional.number,
             errorScore: ow.optional.number,
             maxUsageCount: ow.optional.number,
+            retired: ow.optional.boolean,
             log: ow.optional.object,
+            fingerprint: ow.optional.object,
         }));
-        const { sessionPool, id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, log = defaultLog, } = options;
+        const { id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, retired = false, log = serviceLocator.getLogger(), fingerprint, } = options;
         const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
-        this.log = log.child({ prefix: 'Session' });
-        this._cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
+        this.#log = log.child({ prefix: 'Session' });
+        this.#cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
+        this.#proxyInfo = proxyInfo;
+        this.#fingerprint = fingerprint;
         this.id = id;
-        this.maxAgeSecs = maxAgeSecs;
         this.userData = userData;
-        this._maxErrorScore = maxErrorScore;
-        this._errorScoreDecrement = errorScoreDecrement;
+        this.#maxErrorScore = maxErrorScore;
+        this.#errorScoreDecrement = errorScoreDecrement;
         // Internal
-        this._expiresAt = expiresAt;
-        this._createdAt = createdAt;
-        this._usageCount = usageCount; // indicates how many times the session has been used
-        this._errorScore = errorScore; // indicates number of markBaded request with the session
-        this._maxUsageCount = maxUsageCount;
-        this.sessionPool = sessionPool;
+        this.#expiresAt = expiresAt;
+        this.#createdAt = createdAt;
+        this.#usageCount = usageCount; // indicates how many times the session has been used
+        this.#errorScore = errorScore; // indicates number of markBaded request with the session
+        this.#maxUsageCount = maxUsageCount;
+        this.#retired = retired;
     }
     /**
      * Indicates whether the session is blocked.
@@ -109,21 +127,21 @@ export class Session {
     }
     /**
      * Indicates whether the session can be used for next requests.
-     * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
+     * Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
      */
     isUsable() {
-        return !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
+        return !this.#retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
     }
     /**
      * This method should be called after a successful session usage.
      * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
      */
     markGood() {
-        this._usageCount += 1;
-        if (this._errorScore > 0) {
-            this._errorScore -= this._errorScoreDecrement;
+        this.#usageCount += 1;
+        if (this.#errorScore > 0) {
+            this.#errorScore -= this.#errorScoreDecrement;
         }
-        this._maybeSelfRetire();
+        this.maybeSelfRetire();
     }
     /**
      * Gets session state for persistence in KeyValueStore.
@@ -133,7 +151,9 @@ export class Session {
         return {
             id: this.id,
             cookieJar: this.cookieJar.toJSON(),
+            proxyInfo: this.#proxyInfo,
             userData: this.userData,
+            fingerprint: this.#fingerprint,
             maxErrorScore: this.maxErrorScore,
             errorScoreDecrement: this.errorScoreDecrement,
             expiresAt: this.expiresAt.toISOString(),
@@ -141,89 +161,31 @@ export class Session {
             usageCount: this.usageCount,
             maxUsageCount: this.maxUsageCount,
             errorScore: this.errorScore,
+            retired: this.#retired,
         };
     }
     /**
-     * Marks session as blocked and emits event on the `SessionPool`
-     * This method should be used if the session usage was unsuccessful
-     * and you are sure that it is because of the session configuration and not any external matters.
-     * For example when server returns 403 status code.
-     * If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
+     * Permanently retires the session — `isUsable()` will return `false` from here on,
+     * and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
+     *
+     * Use this when you're confident the session itself is the problem (e.g. a `403` response).
+     * For transient external failures (such as `5XX` responses), use `markBad()` instead.
      */
     retire() {
-        // mark it as an invalid by increasing the error score count.
-        this._errorScore += this._maxErrorScore;
-        this._usageCount += 1;
-        // emit event so we can retire browser in puppeteer pool
-        this.sessionPool.emit(EVENT_SESSION_RETIRED, this);
+        if (this.#retired)
+            return;
+        this.#errorScore += this.#maxErrorScore;
+        this.#usageCount += 1;
+        this.#retired = true;
     }
     /**
      * Increases usage and error count.
      * Should be used when the session has been used unsuccessfully. For example because of timeouts.
      */
     markBad() {
-        this._errorScore += 1;
-        this._usageCount += 1;
-        this._maybeSelfRetire();
-    }
-    /**
-     * With certain status codes: `401`, `403` or `429` we can be certain
-     * that the target website is blocking us. This function helps to do this conveniently
-     * by retiring the session when such code is received. Optionally, the default status
-     * codes can be extended in the second parameter.
-     * @param statusCode HTTP status code.
-     * @returns Whether the session was retired.
-     */
-    retireOnBlockedStatusCodes(statusCode) {
-        // eslint-disable-next-line dot-notation -- accessing private property
-        const isBlocked = this.sessionPool['blockedStatusCodes'].includes(statusCode);
-        if (isBlocked) {
-            this.retire();
-        }
-        return isBlocked;
-    }
-    /**
-     * Saves cookies from an HTTP response to be used with the session.
-     * It expects an object with a `headers` property that's either an `Object`
-     * (typical Node.js responses) or a `Function` (Puppeteer Response).
-     *
-     * It then parses and saves the cookies from the `set-cookie` header, if available.
-     */
-    setCookiesFromResponse(response) {
-        try {
-            const cookies = getCookiesFromResponse(response).filter((c) => c);
-            this._setCookies(cookies, typeof response.url === 'function' ? response.url() : response.url);
-        }
-        catch (e) {
-            const err = e;
-            // if invalid Cookie header is provided just log the exception.
-            this.log.exception(err, 'Could not get cookies from response');
-        }
-    }
-    /**
-     * Saves an array with cookie objects to be used with the session.
-     * The objects should be in the format that
-     * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
-     * but you can also use this function to set cookies manually:
-     *
-     * ```
-     * [
-     *   { name: 'cookie1', value: 'my-cookie' },
-     *   { name: 'cookie2', value: 'your-cookie' }
-     * ]
-     * ```
-     */
-    setCookies(cookies, url) {
-        const normalizedCookies = cookies.map((c) => browserPoolCookieToToughCookie(c, this.maxAgeSecs));
-        this._setCookies(normalizedCookies, url);
-    }
-    /**
-     * Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
-     * @param url website url. Only cookies stored for this url will be returned
-     */
-    getCookies(url) {
-        const cookies = this.cookieJar.getCookiesSync(url);
-        return cookies.map((c) => toughCookieToBrowserPoolCookie(c));
+        this.#errorScore += 1;
+        this.#usageCount += 1;
+        this.maybeSelfRetire();
     }
     /**
      * Returns cookies saved with the session in the typical
@@ -231,41 +193,26 @@ export class Session {
      * a cookie header or elsewhere.
      * @returns Represents `Cookie` header.
      */
-    getCookieString(url) {
-        return this.cookieJar.getCookieStringSync(url, {});
+    async getCookieString(url) {
+        return this.cookieJar.getCookieString(url, {});
     }
     /**
      * Sets a cookie within this session for the specific URL.
      */
-    setCookie(rawCookie, url) {
-        this.cookieJar.setCookieSync(rawCookie, url);
-    }
-    /**
-     * Sets cookies.
-     */
-    _setCookies(cookies, url) {
-        const errorMessages = [];
-        for (const cookie of cookies) {
-            try {
-                this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });
-            }
-            catch (e) {
-                const err = e;
-                errorMessages.push(err.message);
-            }
+    async setCookie(rawCookie, url) {
+        try {
+            await this.cookieJar.setCookie(rawCookie, url);
         }
-        // if invalid cookies are provided just log the exception. No need to retry the request automatically.
-        if (errorMessages.length) {
-            this.log.debug('Could not set cookies.', { errorMessages });
+        catch (e) {
+            this.#log.warning('Could not set cookie.', { url, error: e.message });
         }
     }
     /**
      * Checks if session is not usable. if it is not retires the session.
      */
-    _maybeSelfRetire() {
+    maybeSelfRetire() {
         if (!this.isUsable()) {
             this.retire();
         }
     }
 }
-//# sourceMappingURL=session.js.map

package/session_pool/session_pool.d.ts

@@ -1,24 +1,28 @@
-import { EventEmitter } from 'node:events';
-import type { Log } from '@apify/log';
-import { Configuration } from '../configuration.js';
+import type { ISessionPool } from '@crawlee/types';
 import type { PersistenceOptions } from '../crawlers/statistics.js';
-import type { EventManager } from '../events/event_manager.js';
-import { KeyValueStore } from '../storages/key_value_store.js';
+import type { CrawleeLogger } from '../log.js';
 import type { SessionOptions } from './session.js';
 import { Session } from './session.js';
+declare const SESSION_REUSE_STRATEGIES: readonly ["random", "round-robin", "use-until-failure"];
+export type SessionReuseStrategy = (typeof SESSION_REUSE_STRATEGIES)[number];
 /**
  * Factory user-function which creates customized {@link Session} instances.
  */
 export interface CreateSession {
     /**
-     * @param sessionPool Pool requesting the new session.
-     * @param options
+     * @param options.sessionOptions Per-call session options already merged with the pool-wide defaults.
      */
-    (sessionPool: SessionPool, options?: {
+    (options?: {
         sessionOptions?: SessionOptions;
     }): Session | Promise<Session>;
 }
 export interface SessionPoolOptions {
+    /**
+     * Unique identifier for this session pool instance. Used to generate a unique
+     * persistence key when `persistStateKey` is not provided.
+     * If not provided, an auto-incrementing ID is used.
+     */
+    id?: string | number;
     /**
      * Maximum size of the pool. Indicates how many sessions are rotated.
      * @default 1000
@@ -29,24 +33,26 @@ export interface SessionPoolOptions {
     /** Name or Id of `KeyValueStore` where is the `SessionPool` state stored. */
     persistStateKeyValueStoreId?: string;
     /**
-     * Session pool persists it's state under this key in Key value store.
-     * @default SESSION_POOL_STATE
+     * Session pool persists its state under this key in Key value store.
+     * @default CRAWLEE_SESSION_POOL_STATE_{id}
      */
     persistStateKey?: string;
     /**
-     * Custom function that should return `Session` instance.
-     * Any error thrown from this function will terminate the process.
-     * Function receives `SessionPool` instance as a parameter
+     * Custom function that should return a `Session` instance, or a promise resolving to such instance.
+     * Any error thrown from this function will terminate the process. Receives `{ sessionOptions }`
+     * already merged from the pool-wide defaults and the per-call overrides.
      */
     createSessionFunction?: CreateSession;
     /**
-     * Specifies which response status codes are considered as blocked.
-     * Session connected to such request will be marked as retired.
-     * @default [401, 403, 429]
+     * Strategy for picking sessions from the pool.
+     * - `'random'` (default): fills the pool up to `maxPoolSize`, then picks a random usable session
+     * - `'round-robin'`: fills the pool up to `maxPoolSize`, then reuses sessions cycling through them in order
+     * - `'use-until-failure'`: always reuses the same session until it is retired, then moves to the next one
+     * @default 'random'
      */
-    blockedStatusCodes?: number[];
+    sessionReuseStrategy?: SessionReuseStrategy;
     /** @internal */
-    log?: Log;
+    log?: CrawleeLogger;
     /**
      * Control how and when to persist the state of the session pool.
      */
@@ -58,20 +64,8 @@ export interface SessionPoolOptions {
  * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
  * Learn more in the {@doclink guides/session-management | Session management guide}.
  *
- * You can create one by calling the {@link SessionPool.open} function.
- *
- * Session pool is already integrated into crawlers, and it can significantly improve your scraper
- * performance with just 2 lines of code.
- *
- * **Example usage:**
- *
- * ```javascript
- * const crawler = new CheerioCrawler({
- *     useSessionPool: true,
- *     persistCookiesPerSession: true,
- *     // ...
- * })
- * ```
+ * Session pool is already integrated into crawlers and is always active.
+ * All public methods are lazy-initialized — the pool initializes itself on first use.
  *
  * You can configure the pool with many options. See the {@link SessionPoolOptions}.
  * Session pool is by default persisted in default {@link KeyValueStore}.
@@ -81,7 +75,7 @@ export interface SessionPoolOptions {
  * **Advanced usage:**
  *
  * ```javascript
- * const sessionPool = await SessionPool.open({
+ * const sessionPool = new SessionPool({
  *     maxPoolSize: 25,
  *     sessionOptions:{
  *          maxAgeSecs: 10,
@@ -117,40 +111,31 @@ export interface SessionPoolOptions {
  *
  * @category Scaling
  */
-export declare class SessionPool extends EventEmitter {
-    readonly config: Configuration;
-    protected log: Log;
-    protected maxPoolSize: number;
-    protected createSessionFunction: CreateSession;
-    protected keyValueStore: KeyValueStore;
-    protected sessions: Session[];
-    protected sessionMap: Map<string, Session>;
-    protected sessionOptions: SessionOptions;
-    protected persistStateKeyValueStoreId?: string;
-    protected persistStateKey: string;
-    protected _listener: () => Promise<void>;
-    protected events: EventManager;
-    protected readonly blockedStatusCodes: number[];
-    protected persistenceOptions: PersistenceOptions;
-    protected isInitialized: boolean;
-    private queue;
-    /**
-     * @internal
-     */
-    constructor(options?: SessionPoolOptions, config?: Configuration);
+export declare class SessionPool implements ISessionPool {
+    #private;
+    readonly id: string;
+    private maxPoolSize;
+    private createSessionFunction;
+    private keyValueStore?;
+    private sessionMap;
+    private sessionOptions;
+    private persistStateKeyValueStoreId?;
+    private persistStateKey;
+    constructor(options?: SessionPoolOptions);
     /**
      * Gets count of usable sessions in the pool.
      */
-    get usableSessionsCount(): number;
+    usableSessionsCount(): Promise<number>;
     /**
      * Gets count of retired sessions in the pool.
      */
-    get retiredSessionsCount(): number;
+    retiredSessionsCount(): Promise<number>;
     /**
      * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
-     * It is called automatically by the {@link SessionPool.open} function.
+     * Called automatically on first use of any public method.
      */
-    initialize(): Promise<void>;
+    private ensureInitialized;
+    private setupPool;
     /**
      * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
      * but this allows you to add more sessions once the max pool size is reached.
@@ -158,17 +143,21 @@ export declare class SessionPool extends EventEmitter {
      * @param [options] The configuration options for the session being added to the session pool.
      */
     addSession(options?: Session | SessionOptions): Promise<void>;
+    /**
+     * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
+     * but this allows you to add more sessions once the max pool size is reached.
+     * This also allows you to add session with overridden session options (e.g. with specific session id).
+     * @param [options] The configuration options for the session being added to the session pool.
+     */
+    newSession(sessionOptions?: SessionOptions): Promise<Session>;
     /**
      * Gets session.
      * If there is space for new session, it creates and returns new session.
      * If the session pool is full, it picks a session from the pool,
      * If the picked session is usable it is returned, otherwise it creates and returns a new one.
+     * @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
      */
-    getSession(): Promise<Session>;
-    /**
-     * Gets session based on the provided session id or `undefined.
-     */
-    getSession(sessionId: string): Promise<Session>;
+    getSession(sessionId?: string): Promise<Session | undefined>;
     /**
      * @param options - Override the persistence options provided in the constructor
      */
@@ -177,12 +166,12 @@ export declare class SessionPool extends EventEmitter {
      * Returns an object representing the internal state of the `SessionPool` instance.
      * Note that the object's fields can change in future releases.
      */
-    getState(): {
+    getState(): Promise<{
         usableSessionsCount: number;
         retiredSessionsCount: number;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        sessions: import("./session.js").SessionState[];
-    };
+        sessions: import("@crawlee/types").SessionState[];
+    }>;
     /**
      * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
      * The state is persisted automatically in regular intervals.
@@ -194,58 +183,55 @@ export declare class SessionPool extends EventEmitter {
      * This function should be called after you are done with using the `SessionPool` instance.
      */
     teardown(): Promise<void>;
-    /**
-     * SessionPool should not work before initialization.
-     */
-    protected _throwIfNotInitialized(): void;
     /**
      * Removes retired `Session` instances from `SessionPool`.
      */
-    protected _removeRetiredSessions(): void;
+    private removeRetiredSessions;
     /**
      * Adds `Session` instance to `SessionPool`.
      * @param newSession `Session` instance to be added.
      */
-    protected _addSession(newSession: Session): void;
+    private registerSession;
     /**
      * Gets random index.
      */
-    protected _getRandomIndex(): number;
+    private getRandomIndex;
     /**
      * Creates new session without any extra behavior.
-     * @param sessionPool
      * @param [options]
      * @param [options.sessionOptions] The configuration options for the session being created.
      * @returns New session.
      */
-    protected _defaultCreateSessionFunction(sessionPool: SessionPool, options?: {
-        sessionOptions?: SessionOptions;
-    }): Session;
+    private defaultCreateSessionFunction;
+    /**
+     * Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
+     * the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
+     *
+     * A default {@link SessionFingerprint} is generated up front (host OS as
+     * `platform`, a random valid `browser`/`device` for that platform). Pool-wide
+     * and per-call options override it, and a persisted fingerprint coming
+     * through `maybeLoadSessionPool` naturally wins because it arrives in
+     * `perCallOptions`.
+     */
+    private invokeCreateSessionFunction;
     /**
      * Creates new session and adds it to the pool.
      * @returns Newly created `Session` instance.
      */
-    protected _createSession(): Promise<Session>;
+    private createSession;
     /**
      * Decides whether there is enough space for creating new session.
      */
-    protected _hasSpaceForSession(): boolean;
+    private hasSpaceForSession;
     /**
-     * Picks random session from the `SessionPool`.
-     * @returns Picked `Session`.
+     * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
+     * Returns `undefined` when no session should be reused and a new one should be created instead.
      */
-    protected _pickSession(): Session;
+    private pickSession;
     /**
      * Potentially loads `SessionPool`.
      * If the state was persisted it loads the `SessionPool` from the persisted state.
      */
-    protected _maybeLoadSessionPool(): Promise<void>;
-    /**
-     * Opens a SessionPool and returns a promise resolving to an instance
-     * of the {@link SessionPool} class that is already initialized.
-     *
-     * For more details and code examples, see the {@link SessionPool} class.
-     */
-    static open(options?: SessionPoolOptions, config?: Configuration): Promise<SessionPool>;
+    private maybeLoadSessionPool;
 }
-//# sourceMappingURL=session_pool.d.ts.map
+export {};