@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.130

package/session_pool/session.d.ts

@@ -1,24 +1,6 @@
-import type { Cookie as CookieObject, Dictionary } from '@crawlee/types';
-import type { Cookie, SerializedCookieJar } from 'tough-cookie';
+import type { Dictionary, ISession, ProxyInfo, SessionFingerprint, SessionState } from '@crawlee/types';
 import { CookieJar } from 'tough-cookie';
-import type { Log } from '@apify/log';
-import type { ProxyInfo } from '../proxy_configuration.js';
-/**
- * Persistable {@link Session} state.
- */
-export interface SessionState {
-    id: string;
-    cookieJar: SerializedCookieJar;
-    proxyInfo?: ProxyInfo;
-    userData: object;
-    errorScore: number;
-    maxErrorScore: number;
-    errorScoreDecrement: number;
-    usageCount: number;
-    maxUsageCount: number;
-    expiresAt: string;
-    createdAt: string;
-}
+import type { CrawleeLogger } from '../log.js';
 export interface SessionOptions {
     /** Id of session used for generating fingerprints. It is used as proxy session name. */
     id?: string;
@@ -59,12 +41,22 @@ export interface SessionOptions {
      * @default 50
      */
     maxUsageCount?: number;
-    /** SessionPool instance. Session will emit the `sessionRetired` event on this instance. */
-    sessionPool?: import('./session_pool.js').SessionPool;
-    log?: Log;
+    /**
+     * Marks the session as already retired. Used when restoring a previously persisted session
+     * so that `isUsable()` reflects the terminal state regardless of error score or usage count.
+     * @default false
+     */
+    retired?: boolean;
+    log?: CrawleeLogger;
     errorScore?: number;
     cookieJar?: CookieJar;
     proxyInfo?: ProxyInfo;
+    /**
+     * Browser / HTTP client fingerprint tied to this session. Backends use this to make
+     * repeated requests with the same session look consistent (same user-agent, headers,
+     * TLS profile). See {@link SessionFingerprint}.
+     */
+    fingerprint?: SessionFingerprint;
 }
 /**
  * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
@@ -72,21 +64,10 @@ export interface SessionOptions {
  * Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
  * @category Scaling
  */
-export declare class Session {
+export declare class Session implements ISession {
+    #private;
     readonly id: string;
-    private maxAgeSecs;
-    userData: Dictionary;
-    private _maxErrorScore;
-    private _errorScoreDecrement;
-    private _createdAt;
-    private _expiresAt;
-    private _usageCount;
-    private _maxUsageCount;
-    private sessionPool;
-    private _errorScore;
-    private _proxyInfo?;
-    private _cookieJar;
-    private log;
+    readonly userData: Dictionary;
     get errorScore(): number;
     get usageCount(): number;
     get maxErrorScore(): number;
@@ -96,10 +77,17 @@ export declare class Session {
     get maxUsageCount(): number;
     get cookieJar(): CookieJar;
     get proxyInfo(): ProxyInfo | undefined;
+    get fingerprint(): SessionFingerprint | undefined;
+    set fingerprint(fingerprint: SessionFingerprint | undefined);
+    /**
+     * `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
+     * a retired session is never picked by the pool and cannot be revived via `markGood()`.
+     */
+    get retired(): boolean;
     /**
      * Session configuration.
      */
-    constructor(options: SessionOptions);
+    constructor(options?: SessionOptions);
     /**
      * Indicates whether the session is blocked.
      * Session is blocked once it reaches the `maxErrorScore`.
@@ -118,7 +106,7 @@ export declare class Session {
     isMaxUsageCountReached(): boolean;
     /**
      * Indicates whether the session can be used for next requests.
-     * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
+     * Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
      */
     isUsable(): boolean;
     /**
@@ -132,11 +120,11 @@ export declare class Session {
      */
     getState(): SessionState;
     /**
-     * Marks session as blocked and emits event on the `SessionPool`
-     * This method should be used if the session usage was unsuccessful
-     * and you are sure that it is because of the session configuration and not any external matters.
-     * For example when server returns 403 status code.
-     * If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
+     * Permanently retires the session — `isUsable()` will return `false` from here on,
+     * and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
+     *
+     * Use this when you're confident the session itself is the problem (e.g. a `403` response).
+     * For transient external failures (such as `5XX` responses), use `markBad()` instead.
      */
     retire(): void;
     /**
@@ -144,60 +132,19 @@ export declare class Session {
      * Should be used when the session has been used unsuccessfully. For example because of timeouts.
      */
     markBad(): void;
-    /**
-     * With certain status codes: `401`, `403` or `429` we can be certain
-     * that the target website is blocking us. This function helps to do this conveniently
-     * by retiring the session when such code is received. Optionally, the default status
-     * codes can be extended in the second parameter.
-     * @param statusCode HTTP status code.
-     * @returns Whether the session was retired.
-     */
-    retireOnBlockedStatusCodes(statusCode: number): boolean;
-    /**
-     * Saves cookies from an HTTP response to be used with the session.
-     * It expects an object with a `headers` property that's either an `Object`
-     * (typical Node.js responses) or a `Function` (Puppeteer Response).
-     *
-     * It then parses and saves the cookies from the `set-cookie` header, if available.
-     */
-    setCookiesFromResponse(response: Response): void;
-    /**
-     * Saves an array with cookie objects to be used with the session.
-     * The objects should be in the format that
-     * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
-     * but you can also use this function to set cookies manually:
-     *
-     * ```
-     * [
-     *   { name: 'cookie1', value: 'my-cookie' },
-     *   { name: 'cookie2', value: 'your-cookie' }
-     * ]
-     * ```
-     */
-    setCookies(cookies: CookieObject[], url: string): void;
-    /**
-     * Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
-     * @param url website url. Only cookies stored for this url will be returned
-     */
-    getCookies(url: string): CookieObject[];
     /**
      * Returns cookies saved with the session in the typical
      * key1=value1; key2=value2 format, ready to be used in
      * a cookie header or elsewhere.
      * @returns Represents `Cookie` header.
      */
-    getCookieString(url: string): string;
+    getCookieString(url: string): Promise<string>;
     /**
      * Sets a cookie within this session for the specific URL.
      */
-    setCookie(rawCookie: string, url: string): void;
-    /**
-     * Sets cookies.
-     */
-    protected _setCookies(cookies: Cookie[], url: string): void;
+    setCookie(rawCookie: string, url: string): Promise<void>;
     /**
      * Checks if session is not usable. if it is not retires the session.
      */
-    protected _maybeSelfRetire(): void;
+    private maybeSelfRetire;
 }
-//# sourceMappingURL=session.d.ts.map

package/session_pool/session.js

@@ -1,10 +1,28 @@
-import { EventEmitter } from 'node:events';
-import ow from 'ow';
 import { CookieJar } from 'tough-cookie';
+import { z } from 'zod';
 import { cryptoRandomObjectId } from '@apify/utilities';
-import { browserPoolCookieToToughCookie, getCookiesFromResponse, getDefaultCookieExpirationDate, toughCookieToBrowserPoolCookie, } from '../cookie_utils.js';
-import { log as defaultLog } from '../log.js';
-import { EVENT_SESSION_RETIRED } from './events.js';
+import { getDefaultCookieExpirationDate } from '../cookie_utils.js';
+import { serviceLocator } from '../service_locator.js';
+import { parseArgument, schemas, validators } from '../validators.js';
+// `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
+// copy), so class instances like cookie jars and loggers keep their prototype.
+const sessionOptionsSchema = z.strictObject({
+    id: z.string().default(() => `session_${cryptoRandomObjectId(10)}`),
+    cookieJar: schemas.anyObject.default(() => new CookieJar()),
+    proxyInfo: schemas.anyObject.optional(),
+    maxAgeSecs: schemas.anyNumber.default(3000),
+    userData: schemas.anyObject.default(() => ({})),
+    maxErrorScore: schemas.anyNumber.default(3),
+    errorScoreDecrement: schemas.anyNumber.default(0.5),
+    createdAt: z.date().default(() => new Date()),
+    expiresAt: z.date().optional(),
+    usageCount: schemas.anyNumber.default(0),
+    errorScore: schemas.anyNumber.default(0),
+    maxUsageCount: schemas.anyNumber.default(50),
+    retired: z.boolean().default(false),
+    log: validators.logger.default(() => serviceLocator.getLogger()),
+    fingerprint: schemas.anyObject.optional(),
+});
 /**
  * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
  * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
@@ -13,83 +31,79 @@ import { EVENT_SESSION_RETIRED } from './events.js';
  */
 export class Session {
     id;
-    maxAgeSecs;
     userData;
-    _maxErrorScore;
-    _errorScoreDecrement;
-    _createdAt;
-    _expiresAt;
-    _usageCount;
-    _maxUsageCount;
-    sessionPool;
-    _errorScore;
-    _proxyInfo;
-    _cookieJar;
-    log;
+    #maxErrorScore;
+    #errorScoreDecrement;
+    #createdAt;
+    #expiresAt;
+    #usageCount;
+    #maxUsageCount;
+    #errorScore;
+    #retired = false;
+    #proxyInfo;
+    #cookieJar;
+    #fingerprint;
+    #log;
     get errorScore() {
-        return this._errorScore;
+        return this.#errorScore;
     }
     get usageCount() {
-        return this._usageCount;
+        return this.#usageCount;
     }
     get maxErrorScore() {
-        return this._maxErrorScore;
+        return this.#maxErrorScore;
     }
     get errorScoreDecrement() {
-        return this._errorScoreDecrement;
+        return this.#errorScoreDecrement;
     }
     get expiresAt() {
-        return this._expiresAt;
+        return this.#expiresAt;
     }
     get createdAt() {
-        return this._createdAt;
+        return this.#createdAt;
     }
     get maxUsageCount() {
-        return this._maxUsageCount;
+        return this.#maxUsageCount;
     }
     get cookieJar() {
-        return this._cookieJar;
+        return this.#cookieJar;
     }
     get proxyInfo() {
-        return this._proxyInfo;
+        return this.#proxyInfo;
+    }
+    get fingerprint() {
+        return this.#fingerprint;
+    }
+    set fingerprint(fingerprint) {
+        this.#fingerprint = fingerprint;
+    }
+    /**
+     * `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
+     * a retired session is never picked by the pool and cannot be revived via `markGood()`.
+     */
+    get retired() {
+        return this.#retired;
     }
     /**
      * Session configuration.
      */
-    constructor(options) {
-        ow(options, ow.object.exactShape({
-            sessionPool: ow.object.instanceOf(EventEmitter),
-            id: ow.optional.string,
-            cookieJar: ow.optional.object,
-            proxyInfo: ow.optional.object,
-            maxAgeSecs: ow.optional.number,
-            userData: ow.optional.object,
-            maxErrorScore: ow.optional.number,
-            errorScoreDecrement: ow.optional.number,
-            createdAt: ow.optional.date,
-            expiresAt: ow.optional.date,
-            usageCount: ow.optional.number,
-            errorScore: ow.optional.number,
-            maxUsageCount: ow.optional.number,
-            log: ow.optional.object,
-        }));
-        const { sessionPool, id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, log = defaultLog, } = options;
-        const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
-        this.log = log.child({ prefix: 'Session' });
-        this._cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
-        this._proxyInfo = proxyInfo;
+    constructor(options = {}) {
+        const { id, cookieJar, proxyInfo, maxAgeSecs, userData, maxErrorScore, errorScoreDecrement, createdAt, usageCount, errorScore, maxUsageCount, retired, log, fingerprint, expiresAt = getDefaultCookieExpirationDate(maxAgeSecs), } = parseArgument(options, sessionOptionsSchema);
+        this.#log = log.child({ prefix: 'Session' });
+        this.#cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
+        this.#proxyInfo = proxyInfo;
+        this.#fingerprint = fingerprint;
         this.id = id;
-        this.maxAgeSecs = maxAgeSecs;
         this.userData = userData;
-        this._maxErrorScore = maxErrorScore;
-        this._errorScoreDecrement = errorScoreDecrement;
+        this.#maxErrorScore = maxErrorScore;
+        this.#errorScoreDecrement = errorScoreDecrement;
         // Internal
-        this._expiresAt = expiresAt;
-        this._createdAt = createdAt;
-        this._usageCount = usageCount; // indicates how many times the session has been used
-        this._errorScore = errorScore; // indicates number of markBaded request with the session
-        this._maxUsageCount = maxUsageCount;
-        this.sessionPool = sessionPool;
+        this.#expiresAt = expiresAt;
+        this.#createdAt = createdAt;
+        this.#usageCount = usageCount; // indicates how many times the session has been used
+        this.#errorScore = errorScore; // indicates number of markBaded request with the session
+        this.#maxUsageCount = maxUsageCount;
+        this.#retired = retired;
     }
     /**
      * Indicates whether the session is blocked.
@@ -115,21 +129,21 @@ export class Session {
     }
     /**
      * Indicates whether the session can be used for next requests.
-     * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
+     * Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
      */
     isUsable() {
-        return !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
+        return !this.#retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
     }
     /**
      * This method should be called after a successful session usage.
      * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
      */
     markGood() {
-        this._usageCount += 1;
-        if (this._errorScore > 0) {
-            this._errorScore -= this._errorScoreDecrement;
+        this.#usageCount += 1;
+        if (this.#errorScore > 0) {
+            this.#errorScore -= this.#errorScoreDecrement;
         }
-        this._maybeSelfRetire();
+        this.maybeSelfRetire();
     }
     /**
      * Gets session state for persistence in KeyValueStore.
@@ -139,8 +153,9 @@ export class Session {
         return {
             id: this.id,
             cookieJar: this.cookieJar.toJSON(),
-            proxyInfo: this._proxyInfo,
+            proxyInfo: this.#proxyInfo,
             userData: this.userData,
+            fingerprint: this.#fingerprint,
             maxErrorScore: this.maxErrorScore,
             errorScoreDecrement: this.errorScoreDecrement,
             expiresAt: this.expiresAt.toISOString(),
@@ -148,89 +163,31 @@ export class Session {
             usageCount: this.usageCount,
             maxUsageCount: this.maxUsageCount,
             errorScore: this.errorScore,
+            retired: this.#retired,
         };
     }
     /**
-     * Marks session as blocked and emits event on the `SessionPool`
-     * This method should be used if the session usage was unsuccessful
-     * and you are sure that it is because of the session configuration and not any external matters.
-     * For example when server returns 403 status code.
-     * If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
+     * Permanently retires the session — `isUsable()` will return `false` from here on,
+     * and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
+     *
+     * Use this when you're confident the session itself is the problem (e.g. a `403` response).
+     * For transient external failures (such as `5XX` responses), use `markBad()` instead.
      */
     retire() {
-        // mark it as an invalid by increasing the error score count.
-        this._errorScore += this._maxErrorScore;
-        this._usageCount += 1;
-        // emit event so we can retire browser in puppeteer pool
-        this.sessionPool.emit(EVENT_SESSION_RETIRED, this);
+        if (this.#retired)
+            return;
+        this.#errorScore += this.#maxErrorScore;
+        this.#usageCount += 1;
+        this.#retired = true;
     }
     /**
      * Increases usage and error count.
      * Should be used when the session has been used unsuccessfully. For example because of timeouts.
      */
     markBad() {
-        this._errorScore += 1;
-        this._usageCount += 1;
-        this._maybeSelfRetire();
-    }
-    /**
-     * With certain status codes: `401`, `403` or `429` we can be certain
-     * that the target website is blocking us. This function helps to do this conveniently
-     * by retiring the session when such code is received. Optionally, the default status
-     * codes can be extended in the second parameter.
-     * @param statusCode HTTP status code.
-     * @returns Whether the session was retired.
-     */
-    retireOnBlockedStatusCodes(statusCode) {
-        // eslint-disable-next-line dot-notation -- accessing private property
-        const isBlocked = this.sessionPool['blockedStatusCodes'].includes(statusCode);
-        if (isBlocked) {
-            this.retire();
-        }
-        return isBlocked;
-    }
-    /**
-     * Saves cookies from an HTTP response to be used with the session.
-     * It expects an object with a `headers` property that's either an `Object`
-     * (typical Node.js responses) or a `Function` (Puppeteer Response).
-     *
-     * It then parses and saves the cookies from the `set-cookie` header, if available.
-     */
-    setCookiesFromResponse(response) {
-        try {
-            const cookies = getCookiesFromResponse(response).filter((c) => c);
-            this._setCookies(cookies, response.url);
-        }
-        catch (e) {
-            const err = e;
-            // if invalid Cookie header is provided just log the exception.
-            this.log.exception(err, 'Could not get cookies from response');
-        }
-    }
-    /**
-     * Saves an array with cookie objects to be used with the session.
-     * The objects should be in the format that
-     * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
-     * but you can also use this function to set cookies manually:
-     *
-     * ```
-     * [
-     *   { name: 'cookie1', value: 'my-cookie' },
-     *   { name: 'cookie2', value: 'your-cookie' }
-     * ]
-     * ```
-     */
-    setCookies(cookies, url) {
-        const normalizedCookies = cookies.map((c) => browserPoolCookieToToughCookie(c, this.maxAgeSecs));
-        this._setCookies(normalizedCookies, url);
-    }
-    /**
-     * Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
-     * @param url website url. Only cookies stored for this url will be returned
-     */
-    getCookies(url) {
-        const cookies = this.cookieJar.getCookiesSync(url);
-        return cookies.map((c) => toughCookieToBrowserPoolCookie(c));
+        this.#errorScore += 1;
+        this.#usageCount += 1;
+        this.maybeSelfRetire();
     }
     /**
      * Returns cookies saved with the session in the typical
@@ -238,41 +195,26 @@ export class Session {
      * a cookie header or elsewhere.
      * @returns Represents `Cookie` header.
      */
-    getCookieString(url) {
-        return this.cookieJar.getCookieStringSync(url, {});
+    async getCookieString(url) {
+        return this.cookieJar.getCookieString(url, {});
     }
     /**
      * Sets a cookie within this session for the specific URL.
      */
-    setCookie(rawCookie, url) {
-        this.cookieJar.setCookieSync(rawCookie, url);
-    }
-    /**
-     * Sets cookies.
-     */
-    _setCookies(cookies, url) {
-        const errorMessages = [];
-        for (const cookie of cookies) {
-            try {
-                this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });
-            }
-            catch (e) {
-                const err = e;
-                errorMessages.push(err.message);
-            }
+    async setCookie(rawCookie, url) {
+        try {
+            await this.cookieJar.setCookie(rawCookie, url);
         }
-        // if invalid cookies are provided just log the exception. No need to retry the request automatically.
-        if (errorMessages.length) {
-            this.log.debug('Could not set cookies.', { errorMessages });
+        catch (e) {
+            this.#log.warning('Could not set cookie.', { url, error: e.message });
         }
     }
     /**
      * Checks if session is not usable. if it is not retires the session.
      */
-    _maybeSelfRetire() {
+    maybeSelfRetire() {
         if (!this.isUsable()) {
             this.retire();
         }
     }
 }
-//# sourceMappingURL=session.js.map