@crawlee/core 3.0.0-alpha.2 → 3.0.0-alpha.22

package/session_pool/session.d.ts

@@ -0,0 +1,212 @@
+/// <reference types="node" />
+import { Log } from '@apify/log';
+import { Cookie as BrowserPoolCookie } from '@crawlee/browser-pool';
+import { IncomingMessage } from 'http';
+import { HTTPResponse, Protocol } from 'puppeteer';
+import { Cookie, CookieJar } from 'tough-cookie';
+import { Dictionary } from '../typedefs';
+import { SessionPool } from './session_pool';
+export declare type PuppeteerCookie = Protocol.Network.Cookie | BrowserPoolCookie;
+/**
+ * Persistable {@link Session} state.
+ */
+export interface SessionState {
+    id: string;
+    cookieJar: CookieJar.Serialized;
+    userData: object;
+    errorScore: number;
+    maxErrorScore: number;
+    errorScoreDecrement: number;
+    usageCount: number;
+    maxUsageCount: number;
+    expiresAt: string;
+    createdAt: string;
+}
+export interface SessionOptions {
+    /** Id of session used for generating fingerprints. It is used as proxy session name. */
+    id?: string;
+    /**
+     * Number of seconds after which the session is considered as expired.
+     * @default 3000
+     */
+    maxAgeSecs?: number;
+    /** Object where custom user data can be stored. For example custom headers. */
+    userData?: Dictionary;
+    /**
+     * Maximum number of marking session as blocked usage.
+     * If the `errorScore` reaches the `maxErrorScore` session is marked as block and it is thrown away.
+     * It starts at 0. Calling the `markBad` function increases the `errorScore` by 1.
+     * Calling the `markGood` will decrease the `errorScore` by `errorScoreDecrement`
+     * @default 3
+     */
+    maxErrorScore?: number;
+    /**
+     * It is used for healing the session.
+     * For example: if your session is marked bad two times, but it is successful on the third attempt it's errorScore
+     * is decremented by this number.
+     * @default 0.5
+     */
+    errorScoreDecrement?: number;
+    /** Date of creation. */
+    createdAt?: Date;
+    /** Date of expiration. */
+    expiresAt?: Date;
+    /**
+     * Indicates how many times the session has been used.
+     * @default 0
+     */
+    usageCount?: number;
+    /**
+     * Session should be used only a limited amount of times.
+     * This number indicates how many times the session is going to be used, before it is thrown away.
+     * @default 50
+     */
+    maxUsageCount?: number;
+    /** SessionPool instance. Session will emit the `sessionRetired` event on this instance. */
+    sessionPool?: SessionPool;
+    log?: Log;
+    errorScore?: number;
+    cookieJar?: CookieJar;
+}
+/**
+ * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
+ * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
+ * Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
+ * @category Scaling
+ */
+export declare class Session {
+    readonly id: string;
+    private maxAgeSecs;
+    userData: Dictionary;
+    private maxErrorScore;
+    private errorScoreDecrement;
+    private createdAt;
+    private expiresAt;
+    private usageCount;
+    private maxUsageCount;
+    private sessionPool;
+    private errorScore;
+    private cookieJar;
+    private log;
+    /**
+     * Session configuration.
+     */
+    constructor(options: SessionOptions);
+    /**
+     * Indicates whether the session is blocked.
+     * Session is blocked once it reaches the `maxErrorScore`.
+     */
+    isBlocked(): boolean;
+    /**
+     * Indicates whether the session is expired.
+     * Session expiration is determined by the `maxAgeSecs`.
+     * Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
+     */
+    isExpired(): boolean;
+    /**
+     * Indicates whether the session is used maximum number of times.
+     * Session maximum usage count can be changed by `maxUsageCount` parameter.
+     */
+    isMaxUsageCountReached(): boolean;
+    /**
+     * Indicates whether the session can be used for next requests.
+     * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
+     */
+    isUsable(): boolean;
+    /**
+     * This method should be called after a successful session usage.
+     * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
+     */
+    markGood(): void;
+    /**
+     * Gets session state for persistence in KeyValueStore.
+     * @returns Represents session internal state.
+     */
+    getState(): SessionState;
+    /**
+     * Marks session as blocked and emits event on the `SessionPool`
+     * This method should be used if the session usage was unsuccessful
+     * and you are sure that it is because of the session configuration and not any external matters.
+     * For example when server returns 403 status code.
+     * If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
+     */
+    retire(): void;
+    /**
+     * Increases usage and error count.
+     * Should be used when the session has been used unsuccessfully. For example because of timeouts.
+     */
+    markBad(): void;
+    /**
+     * With certain status codes: `401`, `403` or `429` we can be certain
+     * that the target website is blocking us. This function helps to do this conveniently
+     * by retiring the session when such code is received. Optionally the default status
+     * codes can be extended in the second parameter.
+     * @param statusCode HTTP status code.
+     * @param [blockedStatusCodes] Custom HTTP status codes that means blocking on particular website.
+     * @returns Whether the session was retired.
+     */
+    retireOnBlockedStatusCodes(statusCode: number, blockedStatusCodes?: number[]): boolean;
+    /**
+     * Saves cookies from an HTTP response to be used with the session.
+     * It expects an object with a `headers` property that's either an `Object`
+     * (typical Node.js responses) or a `Function` (Puppeteer Response).
+     *
+     * It then parses and saves the cookies from the `set-cookie` header, if available.
+     */
+    setCookiesFromResponse(response: IncomingMessage | HTTPResponse | {
+        headers: Dictionary<string | string[]>;
+        url: string;
+    }): void;
+    /**
+     * Saves an array with cookie objects to be used with the session.
+     * The objects should be in the format that
+     * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
+     * but you can also use this function to set cookies manually:
+     *
+     * ```
+     * [
+     *   { name: 'cookie1', value: 'my-cookie' },
+     *   { name: 'cookie2', value: 'your-cookie' }
+     * ]
+     * ```
+     */
+    setPuppeteerCookies(cookies: PuppeteerCookie[], url: string): void;
+    /**
+     * Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.
+     * @param url website url. Only cookies stored for this url will be returned
+     */
+    getPuppeteerCookies(url: string): PuppeteerCookie[];
+    /**
+     * Returns cookies saved with the session in the typical
+     * key1=value1; key2=value2 format, ready to be used in
+     * a cookie header or elsewhere.
+     * @returns Represents `Cookie` header.
+     */
+    getCookieString(url: string): string;
+    /**
+     * Transforms puppeteer cookie to tough-cookie.
+     * @param puppeteerCookie Cookie from puppeteer `page.cookies` method.
+     */
+    protected _puppeteerCookieToTough(puppeteerCookie: PuppeteerCookie): Cookie;
+    /**
+     * Transforms tough-cookie to puppeteer cookie.
+     * @param toughCookie Cookie from CookieJar
+     * @return Cookie from Puppeteer
+     */
+    protected _toughCookieToPuppeteer(toughCookie: Cookie): PuppeteerCookie;
+    /**
+     * Sets cookies.
+     */
+    protected _setCookies(cookies: Cookie[], url: string): void;
+    /**
+     * Calculate cookie expiration date
+     * @param maxAgeSecs
+     * @returns Calculated date by session max age seconds.
+     */
+    protected _getDefaultCookieExpirationDate(maxAgeSecs: number): Date;
+    /**
+     * Checks if session is not usable. if it is not retires the session.
+     */
+    protected _maybeSelfRetire(): void;
+}
+//# sourceMappingURL=session.d.ts.map

package/session_pool/session.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEjC,OAAO,EAAE,MAAM,IAAI,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,MAAM,MAAM,CAAC;AAEvC,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGjD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAG7C,oBAAY,eAAe,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC;AAK1E;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAE3B,wFAAwF;IACxF,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,wBAAwB;IACxB,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,2FAA2F;IAC3F,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,SAAS,CAAC;CAEzB;AAED;;;;;GAKG;AACH,qBAAa,OAAO;IAChB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,UAAU,CAAS;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,GAAG,CAAM;IAEjB;;OAEG;gBACS,OAAO,EAAE,cAAc;IAsDnC;;;OAGG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,QAAQ;IAUR;;;OAGG;IACH,QAAQ,IAAI,YAAY;IAexB;;;;;;OAMG;IACH,MAAM;IASN;;;OAGG;IACH,OAAO;IAOP;;;;;;;;OAQG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,EAAE,kBAAkB,GAAE,MAAM,EAAO,GAAG,OAAO;IAQ1F;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAQ,EAAE,eAAe,GAAG,YAAY,GAAG;QAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE;IAWzH;;;;;;;;;;;;OAYG;IACH,mBAAmB,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,GAAG,EAAE,MAAM;IAK3D;;;OAGG;IACH,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,EAAE;IAKnD;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIpC;;;OAGG;IACH,SAAS,CAAC,uBAAuB,CAAC,eAAe,EAAE,eAAe,GAAG,MAAM;IAkB3E;;;;OAIG;IACH,SAAS,CAAC,uBAAuB,CAAC,WAAW,EAAE,MAAM,GAAG,eAAe;IAavE;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAkB3D;;;;OAIG;IACH,SAAS,CAAC,+BAA+B,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAInE;;OAEG;IACH,SAAS,CAAC,gBAAgB,IAAI,IAAI;CAKrC"}

package/session_pool/session.js

@@ -0,0 +1,360 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Session = void 0;
+const tslib_1 = require("tslib");
+const utilities_1 = require("@apify/utilities");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const tough_cookie_1 = require("tough-cookie");
+const constants_1 = require("../constants");
+const log_1 = require("../log");
+const events_1 = require("./events");
+const session_pool_1 = require("./session_pool");
+const session_utils_1 = require("./session_utils");
+// CONSTANTS
+const DEFAULT_SESSION_MAX_AGE_SECS = 3000;
+/**
+ * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
+ * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
+ * Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
+ * @category Scaling
+ */
+class Session {
+    /**
+     * Session configuration.
+     */
+    constructor(options) {
+        Object.defineProperty(this, "id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxAgeSecs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "userData", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxErrorScore", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "errorScoreDecrement", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "createdAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "expiresAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "usageCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxUsageCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "sessionPool", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "errorScore", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "cookieJar", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "log", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        (0, ow_1.default)(options, ow_1.default.object.exactShape({
+            sessionPool: ow_1.default.object.instanceOf(session_pool_1.SessionPool),
+            id: ow_1.default.optional.string,
+            cookieJar: ow_1.default.optional.object,
+            maxAgeSecs: ow_1.default.optional.number,
+            userData: ow_1.default.optional.object,
+            maxErrorScore: ow_1.default.optional.number,
+            errorScoreDecrement: ow_1.default.optional.number,
+            createdAt: ow_1.default.optional.date,
+            expiresAt: ow_1.default.optional.date,
+            usageCount: ow_1.default.optional.number,
+            errorScore: ow_1.default.optional.number,
+            maxUsageCount: ow_1.default.optional.number,
+            log: ow_1.default.optional.object,
+        }));
+        const { sessionPool, id = `session_${(0, utilities_1.cryptoRandomObjectId)(10)}`, cookieJar = new tough_cookie_1.CookieJar(), maxAgeSecs = DEFAULT_SESSION_MAX_AGE_SECS, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, log = log_1.log, } = options;
+        const { expiresAt = this._getDefaultCookieExpirationDate(maxAgeSecs) } = options;
+        this.log = log.child({ prefix: 'Session' });
+        this.cookieJar = cookieJar.setCookie ? cookieJar : tough_cookie_1.CookieJar.fromJSON(JSON.stringify(cookieJar));
+        this.id = id;
+        this.maxAgeSecs = maxAgeSecs;
+        this.userData = userData;
+        this.maxErrorScore = maxErrorScore;
+        this.errorScoreDecrement = errorScoreDecrement;
+        // Internal
+        this.expiresAt = expiresAt;
+        this.createdAt = createdAt;
+        this.usageCount = usageCount; // indicates how many times the session has been used
+        this.errorScore = errorScore; // indicates number of markBaded request with the session
+        this.maxUsageCount = maxUsageCount;
+        this.sessionPool = sessionPool;
+        this._puppeteerCookieToTough = this._puppeteerCookieToTough.bind(this);
+    }
+    /**
+     * Indicates whether the session is blocked.
+     * Session is blocked once it reaches the `maxErrorScore`.
+     */
+    isBlocked() {
+        return this.errorScore >= this.maxErrorScore;
+    }
+    /**
+     * Indicates whether the session is expired.
+     * Session expiration is determined by the `maxAgeSecs`.
+     * Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
+     */
+    isExpired() {
+        return this.expiresAt <= new Date();
+    }
+    /**
+     * Indicates whether the session is used maximum number of times.
+     * Session maximum usage count can be changed by `maxUsageCount` parameter.
+     */
+    isMaxUsageCountReached() {
+        return this.usageCount >= this.maxUsageCount;
+    }
+    /**
+     * Indicates whether the session can be used for next requests.
+     * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
+     */
+    isUsable() {
+        return !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
+    }
+    /**
+     * This method should be called after a successful session usage.
+     * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
+     */
+    markGood() {
+        this.usageCount += 1;
+        if (this.errorScore > 0) {
+            this.errorScore -= this.errorScoreDecrement;
+        }
+        this._maybeSelfRetire();
+    }
+    /**
+     * Gets session state for persistence in KeyValueStore.
+     * @returns Represents session internal state.
+     */
+    getState() {
+        return {
+            id: this.id,
+            cookieJar: this.cookieJar.toJSON(),
+            userData: this.userData,
+            maxErrorScore: this.maxErrorScore,
+            errorScoreDecrement: this.errorScoreDecrement,
+            expiresAt: this.expiresAt.toISOString(),
+            createdAt: this.createdAt.toISOString(),
+            usageCount: this.usageCount,
+            maxUsageCount: this.maxUsageCount,
+            errorScore: this.errorScore,
+        };
+    }
+    /**
+     * Marks session as blocked and emits event on the `SessionPool`
+     * This method should be used if the session usage was unsuccessful
+     * and you are sure that it is because of the session configuration and not any external matters.
+     * For example when server returns 403 status code.
+     * If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
+     */
+    retire() {
+        // mark it as an invalid by increasing the error score count.
+        this.errorScore += this.maxErrorScore;
+        this.usageCount += 1;
+        // emit event so we can retire browser in puppeteer pool
+        this.sessionPool.emit(events_1.EVENT_SESSION_RETIRED, this);
+    }
+    /**
+     * Increases usage and error count.
+     * Should be used when the session has been used unsuccessfully. For example because of timeouts.
+     */
+    markBad() {
+        this.errorScore += 1;
+        this.usageCount += 1;
+        this._maybeSelfRetire();
+    }
+    /**
+     * With certain status codes: `401`, `403` or `429` we can be certain
+     * that the target website is blocking us. This function helps to do this conveniently
+     * by retiring the session when such code is received. Optionally the default status
+     * codes can be extended in the second parameter.
+     * @param statusCode HTTP status code.
+     * @param [blockedStatusCodes] Custom HTTP status codes that means blocking on particular website.
+     * @returns Whether the session was retired.
+     */
+    retireOnBlockedStatusCodes(statusCode, blockedStatusCodes = []) {
+        const isBlocked = constants_1.STATUS_CODES_BLOCKED.concat(blockedStatusCodes).includes(statusCode);
+        if (isBlocked) {
+            this.retire();
+        }
+        return isBlocked;
+    }
+    /**
+     * Saves cookies from an HTTP response to be used with the session.
+     * It expects an object with a `headers` property that's either an `Object`
+     * (typical Node.js responses) or a `Function` (Puppeteer Response).
+     *
+     * It then parses and saves the cookies from the `set-cookie` header, if available.
+     */
+    setCookiesFromResponse(response) {
+        try {
+            const cookies = (0, session_utils_1.getCookiesFromResponse)(response).filter((c) => c);
+            this._setCookies(cookies, typeof response.url === 'function' ? response.url() : response.url);
+        }
+        catch (e) {
+            const err = e;
+            // if invalid Cookie header is provided just log the exception.
+            this.log.exception(err, 'Could not get cookies from response');
+        }
+    }
+    /**
+     * Saves an array with cookie objects to be used with the session.
+     * The objects should be in the format that
+     * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
+     * but you can also use this function to set cookies manually:
+     *
+     * ```
+     * [
+     *   { name: 'cookie1', value: 'my-cookie' },
+     *   { name: 'cookie2', value: 'your-cookie' }
+     * ]
+     * ```
+     */
+    setPuppeteerCookies(cookies, url) {
+        const normalizedCookies = cookies.map(this._puppeteerCookieToTough);
+        this._setCookies(normalizedCookies, url);
+    }
+    /**
+     * Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.
+     * @param url website url. Only cookies stored for this url will be returned
+     */
+    getPuppeteerCookies(url) {
+        const cookies = this.cookieJar.getCookiesSync(url);
+        return cookies.map(this._toughCookieToPuppeteer);
+    }
+    /**
+     * Returns cookies saved with the session in the typical
+     * key1=value1; key2=value2 format, ready to be used in
+     * a cookie header or elsewhere.
+     * @returns Represents `Cookie` header.
+     */
+    getCookieString(url) {
+        return this.cookieJar.getCookieStringSync(url, {});
+    }
+    /**
+     * Transforms puppeteer cookie to tough-cookie.
+     * @param puppeteerCookie Cookie from puppeteer `page.cookies` method.
+     */
+    _puppeteerCookieToTough(puppeteerCookie) {
+        const isExpiresValid = puppeteerCookie.expires && typeof puppeteerCookie.expires === 'number' && puppeteerCookie.expires > 0;
+        const expires = isExpiresValid ? new Date(puppeteerCookie.expires * 1000) : this._getDefaultCookieExpirationDate(this.maxAgeSecs);
+        const domain = typeof puppeteerCookie.domain === 'string' && puppeteerCookie.domain.startsWith('.')
+            ? puppeteerCookie.domain.slice(1)
+            : puppeteerCookie.domain;
+        return new tough_cookie_1.Cookie({
+            key: puppeteerCookie.name,
+            value: puppeteerCookie.value,
+            expires,
+            domain,
+            path: puppeteerCookie.path,
+            secure: puppeteerCookie.secure,
+            httpOnly: puppeteerCookie.httpOnly,
+        });
+    }
+    /**
+     * Transforms tough-cookie to puppeteer cookie.
+     * @param toughCookie Cookie from CookieJar
+     * @return Cookie from Puppeteer
+     */
+    _toughCookieToPuppeteer(toughCookie) {
+        return {
+            name: toughCookie.key,
+            value: toughCookie.value,
+            // Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms
+            expires: new Date(toughCookie.expires).getTime() / 1000,
+            domain: toughCookie.domain,
+            path: toughCookie.path,
+            secure: toughCookie.secure,
+            httpOnly: toughCookie.httpOnly,
+        };
+    }
+    /**
+     * Sets cookies.
+     */
+    _setCookies(cookies, url) {
+        const errorMessages = [];
+        for (const cookie of cookies) {
+            try {
+                this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });
+            }
+            catch (e) {
+                const err = e;
+                errorMessages.push(err.message);
+            }
+        }
+        // if invalid cookies are provided just log the exception. No need to retry the request automatically.
+        if (errorMessages.length) {
+            this.log.debug('Could not set cookies.', { errorMessages });
+        }
+    }
+    /**
+     * Calculate cookie expiration date
+     * @param maxAgeSecs
+     * @returns Calculated date by session max age seconds.
+     */
+    _getDefaultCookieExpirationDate(maxAgeSecs) {
+        return new Date(Date.now() + (maxAgeSecs * 1000));
+    }
+    /**
+     * Checks if session is not usable. if it is not retires the session.
+     */
+    _maybeSelfRetire() {
+        if (!this.isUsable()) {
+            this.retire();
+        }
+    }
+}
+exports.Session = Session;
+//# sourceMappingURL=session.js.map

package/session_pool/session.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";;;;AACA,gDAAwD;AAGxD,oDAAoB;AAEpB,+CAAiD;AACjD,4CAAoD;AACpD,gCAA2C;AAE3C,qCAAiD;AACjD,iDAA6C;AAC7C,mDAAyD;AAIzD,YAAY;AACZ,MAAM,4BAA4B,GAAG,IAAI,CAAC;AA6E1C;;;;;GAKG;AACH,MAAa,OAAO;IAehB;;OAEG;IACH,YAAY,OAAuB;QAjBnC;;;;;WAAoB;QACpB;;;;;WAA2B;QAC3B;;;;;WAAqB;QACrB;;;;;WAA8B;QAC9B;;;;;WAAoC;QACpC;;;;;WAAwB;QACxB;;;;;WAAwB;QACxB;;;;;WAA2B;QAC3B;;;;;WAA8B;QAC9B;;;;;WAAiC;QACjC;;;;;WAA2B;QAC3B;;;;;WAA6B;QAC7B;;;;;WAAiB;QAMb,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,0BAAW,CAAC;YAC9C,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,WAAW,EACX,EAAE,GAAG,WAAW,IAAA,gCAAoB,EAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,wBAAS,EAAE,EAC3B,UAAU,GAAG,4BAA4B,EACzC,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,GAAG,GAAG,SAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC,+BAA+B,CAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAEjF,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,wBAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC5G,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,WAAW;QACX,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACnF,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACvF,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3E,CAAC;IAED;;;OAGG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;YACrB,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,mBAAmB,CAAC;SAC/C;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE;YAClC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,6DAA6D;QAC7D,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;QACtC,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,wDAAwD;QACxD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,8BAAqB,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACrB,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;;OAQG;IACH,0BAA0B,CAAC,UAAkB,EAAE,qBAA+B,EAAE;QAC5E,MAAM,SAAS,GAAG,gCAAoB,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACvF,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAkG;QACrH,IAAI;YACA,MAAM,OAAO,GAAG,IAAA,sCAAsB,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,QAAQ,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAI,CAAC,CAAC;SAClG;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,+DAA+D;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,qCAAqC,CAAC,CAAC;SAClE;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,mBAAmB,CAAC,OAA0B,EAAE,GAAW;QACvD,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACpE,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,GAAW;QAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACrD,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACO,uBAAuB,CAAC,eAAgC;QAC9D,MAAM,cAAc,GAAG,eAAe,CAAC,OAAO,IAAI,OAAO,eAAe,CAAC,OAAO,KAAK,QAAQ,IAAI,eAAe,CAAC,OAAO,GAAG,CAAC,CAAC;QAC7H,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,OAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACnI,MAAM,MAAM,GAAG,OAAO,eAAe,CAAC,MAAM,KAAK,QAAQ,IAAI,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC;YAC/F,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACjC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC;QAE7B,OAAO,IAAI,qBAAM,CAAC;YACd,GAAG,EAAE,eAAe,CAAC,IAAI;YACzB,KAAK,EAAE,eAAe,CAAC,KAAK;YAC5B,OAAO;YACP,MAAM;YACN,IAAI,EAAE,eAAe,CAAC,IAAI;YAC1B,MAAM,EAAE,eAAe,CAAC,MAAM;YAC9B,QAAQ,EAAE,eAAe,CAAC,QAAQ;SACrC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACO,uBAAuB,CAAC,WAAmB;QACjD,OAAO;YACH,IAAI,EAAE,WAAW,CAAC,GAAG;YACrB,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,iFAAiF;YACjF,OAAO,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI;YACvD,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;SACd,CAAC;IACzB,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,OAAiB,EAAE,GAAW;QAChD,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;YAC1B,IAAI;gBACA,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;aACrE;YAAC,OAAO,CAAC,EAAE;gBACR,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;aACnC;SACJ;QAED,sGAAsG;QACtG,IAAI,aAAa,CAAC,MAAM,EAAE;YACtB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;SAC/D;IACL,CAAC;IAED;;;;OAIG;IACO,+BAA+B,CAAC,UAAkB;QACxD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE;YAClB,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;IACL,CAAC;CACJ;AA3TD,0BA2TC"}