@crawlee/core 3.0.0-alpha.2 → 3.0.0-alpha.5

package/session_pool/session.js

@@ -0,0 +1,360 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Session = void 0;
+const tslib_1 = require("tslib");
+const utilities_1 = require("@apify/utilities");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const tough_cookie_1 = require("tough-cookie");
+const constants_1 = require("../constants");
+const log_1 = require("../log");
+const events_1 = require("./events");
+const session_pool_1 = require("./session_pool");
+const session_utils_1 = require("./session_utils");
+// CONSTANTS
+const DEFAULT_SESSION_MAX_AGE_SECS = 3000;
+/**
+ * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
+ * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
+ * Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
+ * @category Scaling
+ */
+class Session {
+    /**
+     * Session configuration.
+     */
+    constructor(options) {
+        Object.defineProperty(this, "id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxAgeSecs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "userData", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxErrorScore", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "errorScoreDecrement", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "createdAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "expiresAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "usageCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxUsageCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "sessionPool", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "errorScore", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "cookieJar", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "log", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        (0, ow_1.default)(options, ow_1.default.object.exactShape({
+            sessionPool: ow_1.default.object.instanceOf(session_pool_1.SessionPool),
+            id: ow_1.default.optional.string,
+            cookieJar: ow_1.default.optional.object,
+            maxAgeSecs: ow_1.default.optional.number,
+            userData: ow_1.default.optional.object,
+            maxErrorScore: ow_1.default.optional.number,
+            errorScoreDecrement: ow_1.default.optional.number,
+            createdAt: ow_1.default.optional.date,
+            expiresAt: ow_1.default.optional.date,
+            usageCount: ow_1.default.optional.number,
+            errorScore: ow_1.default.optional.number,
+            maxUsageCount: ow_1.default.optional.number,
+            log: ow_1.default.optional.object,
+        }));
+        const { sessionPool, id = `session_${(0, utilities_1.cryptoRandomObjectId)(10)}`, cookieJar = new tough_cookie_1.CookieJar(), maxAgeSecs = DEFAULT_SESSION_MAX_AGE_SECS, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, log = log_1.log, } = options;
+        const { expiresAt = this._getDefaultCookieExpirationDate(maxAgeSecs) } = options;
+        this.log = log.child({ prefix: 'Session' });
+        this.cookieJar = cookieJar.setCookie ? cookieJar : tough_cookie_1.CookieJar.fromJSON(JSON.stringify(cookieJar));
+        this.id = id;
+        this.maxAgeSecs = maxAgeSecs;
+        this.userData = userData;
+        this.maxErrorScore = maxErrorScore;
+        this.errorScoreDecrement = errorScoreDecrement;
+        // Internal
+        this.expiresAt = expiresAt;
+        this.createdAt = createdAt;
+        this.usageCount = usageCount; // indicates how many times the session has been used
+        this.errorScore = errorScore; // indicates number of markBaded request with the session
+        this.maxUsageCount = maxUsageCount;
+        this.sessionPool = sessionPool;
+        this._puppeteerCookieToTough = this._puppeteerCookieToTough.bind(this);
+    }
+    /**
+     * Indicates whether the session is blocked.
+     * Session is blocked once it reaches the `maxErrorScore`.
+     */
+    isBlocked() {
+        return this.errorScore >= this.maxErrorScore;
+    }
+    /**
+     * Indicates whether the session is expired.
+     * Session expiration is determined by the `maxAgeSecs`.
+     * Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
+     */
+    isExpired() {
+        return this.expiresAt <= new Date();
+    }
+    /**
+     * Indicates whether the session is used maximum number of times.
+     * Session maximum usage count can be changed by `maxUsageCount` parameter.
+     */
+    isMaxUsageCountReached() {
+        return this.usageCount >= this.maxUsageCount;
+    }
+    /**
+     * Indicates whether the session can be used for next requests.
+     * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
+     */
+    isUsable() {
+        return !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
+    }
+    /**
+     * This method should be called after a successful session usage.
+     * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
+     */
+    markGood() {
+        this.usageCount += 1;
+        if (this.errorScore > 0) {
+            this.errorScore -= this.errorScoreDecrement;
+        }
+        this._maybeSelfRetire();
+    }
+    /**
+     * Gets session state for persistence in KeyValueStore.
+     * @returns Represents session internal state.
+     */
+    getState() {
+        return {
+            id: this.id,
+            cookieJar: this.cookieJar.toJSON(),
+            userData: this.userData,
+            maxErrorScore: this.maxErrorScore,
+            errorScoreDecrement: this.errorScoreDecrement,
+            expiresAt: this.expiresAt.toISOString(),
+            createdAt: this.createdAt.toISOString(),
+            usageCount: this.usageCount,
+            maxUsageCount: this.maxUsageCount,
+            errorScore: this.errorScore,
+        };
+    }
+    /**
+     * Marks session as blocked and emits event on the `SessionPool`
+     * This method should be used if the session usage was unsuccessful
+     * and you are sure that it is because of the session configuration and not any external matters.
+     * For example when server returns 403 status code.
+     * If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
+     */
+    retire() {
+        // mark it as an invalid by increasing the error score count.
+        this.errorScore += this.maxErrorScore;
+        this.usageCount += 1;
+        // emit event so we can retire browser in puppeteer pool
+        this.sessionPool.emit(events_1.EVENT_SESSION_RETIRED, this);
+    }
+    /**
+     * Increases usage and error count.
+     * Should be used when the session has been used unsuccessfully. For example because of timeouts.
+     */
+    markBad() {
+        this.errorScore += 1;
+        this.usageCount += 1;
+        this._maybeSelfRetire();
+    }
+    /**
+     * With certain status codes: `401`, `403` or `429` we can be certain
+     * that the target website is blocking us. This function helps to do this conveniently
+     * by retiring the session when such code is received. Optionally the default status
+     * codes can be extended in the second parameter.
+     * @param statusCode HTTP status code.
+     * @param [blockedStatusCodes] Custom HTTP status codes that means blocking on particular website.
+     * @returns Whether the session was retired.
+     */
+    retireOnBlockedStatusCodes(statusCode, blockedStatusCodes = []) {
+        const isBlocked = constants_1.STATUS_CODES_BLOCKED.concat(blockedStatusCodes).includes(statusCode);
+        if (isBlocked) {
+            this.retire();
+        }
+        return isBlocked;
+    }
+    /**
+     * Saves cookies from an HTTP response to be used with the session.
+     * It expects an object with a `headers` property that's either an `Object`
+     * (typical Node.js responses) or a `Function` (Puppeteer Response).
+     *
+     * It then parses and saves the cookies from the `set-cookie` header, if available.
+     */
+    setCookiesFromResponse(response) {
+        try {
+            const cookies = (0, session_utils_1.getCookiesFromResponse)(response).filter((c) => c);
+            this._setCookies(cookies, typeof response.url === 'function' ? response.url() : response.url);
+        }
+        catch (e) {
+            const err = e;
+            // if invalid Cookie header is provided just log the exception.
+            this.log.exception(err, 'Could not get cookies from response');
+        }
+    }
+    /**
+     * Saves an array with cookie objects to be used with the session.
+     * The objects should be in the format that
+     * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
+     * but you can also use this function to set cookies manually:
+     *
+     * ```
+     * [
+     *   { name: 'cookie1', value: 'my-cookie' },
+     *   { name: 'cookie2', value: 'your-cookie' }
+     * ]
+     * ```
+     */
+    setPuppeteerCookies(cookies, url) {
+        const normalizedCookies = cookies.map(this._puppeteerCookieToTough);
+        this._setCookies(normalizedCookies, url);
+    }
+    /**
+     * Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.
+     * @param url website url. Only cookies stored for this url will be returned
+     */
+    getPuppeteerCookies(url) {
+        const cookies = this.cookieJar.getCookiesSync(url);
+        return cookies.map(this._toughCookieToPuppeteer);
+    }
+    /**
+     * Returns cookies saved with the session in the typical
+     * key1=value1; key2=value2 format, ready to be used in
+     * a cookie header or elsewhere.
+     * @returns Represents `Cookie` header.
+     */
+    getCookieString(url) {
+        return this.cookieJar.getCookieStringSync(url, {});
+    }
+    /**
+     * Transforms puppeteer cookie to tough-cookie.
+     * @param puppeteerCookie Cookie from puppeteer `page.cookies` method.
+     */
+    _puppeteerCookieToTough(puppeteerCookie) {
+        const isExpiresValid = puppeteerCookie.expires && typeof puppeteerCookie.expires === 'number' && puppeteerCookie.expires > 0;
+        const expires = isExpiresValid ? new Date(puppeteerCookie.expires * 1000) : this._getDefaultCookieExpirationDate(this.maxAgeSecs);
+        const domain = typeof puppeteerCookie.domain === 'string' && puppeteerCookie.domain.startsWith('.')
+            ? puppeteerCookie.domain.slice(1)
+            : puppeteerCookie.domain;
+        return new tough_cookie_1.Cookie({
+            key: puppeteerCookie.name,
+            value: puppeteerCookie.value,
+            expires,
+            domain,
+            path: puppeteerCookie.path,
+            secure: puppeteerCookie.secure,
+            httpOnly: puppeteerCookie.httpOnly,
+        });
+    }
+    /**
+     * Transforms tough-cookie to puppeteer cookie.
+     * @param toughCookie Cookie from CookieJar
+     * @return Cookie from Puppeteer
+     */
+    _toughCookieToPuppeteer(toughCookie) {
+        return {
+            name: toughCookie.key,
+            value: toughCookie.value,
+            // Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms
+            expires: new Date(toughCookie.expires).getTime() / 1000,
+            domain: toughCookie.domain,
+            path: toughCookie.path,
+            secure: toughCookie.secure,
+            httpOnly: toughCookie.httpOnly,
+        };
+    }
+    /**
+     * Sets cookies.
+     */
+    _setCookies(cookies, url) {
+        const errorMessages = [];
+        for (const cookie of cookies) {
+            try {
+                this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });
+            }
+            catch (e) {
+                const err = e;
+                errorMessages.push(err.message);
+            }
+        }
+        // if invalid cookies are provided just log the exception. No need to retry the request automatically.
+        if (errorMessages.length) {
+            this.log.debug('Could not set cookies.', { errorMessages });
+        }
+    }
+    /**
+     * Calculate cookie expiration date
+     * @param maxAgeSecs
+     * @returns Calculated date by session max age seconds.
+     */
+    _getDefaultCookieExpirationDate(maxAgeSecs) {
+        return new Date(Date.now() + (maxAgeSecs * 1000));
+    }
+    /**
+     * Checks if session is not usable. if it is not retires the session.
+     */
+    _maybeSelfRetire() {
+        if (!this.isUsable()) {
+            this.retire();
+        }
+    }
+}
+exports.Session = Session;
+//# sourceMappingURL=session.js.map

package/session_pool/session.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";;;;AACA,gDAAwD;AAGxD,oDAAoB;AAEpB,+CAAiD;AACjD,4CAAoD;AACpD,gCAA2C;AAE3C,qCAAiD;AACjD,iDAA6C;AAC7C,mDAAyD;AAIzD,YAAY;AACZ,MAAM,4BAA4B,GAAG,IAAI,CAAC;AA6E1C;;;;;GAKG;AACH,MAAa,OAAO;IAehB;;OAEG;IACH,YAAY,OAAuB;QAjBnC;;;;;WAAoB;QACpB;;;;;WAA2B;QAC3B;;;;;WAAqB;QACrB;;;;;WAA8B;QAC9B;;;;;WAAoC;QACpC;;;;;WAAwB;QACxB;;;;;WAAwB;QACxB;;;;;WAA2B;QAC3B;;;;;WAA8B;QAC9B;;;;;WAAiC;QACjC;;;;;WAA2B;QAC3B;;;;;WAA6B;QAC7B;;;;;WAAiB;QAMb,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,0BAAW,CAAC;YAC9C,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,WAAW,EACX,EAAE,GAAG,WAAW,IAAA,gCAAoB,EAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,wBAAS,EAAE,EAC3B,UAAU,GAAG,4BAA4B,EACzC,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,GAAG,GAAG,SAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC,+BAA+B,CAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAEjF,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,wBAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC5G,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,WAAW;QACX,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACnF,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACvF,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3E,CAAC;IAED;;;OAGG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;YACrB,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,mBAAmB,CAAC;SAC/C;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE;YAClC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,6DAA6D;QAC7D,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;QACtC,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,wDAAwD;QACxD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,8BAAqB,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACrB,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;;OAQG;IACH,0BAA0B,CAAC,UAAkB,EAAE,qBAA+B,EAAE;QAC5E,MAAM,SAAS,GAAG,gCAAoB,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACvF,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAkG;QACrH,IAAI;YACA,MAAM,OAAO,GAAG,IAAA,sCAAsB,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,QAAQ,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAI,CAAC,CAAC;SAClG;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,+DAA+D;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,qCAAqC,CAAC,CAAC;SAClE;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,mBAAmB,CAAC,OAA0B,EAAE,GAAW;QACvD,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACpE,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,GAAW;QAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACrD,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACO,uBAAuB,CAAC,eAAgC;QAC9D,MAAM,cAAc,GAAG,eAAe,CAAC,OAAO,IAAI,OAAO,eAAe,CAAC,OAAO,KAAK,QAAQ,IAAI,eAAe,CAAC,OAAO,GAAG,CAAC,CAAC;QAC7H,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,OAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACnI,MAAM,MAAM,GAAG,OAAO,eAAe,CAAC,MAAM,KAAK,QAAQ,IAAI,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC;YAC/F,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACjC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC;QAE7B,OAAO,IAAI,qBAAM,CAAC;YACd,GAAG,EAAE,eAAe,CAAC,IAAI;YACzB,KAAK,EAAE,eAAe,CAAC,KAAK;YAC5B,OAAO;YACP,MAAM;YACN,IAAI,EAAE,eAAe,CAAC,IAAI;YAC1B,MAAM,EAAE,eAAe,CAAC,MAAM;YAC9B,QAAQ,EAAE,eAAe,CAAC,QAAQ;SACrC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACO,uBAAuB,CAAC,WAAmB;QACjD,OAAO;YACH,IAAI,EAAE,WAAW,CAAC,GAAG;YACrB,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,iFAAiF;YACjF,OAAO,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI;YACvD,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;SACd,CAAC;IACzB,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,OAAiB,EAAE,GAAW;QAChD,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;YAC1B,IAAI;gBACA,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;aACrE;YAAC,OAAO,CAAC,EAAE;gBACR,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;aACnC;SACJ;QAED,sGAAsG;QACtG,IAAI,aAAa,CAAC,MAAM,EAAE;YACtB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;SAC/D;IACL,CAAC;IAED;;;;OAIG;IACO,+BAA+B,CAAC,UAAkB;QACxD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE;YAClB,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;IACL,CAAC;CACJ;AA3TD,0BA2TC"}

package/session_pool/session_pool.d.ts

@@ -0,0 +1,222 @@
+/// <reference types="node" />
+import { Log } from '@apify/log';
+import { EventEmitter } from 'events';
+import { Configuration } from '../configuration';
+import { KeyValueStore } from '../storages/key_value_store';
+import { Session, SessionOptions } from './session';
+/**
+ * Factory user-function which creates customized {@link Session} instances.
+ */
+export interface CreateSession {
+    /**
+     * @param sessionPool Pool requesting the new session.
+     * @param options
+     */
+    (sessionPool: SessionPool, options?: {
+        sessionOptions?: SessionOptions;
+    }): Session | Promise<Session>;
+}
+export interface SessionPoolOptions {
+    /**
+     * Maximum size of the pool. Indicates how many sessions are rotated.
+     * @default 1000
+     */
+    maxPoolSize?: number;
+    /** The configuration options for {@link Session} instances. */
+    sessionOptions?: SessionOptions;
+    /** Name or Id of `KeyValueStore` where is the `SessionPool` state stored. */
+    persistStateKeyValueStoreId?: string;
+    /**
+     * Session pool persists it's state under this key in Key value store.
+     * @default SESSION_POOL_STATE
+     */
+    persistStateKey?: string;
+    /**
+     * Custom function that should return `Session` instance.
+     * Any error thrown from this function will terminate the process.
+     * Function receives `SessionPool` instance as a parameter
+     */
+    createSessionFunction?: CreateSession;
+    /** @internal */
+    log?: Log;
+}
+/**
+ * Handles the rotation, creation and persistence of user-like sessions.
+ * Creates a pool of {@link Session} instances, that are randomly rotated.
+ * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
+ * Learn more in the [Session management guide](../../../docs/guides/session-management).
+ *
+ * You can create one by calling the {@link SessionPool.open} function.
+ *
+ * Session pool is already integrated into crawlers, and it can significantly improve your scraper
+ * performance with just 2 lines of code.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new CheerioCrawler({
+ *     useSessionPool: true,
+ *     persistCookiesPerSession: true,
+ *     // ...
+ * })
+ * ```
+ *
+ * You can configure the pool with many options. See the {@link SessionPoolOptions}.
+ * Session pool is by default persisted in default {@link KeyValueStore}.
+ * If you want to have one pool for all runs you have to specify
+ * {@link SessionPoolOptions.persistStateKeyValueStoreId}.
+ *
+ * **Advanced usage:**
+ *
+ * ```javascript
+ * const sessionPool = await SessionPool.open({
+ *     maxPoolSize: 25,
+ *     sessionOptions:{
+ *          maxAgeSecs: 10,
+ *          maxUsageCount: 150, // for example when you know that the site blocks after 150 requests.
+ *     },
+ *     persistStateKeyValueStoreId: 'my-key-value-store-for-sessions',
+ *     persistStateKey: 'my-session-pool',
+ * });
+ *
+ * // Get random session from the pool
+ * const session1 = await sessionPool.getSession();
+ * const session2 = await sessionPool.getSession();
+ * const session3 = await sessionPool.getSession();
+ *
+ * // Now you can mark the session either failed or successful
+ *
+ * // Marks session as bad after unsuccessful usage -> it increases error count (soft retire)
+ * session1.markBad()
+ *
+ * // Marks as successful.
+ * session2.markGood()
+ *
+ * // Retires session -> session is removed from the pool
+ * session3.retire()
+ *
+ * ```
+ * @category Scaling
+ */
+export declare class SessionPool extends EventEmitter {
+    readonly config: Configuration;
+    log: Log;
+    maxPoolSize: number;
+    createSessionFunction: CreateSession;
+    keyValueStore: KeyValueStore;
+    sessions: Session[];
+    sessionMap: Map<string, Session>;
+    sessionOptions: SessionOptions;
+    persistStateKeyValueStoreId?: string;
+    persistStateKey: string;
+    private _listener;
+    private events;
+    /**
+     * @internal
+     */
+    constructor(options?: SessionPoolOptions, config?: Configuration);
+    /**
+     * Gets count of usable sessions in the pool.
+     */
+    get usableSessionsCount(): number;
+    /**
+     * Gets count of retired sessions in the pool.
+     */
+    get retiredSessionsCount(): number;
+    /**
+     * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
+     * It is called automatically by the {@link SessionPool.open} function.
+     */
+    initialize(): Promise<void>;
+    /**
+     * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
+     * but this allows you to add more sessions once the max pool size is reached.
+     * This also allows you to add session with overridden session options (e.g. with specific session id).
+     * @param [options] The configuration options for the session being added to the session pool.
+     */
+    addSession(options?: Session | SessionOptions): Promise<void>;
+    /**
+     * Gets session.
+     * If there is space for new session, it creates and returns new session.
+     * If the session pool is full, it picks a session from the pool,
+     * If the picked session is usable it is returned, otherwise it creates and returns a new one.
+     */
+    getSession(): Promise<Session>;
+    /**
+     * Gets session based on the provided session id or `undefined.
+     */
+    getSession(sessionId: string): Promise<Session>;
+    /**
+     * Returns an object representing the internal state of the `SessionPool` instance.
+     * Note that the object's fields can change in future releases.
+     */
+    getState(): {
+        usableSessionsCount: number;
+        retiredSessionsCount: number;
+        sessions: import("./session").SessionState[];
+    };
+    /**
+     * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
+     * The state is persisted automatically in regular intervals.
+     */
+    persistState(): Promise<void>;
+    /**
+     * Removes listener from `persistState` event.
+     * This function should be called after you are done with using the `SessionPool` instance.
+     */
+    teardown(): Promise<void>;
+    /**
+     * SessionPool should not work before initialization.
+     */
+    protected _throwIfNotInitialized(): void;
+    /**
+     * Removes retired `Session` instances from `SessionPool`.
+     */
+    protected _removeRetiredSessions(): void;
+    /**
+     * Adds `Session` instance to `SessionPool`.
+     * @param newSession `Session` instance to be added.
+     */
+    protected _addSession(newSession: Session): void;
+    /**
+     * Gets random index.
+     */
+    protected _getRandomIndex(): number;
+    /**
+     * Creates new session without any extra behavior.
+     * @param sessionPool
+     * @param [options]
+     * @param [options.sessionOptions] The configuration options for the session being created.
+     * @returns New session.
+     */
+    protected _defaultCreateSessionFunction(sessionPool: SessionPool, options?: {
+        sessionOptions?: SessionOptions;
+    }): Session;
+    /**
+     * Creates new session and adds it to the pool.
+     * @returns Newly created `Session` instance.
+     */
+    protected _createSession(): Promise<Session>;
+    /**
+     * Decides whether there is enough space for creating new session.
+     */
+    protected _hasSpaceForSession(): boolean;
+    /**
+     * Picks random session from the `SessionPool`.
+     * @returns Picked `Session`.
+     */
+    protected _pickSession(): Session;
+    /**
+     * Potentially loads `SessionPool`.
+     * If the state was persisted it loads the `SessionPool` from the persisted state.
+     */
+    protected _maybeLoadSessionPool(): Promise<void>;
+    /**
+     * Opens a SessionPool and returns a promise resolving to an instance
+     * of the {@link SessionPool} class that is already initialized.
+     *
+     * For more details and code examples, see the {@link SessionPool} class.
+     */
+    static open(options?: SessionPoolOptions): Promise<SessionPool>;
+}
+//# sourceMappingURL=session_pool.d.ts.map

package/session_pool/session_pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEtC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAE5D,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAGpD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;;OAGG;IACH,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,+DAA+D;IAC/D,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,qBAAa,WAAY,SAAQ,YAAY;IAgBK,QAAQ,CAAC,MAAM;IAf7D,GAAG,EAAE,GAAG,CAAC;IACT,WAAW,EAAE,MAAM,CAAC;IACpB,qBAAqB,EAAE,aAAa,CAAC;IACrC,aAAa,EAAG,aAAa,CAAC;IAC9B,QAAQ,EAAE,OAAO,EAAE,CAAM;IACzB,UAAU,uBAA8B;IACxC,cAAc,EAAE,cAAc,CAAC;IAC/B,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,MAAM,CAAe;IAE7B;;OAEG;gBACS,OAAO,GAAE,kBAAuB,EAAW,MAAM,gBAAkC;IA6C/F;;OAEG;IACH,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAiBjC;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBvE;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEpC;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IA8BrD;;;OAGG;IACH,QAAQ;;;;;IAQR;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAQnC;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAK/B;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAIhC;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAWhC;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO;IAKzC;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;;OAMG;IACH,SAAS,CAAC,6BAA6B,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAO,GAAG,OAAO;IAU7H;;;OAGG;cACa,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAQlD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,OAAO;IAIxC;;;OAGG;IACH,SAAS,CAAC,YAAY,IAAI,OAAO;IAIjC;;;OAGG;cACa,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBtD;;;;;OAKG;WACU,IAAI,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC;CAKxE"}