apify 2.3.1-beta.4 → 3.0.0-alpha.0

package/build/session_pool/session.js

@@ -1,355 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Session = void 0;
-const tslib_1 = require("tslib");
-const utilities_1 = require("@apify/utilities");
-const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
-const tough_cookie_1 = require("tough-cookie");
-const events_1 = (0, tslib_1.__importDefault)(require("./events"));
-const constants_1 = require("../constants");
-const session_utils_1 = require("./session_utils");
-const utils_log_1 = (0, tslib_1.__importDefault)(require("../utils_log"));
-// TYPE IMPORTS
-/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order,import/no-cycle */
-const session_pool_1 = require("./session_pool");
-/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order,import/no-cycle */
-/**
- * @typedef {object} PuppeteerCookie
- */
-// CONSTANTS
-const DEFAULT_SESSION_MAX_AGE_SECS = 3000;
-/**
- * Persistable {@link Session} state.
- * @typedef SessionState
- * @property {string} id
- * @property {CookieJar} cookieJar
- * @property {object} userData
- * @property {number} errorScore
- * @property {number} maxErrorScore
- * @property {number} errorScoreDecrement
- * @property {number} usageCount
- * @property {string} expiresAt
- * @property {string} createdAt
- */
-/**
- * @typedef SessionOptions
- * @property {string} [id] - Id of session used for generating fingerprints. It is used as proxy session name.
- * @property {number} [maxAgeSecs=3000] - Number of seconds after which the session is considered as expired.
- * @property {Object<string,*>} [userData] - Object where custom user data can be stored. For example custom headers.
- * @property {number} [maxErrorScore=3] - Maximum number of marking session as blocked usage.
- *   If the `errorScore` reaches the `maxErrorScore` session is marked as block and it is thrown away.
- *   It starts at 0. Calling the `markBad` function increases the `errorScore` by 1.
- *   Calling the `markGood` will decrease the `errorScore` by `errorScoreDecrement`
- * @property {number} [errorScoreDecrement=0.5] - It is used for healing the session.
- *   For example: if your session is marked bad two times, but it is successful on the third attempt it's errorScore is decremented by this
- *   number.
- * @property {Date} [createdAt] - Date of creation.
- * @property {Date} [expiresAt] - Date of expiration.
- * @property {number} [usageCount=0] - Indicates how many times the session has been used.
- * @property {number} [errorCount=0] - Indicates how many times the session is marked bad.
- * @property {number} [maxUsageCount=50] - Session should be used only a limited amount of times.
- *   This number indicates how many times the session is going to be used, before it is thrown away.
- * @property {SessionPool} sessionPool - SessionPool instance. Session will emit the `sessionRetired` event on this instance.
- */
-/**
- *  Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
- *  You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
- *  Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
- *
- */
-class Session {
-    /**
-     * Session configuration.
-     *
-     * @param {SessionOptions} options
-     */
-    constructor(options) {
-        (0, ow_1.default)(options, ow_1.default.object.exactShape({
-            sessionPool: ow_1.default.object.instanceOf(session_pool_1.SessionPool),
-            id: ow_1.default.optional.string,
-            cookieJar: ow_1.default.optional.object,
-            maxAgeSecs: ow_1.default.optional.number,
-            userData: ow_1.default.optional.object,
-            maxErrorScore: ow_1.default.optional.number,
-            errorScoreDecrement: ow_1.default.optional.number,
-            createdAt: ow_1.default.optional.date,
-            expiresAt: ow_1.default.optional.date,
-            usageCount: ow_1.default.optional.number,
-            errorScore: ow_1.default.optional.number,
-            maxUsageCount: ow_1.default.optional.number,
-            log: ow_1.default.optional.object,
-        }));
-        const { sessionPool, id = `session_${(0, utilities_1.cryptoRandomObjectId)(10)}`, cookieJar = new tough_cookie_1.CookieJar(), maxAgeSecs = DEFAULT_SESSION_MAX_AGE_SECS, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, log = utils_log_1.default, } = options;
-        const { expiresAt = this._getDefaultCookieExpirationDate(maxAgeSecs) } = options;
-        this.log = log.child({ prefix: 'Session' });
-        /**
-         * @type {CookieJar}
-         * @private
-         */
-        this.cookieJar = cookieJar.setCookie ? cookieJar : tough_cookie_1.CookieJar.fromJSON(JSON.stringify(cookieJar));
-        this.id = id;
-        this.maxAgeSecs = maxAgeSecs;
-        /** @type {Object<string,*>} */
-        this.userData = userData;
-        this.maxErrorScore = maxErrorScore;
-        this.errorScoreDecrement = errorScoreDecrement;
-        // Internal
-        this.expiresAt = expiresAt;
-        this.createdAt = createdAt;
-        this.usageCount = usageCount; // indicates how many times the session has been used
-        this.errorScore = errorScore; // indicates number of markBaded request with the session
-        this.maxUsageCount = maxUsageCount;
-        this.sessionPool = sessionPool;
-        this._puppeteerCookieToTough = this._puppeteerCookieToTough.bind(this);
-    }
-    /**
-     * indicates whether the session is blocked.
-     * Session is blocked once it reaches the `maxErrorScore`.
-     * @return {boolean}
-     */
-    isBlocked() {
-        return this.errorScore >= this.maxErrorScore;
-    }
-    /**
-     * Indicates whether the session is expired.
-     * Session expiration is determined by the `maxAgeSecs`.
-     * Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
-     * @return {boolean}
-     */
-    isExpired() {
-        return this.expiresAt <= new Date();
-    }
-    /**
-     * Indicates whether the session is used maximum number of times.
-     * Session maximum usage count can be changed by `maxUsageCount` parameter.
-     * @return {boolean}
-     */
-    isMaxUsageCountReached() {
-        return this.usageCount >= this.maxUsageCount;
-    }
-    /**
-     * Indicates whether the session can be used for next requests.
-     * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
-     * @return {boolean}
-     */
-    isUsable() {
-        return !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
-    }
-    /**
-     * This method should be called after a successful session usage.
-     * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
-     */
-    markGood() {
-        this.usageCount += 1;
-        if (this.errorScore > 0) {
-            this.errorScore -= this.errorScoreDecrement;
-        }
-        this._maybeSelfRetire();
-    }
-    /**
-     * Gets session state for persistence in KeyValueStore.
-     * @return {SessionState} represents session internal state.
-     */
-    getState() {
-        return {
-            id: this.id,
-            cookieJar: this.cookieJar.toJSON(),
-            userData: this.userData,
-            maxErrorScore: this.maxErrorScore,
-            errorScoreDecrement: this.errorScoreDecrement,
-            expiresAt: this.expiresAt.toISOString(),
-            createdAt: this.createdAt.toISOString(),
-            usageCount: this.usageCount,
-            maxUsageCount: this.maxUsageCount,
-            errorScore: this.errorScore,
-        };
-    }
-    /**
-     * Marks session as blocked and emits event on the `SessionPool`
-     * This method should be used if the session usage was unsuccessful
-     * and you are sure that it is because of the session configuration and not any external matters.
-     * For example when server returns 403 status code.
-     * If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
-     */
-    retire() {
-        // mark it as an invalid by increasing the error score count.
-        this.errorScore += this.maxErrorScore;
-        this.usageCount += 1;
-        // emit event so we can retire browser in puppeteer pool
-        this.sessionPool.emit(events_1.default.SESSION_RETIRED, this);
-    }
-    /**
-     * Increases usage and error count.
-     * Should be used when the session has been used unsuccessfully. For example because of timeouts.
-     */
-    markBad() {
-        this.errorScore += 1;
-        this.usageCount += 1;
-        this._maybeSelfRetire();
-    }
-    /**
-     * With certain status codes: `401`, `403` or `429` we can be certain
-     * that the target website is blocking us. This function helps to do this conveniently
-     * by retiring the session when such code is received. Optionally the default status
-     * codes can be extended in the second parameter.
-     * @param statusCode {number} - HTTP status code
-     * @param [blockedStatusCodes] {number[]} - Custom HTTP status codes that means blocking on particular website.
-     * @return {boolean} whether the session was retired.
-     */
-    retireOnBlockedStatusCodes(statusCode, blockedStatusCodes = []) {
-        const isBlocked = constants_1.STATUS_CODES_BLOCKED.concat(blockedStatusCodes).includes(statusCode);
-        if (isBlocked) {
-            this.retire();
-        }
-        return isBlocked;
-    }
-    /**
-     * Saves cookies from an HTTP response to be used with the session.
-     * It expects an object with a `headers` property that's either an `Object`
-     * (typical Node.js responses) or a `Function` (Puppeteer Response).
-     *
-     * It then parses and saves the cookies from the `set-cookie` header, if available.
-
-     * @param {(PuppeteerResponse|IncomingMessage)} response
-     */
-    setCookiesFromResponse(response) {
-        try {
-            const cookies = (0, session_utils_1.getCookiesFromResponse)(response).filter((c) => c);
-            this._setCookies(cookies, response.url);
-        }
-        catch (e) {
-            // if invalid Cookie header is provided just log the exception.
-            this.log.exception(e, 'Could not get cookies from response');
-        }
-    }
-    /**
-     * Saves an array with cookie objects to be used with the session.
-     * The objects should be in the format that
-     * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
-     * but you can also use this function to set cookies manually:
-     *
-     * ```
-     * [
-     *   { name: 'cookie1', value: 'my-cookie' },
-     *   { name: 'cookie2', value: 'your-cookie' }
-     * ]
-     * ```
-     *
-     * @param {PuppeteerCookie[]} cookies
-     * @param {string} url
-     */
-    setPuppeteerCookies(cookies, url) {
-        const normalizedCookies = cookies.map(this._puppeteerCookieToTough);
-        this._setCookies(normalizedCookies, url);
-    }
-    /**
-     * Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.
-     * @param {string} url website url. Only cookies stored for this url will be returned
-     * @return {PuppeteerCookie[]}
-     */
-    getPuppeteerCookies(url) {
-        const cookies = this.cookieJar.getCookiesSync(url);
-        return cookies.map(this._toughCookieToPuppeteer);
-    }
-    /**
-     * Returns cookies saved with the session in the typical
-     * key1=value1; key2=value2 format, ready to be used in
-     * a cookie header or elsewhere.
-     * @param {string} url
-     * @return {string} represents `Cookie` header.
-     */
-    getCookieString(url) {
-        return this.cookieJar.getCookieStringSync(url, {});
-    }
-    /**
-     * Transforms puppeteer cookie to tough-cookie.
-     * @param {PuppeteerCookie} puppeteerCookie Cookie from puppeteer `page.cookies method.
-     * @return {Cookie}
-     * @ignore
-     * @protected
-     * @internal
-     */
-    _puppeteerCookieToTough(puppeteerCookie) {
-        const isExpiresValid = puppeteerCookie.expires && typeof puppeteerCookie.expires === 'number' && puppeteerCookie.expires > 0;
-        const expires = isExpiresValid ? new Date(puppeteerCookie.expires * 1000) : this._getDefaultCookieExpirationDate(this.maxAgeSecs);
-        const domain = typeof puppeteerCookie.domain === 'string' && puppeteerCookie.domain.startsWith('.')
-            ? puppeteerCookie.domain.slice(1)
-            : puppeteerCookie.domain;
-        return new tough_cookie_1.Cookie({
-            key: puppeteerCookie.name,
-            value: puppeteerCookie.value,
-            expires,
-            domain,
-            path: puppeteerCookie.path,
-            secure: puppeteerCookie.secure,
-            httpOnly: puppeteerCookie.httpOnly,
-        });
-    }
-    /**
-     * Transforms tough-cookie to puppeteer cookie .
-     * @param {Cookie} toughCookie - Cookie from CookieJar
-     * @return {PuppeteerCookie} - Cookie from Puppeteer
-     * @ignore
-     * @protected
-     * @internal
-     */
-    _toughCookieToPuppeteer(toughCookie) {
-        return {
-            name: toughCookie.key,
-            value: toughCookie.value,
-            // Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms
-            expires: new Date(toughCookie.expires).getTime() / 1000,
-            domain: toughCookie.domain,
-            path: toughCookie.path,
-            secure: toughCookie.secure,
-            httpOnly: toughCookie.httpOnly,
-        };
-    }
-    /**
-     * Sets cookies.
-     * @param {Cookie[]} cookies
-     * @param {string} url
-     * @ignore
-     * @protected
-     * @internal
-     */
-    _setCookies(cookies, url) {
-        const errorMessages = [];
-        for (const cookie of cookies) {
-            try {
-                this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });
-            }
-            catch (e) {
-                errorMessages.push(e.message);
-            }
-        }
-        // if invalid cookies are provided just log the exception. No need to retry the request automatically.
-        if (errorMessages.length) {
-            this.log.debug('Could not set cookies.', { errorMessages });
-        }
-    }
-    /**
-     * Calculate cookie expiration date
-     * @param {number} maxAgeSecs
-     * @return {Date} - calculated date by session max age seconds.
-     * @ignore
-     * @protected
-     * @internal
-     */
-    _getDefaultCookieExpirationDate(maxAgeSecs) {
-        return new Date(Date.now() + (maxAgeSecs * 1000));
-    }
-    /**
-     * Checks if session is not usable. if it is not retires the session.
-     * @ignore
-     * @protected
-     * @internal
-     */
-    _maybeSelfRetire() {
-        if (!this.isUsable()) {
-            this.retire();
-        }
-    }
-}
-exports.Session = Session;
-//# sourceMappingURL=session.js.map

package/build/session_pool/session.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.js"],"names":[],"mappings":";;;;AAAA,gDAAwD;AACxD,yDAAoB;AACpB,+CAAiD;AACjD,mEAA8B;AAC9B,4CAAoD;AACpD,mDAAyD;AACzD,0EAAsC;AAEtC,eAAe;AACf,kGAAkG;AAClG,iDAA6C;AAG7C,iGAAiG;AAEjG;;GAEG;AAEH,YAAY;AACZ,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;;;;;;;;GAYG;AAEH;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;;;;GAKG;AACH,MAAa,OAAO;IAChB;;;;OAIG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,0BAAW,CAAC;YAC9C,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,WAAW,EACX,EAAE,GAAG,WAAW,IAAA,gCAAoB,EAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,wBAAS,EAAE,EAC3B,UAAU,GAAG,4BAA4B,EACzC,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,GAAG,GAAG,mBAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC,+BAA+B,CAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAEjF,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C;;;WAGG;QACH,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,wBAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QACjG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,+BAA+B;QAC/B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,WAAW;QACX,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACnF,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACvF,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3E,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;;OAKG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;;OAIG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;YACrB,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,mBAAmB,CAAC;SAC/C;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE;YAClC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,6DAA6D;QAC7D,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;QACtC,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,wDAAwD;QACxD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,gBAAM,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IACxD,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACrB,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;;OAQG;IACH,0BAA0B,CAAC,UAAU,EAAE,kBAAkB,GAAG,EAAE;QAC1D,MAAM,SAAS,GAAG,gCAAoB,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACvF,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACH,sBAAsB,CAAC,QAAQ;QAC3B,IAAI;YACA,MAAM,OAAO,GAAG,IAAA,sCAAsB,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAElE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;SAC3C;QAAC,OAAO,CAAC,EAAE;YACR,+DAA+D;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,qCAAqC,CAAC,CAAC;SAChE;IACL,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,mBAAmB,CAAC,OAAO,EAAE,GAAG;QAC5B,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACpE,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACH,mBAAmB,CAAC,GAAG;QACnB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAEnD,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACrD,CAAC;IAED;;;;;;OAMG;IACH,eAAe,CAAC,GAAG;QACf,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;;;;;;OAOG;IACH,uBAAuB,CAAC,eAAe;QACnC,MAAM,cAAc,GAAG,eAAe,CAAC,OAAO,IAAI,OAAO,eAAe,CAAC,OAAO,KAAK,QAAQ,IAAI,eAAe,CAAC,OAAO,GAAG,CAAC,CAAC;QAC7H,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAClI,MAAM,MAAM,GAAG,OAAO,eAAe,CAAC,MAAM,KAAK,QAAQ,IAAI,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC;YAC/F,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACjC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC;QAC7B,OAAO,IAAI,qBAAM,CAAC;YACd,GAAG,EAAE,eAAe,CAAC,IAAI;YACzB,KAAK,EAAE,eAAe,CAAC,KAAK;YAC5B,OAAO;YACP,MAAM;YACN,IAAI,EAAE,eAAe,CAAC,IAAI;YAC1B,MAAM,EAAE,eAAe,CAAC,MAAM;YAC9B,QAAQ,EAAE,eAAe,CAAC,QAAQ;SACrC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACH,uBAAuB,CAAC,WAAW;QAC/B,OAAO;YACH,IAAI,EAAE,WAAW,CAAC,GAAG;YACrB,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,iFAAiF;YACjF,OAAO,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI;YACvD,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;SACjC,CAAC;IACN,CAAC;IAED;;;;;;;OAOG;IACH,WAAW,CAAC,OAAO,EAAE,GAAG;QACpB,MAAM,aAAa,GAAG,EAAE,CAAC;QACzB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;YAC1B,IAAI;gBACA,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;aACrE;YAAC,OAAO,CAAC,EAAE;gBACR,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;aACjC;SACJ;QACD,sGAAsG;QACtG,IAAI,aAAa,CAAC,MAAM,EAAE;YACtB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;SAC/D;IACL,CAAC;IAED;;;;;;;OAOG;IACH,+BAA+B,CAAC,UAAU;QACtC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC;IACtD,CAAC;IAED;;;;;OAKG;IACH,gBAAgB;QACZ,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE;YAClB,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;IACL,CAAC;CACJ;AA9UD,0BA8UC"}

package/build/session_pool/session_pool.d.ts

@@ -1,280 +0,0 @@
-/**
- * Factory user-function which creates customized {@link Session} instances.
- * @callback CreateSession
- * @param {SessionPool} sessionPool Pool requesting the new session.
- * @returns {Promise<Session>}
- */
-/**
- * @typedef SessionPoolOptions
- * @property {number} [maxPoolSize=1000] - Maximum size of the pool.
- * Indicates how many sessions are rotated.
- * @property {SessionOptions} [sessionOptions] The configuration options for {@link Session} instances.
- * @property {string} [persistStateKeyValueStoreId] - Name or Id of `KeyValueStore` where is the `SessionPool` state stored.
- * @property {string} [persistStateKey="SESSION_POOL_STATE"] - Session pool persists it's state under this key in Key value store.
- * @property {CreateSession} [createSessionFunction] - Custom function that should return `Session` instance.
- * Any error thrown from this function will terminate the process.
- * Function receives `SessionPool` instance as a parameter
- * @property {boolean} [forceCloud=false] If set to `true` then the function uses cloud storage usage even if the `APIFY_LOCAL_STORAGE_DIR`
- * environment variable is set. This way it is possible to combine local and cloud storage.
- *
- * **Note:** If you use `forceCloud`, it is recommended to also set the `persistStateKeyValueStoreId` option, as otherwise the
- * `KeyValueStore` will be unnamed!
- */
-/**
- * Handles the rotation, creation and persistence of user-like sessions.
- * Creates a pool of {@link Session} instances, that are randomly rotated.
- * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
- * Learn more in the [`Session management guide`](../guides/session-management).
- *
- * You can create one by calling the {@link Apify.openSessionPool} function.
- *
- * Session pool is already integrated into crawlers, and it can significantly improve your scraper
- * performance with just 2 lines of code.
- *
- * **Example usage:**
- *
- * ```javascript
- * const crawler = new Apify.CheerioCrawler({
- *     useSessionPool: true,
- *     persistCookiesPerSession: true,
- *     // ...
- * })
- * ```
- *
- * You can configure the pool with many options. See the {@link SessionPoolOptions}.
- * Session pool is by default persisted in default {@link KeyValueStore}.
- * If you want to have one pool for all runs you have to specify
- * {@link SessionPoolOptions.persistStateKeyValueStoreId}.
- *
- * **Advanced usage:**
- *
- * ```javascript
- * const sessionPool = await Apify.openSessionPool({
- *     maxPoolSize: 25,
- *     sessionOptions:{
- *          maxAgeSecs: 10,
- *          maxUsageCount: 150, // for example when you know that the site blocks after 150 requests.
- *     },
- *     persistStateKeyValueStoreId: 'my-key-value-store-for-sessions',
- *     persistStateKey: 'my-session-pool',
- * });
- *
- * // Get random session from the pool
- * const session1 = await sessionPool.getSession();
- * const session2 = await sessionPool.getSession();
- * const session3 = await sessionPool.getSession();
- *
- * // Now you can mark the session either failed or successful
- *
- * // Marks session as bad after unsuccessful usage -> it increases error count (soft retire)
- * session1.markBad()
- *
- * // Marks as successful.
- * session2.markGood()
- *
- * // Retires session -> session is removed from the pool
- * session3.retire()
- *
- * ```
- * @hideconstructor
- */
-export class SessionPool extends EventEmitter {
-    /**
-     * Session pool configuration.
-     * @param {SessionPoolOptions} [options] All `SessionPool` configuration options.
-     * @param {Configuration} [config]
-     */
-    constructor(options?: SessionPoolOptions | undefined, config?: Configuration | undefined);
-    config: Configuration;
-    log: any;
-    maxPoolSize: number;
-    createSessionFunction: (CreateSession & Function) | ((sessionPool: SessionPool, options?: {
-        sessionOptions?: SessionOptions | undefined;
-    } | undefined) => Session);
-    sessionOptions: {
-        log: any;
-    };
-    persistStateKeyValueStoreId: string | undefined;
-    persistStateKey: string;
-    keyValueStore: import("../storages/key_value_store").KeyValueStore | null;
-    /** @type {Session[]} */
-    sessions: Session[];
-    sessionMap: Map<any, any>;
-    /** @type {boolean} */
-    forceCloud: boolean;
-    /**
-     * Gets count of usable sessions in the pool.
-     * @return {number}
-     */
-    get usableSessionsCount(): number;
-    /**
-     * Gets count of retired sessions in the pool.
-     * @return {number}
-     */
-    get retiredSessionsCount(): number;
-    /**
-     * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
-     * It is called automatically by the {@link Apify.openSessionPool} function.
-     *
-     * @return {Promise<void>}
-     */
-    initialize(): Promise<void>;
-    _listener: (() => Promise<void>) | undefined;
-    /**
-     * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
-     * but this allows you to add more sessions once the max pool size is reached.
-     * This also allows you to add session with overridden session options (e.g. with specific session id).
-     * @param {Session|SessionOptions} [options] - The configuration options for the session being added to the session pool.
-     */
-    addSession(options?: Session | SessionOptions | undefined): Promise<void>;
-    /**
-     * Gets session.
-     * If there is space for new session, it creates and returns new session.
-     * If the session pool is full, it picks a session from the pool,
-     * If the picked session is usable it is returned, otherwise it creates and returns a new one.
-     *
-     * @param {String} [sessionId] - If provided, it returns the usable session with this id, `undefined` otherwise.
-     * @return {Promise<Session>}
-     */
-    getSession(sessionId?: string | undefined): Promise<Session>;
-    /**
-     * Returns an object representing the internal state of the `SessionPool` instance.
-     * Note that the object's fields can change in future releases.
-     */
-    getState(): {
-        usableSessionsCount: number;
-        retiredSessionsCount: number;
-        sessions: import("./session").SessionState[];
-    };
-    /**
-     * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
-     * The state is persisted automatically in regular intervals.
-     *
-     * @return {Promise<void>}
-     */
-    persistState(): Promise<void>;
-    /**
-     * Removes listener from `persistState` event.
-     * This function should be called after you are done with using the `SessionPool` instance.
-     */
-    teardown(): Promise<void>;
-    /**
-     * SessionPool should not work before initialization.
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _throwIfNotInitialized(): void;
-    /**
-     * Removes retired `Session` instances from `SessionPool`.
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _removeRetiredSessions(): void;
-    /**
-     * Adds `Session` instance to `SessionPool`.
-     * @param {Session} newSession `Session` instance to be added.
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _addSession(newSession: Session): void;
-    /**
-     * Gets random index.
-     * @return {number}
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _getRandomIndex(): number;
-    /**
-     * Creates new session without any extra behavior.
-     * @param {SessionPool} sessionPool
-     * @param {Object} [options]
-     * @param {SessionOptions} [options.sessionOptions] - The configuration options for the session being created
-     * @return {Session} - New session.
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _defaultCreateSessionFunction(sessionPool: SessionPool, options?: {
-        sessionOptions?: SessionOptions | undefined;
-    } | undefined): Session;
-    /**
-     * Creates new session and adds it to the pool.
-     * @return {Promise<Session>} - Newly created `Session` instance.
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _createSession(): Promise<Session>;
-    /**
-     * Decides whether there is enough space for creating new session.
-     * @return {boolean}
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _hasSpaceForSession(): boolean;
-    /**
-     * Picks random session from the `SessionPool`.
-     * @return {Session} - Picked `Session`
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _pickSession(): Session;
-    /**
-     * Potentially loads `SessionPool`.
-     * If the state was persisted it loads the `SessionPool` from the persisted state.
-     * @return {Promise<void>}
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _maybeLoadSessionPool(): Promise<void>;
-}
-export function openSessionPool(sessionPoolOptions: SessionPoolOptions): Promise<SessionPool>;
-/**
- * Factory user-function which creates customized {@link Session } instances.
- */
-export type CreateSession = (sessionPool: SessionPool) => Promise<Session>;
-export type SessionPoolOptions = {
-    /**
-     * - Maximum size of the pool.
-     * Indicates how many sessions are rotated.
-     */
-    maxPoolSize?: number | undefined;
-    /**
-     * The configuration options for {@link Session } instances.
-     */
-    sessionOptions?: SessionOptions | undefined;
-    /**
-     * - Name or Id of `KeyValueStore` where is the `SessionPool` state stored.
-     */
-    persistStateKeyValueStoreId?: string | undefined;
-    /**
-     * - Session pool persists it's state under this key in Key value store.
-     */
-    persistStateKey?: string | undefined;
-    /**
-     * - Custom function that should return `Session` instance.
-     * Any error thrown from this function will terminate the process.
-     * Function receives `SessionPool` instance as a parameter
-     */
-    createSessionFunction?: CreateSession | undefined;
-    /**
-     * If set to `true` then the function uses cloud storage usage even if the `APIFY_LOCAL_STORAGE_DIR`
-     * environment variable is set. This way it is possible to combine local and cloud storage.
-     *
-     * **Note:** If you use `forceCloud`, it is recommended to also set the `persistStateKeyValueStoreId` option, as otherwise the
-     * `KeyValueStore` will be unnamed!
-     */
-    forceCloud?: boolean | undefined;
-};
-import { EventEmitter } from "events";
-import { Configuration } from "../configuration";
-import { SessionOptions } from "./session";
-import { Session } from "./session";
-//# sourceMappingURL=session_pool.d.ts.map

package/build/session_pool/session_pool.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.js"],"names":[],"mappings":"AASA;;;;;GAKG;AAEH;;;;;;;;;;;;;;;GAeG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH;IACI;;;;OAIG;IACH,0FAqDC;IA1BG,sBAAoB;IACpB,SAA+C;IAG/C,oBAA8B;IAC9B,mEA6MO,WAAW;;sBAGV,OAAO,EAhNyE;IAGxF;;MAKC;IAGD,gDAA8D;IAC9D,wBAAsC;IAGtC,0EAAyB;IACzB,wBAAwB;IACxB,UADW,OAAO,EAAE,CACF;IAClB,0BAA2B;IAC3B,sBAAsB;IACtB,YADW,OAAO,CACU;IAGhC;;;OAGG;IACH,kCAEC;IAED;;;OAGG;IACH,mCAEC;IAED;;;;;OAKG;IACH,cAFY,QAAQ,IAAI,CAAC,CAgBxB;IAHG,kBA+EQ,QAAQ,IAAI,CAAC,cA/EwB;IAKjD;;;;;OAKG;IACH,0EAoBC;IAED;;;;;;;;OAQG;IACH,4CAFY,QAAQ,OAAO,CAAC,CAqB3B;IAED;;;OAGG;IACH;;;;MAMC;IAED;;;;;OAKG;IACH,gBAFY,QAAQ,IAAI,CAAC,CASxB;IAED;;;OAGG;IACH,0BAGC;IAED;;;;;OAKG;IACH,yCAEC;IAED;;;;;OAKG;IACH,yCAOC;IAED;;;;;;OAMG;IACH,kCALW,OAAO,QAQjB;IAED;;;;;;OAMG;IACH,6BALY,MAAM,CAOjB;IAED;;;;;;;;;OASG;IACH,qDARW,WAAW;;oBAGV,OAAO,CAalB;IAED;;;;;;OAMG;IACH,4BALY,QAAQ,OAAO,CAAC,CAY3B;IAED;;;;;;OAMG;IACH,iCALY,OAAO,CAOlB;IAED;;;;;;OAMG;IACH,0BALY,OAAO,CAOlB;IAED;;;;;;;OAOG;IACH,mCALY,QAAQ,IAAI,CAAC,CA2BxB;CACJ;AAcM,oDANI,kBAAkB,GACjB,QAAQ,WAAW,CAAC,CAS/B;;;;0CA3aU,WAAW,KACT,QAAQ,OAAO,CAAC"}