@crawlee/core 4.0.0-beta.28 → 4.0.0-beta.280

package/session_pool/session.js

@@ -1,278 +0,0 @@
-import { EventEmitter } from 'node:events';
-import ow from 'ow';
-import { CookieJar } from 'tough-cookie';
-import { cryptoRandomObjectId } from '@apify/utilities';
-import { browserPoolCookieToToughCookie, getCookiesFromResponse, getDefaultCookieExpirationDate, toughCookieToBrowserPoolCookie, } from '../cookie_utils.js';
-import { log as defaultLog } from '../log.js';
-import { EVENT_SESSION_RETIRED } from './events.js';
-/**
- * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
- * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
- * Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
- * @category Scaling
- */
-export class Session {
-    id;
-    maxAgeSecs;
-    userData;
-    _maxErrorScore;
-    _errorScoreDecrement;
-    _createdAt;
-    _expiresAt;
-    _usageCount;
-    _maxUsageCount;
-    sessionPool;
-    _errorScore;
-    _proxyInfo;
-    _cookieJar;
-    log;
-    get errorScore() {
-        return this._errorScore;
-    }
-    get usageCount() {
-        return this._usageCount;
-    }
-    get maxErrorScore() {
-        return this._maxErrorScore;
-    }
-    get errorScoreDecrement() {
-        return this._errorScoreDecrement;
-    }
-    get expiresAt() {
-        return this._expiresAt;
-    }
-    get createdAt() {
-        return this._createdAt;
-    }
-    get maxUsageCount() {
-        return this._maxUsageCount;
-    }
-    get cookieJar() {
-        return this._cookieJar;
-    }
-    get proxyInfo() {
-        return this._proxyInfo;
-    }
-    /**
-     * Session configuration.
-     */
-    constructor(options) {
-        ow(options, ow.object.exactShape({
-            sessionPool: ow.object.instanceOf(EventEmitter),
-            id: ow.optional.string,
-            cookieJar: ow.optional.object,
-            proxyInfo: ow.optional.object,
-            maxAgeSecs: ow.optional.number,
-            userData: ow.optional.object,
-            maxErrorScore: ow.optional.number,
-            errorScoreDecrement: ow.optional.number,
-            createdAt: ow.optional.date,
-            expiresAt: ow.optional.date,
-            usageCount: ow.optional.number,
-            errorScore: ow.optional.number,
-            maxUsageCount: ow.optional.number,
-            log: ow.optional.object,
-        }));
-        const { sessionPool, id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, log = defaultLog, } = options;
-        const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
-        this.log = log.child({ prefix: 'Session' });
-        this._cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
-        this._proxyInfo = proxyInfo;
-        this.id = id;
-        this.maxAgeSecs = maxAgeSecs;
-        this.userData = userData;
-        this._maxErrorScore = maxErrorScore;
-        this._errorScoreDecrement = errorScoreDecrement;
-        // Internal
-        this._expiresAt = expiresAt;
-        this._createdAt = createdAt;
-        this._usageCount = usageCount; // indicates how many times the session has been used
-        this._errorScore = errorScore; // indicates number of markBaded request with the session
-        this._maxUsageCount = maxUsageCount;
-        this.sessionPool = sessionPool;
-    }
-    /**
-     * Indicates whether the session is blocked.
-     * Session is blocked once it reaches the `maxErrorScore`.
-     */
-    isBlocked() {
-        return this.errorScore >= this.maxErrorScore;
-    }
-    /**
-     * Indicates whether the session is expired.
-     * Session expiration is determined by the `maxAgeSecs`.
-     * Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
-     */
-    isExpired() {
-        return this.expiresAt <= new Date();
-    }
-    /**
-     * Indicates whether the session is used maximum number of times.
-     * Session maximum usage count can be changed by `maxUsageCount` parameter.
-     */
-    isMaxUsageCountReached() {
-        return this.usageCount >= this.maxUsageCount;
-    }
-    /**
-     * Indicates whether the session can be used for next requests.
-     * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
-     */
-    isUsable() {
-        return !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
-    }
-    /**
-     * This method should be called after a successful session usage.
-     * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
-     */
-    markGood() {
-        this._usageCount += 1;
-        if (this._errorScore > 0) {
-            this._errorScore -= this._errorScoreDecrement;
-        }
-        this._maybeSelfRetire();
-    }
-    /**
-     * Gets session state for persistence in KeyValueStore.
-     * @returns Represents session internal state.
-     */
-    getState() {
-        return {
-            id: this.id,
-            cookieJar: this.cookieJar.toJSON(),
-            proxyInfo: this._proxyInfo,
-            userData: this.userData,
-            maxErrorScore: this.maxErrorScore,
-            errorScoreDecrement: this.errorScoreDecrement,
-            expiresAt: this.expiresAt.toISOString(),
-            createdAt: this.createdAt.toISOString(),
-            usageCount: this.usageCount,
-            maxUsageCount: this.maxUsageCount,
-            errorScore: this.errorScore,
-        };
-    }
-    /**
-     * Marks session as blocked and emits event on the `SessionPool`
-     * This method should be used if the session usage was unsuccessful
-     * and you are sure that it is because of the session configuration and not any external matters.
-     * For example when server returns 403 status code.
-     * If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
-     */
-    retire() {
-        // mark it as an invalid by increasing the error score count.
-        this._errorScore += this._maxErrorScore;
-        this._usageCount += 1;
-        // emit event so we can retire browser in puppeteer pool
-        this.sessionPool.emit(EVENT_SESSION_RETIRED, this);
-    }
-    /**
-     * Increases usage and error count.
-     * Should be used when the session has been used unsuccessfully. For example because of timeouts.
-     */
-    markBad() {
-        this._errorScore += 1;
-        this._usageCount += 1;
-        this._maybeSelfRetire();
-    }
-    /**
-     * With certain status codes: `401`, `403` or `429` we can be certain
-     * that the target website is blocking us. This function helps to do this conveniently
-     * by retiring the session when such code is received. Optionally, the default status
-     * codes can be extended in the second parameter.
-     * @param statusCode HTTP status code.
-     * @returns Whether the session was retired.
-     */
-    retireOnBlockedStatusCodes(statusCode) {
-        // eslint-disable-next-line dot-notation -- accessing private property
-        const isBlocked = this.sessionPool['blockedStatusCodes'].includes(statusCode);
-        if (isBlocked) {
-            this.retire();
-        }
-        return isBlocked;
-    }
-    /**
-     * Saves cookies from an HTTP response to be used with the session.
-     * It expects an object with a `headers` property that's either an `Object`
-     * (typical Node.js responses) or a `Function` (Puppeteer Response).
-     *
-     * It then parses and saves the cookies from the `set-cookie` header, if available.
-     */
-    setCookiesFromResponse(response) {
-        try {
-            const cookies = getCookiesFromResponse(response).filter((c) => c);
-            this._setCookies(cookies, response.url);
-        }
-        catch (e) {
-            const err = e;
-            // if invalid Cookie header is provided just log the exception.
-            this.log.exception(err, 'Could not get cookies from response');
-        }
-    }
-    /**
-     * Saves an array with cookie objects to be used with the session.
-     * The objects should be in the format that
-     * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
-     * but you can also use this function to set cookies manually:
-     *
-     * ```
-     * [
-     *   { name: 'cookie1', value: 'my-cookie' },
-     *   { name: 'cookie2', value: 'your-cookie' }
-     * ]
-     * ```
-     */
-    setCookies(cookies, url) {
-        const normalizedCookies = cookies.map((c) => browserPoolCookieToToughCookie(c, this.maxAgeSecs));
-        this._setCookies(normalizedCookies, url);
-    }
-    /**
-     * Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
-     * @param url website url. Only cookies stored for this url will be returned
-     */
-    getCookies(url) {
-        const cookies = this.cookieJar.getCookiesSync(url);
-        return cookies.map((c) => toughCookieToBrowserPoolCookie(c));
-    }
-    /**
-     * Returns cookies saved with the session in the typical
-     * key1=value1; key2=value2 format, ready to be used in
-     * a cookie header or elsewhere.
-     * @returns Represents `Cookie` header.
-     */
-    getCookieString(url) {
-        return this.cookieJar.getCookieStringSync(url, {});
-    }
-    /**
-     * Sets a cookie within this session for the specific URL.
-     */
-    setCookie(rawCookie, url) {
-        this.cookieJar.setCookieSync(rawCookie, url);
-    }
-    /**
-     * Sets cookies.
-     */
-    _setCookies(cookies, url) {
-        const errorMessages = [];
-        for (const cookie of cookies) {
-            try {
-                this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });
-            }
-            catch (e) {
-                const err = e;
-                errorMessages.push(err.message);
-            }
-        }
-        // if invalid cookies are provided just log the exception. No need to retry the request automatically.
-        if (errorMessages.length) {
-            this.log.debug('Could not set cookies.', { errorMessages });
-        }
-    }
-    /**
-     * Checks if session is not usable. if it is not retires the session.
-     */
-    _maybeSelfRetire() {
-        if (!this.isUsable()) {
-            this.retire();
-        }
-    }
-}
-//# sourceMappingURL=session.js.map

package/session_pool/session.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EACH,8BAA8B,EAC9B,sBAAsB,EACtB,8BAA8B,EAC9B,8BAA8B,GACjC,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AA4DpD;;;;;GAKG;AACH,MAAM,OAAO,OAAO;IACP,EAAE,CAAS;IACZ,UAAU,CAAS;IAC3B,QAAQ,CAAa;IACb,cAAc,CAAS;IACvB,oBAAoB,CAAS;IAC7B,UAAU,CAAO;IACjB,UAAU,CAAO;IACjB,WAAW,CAAS;IACpB,cAAc,CAAS;IACvB,WAAW,CAA0C;IACrD,WAAW,CAAS;IACpB,UAAU,CAAa;IACvB,UAAU,CAAY;IACtB,GAAG,CAAM;IAEjB,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACrC,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,YAAY,OAAuB;QAC/B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC;YAC/C,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CACL,CAAC;QAEF,MAAM,EACF,WAAW,EACX,EAAE,GAAG,WAAW,oBAAoB,CAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,SAAS,EAAE,EAC3B,SAAS,GAAG,SAAS,EACrB,UAAU,GAAG,IAAI,EACjB,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,GAAG,GAAG,UAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,8BAA8B,CAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAE3E,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,UAAU,GAAI,SAAS,CAAC,SAAqB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/G,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,oBAAoB,GAAG,mBAAmB,CAAC;QAEhD,WAAW;QACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACpF,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACxF,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAClD,CAAC;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAG;YACnC,SAAS,EAAE,IAAI,CAAC,UAAU;YAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,6DAA6D;QAC7D,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,cAAc,CAAC;QACxC,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,wDAAwD;QACxD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;OAOG;IACH,0BAA0B,CAAC,UAAkB;QACzC,sEAAsE;QACtE,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC9E,IAAI,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAkB;QACrC,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,+DAA+D;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,qCAAqC,CAAC,CAAC;QACnE,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAuB,EAAE,GAAW;QAC3C,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACjG,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,UAAU,CAAC,GAAW;QAClB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,SAAiB,EAAE,GAAW;QACpC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,OAAiB,EAAE,GAAW;QAChD,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACD,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YACtE,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;QACL,CAAC;QAED,sGAAsG;QACtG,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;QAChE,CAAC;IACL,CAAC;IAED;;OAEG;IACO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;YACnB,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;CACJ"}

package/session_pool/session_pool.d.ts

@@ -1,256 +0,0 @@
-import { EventEmitter } from 'node:events';
-import type { Log } from '@apify/log';
-import type { PersistenceOptions } from '../crawlers/statistics.js';
-import type { EventManager } from '../events/event_manager.js';
-import { KeyValueStore } from '../storages/key_value_store.js';
-import type { SessionOptions } from './session.js';
-import { Session } from './session.js';
-/**
- * Factory user-function which creates customized {@link Session} instances.
- */
-export interface CreateSession {
-    /**
-     * @param sessionPool Pool requesting the new session.
-     * @param options
-     */
-    (sessionPool: SessionPool, options?: {
-        sessionOptions?: SessionOptions;
-    }): Session | Promise<Session>;
-}
-export interface SessionPoolOptions {
-    /**
-     * Maximum size of the pool. Indicates how many sessions are rotated.
-     * @default 1000
-     */
-    maxPoolSize?: number;
-    /** The configuration options for {@link Session} instances. */
-    sessionOptions?: SessionOptions;
-    /** Name or Id of `KeyValueStore` where is the `SessionPool` state stored. */
-    persistStateKeyValueStoreId?: string;
-    /**
-     * Session pool persists it's state under this key in Key value store.
-     * @default SESSION_POOL_STATE
-     */
-    persistStateKey?: string;
-    /**
-     * Custom function that should return a `Session` instance, or a promise resolving to such instance.
-     * Any error thrown from this function will terminate the process.
-     * Function receives `SessionPool` instance as a parameter
-     */
-    createSessionFunction?: CreateSession;
-    /**
-     * Specifies which response status codes are considered as blocked.
-     * Session connected to such request will be marked as retired.
-     * @default [401, 403, 429]
-     */
-    blockedStatusCodes?: number[];
-    /** @internal */
-    log?: Log;
-    /**
-     * Control how and when to persist the state of the session pool.
-     */
-    persistenceOptions?: PersistenceOptions;
-}
-/**
- * Handles the rotation, creation and persistence of user-like sessions.
- * Creates a pool of {@link Session} instances, that are randomly rotated.
- * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
- * Learn more in the {@doclink guides/session-management | Session management guide}.
- *
- * You can create one by calling the {@link SessionPool.open} function.
- *
- * Session pool is already integrated into crawlers, and it can significantly improve your scraper
- * performance with just 2 lines of code.
- *
- * **Example usage:**
- *
- * ```javascript
- * const crawler = new CheerioCrawler({
- *     useSessionPool: true,
- *     persistCookiesPerSession: true,
- *     // ...
- * })
- * ```
- *
- * You can configure the pool with many options. See the {@link SessionPoolOptions}.
- * Session pool is by default persisted in default {@link KeyValueStore}.
- * If you want to have one pool for all runs you have to specify
- * {@link SessionPoolOptions.persistStateKeyValueStoreId}.
- *
- * **Advanced usage:**
- *
- * ```javascript
- * const sessionPool = await SessionPool.open({
- *     maxPoolSize: 25,
- *     sessionOptions:{
- *          maxAgeSecs: 10,
- *          maxUsageCount: 150, // for example when you know that the site blocks after 150 requests.
- *     },
- *     persistStateKeyValueStoreId: 'my-key-value-store-for-sessions',
- *     persistStateKey: 'my-session-pool',
- * });
- *
- * // Get random session from the pool
- * const session1 = await sessionPool.getSession();
- * const session2 = await sessionPool.getSession();
- * const session3 = await sessionPool.getSession();
- *
- * // Now you can mark the session either failed or successful
- *
- * // Marks session as bad after unsuccessful usage -> it increases error count (soft retire)
- * session1.markBad()
- *
- * // Marks as successful.
- * session2.markGood()
- *
- * // Retires session -> session is removed from the pool
- * session3.retire()
- *
- * ```
- *
- * **Default session allocation flow:*
- * 1. Until the `SessionPool` reaches `maxPoolSize`, new sessions are created, provided to the user and added to the pool
- * 2. Blocked/retired sessions stay in the pool but are never provided to the user
- * 3. Once the pool is full (live plus blocked session count reaches `maxPoolSize`), a random session from the pool is provided.
- * 4. If a blocked session would be picked, instead all blocked sessions are evicted from the pool and a new session is created and provided
- *
- * @category Scaling
- */
-export declare class SessionPool extends EventEmitter {
-    protected log: Log;
-    protected maxPoolSize: number;
-    protected createSessionFunction: CreateSession;
-    protected keyValueStore: KeyValueStore;
-    protected sessions: Session[];
-    protected sessionMap: Map<string, Session>;
-    protected sessionOptions: SessionOptions;
-    protected persistStateKeyValueStoreId?: string;
-    protected persistStateKey: string;
-    protected _listener: () => Promise<void>;
-    protected events: EventManager;
-    protected readonly blockedStatusCodes: number[];
-    protected persistenceOptions: PersistenceOptions;
-    protected isInitialized: boolean;
-    private queue;
-    /**
-     * @internal
-     */
-    constructor(options?: SessionPoolOptions);
-    /**
-     * Gets count of usable sessions in the pool.
-     */
-    get usableSessionsCount(): number;
-    /**
-     * Gets count of retired sessions in the pool.
-     */
-    get retiredSessionsCount(): number;
-    /**
-     * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
-     * It is called automatically by the {@link SessionPool.open} function.
-     */
-    initialize(): Promise<void>;
-    /**
-     * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
-     * but this allows you to add more sessions once the max pool size is reached.
-     * This also allows you to add session with overridden session options (e.g. with specific session id).
-     * @param [options] The configuration options for the session being added to the session pool.
-     */
-    addSession(options?: Session | SessionOptions): Promise<void>;
-    /**
-     * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
-     * but this allows you to add more sessions once the max pool size is reached.
-     * This also allows you to add session with overridden session options (e.g. with specific session id).
-     * @param [options] The configuration options for the session being added to the session pool.
-     */
-    newSession(sessionOptions?: SessionOptions): Promise<Session>;
-    /**
-     * Gets session.
-     * If there is space for new session, it creates and returns new session.
-     * If the session pool is full, it picks a session from the pool,
-     * If the picked session is usable it is returned, otherwise it creates and returns a new one.
-     */
-    getSession(): Promise<Session>;
-    /**
-     * Gets session based on the provided session id or `undefined.
-     */
-    getSession(sessionId: string): Promise<Session>;
-    /**
-     * @param options - Override the persistence options provided in the constructor
-     */
-    resetStore(options?: PersistenceOptions): Promise<void>;
-    /**
-     * Returns an object representing the internal state of the `SessionPool` instance.
-     * Note that the object's fields can change in future releases.
-     */
-    getState(): {
-        usableSessionsCount: number;
-        retiredSessionsCount: number;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        sessions: import("@crawlee/types").SessionState[];
-    };
-    /**
-     * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
-     * The state is persisted automatically in regular intervals.
-     * @param options - Override the persistence options provided in the constructor
-     */
-    persistState(options?: PersistenceOptions): Promise<void>;
-    /**
-     * Removes listener from `persistState` event.
-     * This function should be called after you are done with using the `SessionPool` instance.
-     */
-    teardown(): Promise<void>;
-    /**
-     * SessionPool should not work before initialization.
-     */
-    protected _throwIfNotInitialized(): void;
-    /**
-     * Removes retired `Session` instances from `SessionPool`.
-     */
-    protected _removeRetiredSessions(): void;
-    /**
-     * Adds `Session` instance to `SessionPool`.
-     * @param newSession `Session` instance to be added.
-     */
-    protected _addSession(newSession: Session): void;
-    /**
-     * Gets random index.
-     */
-    protected _getRandomIndex(): number;
-    /**
-     * Creates new session without any extra behavior.
-     * @param sessionPool
-     * @param [options]
-     * @param [options.sessionOptions] The configuration options for the session being created.
-     * @returns New session.
-     */
-    protected _defaultCreateSessionFunction(sessionPool: SessionPool, options?: {
-        sessionOptions?: SessionOptions;
-    }): Promise<Session>;
-    /**
-     * Creates new session and adds it to the pool.
-     * @returns Newly created `Session` instance.
-     */
-    protected _createSession(): Promise<Session>;
-    /**
-     * Decides whether there is enough space for creating new session.
-     */
-    protected _hasSpaceForSession(): boolean;
-    /**
-     * Picks random session from the `SessionPool`.
-     * @returns Picked `Session`.
-     */
-    protected _pickSession(): Session;
-    /**
-     * Potentially loads `SessionPool`.
-     * If the state was persisted it loads the `SessionPool` from the persisted state.
-     */
-    protected _maybeLoadSessionPool(): Promise<void>;
-    /**
-     * Opens a SessionPool and returns a promise resolving to an instance
-     * of the {@link SessionPool} class that is already initialized.
-     *
-     * For more details and code examples, see the {@link SessionPool} class.
-     */
-    static open(options?: SessionPoolOptions): Promise<SessionPool>;
-}
-//# sourceMappingURL=session_pool.d.ts.map

package/session_pool/session_pool.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAM3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAI/D,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAE/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;;OAGG;IACH,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,kEAAkE;IAClE,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE9B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,qBAAa,WAAY,SAAQ,YAAY;IACzC,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC;IACnB,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,qBAAqB,EAAE,aAAa,CAAC;IAC/C,SAAS,CAAC,aAAa,EAAG,aAAa,CAAC;IACxC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAM;IACnC,SAAS,CAAC,UAAU,uBAA8B;IAClD,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IACzC,SAAS,CAAC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAC/C,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,SAAS,EAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAChD,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,aAAa,UAAS;IAEhC,OAAO,CAAC,KAAK,CAAoB;IAEjC;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IAoD5C;;OAEG;IACH,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA4BjC;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBvE;;;;;OAKG;IACG,UAAU,CAAC,cAAc,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC;IASnE;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEpC;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAqCrD;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAQ7C;;;OAGG;IACH,QAAQ;;;;;IAQR;;;;OAIG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB/D;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAK/B;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAIhC;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAWhC;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO;IAKzC;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;;OAMG;cACa,6BAA6B,CACzC,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAO,GAClD,OAAO,CAAC,OAAO,CAAC;IAWnB;;;OAGG;cACa,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAQlD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,OAAO;IAIxC;;;OAGG;IACH,SAAS,CAAC,YAAY,IAAI,OAAO;IAIjC;;;OAGG;cACa,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBtD;;;;;OAKG;WACU,IAAI,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC;CAKxE"}