@crawlee/core 3.0.0-alpha.0 → 3.0.0-alpha.3

package/dist/session_pool/session_pool.js

@@ -0,0 +1,389 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SessionPool = void 0;
+const tslib_1 = require("tslib");
+const events_1 = require("events");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const configuration_1 = require("../configuration");
+const log_1 = require("../log");
+const key_value_store_1 = require("../storages/key_value_store");
+const session_1 = require("./session");
+/**
+ * Handles the rotation, creation and persistence of user-like sessions.
+ * Creates a pool of {@link Session} instances, that are randomly rotated.
+ * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
+ * Learn more in the [Session management guide](../../../docs/guides/session-management).
+ *
+ * You can create one by calling the {@link SessionPool.open} function.
+ *
+ * Session pool is already integrated into crawlers, and it can significantly improve your scraper
+ * performance with just 2 lines of code.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new CheerioCrawler({
+ *     useSessionPool: true,
+ *     persistCookiesPerSession: true,
+ *     // ...
+ * })
+ * ```
+ *
+ * You can configure the pool with many options. See the {@link SessionPoolOptions}.
+ * Session pool is by default persisted in default {@link KeyValueStore}.
+ * If you want to have one pool for all runs you have to specify
+ * {@link SessionPoolOptions.persistStateKeyValueStoreId}.
+ *
+ * **Advanced usage:**
+ *
+ * ```javascript
+ * const sessionPool = await SessionPool.open({
+ *     maxPoolSize: 25,
+ *     sessionOptions:{
+ *          maxAgeSecs: 10,
+ *          maxUsageCount: 150, // for example when you know that the site blocks after 150 requests.
+ *     },
+ *     persistStateKeyValueStoreId: 'my-key-value-store-for-sessions',
+ *     persistStateKey: 'my-session-pool',
+ * });
+ *
+ * // Get random session from the pool
+ * const session1 = await sessionPool.getSession();
+ * const session2 = await sessionPool.getSession();
+ * const session3 = await sessionPool.getSession();
+ *
+ * // Now you can mark the session either failed or successful
+ *
+ * // Marks session as bad after unsuccessful usage -> it increases error count (soft retire)
+ * session1.markBad()
+ *
+ * // Marks as successful.
+ * session2.markGood()
+ *
+ * // Retires session -> session is removed from the pool
+ * session3.retire()
+ *
+ * ```
+ * @category Scaling
+ */
+class SessionPool extends events_1.EventEmitter {
+    /**
+     * @internal
+     */
+    constructor(options = {}, config = configuration_1.Configuration.getGlobalConfig()) {
+        super();
+        Object.defineProperty(this, "config", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: config
+        });
+        Object.defineProperty(this, "log", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxPoolSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "createSessionFunction", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "keyValueStore", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "sessions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "sessionMap", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+        Object.defineProperty(this, "sessionOptions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "persistStateKeyValueStoreId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "persistStateKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_listener", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "events", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        (0, ow_1.default)(options, ow_1.default.object.exactShape({
+            maxPoolSize: ow_1.default.optional.number,
+            persistStateKeyValueStoreId: ow_1.default.optional.string,
+            persistStateKey: ow_1.default.optional.string,
+            createSessionFunction: ow_1.default.optional.function,
+            sessionOptions: ow_1.default.optional.object,
+            log: ow_1.default.optional.object,
+        }));
+        const { maxPoolSize = 1000, persistStateKeyValueStoreId, persistStateKey = 'SDK_SESSION_POOL_STATE', createSessionFunction, sessionOptions = {}, log = log_1.log, } = options;
+        this.config = config;
+        this.events = config.getEventManager();
+        this.log = log.child({ prefix: 'SessionPool' });
+        // Pool Configuration
+        this.maxPoolSize = maxPoolSize;
+        this.createSessionFunction = createSessionFunction || this._defaultCreateSessionFunction;
+        // Session configuration
+        this.sessionOptions = {
+            ...sessionOptions,
+            // the log needs to propagate to createSessionFunction as in "new Session({ ...sessionPool.sessionOptions })"
+            // and can't go inside _defaultCreateSessionFunction
+            log: this.log,
+        };
+        // Session keyValueStore
+        this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
+        this.persistStateKey = persistStateKey;
+    }
+    /**
+     * Gets count of usable sessions in the pool.
+     */
+    get usableSessionsCount() {
+        return this.sessions.filter((session) => session.isUsable()).length;
+    }
+    /**
+     * Gets count of retired sessions in the pool.
+     */
+    get retiredSessionsCount() {
+        return this.sessions.filter((session) => !session.isUsable()).length;
+    }
+    /**
+     * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
+     * It is called automatically by the {@link SessionPool.open} function.
+     */
+    async initialize() {
+        this.keyValueStore = await key_value_store_1.KeyValueStore.open(this.persistStateKeyValueStoreId, { config: this.config });
+        // FIXME should we still validate this somehow somewhere?
+        // if (this.forceCloud && !this.persistStateKeyValueStoreId) {
+        //     this.log.warning(`You enabled 'forceCloud' in the session pool options but you haven't specified a 'persistStateKeyValueStoreId'!`);
+        //     this.log.warning(`This session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
+        // }
+        // in case of migration happened and SessionPool state should be restored from the keyValueStore.
+        await this._maybeLoadSessionPool();
+        this._listener = this.persistState.bind(this);
+        this.events.on("persistState" /* PERSIST_STATE */, this._listener);
+    }
+    /**
+     * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
+     * but this allows you to add more sessions once the max pool size is reached.
+     * This also allows you to add session with overridden session options (e.g. with specific session id).
+     * @param [options] The configuration options for the session being added to the session pool.
+     */
+    async addSession(options = {}) {
+        this._throwIfNotInitialized();
+        const { id } = options;
+        if (id) {
+            const sessionExists = this.sessionMap.has(id);
+            if (sessionExists) {
+                throw new Error(`Cannot add session with id '${id}' as it already exists in the pool`);
+            }
+        }
+        if (!this._hasSpaceForSession()) {
+            this._removeRetiredSessions();
+        }
+        const newSession = options instanceof session_1.Session
+            ? options
+            : await this.createSessionFunction(this, { sessionOptions: options });
+        this.log.debug(`Adding new Session - ${newSession.id}`);
+        this._addSession(newSession);
+    }
+    /**
+     * Gets session.
+     * If there is space for new session, it creates and returns new session.
+     * If the session pool is full, it picks a session from the pool,
+     * If the picked session is usable it is returned, otherwise it creates and returns a new one.
+     * @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
+     */
+    async getSession(sessionId) {
+        this._throwIfNotInitialized();
+        if (sessionId) {
+            const session = this.sessionMap.get(sessionId);
+            if (session && session.isUsable())
+                return session;
+            return undefined;
+        }
+        if (this._hasSpaceForSession()) {
+            return this._createSession();
+        }
+        const pickedSession = this._pickSession();
+        if (pickedSession.isUsable()) {
+            return pickedSession;
+        }
+        this._removeRetiredSessions();
+        return this._createSession();
+    }
+    /**
+     * Returns an object representing the internal state of the `SessionPool` instance.
+     * Note that the object's fields can change in future releases.
+     */
+    getState() {
+        return {
+            usableSessionsCount: this.usableSessionsCount,
+            retiredSessionsCount: this.retiredSessionsCount,
+            sessions: this.sessions.map((session) => session.getState()),
+        };
+    }
+    /**
+     * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
+     * The state is persisted automatically in regular intervals.
+     */
+    async persistState() {
+        this.log.debug('Persisting state', {
+            persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
+            persistStateKey: this.persistStateKey,
+        });
+        await this.keyValueStore.setValue(this.persistStateKey, this.getState());
+    }
+    /**
+     * Removes listener from `persistState` event.
+     * This function should be called after you are done with using the `SessionPool` instance.
+     */
+    async teardown() {
+        this.events.off("persistState" /* PERSIST_STATE */, this._listener);
+        await this.persistState();
+    }
+    /**
+     * SessionPool should not work before initialization.
+     */
+    _throwIfNotInitialized() {
+        if (!this._listener)
+            throw new Error('SessionPool is not initialized.');
+    }
+    /**
+     * Removes retired `Session` instances from `SessionPool`.
+     */
+    _removeRetiredSessions() {
+        this.sessions = this.sessions.filter((storedSession) => {
+            if (storedSession.isUsable())
+                return true;
+            this.sessionMap.delete(storedSession.id);
+            this.log.debug(`Removed Session - ${storedSession.id}`);
+            return false;
+        });
+    }
+    /**
+     * Adds `Session` instance to `SessionPool`.
+     * @param newSession `Session` instance to be added.
+     */
+    _addSession(newSession) {
+        this.sessions.push(newSession);
+        this.sessionMap.set(newSession.id, newSession);
+    }
+    /**
+     * Gets random index.
+     */
+    _getRandomIndex() {
+        return Math.floor(Math.random() * this.sessions.length);
+    }
+    /**
+     * Creates new session without any extra behavior.
+     * @param sessionPool
+     * @param [options]
+     * @param [options.sessionOptions] The configuration options for the session being created.
+     * @returns New session.
+     */
+    _defaultCreateSessionFunction(sessionPool, options = {}) {
+        (0, ow_1.default)(options, ow_1.default.object.exactShape({ sessionOptions: ow_1.default.optional.object }));
+        const { sessionOptions = {} } = options;
+        return new session_1.Session({
+            ...this.sessionOptions,
+            ...sessionOptions,
+            sessionPool,
+        });
+    }
+    /**
+     * Creates new session and adds it to the pool.
+     * @returns Newly created `Session` instance.
+     */
+    async _createSession() {
+        const newSession = await this.createSessionFunction(this);
+        this._addSession(newSession);
+        this.log.debug(`Created new Session - ${newSession.id}`);
+        return newSession;
+    }
+    /**
+     * Decides whether there is enough space for creating new session.
+     */
+    _hasSpaceForSession() {
+        return this.sessions.length < this.maxPoolSize;
+    }
+    /**
+     * Picks random session from the `SessionPool`.
+     * @returns Picked `Session`.
+     */
+    _pickSession() {
+        return this.sessions[this._getRandomIndex()]; // Or maybe we should let the developer to customize the picking algorithm
+    }
+    /**
+     * Potentially loads `SessionPool`.
+     * If the state was persisted it loads the `SessionPool` from the persisted state.
+     */
+    async _maybeLoadSessionPool() {
+        const loadedSessionPool = await this.keyValueStore.getValue(this.persistStateKey);
+        if (!loadedSessionPool)
+            return;
+        // Invalidate old sessions and load active sessions only
+        this.log.debug('Recreating state from KeyValueStore', {
+            persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
+            persistStateKey: this.persistStateKey,
+        });
+        for (const sessionObject of loadedSessionPool.sessions) {
+            sessionObject.sessionPool = this;
+            sessionObject.createdAt = new Date(sessionObject.createdAt);
+            sessionObject.expiresAt = new Date(sessionObject.expiresAt);
+            const recreatedSession = new session_1.Session(sessionObject);
+            if (recreatedSession.isUsable()) {
+                this._addSession(recreatedSession);
+            }
+        }
+        this.log.debug(`${this.usableSessionsCount} active sessions loaded from KeyValueStore`);
+    }
+    /**
+     * Opens a SessionPool and returns a promise resolving to an instance
+     * of the {@link SessionPool} class that is already initialized.
+     *
+     * For more details and code examples, see the {@link SessionPool} class.
+     */
+    static async open(options) {
+        const sessionPool = new SessionPool(options);
+        await sessionPool.initialize();
+        return sessionPool;
+    }
+}
+exports.SessionPool = SessionPool;
+//# sourceMappingURL=session_pool.js.map

package/dist/session_pool/session_pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":";;;;AACA,mCAAsC;AACtC,oDAAoB;AACpB,oDAAiD;AACjD,gCAA2C;AAC3C,iEAA4D;AAE5D,uCAAoD;AA4CpD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,MAAa,WAAY,SAAQ,qBAAY;IAazC;;OAEG;IACH,YAAY,UAA8B,EAAE,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;QAC3F,KAAK,EAAE,CAAC;;;;;mBAD2C;;QAfvD;;;;;WAAS;QACT;;;;;WAAoB;QACpB;;;;;WAAqC;QACrC;;;;;WAA8B;QAC9B;;;;mBAAsB,EAAE;WAAC;QACzB;;;;mBAAa,IAAI,GAAG,EAAmB;WAAC;QACxC;;;;;WAA+B;QAC/B;;;;;WAAqC;QACrC;;;;;WAAwB;QACxB;;;;;WAAwC;QACxC;;;;;WAA6B;QAQzB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC/B,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC/C,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;YAC3C,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,WAAW,GAAG,IAAI,EAElB,2BAA2B,EAC3B,eAAe,GAAG,wBAAwB,EAE1C,qBAAqB,EACrB,cAAc,GAAG,EAAE,EAEnB,GAAG,GAAG,SAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QACvC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAEhD,qBAAqB;QACrB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,IAAI,IAAI,CAAC,6BAA6B,CAAC;QAEzF,wBAAwB;QACxB,IAAI,CAAC,cAAc,GAAG;YAClB,GAAG,cAAc;YACjB,6GAA6G;YAC7G,oDAAoD;YACpD,GAAG,EAAE,IAAI,CAAC,GAAG;SAChB,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACxE,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACzE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,CAAC,aAAa,GAAG,MAAM,+BAAa,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAEzG,yDAAyD;QACzD,8DAA8D;QAC9D,2IAA2I;QAC3I,8HAA8H;QAC9H,IAAI;QAEJ,iGAAiG;QACjG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAEnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,CAAC,MAAM,CAAC,EAAE,qCAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,UAAoC,EAAE;QACnD,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;QACvB,IAAI,EAAE,EAAE;YACJ,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,aAAa,EAAE;gBACf,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,oCAAoC,CAAC,CAAC;aAC1F;SACJ;QAED,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE;YAC7B,IAAI,CAAC,sBAAsB,EAAE,CAAC;SACjC;QAED,MAAM,UAAU,GAAG,OAAO,YAAY,iBAAO;YACzC,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1E,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAExD,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAeD;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,SAAkB;QAC/B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,IAAI,SAAS,EAAE;YACX,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/C,IAAI,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE;gBAAE,OAAO,OAAO,CAAC;YAClD,OAAO,SAAS,CAAC;SACpB;QAED,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE;YAC5B,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;SAChC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,aAAa,CAAC,QAAQ,EAAE,EAAE;YAC1B,OAAO,aAAa,CAAC;SACxB;QAED,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;YAC/C,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;SAC/D,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE;YAC/B,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,CAAC,GAAG,qCAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;QACzD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IAC5E,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,EAAE;YACnD,IAAI,aAAa,CAAC,QAAQ,EAAE;gBAAE,OAAO,IAAI,CAAC;YAE1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qBAAqB,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;YAExD,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,WAAW,CAAC,UAAmB;QACrC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACO,eAAe;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;;OAMG;IACO,6BAA6B,CAAC,WAAwB,EAAE,UAA+C,EAAE;QAC/G,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACxC,OAAO,IAAI,iBAAO,CAAC;YACf,GAAG,IAAI,CAAC,cAAc;YACtB,GAAG,cAAc;YACjB,WAAW;SACd,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,cAAc;QAC1B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAAyB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAEzD,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;OAEG;IACO,mBAAmB;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;IACnD,CAAC;IAED;;;OAGG;IACO,YAAY;QAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,0EAA0E;IAC5H,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,qBAAqB;QACjC,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA6B,IAAI,CAAC,eAAe,CAAC,CAAC;QAE9G,IAAI,CAAC,iBAAiB;YAAE,OAAO;QAE/B,wDAAwD;QACxD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE;YAClD,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,KAAK,MAAM,aAAa,IAAI,iBAAiB,CAAC,QAAQ,EAAE;YACpD,aAAa,CAAC,WAAW,GAAG,IAAI,CAAC;YACjC,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,MAAM,gBAAgB,GAAG,IAAI,iBAAO,CAAC,aAAa,CAAC,CAAC;YAEpD,IAAI,gBAAgB,CAAC,QAAQ,EAAE,EAAE;gBAC7B,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;aACtC;SACJ;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,mBAAmB,4CAA4C,CAAC,CAAC;IAC5F,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAA4B;QAC1C,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;QAC7C,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC;QAC/B,OAAO,WAAW,CAAC;IACvB,CAAC;CACJ;AA/TD,kCA+TC"}

package/dist/session_pool/session_utils.d.ts

@@ -0,0 +1,9 @@
+/// <reference types="node" />
+import { IncomingMessage } from 'http';
+import { HTTPResponse } from 'puppeteer';
+import { Cookie } from 'tough-cookie';
+import { Dictionary } from '../typedefs';
+export declare function getCookiesFromResponse(response: IncomingMessage | HTTPResponse | {
+    headers: Dictionary<string | string[]>;
+}): Cookie[];
+//# sourceMappingURL=session_utils.d.ts.map

package/dist/session_pool/session_utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"session_utils.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,MAAM,CAAC;AACvC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,eAAe,GAAG,YAAY,GAAG;IAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAA;CAAE,GAAG,MAAM,EAAE,CAWtI"}

package/dist/session_pool/session_utils.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getCookiesFromResponse = void 0;
+const tough_cookie_1 = require("tough-cookie");
+const errors_1 = require("./errors");
+function getCookiesFromResponse(response) {
+    const headers = typeof response.headers === 'function' ? response.headers() : response.headers;
+    const cookieHeader = headers['set-cookie'] || '';
+    try {
+        return Array.isArray(cookieHeader)
+            ? cookieHeader.map((cookie) => tough_cookie_1.Cookie.parse(cookie))
+            : [tough_cookie_1.Cookie.parse(cookieHeader)];
+    }
+    catch (e) {
+        throw new errors_1.CookieParseError(cookieHeader);
+    }
+}
+exports.getCookiesFromResponse = getCookiesFromResponse;
+//# sourceMappingURL=session_utils.js.map

package/dist/session_pool/session_utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"session_utils.js","sourceRoot":"","sources":["../../src/session_pool/session_utils.ts"],"names":[],"mappings":";;;AAEA,+CAAsC;AAEtC,qCAA4C;AAE5C,SAAgB,sBAAsB,CAAC,QAAqF;IACxH,MAAM,OAAO,GAAG,OAAO,QAAQ,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;IAC/F,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAEjD,IAAI;QACA,OAAO,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;YAC9B,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;YACrD,CAAC,CAAC,CAAC,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC,CAAC;KACvC;IAAC,OAAO,CAAC,EAAE;QACR,MAAM,IAAI,yBAAgB,CAAC,YAAY,CAAC,CAAC;KAC5C;AACL,CAAC;AAXD,wDAWC"}