@crawlee/core 3.0.0-alpha.2 → 3.0.0-alpha.3

package/dist/serialization.d.ts

@@ -0,0 +1,33 @@
+/// <reference types="node" />
+import { Readable } from 'stream';
+/**
+ * Uses Gzip compression to take an array of values, which can be anything
+ * from entries in a Dataset to Requests in a RequestList and compresses
+ * them to a Buffer in a memory-efficient way (streaming one by one). Ideally,
+ * the largest chunk of memory consumed will be the final compressed Buffer.
+ * This could be further improved by outputting a Stream, if and when
+ * apify-client supports streams.
+ * @internal
+ */
+export declare function serializeArray<T>(data: T[]): Promise<Buffer>;
+/**
+ * Decompresses a Buffer previously created with compressData (technically,
+ * any JSON that is an Array) and collects it into an Array of values
+ * in a memory-efficient way (streaming the array items one by one instead
+ * of creating a fully decompressed buffer -> full JSON -> full Array all
+ * in memory at once. Could be further optimized to ingest a Stream if and
+ * when apify-client supports streams.
+ * @internal
+ */
+export declare function deserializeArray<T extends string | Buffer>(compressedData: Buffer): Promise<T[]>;
+/**
+ * Creates a stream that decompresses a Buffer previously created with
+ * compressData (technically, any JSON that is an Array) and collects it
+ * into an Array of values in a memory-efficient way (streaming the array
+ * items one by one instead of creating a fully decompressed buffer
+ * -> full JSON -> full Array all in memory at once. Could be further
+ * optimized to ingest a Stream if and when apify-client supports streams.
+ * @internal
+ */
+export declare function createDeserialize(compressedData: Buffer): Readable;
+//# sourceMappingURL=serialization.d.ts.map

package/dist/serialization.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"serialization.d.ts","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":";AACA,OAAO,EAA8B,QAAQ,EAAY,MAAM,QAAQ,CAAC;AAkDxE;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAUlE;AAED;;;;;;;;GAQG;AACH,wBAAsB,gBAAgB,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAWtG;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,cAAc,EAAE,MAAM,GAAG,QAAQ,CAelE"}

package/dist/serialization.js

@@ -0,0 +1,146 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createDeserialize = exports.deserializeArray = exports.serializeArray = void 0;
+const tslib_1 = require("tslib");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const stream_1 = require("stream");
+const StreamArray_1 = tslib_1.__importDefault(require("stream-json/streamers/StreamArray"));
+const util_1 = tslib_1.__importDefault(require("util"));
+const zlib_1 = tslib_1.__importDefault(require("zlib"));
+const pipeline = util_1.default.promisify(stream_1.pipeline);
+/**
+ * Transforms an array of items to a JSON in a streaming
+ * fashion to save memory. It operates in batches to speed
+ * up the process.
+ * @internal
+ */
+class ArrayToJson extends stream_1.Readable {
+    constructor(data, options = {}) {
+        super({
+            ...options,
+            autoDestroy: true,
+            emitClose: true,
+        });
+        Object.defineProperty(this, "data", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: data
+        });
+        Object.defineProperty(this, "offset", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "batchSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const { batchSize = 10000 } = options;
+        this.batchSize = batchSize;
+        this.data = data;
+        this.push('[');
+    }
+    _read() {
+        try {
+            const items = this.data.slice(this.offset, this.offset + this.batchSize);
+            if (items.length) {
+                const json = JSON.stringify(items);
+                // Strip brackets to flatten the batch.
+                const itemString = json.substring(1, json.length - 1);
+                if (this.offset > 0)
+                    this.push(',', 'utf8');
+                this.push(itemString, 'utf8');
+                this.offset += this.batchSize;
+            }
+            else {
+                this.push(']');
+                this.push(null);
+            }
+        }
+        catch (err) {
+            this.emit('error', err);
+        }
+    }
+}
+/**
+ * Uses Gzip compression to take an array of values, which can be anything
+ * from entries in a Dataset to Requests in a RequestList and compresses
+ * them to a Buffer in a memory-efficient way (streaming one by one). Ideally,
+ * the largest chunk of memory consumed will be the final compressed Buffer.
+ * This could be further improved by outputting a Stream, if and when
+ * apify-client supports streams.
+ * @internal
+ */
+async function serializeArray(data) {
+    (0, ow_1.default)(data, ow_1.default.array);
+    const { chunks, collector } = createChunkCollector();
+    await pipeline(new ArrayToJson(data), zlib_1.default.createGzip(), collector);
+    return Buffer.concat(chunks);
+}
+exports.serializeArray = serializeArray;
+/**
+ * Decompresses a Buffer previously created with compressData (technically,
+ * any JSON that is an Array) and collects it into an Array of values
+ * in a memory-efficient way (streaming the array items one by one instead
+ * of creating a fully decompressed buffer -> full JSON -> full Array all
+ * in memory at once. Could be further optimized to ingest a Stream if and
+ * when apify-client supports streams.
+ * @internal
+ */
+async function deserializeArray(compressedData) {
+    (0, ow_1.default)(compressedData, ow_1.default.buffer);
+    const { chunks, collector } = createChunkCollector({ fromValuesStream: true });
+    await pipeline(stream_1.Readable.from([compressedData]), zlib_1.default.createGunzip(), StreamArray_1.default.withParser(), collector);
+    return chunks;
+}
+exports.deserializeArray = deserializeArray;
+/**
+ * Creates a stream that decompresses a Buffer previously created with
+ * compressData (technically, any JSON that is an Array) and collects it
+ * into an Array of values in a memory-efficient way (streaming the array
+ * items one by one instead of creating a fully decompressed buffer
+ * -> full JSON -> full Array all in memory at once. Could be further
+ * optimized to ingest a Stream if and when apify-client supports streams.
+ * @internal
+ */
+function createDeserialize(compressedData) {
+    (0, ow_1.default)(compressedData, ow_1.default.buffer);
+    const streamArray = StreamArray_1.default.withParser();
+    const destination = pluckValue(streamArray);
+    (0, stream_1.pipeline)(stream_1.Readable.from([compressedData]), zlib_1.default.createGunzip(), destination, 
+    // TODO: figure this out
+    // @ts-expect-error Somethings wrong here, the types are wrong but tests fail if we correct the code to make them right
+    (err) => destination.emit(err));
+    return destination;
+}
+exports.createDeserialize = createDeserialize;
+function createChunkCollector(options = {}) {
+    const { fromValuesStream = false } = options;
+    const chunks = [];
+    const collector = new stream_1.Writable({
+        decodeStrings: false,
+        objectMode: fromValuesStream,
+        write(chunk, _nil, callback) {
+            chunks.push(fromValuesStream ? chunk.value : chunk);
+            callback();
+        },
+        writev(chunkObjects, callback) {
+            const buffers = chunkObjects.map(({ chunk }) => {
+                return fromValuesStream ? chunk.value : chunk;
+            });
+            chunkObjects.push(...buffers);
+            callback();
+        },
+    });
+    return { collector, chunks };
+}
+function pluckValue(streamArray) {
+    const realPush = streamArray.push.bind(streamArray);
+    streamArray.push = (obj) => realPush(obj && obj.value);
+    return streamArray;
+}
+//# sourceMappingURL=serialization.js.map

package/dist/serialization.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"serialization.js","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,mCAAwE;AAExE,4FAA4D;AAC5D,wDAAwB;AACxB,wDAAwB;AAExB,MAAM,QAAQ,GAAG,cAAI,CAAC,SAAS,CAAC,iBAAc,CAAC,CAAC;AAEhD;;;;;GAKG;AACH,MAAM,WAAe,SAAQ,iBAAQ;IAIjC,YAAoB,IAAS,EAAE,UAAkC,EAAE;QAC/D,KAAK,CAAC;YACF,GAAG,OAAO;YACV,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;;;;;mBALa;;QAHpB;;;;mBAAiB,CAAC;WAAC;QACnB;;;;;WAAmC;QAQ/B,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAEQ,KAAK;QACV,IAAI;YACA,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC,MAAM,EAAE;gBACd,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACnC,uCAAuC;gBACvC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC;aACjC;iBAAM;gBACH,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACnB;SACJ;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;SAC3B;IACL,CAAC;CACJ;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,cAAc,CAAI,IAAS;IAC7C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,EAAE,CAAC;IACrD,MAAM,QAAQ,CACV,IAAI,WAAW,CAAC,IAAI,CAAC,EACrB,cAAI,CAAC,UAAU,EAAE,EACjB,SAAS,CACZ,CAAC;IAEF,OAAO,MAAM,CAAC,MAAM,CAAC,MAAkB,CAAC,CAAC;AAC7C,CAAC;AAVD,wCAUC;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,gBAAgB,CAA4B,cAAsB;IACpF,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,CAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAClF,MAAM,QAAQ,CACV,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,cAAI,CAAC,YAAY,EAAE,EACnB,qBAAW,CAAC,UAAU,EAAE,EACxB,SAAS,CACZ,CAAC;IAEF,OAAO,MAAa,CAAC;AACzB,CAAC;AAXD,4CAWC;AAED;;;;;;;;GAQG;AACH,SAAgB,iBAAiB,CAAC,cAAsB;IACpD,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,WAAW,GAAG,qBAAW,CAAC,UAAU,EAAE,CAAC;IAC7C,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IAE5C,IAAA,iBAAc,EACV,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,cAAI,CAAC,YAAY,EAAE,EACnB,WAAW;IACX,wBAAwB;IACxB,uHAAuH;IACvH,CAAC,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CACjC,CAAC;IAEF,OAAO,WAAW,CAAC;AACvB,CAAC;AAfD,8CAeC;AAED,SAAS,oBAAoB,CAA4B,UAA0C,EAAE;IACjG,MAAM,EAAE,gBAAgB,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,MAAM,GAAQ,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,iBAAQ,CAAC;QAC3B,aAAa,EAAE,KAAK;QACpB,UAAU,EAAE,gBAAgB;QAC5B,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ;YACvB,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpD,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,MAAM,CAAC,YAAY,EAAE,QAAQ;YACzB,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC3C,OAAO,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YAClD,CAAC,CAAC,CAAC;YACH,YAAY,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AACjC,CAAC;AAED,SAAS,UAAU,CAAC,WAAkB;IAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpD,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IACvD,OAAO,WAAW,CAAC;AACvB,CAAC"}

package/dist/session_pool/errors.d.ts

@@ -0,0 +1,8 @@
+/**
+ * @ignore
+ */
+export declare class CookieParseError extends Error {
+    readonly cookieHeaderString: unknown;
+    constructor(cookieHeaderString: unknown);
+}
+//# sourceMappingURL=errors.d.ts.map

package/dist/session_pool/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/session_pool/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,gBAAiB,SAAQ,KAAK;IAC3B,QAAQ,CAAC,kBAAkB,EAAE,OAAO;gBAA3B,kBAAkB,EAAE,OAAO;CAInD"}

package/dist/session_pool/errors.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CookieParseError = void 0;
+/**
+ * @ignore
+ */
+class CookieParseError extends Error {
+    constructor(cookieHeaderString) {
+        super(`Could not parse cookie header string: ${cookieHeaderString}`);
+        Object.defineProperty(this, "cookieHeaderString", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: cookieHeaderString
+        });
+        Error.captureStackTrace(this, CookieParseError);
+    }
+}
+exports.CookieParseError = CookieParseError;
+//# sourceMappingURL=errors.js.map

package/dist/session_pool/errors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/session_pool/errors.ts"],"names":[],"mappings":";;;AAAA;;GAEG;AACH,MAAa,gBAAiB,SAAQ,KAAK;IACvC,YAAqB,kBAA2B;QAC5C,KAAK,CAAC,yCAAyC,kBAAkB,EAAE,CAAC,CAAC;;;;;mBADpD;;QAEjB,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;IACpD,CAAC;CACJ;AALD,4CAKC"}

package/dist/session_pool/events.d.ts

@@ -0,0 +1,3 @@
+/** @internal */
+export declare const EVENT_SESSION_RETIRED = "sessionRetired";
+//# sourceMappingURL=events.d.ts.map

package/dist/session_pool/events.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/session_pool/events.ts"],"names":[],"mappings":"AAAA,gBAAgB;AAChB,eAAO,MAAM,qBAAqB,mBAAmB,CAAC"}

package/dist/session_pool/events.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EVENT_SESSION_RETIRED = void 0;
+/** @internal */
+exports.EVENT_SESSION_RETIRED = 'sessionRetired';
+//# sourceMappingURL=events.js.map

package/dist/session_pool/events.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"events.js","sourceRoot":"","sources":["../../src/session_pool/events.ts"],"names":[],"mappings":";;;AAAA,gBAAgB;AACH,QAAA,qBAAqB,GAAG,gBAAgB,CAAC"}

package/dist/session_pool/index.d.ts

@@ -0,0 +1,6 @@
+export * from './errors';
+export * from './events';
+export * from './session';
+export * from './session_pool';
+export * from './session_utils';
+//# sourceMappingURL=index.d.ts.map

package/dist/session_pool/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC"}

package/dist/session_pool/index.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const tslib_1 = require("tslib");
+tslib_1.__exportStar(require("./errors"), exports);
+tslib_1.__exportStar(require("./events"), exports);
+tslib_1.__exportStar(require("./session"), exports);
+tslib_1.__exportStar(require("./session_pool"), exports);
+tslib_1.__exportStar(require("./session_utils"), exports);
+//# sourceMappingURL=index.js.map

package/dist/session_pool/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":";;;AAAA,mDAAyB;AACzB,mDAAyB;AACzB,oDAA0B;AAC1B,yDAA+B;AAC/B,0DAAgC"}

package/dist/session_pool/session.d.ts

@@ -0,0 +1,212 @@
+/// <reference types="node" />
+import { Log } from '@apify/log';
+import { Cookie as BrowserPoolCookie } from '@crawlee/browser-pool';
+import { IncomingMessage } from 'http';
+import { HTTPResponse, Protocol } from 'puppeteer';
+import { Cookie, CookieJar } from 'tough-cookie';
+import { Dictionary } from '../typedefs';
+import { SessionPool } from './session_pool';
+export declare type PuppeteerCookie = Protocol.Network.Cookie | BrowserPoolCookie;
+/**
+ * Persistable {@link Session} state.
+ */
+export interface SessionState {
+    id: string;
+    cookieJar: CookieJar.Serialized;
+    userData: object;
+    errorScore: number;
+    maxErrorScore: number;
+    errorScoreDecrement: number;
+    usageCount: number;
+    maxUsageCount: number;
+    expiresAt: string;
+    createdAt: string;
+}
+export interface SessionOptions {
+    /** Id of session used for generating fingerprints. It is used as proxy session name. */
+    id?: string;
+    /**
+     * Number of seconds after which the session is considered as expired.
+     * @default 3000
+     */
+    maxAgeSecs?: number;
+    /** Object where custom user data can be stored. For example custom headers. */
+    userData?: Dictionary;
+    /**
+     * Maximum number of marking session as blocked usage.
+     * If the `errorScore` reaches the `maxErrorScore` session is marked as block and it is thrown away.
+     * It starts at 0. Calling the `markBad` function increases the `errorScore` by 1.
+     * Calling the `markGood` will decrease the `errorScore` by `errorScoreDecrement`
+     * @default 3
+     */
+    maxErrorScore?: number;
+    /**
+     * It is used for healing the session.
+     * For example: if your session is marked bad two times, but it is successful on the third attempt it's errorScore
+     * is decremented by this number.
+     * @default 0.5
+     */
+    errorScoreDecrement?: number;
+    /** Date of creation. */
+    createdAt?: Date;
+    /** Date of expiration. */
+    expiresAt?: Date;
+    /**
+     * Indicates how many times the session has been used.
+     * @default 0
+     */
+    usageCount?: number;
+    /**
+     * Session should be used only a limited amount of times.
+     * This number indicates how many times the session is going to be used, before it is thrown away.
+     * @default 50
+     */
+    maxUsageCount?: number;
+    /** SessionPool instance. Session will emit the `sessionRetired` event on this instance. */
+    sessionPool?: SessionPool;
+    log?: Log;
+    errorScore?: number;
+    cookieJar?: CookieJar;
+}
+/**
+ * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
+ * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
+ * Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
+ * @category Scaling
+ */
+export declare class Session {
+    readonly id: string;
+    private maxAgeSecs;
+    userData: Dictionary;
+    private maxErrorScore;
+    private errorScoreDecrement;
+    private createdAt;
+    private expiresAt;
+    private usageCount;
+    private maxUsageCount;
+    private sessionPool;
+    private errorScore;
+    private cookieJar;
+    private log;
+    /**
+     * Session configuration.
+     */
+    constructor(options: SessionOptions);
+    /**
+     * Indicates whether the session is blocked.
+     * Session is blocked once it reaches the `maxErrorScore`.
+     */
+    isBlocked(): boolean;
+    /**
+     * Indicates whether the session is expired.
+     * Session expiration is determined by the `maxAgeSecs`.
+     * Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
+     */
+    isExpired(): boolean;
+    /**
+     * Indicates whether the session is used maximum number of times.
+     * Session maximum usage count can be changed by `maxUsageCount` parameter.
+     */
+    isMaxUsageCountReached(): boolean;
+    /**
+     * Indicates whether the session can be used for next requests.
+     * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
+     */
+    isUsable(): boolean;
+    /**
+     * This method should be called after a successful session usage.
+     * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
+     */
+    markGood(): void;
+    /**
+     * Gets session state for persistence in KeyValueStore.
+     * @returns Represents session internal state.
+     */
+    getState(): SessionState;
+    /**
+     * Marks session as blocked and emits event on the `SessionPool`
+     * This method should be used if the session usage was unsuccessful
+     * and you are sure that it is because of the session configuration and not any external matters.
+     * For example when server returns 403 status code.
+     * If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
+     */
+    retire(): void;
+    /**
+     * Increases usage and error count.
+     * Should be used when the session has been used unsuccessfully. For example because of timeouts.
+     */
+    markBad(): void;
+    /**
+     * With certain status codes: `401`, `403` or `429` we can be certain
+     * that the target website is blocking us. This function helps to do this conveniently
+     * by retiring the session when such code is received. Optionally the default status
+     * codes can be extended in the second parameter.
+     * @param statusCode HTTP status code.
+     * @param [blockedStatusCodes] Custom HTTP status codes that means blocking on particular website.
+     * @returns Whether the session was retired.
+     */
+    retireOnBlockedStatusCodes(statusCode: number, blockedStatusCodes?: number[]): boolean;
+    /**
+     * Saves cookies from an HTTP response to be used with the session.
+     * It expects an object with a `headers` property that's either an `Object`
+     * (typical Node.js responses) or a `Function` (Puppeteer Response).
+     *
+     * It then parses and saves the cookies from the `set-cookie` header, if available.
+     */
+    setCookiesFromResponse(response: IncomingMessage | HTTPResponse | {
+        headers: Dictionary<string | string[]>;
+        url: string;
+    }): void;
+    /**
+     * Saves an array with cookie objects to be used with the session.
+     * The objects should be in the format that
+     * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
+     * but you can also use this function to set cookies manually:
+     *
+     * ```
+     * [
+     *   { name: 'cookie1', value: 'my-cookie' },
+     *   { name: 'cookie2', value: 'your-cookie' }
+     * ]
+     * ```
+     */
+    setPuppeteerCookies(cookies: PuppeteerCookie[], url: string): void;
+    /**
+     * Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.
+     * @param url website url. Only cookies stored for this url will be returned
+     */
+    getPuppeteerCookies(url: string): PuppeteerCookie[];
+    /**
+     * Returns cookies saved with the session in the typical
+     * key1=value1; key2=value2 format, ready to be used in
+     * a cookie header or elsewhere.
+     * @returns Represents `Cookie` header.
+     */
+    getCookieString(url: string): string;
+    /**
+     * Transforms puppeteer cookie to tough-cookie.
+     * @param puppeteerCookie Cookie from puppeteer `page.cookies` method.
+     */
+    protected _puppeteerCookieToTough(puppeteerCookie: PuppeteerCookie): Cookie;
+    /**
+     * Transforms tough-cookie to puppeteer cookie.
+     * @param toughCookie Cookie from CookieJar
+     * @return Cookie from Puppeteer
+     */
+    protected _toughCookieToPuppeteer(toughCookie: Cookie): PuppeteerCookie;
+    /**
+     * Sets cookies.
+     */
+    protected _setCookies(cookies: Cookie[], url: string): void;
+    /**
+     * Calculate cookie expiration date
+     * @param maxAgeSecs
+     * @returns Calculated date by session max age seconds.
+     */
+    protected _getDefaultCookieExpirationDate(maxAgeSecs: number): Date;
+    /**
+     * Checks if session is not usable. if it is not retires the session.
+     */
+    protected _maybeSelfRetire(): void;
+}
+//# sourceMappingURL=session.d.ts.map

package/dist/session_pool/session.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEjC,OAAO,EAAE,MAAM,IAAI,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,MAAM,MAAM,CAAC;AAEvC,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGjD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAG7C,oBAAY,eAAe,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC;AAK1E;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAE3B,wFAAwF;IACxF,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,wBAAwB;IACxB,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,2FAA2F;IAC3F,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,SAAS,CAAC;CAEzB;AAED;;;;;GAKG;AACH,qBAAa,OAAO;IAChB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,UAAU,CAAS;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,GAAG,CAAM;IAEjB;;OAEG;gBACS,OAAO,EAAE,cAAc;IAsDnC;;;OAGG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,QAAQ;IAUR;;;OAGG;IACH,QAAQ,IAAI,YAAY;IAexB;;;;;;OAMG;IACH,MAAM;IASN;;;OAGG;IACH,OAAO;IAOP;;;;;;;;OAQG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,EAAE,kBAAkB,GAAE,MAAM,EAAO,GAAG,OAAO;IAQ1F;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAQ,EAAE,eAAe,GAAG,YAAY,GAAG;QAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE;IAWzH;;;;;;;;;;;;OAYG;IACH,mBAAmB,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,GAAG,EAAE,MAAM;IAK3D;;;OAGG;IACH,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,EAAE;IAKnD;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIpC;;;OAGG;IACH,SAAS,CAAC,uBAAuB,CAAC,eAAe,EAAE,eAAe,GAAG,MAAM;IAkB3E;;;;OAIG;IACH,SAAS,CAAC,uBAAuB,CAAC,WAAW,EAAE,MAAM,GAAG,eAAe;IAavE;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAkB3D;;;;OAIG;IACH,SAAS,CAAC,+BAA+B,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAInE;;OAEG;IACH,SAAS,CAAC,gBAAgB,IAAI,IAAI;CAKrC"}