@crawlee/core 3.0.0-alpha.2 → 3.0.0-alpha.22

package/request.js

@@ -0,0 +1,330 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Request = void 0;
+const tslib_1 = require("tslib");
+const utilities_1 = require("@apify/utilities");
+const crypto_1 = tslib_1.__importDefault(require("crypto"));
+const ow_1 = tslib_1.__importStar(require("ow"));
+const util_1 = tslib_1.__importDefault(require("util"));
+const log_1 = require("./log");
+const typedefs_1 = require("./typedefs");
+// new properties on the Request object breaks serialization
+const log = log_1.log.child({ prefix: 'Request' });
+const requestOptionalPredicates = {
+    id: ow_1.default.optional.string,
+    loadedUrl: ow_1.default.optional.string.url,
+    uniqueKey: ow_1.default.optional.string,
+    method: ow_1.default.optional.string,
+    payload: ow_1.default.optional.any(ow_1.default.string, ow_1.default.buffer),
+    noRetry: ow_1.default.optional.boolean,
+    retryCount: ow_1.default.optional.number,
+    errorMessages: ow_1.default.optional.array.ofType(ow_1.default.string),
+    headers: ow_1.default.optional.object,
+    userData: ow_1.default.optional.object,
+    handledAt: ow_1.default.optional.any(ow_1.default.string.date, ow_1.default.date),
+    keepUrlFragment: ow_1.default.optional.boolean,
+    useExtendedUniqueKey: ow_1.default.optional.boolean,
+    skipNavigation: ow_1.default.optional.boolean,
+};
+/**
+ * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
+ * The `Request` object also stores information about errors that occurred during processing of the request.
+ *
+ * Each `Request` instance has the `uniqueKey` property, which can be either specified
+ * manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
+ * are considered as pointing to the same web resource. This behavior applies to all Apify SDK classes,
+ * such as {@link RequestList}, {@link RequestQueue}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
+ *
+ * Example use:
+ *
+ * ```javascript
+ * const request = new Request({
+ *     url: 'http://www.example.com',
+ *     headers: { Accept: 'application/json' },
+ * });
+ *
+ * ...
+ *
+ * request.userData.foo = 'bar';
+ * request.pushErrorMessage(new Error('Request failed!'));
+ *
+ * ...
+ *
+ * const foo = request.userData.foo;
+ * ```
+ * @category Sources
+ */
+class Request {
+    /**
+     * `Request` parameters including the URL, HTTP method and headers, and others.
+     */
+    constructor(options) {
+        /** Request ID */
+        Object.defineProperty(this, "id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** URL of the web page to crawl. */
+        Object.defineProperty(this, "url", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * An actually loaded URL after redirects, if present. HTTP redirects are guaranteed
+         * to be included.
+         *
+         * When using {@link PuppeteerCrawler} or {@link PlaywrightCrawler}, meta tag and JavaScript redirects may,
+         * or may not be included, depending on their nature. This generally means that redirects,
+         * which happen immediately will most likely be included, but delayed redirects will not.
+         */
+        Object.defineProperty(this, "loadedUrl", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * A unique key identifying the request.
+         * Two requests with the same `uniqueKey` are considered as pointing to the same URL.
+         */
+        Object.defineProperty(this, "uniqueKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** HTTP method, e.g. `GET` or `POST`. */
+        Object.defineProperty(this, "method", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** HTTP request payload, e.g. for POST requests. */
+        Object.defineProperty(this, "payload", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** The `true` value indicates that the request will not be automatically retried on error. */
+        Object.defineProperty(this, "noRetry", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** Indicates the number of times the crawling of the request has been retried on error. */
+        Object.defineProperty(this, "retryCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** An array of error messages from request processing. */
+        Object.defineProperty(this, "errorMessages", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** Object with HTTP headers. Key is header name, value is the value. */
+        Object.defineProperty(this, "headers", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** Private store for the custom user data assigned to the request. */
+        Object.defineProperty(this, "_userData", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        /** Custom user data assigned to the request. */
+        Object.defineProperty(this, "userData", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        /**
+         * ISO datetime string that indicates the time when the request has been processed.
+         * Is `null` if the request has not been crawled yet.
+         */
+        Object.defineProperty(this, "handledAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        (0, ow_1.default)(options, 'RequestOptions', ow_1.default.object);
+        (0, ow_1.default)(options.url, 'RequestOptions.url', ow_1.default.string);
+        // 'ow' validation is slow, because it checks all predicates
+        // even if the validated object has only 1 property.
+        // This custom validation loop iterates only over existing
+        // properties and speeds up the validation cca 3-fold.
+        // See https://github.com/sindresorhus/ow/issues/193
+        (0, typedefs_1.keys)(options).forEach((prop) => {
+            const predicate = requestOptionalPredicates[prop];
+            const value = options[prop];
+            if (predicate) {
+                (0, ow_1.default)(value, `RequestOptions.${prop}`, predicate);
+                // 'url' is checked above because it's not optional
+            }
+            else if (prop !== 'url') {
+                const msg = `Did not expect property \`${prop}\` to exist, got \`${value}\` in object \`RequestOptions\``;
+                throw new ow_1.ArgumentError(msg, this.constructor);
+            }
+        });
+        const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, errorMessages = [], headers = {}, userData = {}, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, } = options;
+        let { method = 'GET', } = options;
+        method = method.toUpperCase();
+        if (method === 'GET' && payload)
+            throw new Error('Request with GET method cannot have a payload.');
+        this.id = id;
+        this.url = url;
+        this.loadedUrl = loadedUrl;
+        this.uniqueKey = uniqueKey || this._computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
+        this.method = method;
+        this.payload = payload;
+        this.noRetry = noRetry;
+        this.retryCount = retryCount;
+        this.errorMessages = [...errorMessages];
+        this.headers = { ...headers };
+        this.handledAt = handledAt instanceof Date ? handledAt.toISOString() : handledAt;
+        Object.defineProperties(this, {
+            _userData: {
+                value: { __crawlee: {}, ...userData },
+                enumerable: false,
+                writable: true,
+            },
+            userData: {
+                get: () => this._userData,
+                set: (value) => {
+                    Object.defineProperties(value, {
+                        __crawlee: {
+                            // eslint-disable-next-line no-underscore-dangle
+                            value: this._userData.__crawlee,
+                            enumerable: false,
+                            writable: true,
+                        },
+                        toJSON: {
+                            // eslint-disable-next-line no-underscore-dangle
+                            value: () => {
+                                if (Object.keys(this._userData.__crawlee).length > 0) {
+                                    return ({
+                                        ...this._userData,
+                                        __crawlee: this._userData.__crawlee,
+                                    });
+                                }
+                                return this._userData;
+                            },
+                            enumerable: false,
+                            writable: true,
+                        },
+                    });
+                    this._userData = value;
+                },
+                enumerable: true,
+            },
+        });
+        // reassign userData to ensure internal `__crawlee` object is non-enumerable
+        this.userData = userData;
+        if (skipNavigation != null)
+            this.skipNavigation = skipNavigation;
+    }
+    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
+    get skipNavigation() {
+        // eslint-disable-next-line no-underscore-dangle
+        return this.userData.__crawlee?.skipNavigation ?? false;
+    }
+    set skipNavigation(value) {
+        // eslint-disable-next-line no-underscore-dangle
+        if (!this.userData.__crawlee)
+            this.userData.__crawlee = { skipNavigation: value };
+        // eslint-disable-next-line no-underscore-dangle
+        else
+            this.userData.__crawlee.skipNavigation = value;
+    }
+    /**
+     * Stores information about an error that occurred during processing of this request.
+     *
+     * You should always use Error instances when throwing errors in JavaScript.
+     *
+     * Nevertheless, to improve the debugging experience when using third party libraries
+     * that may not always throw an Error instance, the function performs a type
+     * inspection of the passed argument and attempts to extract as much information
+     * as possible, since just throwing a bad type error makes any debugging rather difficult.
+     *
+     * @param errorOrMessage Error object or error message to be stored in the request.
+     * @param [options]
+     */
+    pushErrorMessage(errorOrMessage, options = {}) {
+        const { omitStack } = options;
+        let message;
+        const type = typeof errorOrMessage;
+        if (type === 'object') {
+            if (!errorOrMessage) {
+                message = 'null';
+            }
+            else if (errorOrMessage instanceof Error) {
+                message = omitStack
+                    ? errorOrMessage.message
+                    // .stack includes the message
+                    : errorOrMessage.stack;
+            }
+            else if (Reflect.has(Object(errorOrMessage), 'message')) {
+                message = Reflect.get(Object(errorOrMessage), 'message');
+            }
+            else if (errorOrMessage.toString() !== '[object Object]') {
+                message = errorOrMessage.toString();
+            }
+            else {
+                try {
+                    message = util_1.default.inspect(errorOrMessage);
+                }
+                catch (err) {
+                    message = 'Unable to extract any message from the received object.';
+                }
+            }
+        }
+        else if (type === 'undefined') {
+            message = 'undefined';
+        }
+        else {
+            message = errorOrMessage.toString();
+        }
+        this.errorMessages.push(message);
+    }
+    _computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }) {
+        const normalizedMethod = method.toUpperCase();
+        const normalizedUrl = (0, utilities_1.normalizeUrl)(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
+        if (!useExtendedUniqueKey) {
+            if (normalizedMethod !== 'GET' && payload) {
+                // Using log.deprecated to log only once. We should add log.once or some such.
+                log.deprecated(`We've encountered a ${normalizedMethod} Request with a payload. `
+                    + 'This is fine. Just letting you know that if your requests point to the same URL '
+                    + 'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.');
+            }
+            return normalizedUrl;
+        }
+        const payloadHash = payload ? this._hashPayload(payload) : '';
+        return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
+    }
+    _hashPayload(payload) {
+        return crypto_1.default
+            .createHash('sha256')
+            .update(payload)
+            .digest('base64')
+            .replace(/[+/=]/g, '')
+            .substring(0, 8);
+    }
+}
+exports.Request = Request;
+//# sourceMappingURL=request.js.map

package/request.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";;;;AAAA,gDAAgD;AAChD,4DAA4C;AAC5C,iDAAsD;AACtD,wDAAwB;AACxB,+BAA0C;AAC1C,yCAAsD;AAEtD,4DAA4D;AAC5D,MAAM,GAAG,GAAG,SAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAEpD,MAAM,yBAAyB,GAAG;IAC9B,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;IACjC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC1B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC;IAC9C,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IAC5B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;IAClD,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC5B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,CAAC,IAAI,EAAE,YAAE,CAAC,IAAI,CAAC;IACnD,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IACpC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IACzC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;CACtC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAa,OAAO;IAqDhB;;OAEG;IACH,YAAY,OAAuB;QAvDnC,iBAAiB;QACjB;;;;;WAAY;QAEZ,oCAAoC;QACpC;;;;;WAAY;QAEZ;;;;;;;WAOG;QACH;;;;;WAAmB;QAEnB;;;WAGG;QACH;;;;;WAAkB;QAElB,yCAAyC;QACzC;;;;;WAA2B;QAE3B,oDAAoD;QACpD;;;;;WAAiB;QAEjB,8FAA8F;QAC9F;;;;;WAAiB;QAEjB,2FAA2F;QAC3F;;;;;WAAmB;QAEnB,0DAA0D;QAC1D;;;;;WAAwB;QAExB,wEAAwE;QACxE;;;;;WAAiC;QAEjC,sEAAsE;QACtE;;;;mBAAyC,EAAE;WAAC;QAE5C,gDAAgD;QAChD;;;;mBAAgC,EAAE;WAAC;QAEnC;;;WAGG;QACH;;;;;WAAmB;QAMf,IAAA,YAAE,EAAC,OAAO,EAAE,gBAAgB,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACzC,IAAA,YAAE,EAAC,OAAO,CAAC,GAAG,EAAE,oBAAoB,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACjD,4DAA4D;QAC5D,oDAAoD;QACpD,0DAA0D;QAC1D,sDAAsD;QACtD,oDAAoD;QACpD,IAAA,eAAI,EAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,MAAM,SAAS,GAAG,yBAAyB,CAAC,IAA8C,CAAC,CAAC;YAC5F,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,SAAS,EAAE;gBACX,IAAA,YAAE,EAAC,KAAK,EAAE,kBAAkB,IAAI,EAAE,EAAE,SAA0B,CAAC,CAAC;gBAChE,mDAAmD;aACtD;iBAAM,IAAI,IAAI,KAAK,KAAK,EAAE;gBACvB,MAAM,GAAG,GAAG,6BAA6B,IAAI,sBAAsB,KAAK,iCAAiC,CAAC;gBAC1G,MAAM,IAAI,kBAAa,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;aAClD;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,EACF,EAAE,EACF,GAAG,EACH,SAAS,EACT,SAAS,EACT,OAAO,EACP,OAAO,GAAG,KAAK,EACf,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,OAAO,GAAG,EAAE,EACZ,QAAQ,GAAG,EAAE,EACb,SAAS,EACT,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,EAC5B,cAAc,GACjB,GAAG,OAA4H,CAAC;QAEjI,IAAI,EACA,MAAM,GAAG,KAAK,GACjB,GAAG,OAAO,CAAC;QAEZ,MAAM,GAAG,MAAM,CAAC,WAAW,EAAwB,CAAC;QAEpD,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAEnG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACtH,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,SAAoB,YAAY,IAAI,CAAC,CAAC,CAAE,SAAkB,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAU,CAAC;QAEvG,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;YAC1B,SAAS,EAAE;gBACP,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,QAAQ,EAAE;gBACrC,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,IAAI;aACjB;YACD,QAAQ,EAAE;gBACN,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS;gBACzB,GAAG,EAAE,CAAC,KAA0B,EAAE,EAAE;oBAChC,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE;wBAC3B,SAAS,EAAE;4BACP,gDAAgD;4BAChD,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;4BAC/B,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;wBACD,MAAM,EAAE;4BACJ,gDAAgD;4BAChD,KAAK,EAAE,GAAG,EAAE;gCACR,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;oCAClD,OAAO,CAAC;wCACJ,GAAG,IAAI,CAAC,SAAS;wCACjB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;qCACtC,CAAC,CAAC;iCACN;gCAED,OAAO,IAAI,CAAC,SAAS,CAAC;4BAC1B,CAAC;4BACD,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;qBACJ,CAAC,CAAC;oBACH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBACD,UAAU,EAAE,IAAI;aACnB;SACJ,CAAC,CAAC;QAEH,4EAA4E;QAC5E,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEzB,IAAI,cAAc,IAAI,IAAI;YAAE,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACrE,CAAC;IAED,yGAAyG;IACzG,IAAI,cAAc;QACd,gDAAgD;QAChD,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,cAAc,IAAI,KAAK,CAAC;IAC5D,CAAC;IAED,IAAI,cAAc,CAAC,KAAc;QAC7B,gDAAgD;QAChD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS;YAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QAClF,gDAAgD;;YAC3C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,cAAc,GAAG,KAAK,CAAC;IACxD,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAuB,EAAE,UAAmC,EAAE;QAC3E,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,cAAc,CAAC;QACnC,IAAI,IAAI,KAAK,QAAQ,EAAE;YACnB,IAAI,CAAC,cAAc,EAAE;gBACjB,OAAO,GAAG,MAAM,CAAC;aACpB;iBAAM,IAAI,cAAc,YAAY,KAAK,EAAE;gBACxC,OAAO,GAAG,SAAS;oBACf,CAAC,CAAC,cAAc,CAAC,OAAO;oBACxB,8BAA8B;oBAC9B,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC;aAC9B;iBAAM,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,EAAE;gBACvD,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,CAAC;aAC5D;iBAAM,IAAK,cAAyB,CAAC,QAAQ,EAAE,KAAK,iBAAiB,EAAE;gBACpE,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;aACnD;iBAAM;gBACH,IAAI;oBACA,OAAO,GAAG,cAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;iBAC1C;gBAAC,OAAO,GAAG,EAAE;oBACV,OAAO,GAAG,yDAAyD,CAAC;iBACvE;aACJ;SACJ;aAAM,IAAI,IAAI,KAAK,WAAW,EAAE;YAC7B,OAAO,GAAG,WAAW,CAAC;SACzB;aAAM;YACH,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;SACnD;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAES,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAA2B;QAChH,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,IAAA,wBAAY,EAAC,GAAG,EAAE,eAAe,CAAC,IAAI,GAAG,CAAC,CAAC,6DAA6D;QAC9H,IAAI,CAAC,oBAAoB,EAAE;YACvB,IAAI,gBAAgB,KAAK,KAAK,IAAI,OAAO,EAAE;gBACvC,8EAA8E;gBAC9E,GAAG,CAAC,UAAU,CAAC,uBAAuB,gBAAgB,2BAA2B;sBAC3E,kFAAkF;sBAClF,iHAAiH,CAAC,CAAC;aAC5H;YACD,OAAO,aAAa,CAAC;SACxB;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,OAAO,GAAG,gBAAgB,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;IAClE,CAAC;IAES,YAAY,CAAC,OAAmB;QACtC,OAAO,gBAAM;aACR,UAAU,CAAC,QAAQ,CAAC;aACpB,MAAM,CAAC,OAAO,CAAC;aACf,MAAM,CAAC,QAAQ,CAAC;aAChB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;aACrB,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzB,CAAC;CACJ;AA/OD,0BA+OC"}

package/serialization.d.ts

@@ -0,0 +1,33 @@
+/// <reference types="node" />
+import { Readable } from 'stream';
+/**
+ * Uses Gzip compression to take an array of values, which can be anything
+ * from entries in a Dataset to Requests in a RequestList and compresses
+ * them to a Buffer in a memory-efficient way (streaming one by one). Ideally,
+ * the largest chunk of memory consumed will be the final compressed Buffer.
+ * This could be further improved by outputting a Stream, if and when
+ * apify-client supports streams.
+ * @internal
+ */
+export declare function serializeArray<T>(data: T[]): Promise<Buffer>;
+/**
+ * Decompresses a Buffer previously created with compressData (technically,
+ * any JSON that is an Array) and collects it into an Array of values
+ * in a memory-efficient way (streaming the array items one by one instead
+ * of creating a fully decompressed buffer -> full JSON -> full Array all
+ * in memory at once. Could be further optimized to ingest a Stream if and
+ * when apify-client supports streams.
+ * @internal
+ */
+export declare function deserializeArray<T extends string | Buffer>(compressedData: Buffer): Promise<T[]>;
+/**
+ * Creates a stream that decompresses a Buffer previously created with
+ * compressData (technically, any JSON that is an Array) and collects it
+ * into an Array of values in a memory-efficient way (streaming the array
+ * items one by one instead of creating a fully decompressed buffer
+ * -> full JSON -> full Array all in memory at once. Could be further
+ * optimized to ingest a Stream if and when apify-client supports streams.
+ * @internal
+ */
+export declare function createDeserialize(compressedData: Buffer): Readable;
+//# sourceMappingURL=serialization.d.ts.map

package/serialization.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"serialization.d.ts","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":";AACA,OAAO,EAA8B,QAAQ,EAAY,MAAM,QAAQ,CAAC;AAkDxE;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAUlE;AAED;;;;;;;;GAQG;AACH,wBAAsB,gBAAgB,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAWtG;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,cAAc,EAAE,MAAM,GAAG,QAAQ,CAelE"}

package/serialization.js

@@ -0,0 +1,146 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createDeserialize = exports.deserializeArray = exports.serializeArray = void 0;
+const tslib_1 = require("tslib");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const stream_1 = require("stream");
+const StreamArray_1 = tslib_1.__importDefault(require("stream-json/streamers/StreamArray"));
+const util_1 = tslib_1.__importDefault(require("util"));
+const zlib_1 = tslib_1.__importDefault(require("zlib"));
+const pipeline = util_1.default.promisify(stream_1.pipeline);
+/**
+ * Transforms an array of items to a JSON in a streaming
+ * fashion to save memory. It operates in batches to speed
+ * up the process.
+ * @internal
+ */
+class ArrayToJson extends stream_1.Readable {
+    constructor(data, options = {}) {
+        super({
+            ...options,
+            autoDestroy: true,
+            emitClose: true,
+        });
+        Object.defineProperty(this, "data", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: data
+        });
+        Object.defineProperty(this, "offset", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "batchSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const { batchSize = 10000 } = options;
+        this.batchSize = batchSize;
+        this.data = data;
+        this.push('[');
+    }
+    _read() {
+        try {
+            const items = this.data.slice(this.offset, this.offset + this.batchSize);
+            if (items.length) {
+                const json = JSON.stringify(items);
+                // Strip brackets to flatten the batch.
+                const itemString = json.substring(1, json.length - 1);
+                if (this.offset > 0)
+                    this.push(',', 'utf8');
+                this.push(itemString, 'utf8');
+                this.offset += this.batchSize;
+            }
+            else {
+                this.push(']');
+                this.push(null);
+            }
+        }
+        catch (err) {
+            this.emit('error', err);
+        }
+    }
+}
+/**
+ * Uses Gzip compression to take an array of values, which can be anything
+ * from entries in a Dataset to Requests in a RequestList and compresses
+ * them to a Buffer in a memory-efficient way (streaming one by one). Ideally,
+ * the largest chunk of memory consumed will be the final compressed Buffer.
+ * This could be further improved by outputting a Stream, if and when
+ * apify-client supports streams.
+ * @internal
+ */
+async function serializeArray(data) {
+    (0, ow_1.default)(data, ow_1.default.array);
+    const { chunks, collector } = createChunkCollector();
+    await pipeline(new ArrayToJson(data), zlib_1.default.createGzip(), collector);
+    return Buffer.concat(chunks);
+}
+exports.serializeArray = serializeArray;
+/**
+ * Decompresses a Buffer previously created with compressData (technically,
+ * any JSON that is an Array) and collects it into an Array of values
+ * in a memory-efficient way (streaming the array items one by one instead
+ * of creating a fully decompressed buffer -> full JSON -> full Array all
+ * in memory at once. Could be further optimized to ingest a Stream if and
+ * when apify-client supports streams.
+ * @internal
+ */
+async function deserializeArray(compressedData) {
+    (0, ow_1.default)(compressedData, ow_1.default.buffer);
+    const { chunks, collector } = createChunkCollector({ fromValuesStream: true });
+    await pipeline(stream_1.Readable.from([compressedData]), zlib_1.default.createGunzip(), StreamArray_1.default.withParser(), collector);
+    return chunks;
+}
+exports.deserializeArray = deserializeArray;
+/**
+ * Creates a stream that decompresses a Buffer previously created with
+ * compressData (technically, any JSON that is an Array) and collects it
+ * into an Array of values in a memory-efficient way (streaming the array
+ * items one by one instead of creating a fully decompressed buffer
+ * -> full JSON -> full Array all in memory at once. Could be further
+ * optimized to ingest a Stream if and when apify-client supports streams.
+ * @internal
+ */
+function createDeserialize(compressedData) {
+    (0, ow_1.default)(compressedData, ow_1.default.buffer);
+    const streamArray = StreamArray_1.default.withParser();
+    const destination = pluckValue(streamArray);
+    (0, stream_1.pipeline)(stream_1.Readable.from([compressedData]), zlib_1.default.createGunzip(), destination, 
+    // TODO: figure this out
+    // @ts-expect-error Somethings wrong here, the types are wrong but tests fail if we correct the code to make them right
+    (err) => destination.emit(err));
+    return destination;
+}
+exports.createDeserialize = createDeserialize;
+function createChunkCollector(options = {}) {
+    const { fromValuesStream = false } = options;
+    const chunks = [];
+    const collector = new stream_1.Writable({
+        decodeStrings: false,
+        objectMode: fromValuesStream,
+        write(chunk, _nil, callback) {
+            chunks.push(fromValuesStream ? chunk.value : chunk);
+            callback();
+        },
+        writev(chunkObjects, callback) {
+            const buffers = chunkObjects.map(({ chunk }) => {
+                return fromValuesStream ? chunk.value : chunk;
+            });
+            chunkObjects.push(...buffers);
+            callback();
+        },
+    });
+    return { collector, chunks };
+}
+function pluckValue(streamArray) {
+    const realPush = streamArray.push.bind(streamArray);
+    streamArray.push = (obj) => realPush(obj && obj.value);
+    return streamArray;
+}
+//# sourceMappingURL=serialization.js.map

package/serialization.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"serialization.js","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,mCAAwE;AAExE,4FAA4D;AAC5D,wDAAwB;AACxB,wDAAwB;AAExB,MAAM,QAAQ,GAAG,cAAI,CAAC,SAAS,CAAC,iBAAc,CAAC,CAAC;AAEhD;;;;;GAKG;AACH,MAAM,WAAe,SAAQ,iBAAQ;IAIjC,YAAoB,IAAS,EAAE,UAAkC,EAAE;QAC/D,KAAK,CAAC;YACF,GAAG,OAAO;YACV,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;;;;;mBALa;;QAHpB;;;;mBAAiB,CAAC;WAAC;QACnB;;;;;WAAmC;QAQ/B,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAEQ,KAAK;QACV,IAAI;YACA,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC,MAAM,EAAE;gBACd,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACnC,uCAAuC;gBACvC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC;aACjC;iBAAM;gBACH,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACnB;SACJ;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;SAC3B;IACL,CAAC;CACJ;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,cAAc,CAAI,IAAS;IAC7C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,EAAE,CAAC;IACrD,MAAM,QAAQ,CACV,IAAI,WAAW,CAAC,IAAI,CAAC,EACrB,cAAI,CAAC,UAAU,EAAE,EACjB,SAAS,CACZ,CAAC;IAEF,OAAO,MAAM,CAAC,MAAM,CAAC,MAAkB,CAAC,CAAC;AAC7C,CAAC;AAVD,wCAUC;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,gBAAgB,CAA4B,cAAsB;IACpF,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,CAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAClF,MAAM,QAAQ,CACV,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,cAAI,CAAC,YAAY,EAAE,EACnB,qBAAW,CAAC,UAAU,EAAE,EACxB,SAAS,CACZ,CAAC;IAEF,OAAO,MAAa,CAAC;AACzB,CAAC;AAXD,4CAWC;AAED;;;;;;;;GAQG;AACH,SAAgB,iBAAiB,CAAC,cAAsB;IACpD,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,WAAW,GAAG,qBAAW,CAAC,UAAU,EAAE,CAAC;IAC7C,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IAE5C,IAAA,iBAAc,EACV,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,cAAI,CAAC,YAAY,EAAE,EACnB,WAAW;IACX,wBAAwB;IACxB,uHAAuH;IACvH,CAAC,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CACjC,CAAC;IAEF,OAAO,WAAW,CAAC;AACvB,CAAC;AAfD,8CAeC;AAED,SAAS,oBAAoB,CAA4B,UAA0C,EAAE;IACjG,MAAM,EAAE,gBAAgB,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,MAAM,GAAQ,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,iBAAQ,CAAC;QAC3B,aAAa,EAAE,KAAK;QACpB,UAAU,EAAE,gBAAgB;QAC5B,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ;YACvB,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpD,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,MAAM,CAAC,YAAY,EAAE,QAAQ;YACzB,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC3C,OAAO,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YAClD,CAAC,CAAC,CAAC;YACH,YAAY,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AACjC,CAAC;AAED,SAAS,UAAU,CAAC,WAAkB;IAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpD,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IACvD,OAAO,WAAW,CAAC;AACvB,CAAC"}

package/session_pool/errors.d.ts

@@ -0,0 +1,8 @@
+/**
+ * @ignore
+ */
+export declare class CookieParseError extends Error {
+    readonly cookieHeaderString: unknown;
+    constructor(cookieHeaderString: unknown);
+}
+//# sourceMappingURL=errors.d.ts.map

package/session_pool/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/session_pool/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,gBAAiB,SAAQ,KAAK;IAC3B,QAAQ,CAAC,kBAAkB,EAAE,OAAO;gBAA3B,kBAAkB,EAAE,OAAO;CAInD"}

package/session_pool/errors.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CookieParseError = void 0;
+/**
+ * @ignore
+ */
+class CookieParseError extends Error {
+    constructor(cookieHeaderString) {
+        super(`Could not parse cookie header string: ${cookieHeaderString}`);
+        Object.defineProperty(this, "cookieHeaderString", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: cookieHeaderString
+        });
+        Error.captureStackTrace(this, CookieParseError);
+    }
+}
+exports.CookieParseError = CookieParseError;
+//# sourceMappingURL=errors.js.map

package/session_pool/errors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/session_pool/errors.ts"],"names":[],"mappings":";;;AAAA;;GAEG;AACH,MAAa,gBAAiB,SAAQ,KAAK;IACvC,YAAqB,kBAA2B;QAC5C,KAAK,CAAC,yCAAyC,kBAAkB,EAAE,CAAC,CAAC;;;;;mBADpD;;QAEjB,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;IACpD,CAAC;CACJ;AALD,4CAKC"}

package/session_pool/events.d.ts

@@ -0,0 +1,3 @@
+/** @internal */
+export declare const EVENT_SESSION_RETIRED = "sessionRetired";
+//# sourceMappingURL=events.d.ts.map

package/session_pool/events.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/session_pool/events.ts"],"names":[],"mappings":"AAAA,gBAAgB;AAChB,eAAO,MAAM,qBAAqB,mBAAmB,CAAC"}

package/session_pool/events.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EVENT_SESSION_RETIRED = void 0;
+/** @internal */
+exports.EVENT_SESSION_RETIRED = 'sessionRetired';
+//# sourceMappingURL=events.js.map

package/session_pool/events.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"events.js","sourceRoot":"","sources":["../../src/session_pool/events.ts"],"names":[],"mappings":";;;AAAA,gBAAgB;AACH,QAAA,qBAAqB,GAAG,gBAAgB,CAAC"}

package/session_pool/index.d.ts

@@ -0,0 +1,6 @@
+export * from './errors';
+export * from './events';
+export * from './session';
+export * from './session_pool';
+export * from './session_utils';
+//# sourceMappingURL=index.d.ts.map

package/session_pool/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC"}

package/session_pool/index.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const tslib_1 = require("tslib");
+tslib_1.__exportStar(require("./errors"), exports);
+tslib_1.__exportStar(require("./events"), exports);
+tslib_1.__exportStar(require("./session"), exports);
+tslib_1.__exportStar(require("./session_pool"), exports);
+tslib_1.__exportStar(require("./session_utils"), exports);
+//# sourceMappingURL=index.js.map

package/session_pool/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":";;;AAAA,mDAAyB;AACzB,mDAAyB;AACzB,oDAA0B;AAC1B,yDAA+B;AAC/B,0DAAgC"}