@crawlee/core 3.10.6-beta.8 → 3.11.0

package/storages/sitemap_request_list.js

@@ -0,0 +1,388 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SitemapRequestList = void 0;
+const tslib_1 = require("tslib");
+const node_stream_1 = require("node:stream");
+const log_1 = tslib_1.__importDefault(require("@apify/log"));
+const utils_1 = require("@crawlee/utils");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const key_value_store_1 = require("./key_value_store");
+const utils_2 = require("./utils");
+const request_1 = require("../request");
+/**
+ * A list of URLs to crawl parsed from a sitemap.
+ *
+ * The loading of the sitemap is performed in the background so that crawling can start before the sitemap is fully loaded.
+ */
+class SitemapRequestList {
+    /** @internal */
+    constructor(options) {
+        /**
+         * Set of URLs that were returned by `fetchNextRequest()` and not marked as handled yet.
+         * @internal
+         */
+        Object.defineProperty(this, "inProgress", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Set()
+        });
+        /** Set of URLs for which `reclaimRequest()` was called. */
+        Object.defineProperty(this, "reclaimed", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Set()
+        });
+        /**
+         * Map of returned Request objects that have not been marked as handled yet.
+         *
+         * We use this to persist custom user fields on the in-progress (or reclaimed) requests.
+         */
+        Object.defineProperty(this, "requestData", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+        /**
+         * Object for keeping track of the sitemap parsing progress.
+         */
+        Object.defineProperty(this, "sitemapParsingProgress", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {
+                /**
+                 * URL of the sitemap that is currently being parsed. `null` if no sitemap is being parsed.
+                 */
+                inProgressSitemapUrl: null,
+                /**
+                 * Buffer for URLs from the currently parsed sitemap. Used for tracking partially loaded sitemaps across migrations.
+                 */
+                inProgressEntries: new Set(),
+                /**
+                 * Set of sitemap URLs that have not been fully parsed yet. If the set is empty and `inProgressSitemapUrl` is `null`, the sitemap loading is finished.
+                 */
+                pendingSitemapUrls: new Set(),
+            }
+        });
+        /**
+         * Object stream of URLs parsed from the sitemaps.
+         * Using `highWaterMark`, this can manage the speed of the sitemap loading.
+         *
+         * Fetch the next URL to be processed using `fetchNextRequest()`.
+         */
+        Object.defineProperty(this, "urlQueueStream", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * Indicates whether the request list sitemap loading was aborted.
+         *
+         * If the loading was aborted before the sitemaps were fully loaded, the request list might be missing some URLs.
+         * The `isSitemapFullyLoaded` method can be used to check if the sitemaps were fully loaded.
+         *
+         * If the loading is aborted and all the requests are handled, `isFinished()` will return `true`.
+         */
+        Object.defineProperty(this, "abortLoading", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        /** Number of URLs that were marked as handled */
+        Object.defineProperty(this, "handledUrlCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "persistStateKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "store", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * Proxy URL to be used for sitemap loading.
+         */
+        Object.defineProperty(this, "proxyUrl", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * Logger instance.
+         */
+        Object.defineProperty(this, "log", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: log_1.default.child({ prefix: 'SitemapRequestList' })
+        });
+        (0, ow_1.default)(options, ow_1.default.object.exactShape({
+            sitemapUrls: ow_1.default.array.ofType(ow_1.default.string),
+            proxyUrl: ow_1.default.optional.string,
+            persistStateKey: ow_1.default.optional.string,
+            signal: ow_1.default.optional.any(),
+            timeoutMillis: ow_1.default.optional.number,
+            maxBufferSize: ow_1.default.optional.number,
+        }));
+        this.persistStateKey = options.persistStateKey;
+        this.proxyUrl = options.proxyUrl;
+        this.urlQueueStream = new node_stream_1.Transform({
+            objectMode: true,
+            highWaterMark: options.maxBufferSize ?? 200,
+        });
+        this.urlQueueStream.pause();
+        this.sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);
+    }
+    /**
+     * Adds a URL to the queue of parsed URLs.
+     *
+     * Blocks if the stream is full until it is drained.
+     */
+    async pushNextUrl(url) {
+        return new Promise((resolve) => {
+            if (!this.urlQueueStream.push(url)) {
+                // This doesn't work with the 'drain' event (it's not emitted for some reason).
+                this.urlQueueStream.once('readdata', () => {
+                    resolve();
+                });
+            }
+            else {
+                resolve();
+            }
+        });
+    }
+    /**
+     * Reads the next URL from the queue of parsed URLs.
+     *
+     * If the stream is empty, blocks until a new URL is pushed.
+     * @returns The next URL from the queue or `null` if we have read all URLs.
+     */
+    async readNextUrl() {
+        return new Promise((resolve) => {
+            const result = this.urlQueueStream.read();
+            if (!result && !this.isSitemapFullyLoaded()) {
+                this.urlQueueStream.once('readable', () => {
+                    const nextUrl = this.urlQueueStream.read();
+                    resolve(nextUrl);
+                });
+            }
+            else {
+                resolve(result);
+            }
+            this.urlQueueStream.emit('readdata');
+        });
+    }
+    /**
+     * Indicates whether the background processing of sitemap contents has successfully finished.
+     *
+     * If this is `false`, the background processing is either still in progress or was aborted.
+     */
+    isSitemapFullyLoaded() {
+        return (this.sitemapParsingProgress.inProgressSitemapUrl === null &&
+            this.sitemapParsingProgress.pendingSitemapUrls.size === 0);
+    }
+    /**
+     * Start processing the sitemaps and loading the URLs.
+     *
+     * Resolves once all the sitemaps URLs have been fully loaded (sets `isSitemapFullyLoaded` to `true`).
+     */
+    async load() {
+        while (!this.isSitemapFullyLoaded() && !this.abortLoading) {
+            const sitemapUrl = this.sitemapParsingProgress.inProgressSitemapUrl ??
+                this.sitemapParsingProgress.pendingSitemapUrls.values().next().value;
+            try {
+                for await (const item of (0, utils_1.parseSitemap)([{ type: 'url', url: sitemapUrl }], this.proxyUrl, {
+                    maxDepth: 0,
+                    emitNestedSitemaps: true,
+                })) {
+                    if (!item.originSitemapUrl) {
+                        // This is a nested sitemap
+                        this.sitemapParsingProgress.pendingSitemapUrls.add(item.loc);
+                        continue;
+                    }
+                    if (!this.sitemapParsingProgress.inProgressEntries.has(item.loc)) {
+                        await this.pushNextUrl(item.loc);
+                        this.sitemapParsingProgress.inProgressEntries.add(item.loc);
+                    }
+                }
+            }
+            catch (e) {
+                this.log.error('Error loading sitemap contents:', e);
+            }
+            this.sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
+            this.sitemapParsingProgress.inProgressEntries.clear();
+            this.sitemapParsingProgress.inProgressSitemapUrl = null;
+        }
+        await this.pushNextUrl(null);
+    }
+    /**
+     * Open a sitemap and start processing it.
+     *
+     * Resolves to a new instance of `SitemapRequestList`, which **might not be fully loaded yet** - i.e. the sitemap might still be loading in the background.
+     *
+     * Track the loading progress using the `isSitemapFullyLoaded` property.
+     */
+    static async open(options) {
+        const requestList = new SitemapRequestList(options);
+        await requestList.restoreState();
+        void requestList.load();
+        options?.signal?.addEventListener('abort', () => {
+            requestList.abortLoading = true;
+        });
+        if (options.timeoutMillis) {
+            setTimeout(() => {
+                requestList.abortLoading = true;
+            }, options.timeoutMillis);
+        }
+        return requestList;
+    }
+    /**
+     * @inheritDoc
+     */
+    length() {
+        return this.urlQueueStream.readableLength + this.handledUrlCount - this.inProgress.size - this.reclaimed.size;
+    }
+    /**
+     * @inheritDoc
+     */
+    async isFinished() {
+        return ((await this.isEmpty()) && this.inProgress.size === 0 && (this.isSitemapFullyLoaded() || this.abortLoading));
+    }
+    /**
+     * @inheritDoc
+     */
+    async isEmpty() {
+        return this.reclaimed.size === 0 && this.urlQueueStream.readableLength === 0;
+    }
+    /**
+     * @inheritDoc
+     */
+    handledCount() {
+        return this.handledUrlCount;
+    }
+    /**
+     * @inheritDoc
+     */
+    async persistState() {
+        if (this.persistStateKey === undefined) {
+            return;
+        }
+        this.store ?? (this.store = await key_value_store_1.KeyValueStore.open());
+        const urlQueue = [];
+        while (this.urlQueueStream.readableLength > 0) {
+            const url = this.urlQueueStream.read();
+            if (url === null) {
+                break;
+            }
+            urlQueue.push(url);
+        }
+        for (const url of urlQueue) {
+            this.urlQueueStream.push(url);
+        }
+        await this.store.setValue(this.persistStateKey, {
+            sitemapParsingProgress: {
+                pendingSitemapUrls: Array.from(this.sitemapParsingProgress.pendingSitemapUrls),
+                inProgressSitemapUrl: this.sitemapParsingProgress.inProgressSitemapUrl,
+                inProgressEntries: Array.from(this.sitemapParsingProgress.inProgressEntries),
+            },
+            urlQueue,
+            reclaimed: [...this.inProgress, ...this.reclaimed], // In-progress and reclaimed requests will be both retried if state is restored
+            requestData: Array.from(this.requestData.entries()),
+            abortLoading: this.abortLoading,
+        });
+    }
+    async restoreState() {
+        await (0, utils_2.purgeDefaultStorages)({ onlyPurgeOnce: true });
+        if (this.persistStateKey === undefined) {
+            return;
+        }
+        this.store ?? (this.store = await key_value_store_1.KeyValueStore.open());
+        const state = await this.store.getValue(this.persistStateKey);
+        if (state === null) {
+            return;
+        }
+        this.reclaimed = new Set(state.reclaimed);
+        this.sitemapParsingProgress = {
+            pendingSitemapUrls: new Set(state.sitemapParsingProgress.pendingSitemapUrls),
+            inProgressSitemapUrl: state.sitemapParsingProgress.inProgressSitemapUrl,
+            inProgressEntries: new Set(state.sitemapParsingProgress.inProgressEntries),
+        };
+        this.requestData = new Map(state.requestData ?? []);
+        for (const url of state.urlQueue) {
+            this.urlQueueStream.push(url);
+        }
+        this.abortLoading = state.abortLoading;
+    }
+    /**
+     * @inheritDoc
+     */
+    async fetchNextRequest() {
+        // Try to return a reclaimed request first
+        let nextUrl = this.reclaimed.values().next().value;
+        if (nextUrl) {
+            this.reclaimed.delete(nextUrl);
+        }
+        else {
+            // Otherwise read next url from the stream
+            nextUrl = await this.readNextUrl();
+            if (!nextUrl) {
+                return null;
+            }
+            this.requestData.set(nextUrl, new request_1.Request({ url: nextUrl }));
+        }
+        this.inProgress.add(nextUrl);
+        return this.requestData.get(nextUrl);
+    }
+    /**
+     * @inheritDoc
+     */
+    async *[Symbol.asyncIterator]() {
+        while ((!this.isSitemapFullyLoaded() && !this.abortLoading) || !(await this.isEmpty())) {
+            const request = await this.fetchNextRequest();
+            if (!request)
+                break;
+            yield request;
+        }
+    }
+    /**
+     * @inheritDoc
+     */
+    async reclaimRequest(request) {
+        this.ensureInProgressAndNotReclaimed(request.url);
+        this.reclaimed.add(request.url);
+        this.inProgress.delete(request.url);
+    }
+    /**
+     * @inheritDoc
+     */
+    async markRequestHandled(request) {
+        this.handledUrlCount += 1;
+        this.ensureInProgressAndNotReclaimed(request.url);
+        this.inProgress.delete(request.url);
+        this.requestData.delete(request.url);
+    }
+    ensureInProgressAndNotReclaimed(url) {
+        if (!this.inProgress.has(url)) {
+            throw new Error(`The request is not being processed (url: ${url})`);
+        }
+        if (this.reclaimed.has(url)) {
+            throw new Error(`The request was already reclaimed (url: ${url})`);
+        }
+    }
+}
+exports.SitemapRequestList = SitemapRequestList;
+//# sourceMappingURL=sitemap_request_list.js.map

package/storages/sitemap_request_list.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap_request_list.js","sourceRoot":"","sources":["../../src/storages/sitemap_request_list.ts"],"names":[],"mappings":";;;;AAAA,6CAAwC;AAExC,6DAAoC;AACpC,0CAA8C;AAC9C,oDAAoB;AAEpB,uDAAkD;AAElD,mCAA+C;AAC/C,wCAAqC;AA8CrC;;;;GAIG;AACH,MAAa,kBAAkB;IAsE3B,gBAAgB;IAChB,YAAoB,OAAkC;QAtEtD;;;WAGG;QACH;;;;mBAAa,IAAI,GAAG,EAAU;WAAC;QAE/B,2DAA2D;QACnD;;;;mBAAY,IAAI,GAAG,EAAU;WAAC;QAEtC;;;;WAIG;QACK;;;;mBAAc,IAAI,GAAG,EAAmB;WAAC;QAEjD;;WAEG;QACK;;;;mBAAiD;gBACrD;;mBAEG;gBACH,oBAAoB,EAAE,IAAI;gBAC1B;;mBAEG;gBACH,iBAAiB,EAAE,IAAI,GAAG,EAAU;gBACpC;;mBAEG;gBACH,kBAAkB,EAAE,IAAI,GAAG,EAAU;aACxC;WAAC;QAEF;;;;;WAKG;QACK;;;;;WAA0B;QAElC;;;;;;;WAOG;QACK;;;;mBAAwB,KAAK;WAAC;QAEtC,iDAAiD;QACzC;;;;mBAAkB,CAAC;WAAC;QAEpB;;;;;WAAyB;QAEzB;;;;;WAAsB;QAE9B;;WAEG;QACK;;;;;WAA6B;QAErC;;WAEG;QACK;;;;mBAAM,aAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;WAAC;QAI7D,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;YACvC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,EAAE;YACzB,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACpC,CAAC,CACL,CAAC;QAEF,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEjC,IAAI,CAAC,cAAc,GAAG,IAAI,uBAAS,CAAC;YAChC,UAAU,EAAE,IAAI;YAChB,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,GAAG;SAC9C,CAAC,CAAC;QACH,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAE5B,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAClF,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CAAC,GAAkB;QACxC,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACjC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjC,+EAA+E;gBAC/E,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,OAAO,EAAE,CAAC;gBACd,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,EAAE,CAAC;YACd,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAE1C,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;gBAC1C,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;oBAC3C,OAAO,CAAC,OAAO,CAAC,CAAC;gBACrB,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACH,oBAAoB;QAChB,OAAO,CACH,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,KAAK,IAAI;YACzD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,IAAI,KAAK,CAAC,CAC5D,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,IAAI;QACd,OAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACxD,MAAM,UAAU,GACZ,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBAChD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;YAEzE,IAAI,CAAC;gBACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAA,oBAAY,EAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACrF,QAAQ,EAAE,CAAC;oBACX,kBAAkB,EAAE,IAAI;iBAC3B,CAAC,EAAE,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBACzB,2BAA2B;wBAC3B,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBAC7D,SAAS;oBACb,CAAC;oBAED,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC/D,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAChE,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iCAAiC,EAAE,CAAC,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAClE,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;YACtD,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,GAAG,IAAI,CAAC;QAC5D,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAkC;QAChD,MAAM,WAAW,GAAG,IAAI,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACpD,MAAM,WAAW,CAAC,YAAY,EAAE,CAAC;QACjC,KAAK,WAAW,CAAC,IAAI,EAAE,CAAC;QAExB,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;YAC5C,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YACxB,UAAU,CAAC,GAAG,EAAE;gBACZ,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;YACpC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,MAAM;QACF,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;IAClH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,OAAO,CACH,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,CAC7G,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,KAAK,CAAC,CAAC;IACjF,CAAC;IAED;;OAEG;IACH,YAAY;QACR,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAV,IAAI,CAAC,KAAK,GAAK,MAAM,+BAAa,CAAC,IAAI,EAAE,EAAC;QAE1C,MAAM,QAAQ,GAAG,EAAE,CAAC;QAEpB,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YACvC,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACf,MAAM;YACV,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE;YAC5C,sBAAsB,EAAE;gBACpB,kBAAkB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;gBAC9E,oBAAoB,EAAE,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBACtE,iBAAiB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;aAC/E;YACD,QAAQ;YACR,SAAS,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,+EAA+E;YACnI,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;YACnD,YAAY,EAAE,IAAI,CAAC,YAAY;SACA,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,YAAY;QACtB,MAAM,IAAA,4BAAoB,EAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpD,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAV,IAAI,CAAC,KAAK,GAAK,MAAM,+BAAa,CAAC,IAAI,EAAE,EAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEvF,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,sBAAsB,GAAG;YAC1B,kBAAkB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;YAC5E,oBAAoB,EAAE,KAAK,CAAC,sBAAsB,CAAC,oBAAoB;YACvE,iBAAiB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;SAC7E,CAAC;QAEF,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QAEpD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB;QAClB,0CAA0C;QAC1C,IAAI,OAAO,GAAkB,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QAClE,IAAI,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACJ,0CAA0C;YAC1C,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,OAAO,IAAI,CAAC;YAChB,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,iBAAO,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,OAAO,CAAC,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;YACrF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9C,IAAI,CAAC,OAAO;gBAAE,MAAM;YAEpB,MAAM,OAAO,CAAC;QAClB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,OAAgB;QACjC,IAAI,CAAC,+BAA+B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC;QAC1B,IAAI,CAAC,+BAA+B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAEO,+BAA+B,CAAC,GAAW;QAC/C,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,4CAA4C,GAAG,GAAG,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,GAAG,GAAG,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;CACJ;AAjXD,gDAiXC"}

package/storages/storage_manager.d.ts

@@ -1,4 +1,4 @@
-import type { Dictionary, StorageClient } from '@crawlee/types';
+import type { StorageClient } from '@crawlee/types';
 import { Configuration } from '../configuration';
 import type { ProxyConfiguration } from '../proxy_configuration';
 import type { Constructor } from '../typedefs';
@@ -33,7 +33,7 @@ export declare class StorageManager<T extends IStorage = IStorage> {
     protected _getOrCreateStorage(storageIdOrName: string, storageConstructorName: string, apiClient: StorageClient): Promise<import("@crawlee/types").DatasetCollectionData>;
     protected _getStorageClientFactories(client: StorageClient, storageConstructorName: string): {
 // @ts-ignore optional peer dependency or compatibility with es2022
-        createStorageClient: ((id: string) => import("@crawlee/types").DatasetClient<Dictionary>) | ((id: string) => import("@crawlee/types").KeyValueStoreClient) | ((id: string, options?: import("@crawlee/types").RequestQueueOptions | undefined) => import("@crawlee/types").RequestQueueClient);
+        createStorageClient: ((id: string) => import("@crawlee/types").DatasetClient) | ((id: string) => import("@crawlee/types").KeyValueStoreClient) | ((id: string, options?: import("@crawlee/types").RequestQueueOptions) => import("@crawlee/types").RequestQueueClient);
 // @ts-ignore optional peer dependency or compatibility with es2022
         createStorageCollectionClient: (() => import("@crawlee/types").DatasetCollectionClient) | (() => import("@crawlee/types").KeyValueStoreCollectionClient) | (() => import("@crawlee/types").RequestQueueCollectionClient);
     };

package/storages/storage_manager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"storage_manager.d.ts","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGhE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAQ/C,MAAM,WAAW,QAAQ;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,qBAAa,cAAc,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IAQjD,OAAO,CAAC,QAAQ,CAAC,MAAM;IAP3B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA+C;IACpE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAoC;IACvE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAwB;IAC9C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAoB;gBAGjD,kBAAkB,EAAE,WAAW,CAAC,CAAC,CAAC,EACjB,MAAM,gBAAkC;WAMhD,WAAW,CAAC,CAAC,SAAS,QAAQ,EACvC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,EAC5B,QAAQ,CAAC,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,aAAa,EACtB,MAAM,gBAAkC,GACzC,OAAO,CAAC,CAAC,CAAC;IAIb,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,QAAQ,EAChC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,EAC5B,MAAM,gBAAkC,GACzC,cAAc,CAAC,CAAC,CAAC;IASpB,gBAAgB;IAChB,MAAM,CAAC,UAAU,CAAC,MAAM,gBAAkC,GAAG,IAAI;IAW3D,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC;IA4B/E,YAAY,CAAC,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAU1D;;OAEG;cACa,mBAAmB,CAC/B,eAAe,EAAE,MAAM,EACvB,sBAAsB,EAAE,MAAM,EAC9B,SAAS,EAAE,aAAa;IAe5B,SAAS,CAAC,0BAA0B,CAAC,MAAM,EAAE,aAAa,EAAE,sBAAsB,EAAE,MAAM;;;;IAY1F,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI;CASjD;AAKD,MAAM,WAAW,qBAAqB;IAClC;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C"}
+{"version":3,"file":"storage_manager.d.ts","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGhE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAQ/C,MAAM,WAAW,QAAQ;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,qBAAa,cAAc,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IAQjD,OAAO,CAAC,QAAQ,CAAC,MAAM;IAP3B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA+C;IACpE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAoC;IACvE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAwB;IAC9C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAoB;gBAGjD,kBAAkB,EAAE,WAAW,CAAC,CAAC,CAAC,EACjB,MAAM,gBAAkC;WAMhD,WAAW,CAAC,CAAC,SAAS,QAAQ,EACvC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,EAC5B,QAAQ,CAAC,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,aAAa,EACtB,MAAM,gBAAkC,GACzC,OAAO,CAAC,CAAC,CAAC;IAIb,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,QAAQ,EAChC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,EAC5B,MAAM,gBAAkC,GACzC,cAAc,CAAC,CAAC,CAAC;IASpB,gBAAgB;IAChB,MAAM,CAAC,UAAU,CAAC,MAAM,gBAAkC,GAAG,IAAI;IAW3D,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC;IA4B/E,YAAY,CAAC,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAU1D;;OAEG;cACa,mBAAmB,CAC/B,eAAe,EAAE,MAAM,EACvB,sBAAsB,EAAE,MAAM,EAC9B,SAAS,EAAE,aAAa;IAe5B,SAAS,CAAC,0BAA0B,CAAC,MAAM,EAAE,aAAa,EAAE,sBAAsB,EAAE,MAAM;;;;IAY1F,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI;CASjD;AAKD,MAAM,WAAW,qBAAqB;IAClC;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C"}

package/storages/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGhE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD;;GAEG;AACH,UAAU,0BAA0B;IAChC;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,oBAAoB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAChG;;;;;;;;;;GAUG;AACH,wBAAsB,oBAAoB,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAwB1G,MAAM,WAAW,eAAe;IAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAChE,IAAI,CAAC,EAAE,MAAM,EACb,YAAY,QAAc,EAC1B,OAAO,CAAC,EAAE,eAAe,kBAM5B;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,UAI7C;AAED;;;GAGG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,OAAO,CAAC;AAErD,gBAAgB;AAChB,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAE1D;;;GAGG;AACH,eAAO,MAAM,2BAA2B,IAAI,CAAC"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGhE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD;;GAEG;AACH,UAAU,0BAA0B;IAChC;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,oBAAoB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAChG;;;;;;;;;;GAUG;AACH,wBAAsB,oBAAoB,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAwB1G,MAAM,WAAW,eAAe;IAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAChE,IAAI,CAAC,EAAE,MAAM,EACb,YAAY,GAAS,KAAK,EAC1B,OAAO,CAAC,EAAE,eAAe,kBAM5B;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,UAI7C;AAED;;;GAGG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,OAAO,CAAC;AAErD,gBAAgB;AAChB,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAE1D;;;GAGG;AACH,eAAO,MAAM,2BAA2B,IAAI,CAAC"}

package/storages/utils.js

@@ -1,6 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MAX_QUERIES_FOR_CONSISTENCY = exports.API_PROCESSED_REQUESTS_DELAY_MILLIS = exports.QUERY_HEAD_BUFFER = exports.STORAGE_CONSISTENCY_DELAY_MILLIS = exports.QUERY_HEAD_MIN_LENGTH = exports.getRequestId = exports.useState = exports.purgeDefaultStorages = void 0;
+exports.MAX_QUERIES_FOR_CONSISTENCY = exports.API_PROCESSED_REQUESTS_DELAY_MILLIS = exports.QUERY_HEAD_BUFFER = exports.STORAGE_CONSISTENCY_DELAY_MILLIS = exports.QUERY_HEAD_MIN_LENGTH = void 0;
+exports.purgeDefaultStorages = purgeDefaultStorages;
+exports.useState = useState;
+exports.getRequestId = getRequestId;
 const tslib_1 = require("tslib");
 const node_crypto_1 = tslib_1.__importDefault(require("node:crypto"));
 const key_value_store_1 = require("./key_value_store");
@@ -21,7 +24,6 @@ async function purgeDefaultStorages(configOrOptions, client) {
         await casted.purge?.();
     }
 }
-exports.purgeDefaultStorages = purgeDefaultStorages;
 /**
  * Easily create and manage state values. All state values are automatically persisted.
  *
@@ -37,7 +39,6 @@ async function useState(name, defaultValue = {}, options) {
     });
     return kvStore.getAutoSavedValue(name || 'CRAWLEE_GLOBAL_STATE', defaultValue);
 }
-exports.useState = useState;
 /**
  * Helper function that creates ID from uniqueKey for local emulation of request queue.
  * It's also used for local cache of remote request queue.
@@ -51,7 +52,6 @@ function getRequestId(uniqueKey) {
     const str = node_crypto_1.default.createHash('sha256').update(uniqueKey).digest('base64').replace(/[+/=]/g, '');
     return str.slice(0, 15);
 }
-exports.getRequestId = getRequestId;
 /**
  * When requesting queue head we always fetch requestsInProgressCount * QUERY_HEAD_BUFFER number of requests.
  * @internal

package/storages/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":";;;;AAAA,sEAAiC;AAIjC,uDAAkD;AAClD,oDAAiD;AAwC1C,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GACT,eAAe,YAAY,6BAAa;QACpC,CAAC,CAAC;YACI,MAAM;YACN,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC;IAChC,MAAM,EAAE,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACpF,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AArBD,oDAqBC;AAWD;;;;;;;;GAQG;AACI,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,+BAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,EAAE;QACjE,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,6BAAa,CAAC,eAAe,EAAE;KAC7D,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AATD,4BASC;AAED;;;;;;;;GAQG;AACH,SAAgB,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,qBAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAJD,oCAIC;AAED;;;GAGG;AACU,QAAA,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACU,QAAA,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AACH,QAAA,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACU,QAAA,mCAAmC,GAAG,KAAM,CAAC;AAE1D;;;GAGG;AACU,QAAA,2BAA2B,GAAG,CAAC,CAAC"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":";;;AA6CA,oDAqBC;AAoBD,4BASC;AAWD,oCAIC;;AA9GD,sEAAiC;AAIjC,uDAAkD;AAClD,oDAAiD;AAwC1C,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GACT,eAAe,YAAY,6BAAa;QACpC,CAAC,CAAC;YACI,MAAM;YACN,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC;IAChC,MAAM,EAAE,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACpF,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AAWD;;;;;;;;GAQG;AACI,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,+BAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,EAAE;QACjE,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,6BAAa,CAAC,eAAe,EAAE;KAC7D,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AAED;;;;;;;;GAQG;AACH,SAAgB,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,qBAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACU,QAAA,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACU,QAAA,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AACH,QAAA,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACU,QAAA,mCAAmC,GAAG,KAAM,CAAC;AAE1D;;;GAGG;AACU,QAAA,2BAA2B,GAAG,CAAC,CAAC"}