@crawlee/core 4.0.0-beta.66 → 4.0.0-beta.68

package/memory-storage/resource-clients/request-queue.js

@@ -0,0 +1,422 @@
+import { randomUUID } from 'node:crypto';
+import { AsyncQueue } from '@sapphire/async-queue';
+import { s } from '@sapphire/shapeshift';
+import { purgeNullsFromObject, uniqueKeyToRequestId } from '../utils.js';
+import { BaseClient } from './common/base-client.js';
+const requestShape = s
+    .object({
+    id: s.string(),
+    url: s.string().url({ allowedProtocols: ['http:', 'https:'] }),
+    uniqueKey: s.string(),
+    method: s.string().optional(),
+    retryCount: s.number().int().optional(),
+    handledAt: s.union([s.string(), s.date().valid()]).optional(),
+})
+    .passthrough();
+const requestShapeWithoutId = requestShape.omit(['id']);
+const batchRequestShapeWithoutId = requestShapeWithoutId.array();
+const requestOptionsShape = s.object({
+    forefront: s.boolean().optional(),
+});
+export class RequestQueueBackend extends BaseClient {
+    name;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey;
+    createdAt = new Date();
+    accessedAt = new Date();
+    modifiedAt = new Date();
+    handledRequestCount = 0;
+    pendingRequestCount = 0;
+    /**
+     * Serializes every operation that reads-then-writes this backend's shared queue state — the
+     * `requests` map, the `forefrontRequestIds` array, the `inProgressRequestIds` set and the request
+     * counts. Those mutations span `await` points, so without this mutex a concurrent operation could
+     * interleave and corrupt them (e.g. a head scan pruning `forefrontRequestIds` while
+     * `addBatchOfRequests` pushes to it). Held by every mutating method as well as by `isEmpty`/
+     * `isFinished`, whose head scan also prunes `forefrontRequestIds`.
+     */
+    queueStateMutex = new AsyncQueue();
+    forefrontRequestIds = [];
+    /**
+     * IDs of requests currently fetched but not yet handled or reclaimed. A request in this set is
+     * "in progress" and will not be handed out again by {@link fetchNextRequest}.
+     *
+     * Unlike the file-system / platform clients, the in-memory queue lives entirely within a single
+     * process and is never shared with another consumer, so there is no need for an expiring,
+     * cross-process-visible lock — tracking in-progress requests in this set is enough.
+     */
+    inProgressRequestIds = new Set();
+    requests = new Map();
+    storageBackend;
+    constructor(options) {
+        super(options.id ?? randomUUID());
+        this.name = options.name;
+        this.cacheKey = options.cacheKey ?? this.name ?? this.id;
+        this.storageBackend = options.storageBackend;
+    }
+    async getMetadata() {
+        this.updateTimestamps(false);
+        return this.toRequestQueueInfo();
+    }
+    async drop() {
+        // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so a concurrent
+        // operation cannot observe half-cleared state — e.g. a forefront id whose request has already been
+        // removed, which `listPendingHead` would then dereference as `undefined`.
+        await this.queueStateMutex.wait();
+        try {
+            const storeIndex = this.storageBackend.requestQueueBackendCache.findIndex((queue) => queue.id === this.id);
+            if (storeIndex !== -1) {
+                const [oldBackend] = this.storageBackend.requestQueueBackendCache.splice(storeIndex, 1);
+                oldBackend.pendingRequestCount = 0;
+                // Clear all in-memory state, consistent with `purge`. Clearing `requests` alone would
+                // leave dangling ids in `forefrontRequestIds`/`inProgressRequestIds`, which a later head
+                // scan would resolve to a missing request and dereference.
+                oldBackend.requests.clear();
+                oldBackend.forefrontRequestIds = [];
+                oldBackend.inProgressRequestIds.clear();
+            }
+        }
+        finally {
+            this.queueStateMutex.shift();
+        }
+    }
+    async purge() {
+        // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so a concurrent
+        // operation cannot observe or repopulate half-cleared state across the `await` below.
+        await this.queueStateMutex.wait();
+        try {
+            // Clear all in-memory state
+            this.requests.clear();
+            this.forefrontRequestIds = [];
+            this.inProgressRequestIds.clear();
+            this.handledRequestCount = 0;
+            this.pendingRequestCount = 0;
+            this.updateTimestamps(true);
+        }
+        finally {
+            this.queueStateMutex.shift();
+        }
+    }
+    *requestKeyIterator() {
+        for (let i = this.forefrontRequestIds.length - 1; i >= 0; i--) {
+            yield this.forefrontRequestIds[i];
+        }
+        for (const key of this.requests.keys()) {
+            yield key;
+        }
+    }
+    /**
+     * Scans the queue and returns the pending head — requests that are neither handled nor currently
+     * in progress — ordered by `orderNo`, deduplicated.
+     *
+     * When `detectInProgressRequests` is set, the result also carries an `hasInProgressRequests` flag
+     * telling whether any unhandled-but-in-progress request was skipped along the way. It lets
+     * {@link isFinished} distinguish "no work left at all" from "work remains, but it is currently being
+     * processed". Without it, a consumer with concurrency could consider the queue finished and shut the
+     * crawler down while it is still handling the last requests.
+     *
+     * Computing the flag is expensive: because an in-progress request may sit anywhere in the queue, it
+     * forces a scan of every pending entry even when only `limit` items are wanted. Callers that only
+     * need the head (e.g. {@link fetchNextRequest}, {@link isEmpty}) leave it off so the scan can stop as
+     * soon as the page is filled, keeping those calls O(head) instead of O(N).
+     */
+    async listPendingHead(limit, detectInProgressRequests = false) {
+        const items = [];
+        let hasInProgressRequests = false;
+        // Tracks processed request IDs to avoid duplicates (request in both `forefrontRequestIds` and `requests`).
+        const seenRequestIds = new Set();
+        // Tracks handled request IDs from `forefrontRequestIds` to be removed.
+        const handledForefrontIds = new Set();
+        for (const requestId of this.requestKeyIterator()) {
+            // Once the requested page is filled we can stop — unless the caller asked us to detect
+            // in-progress requests and we have not yet seen one, in which case we must keep scanning.
+            if (items.length >= limit && (!detectInProgressRequests || hasInProgressRequests)) {
+                break;
+            }
+            if (seenRequestIds.has(requestId)) {
+                continue;
+            }
+            seenRequestIds.add(requestId);
+            const request = this.requests.get(requestId);
+            // Permanently-handled requests (`orderNo === null`) are in a terminal state and can be skipped.
+            if (request.orderNo === null) {
+                if (this.forefrontRequestIds.includes(requestId)) {
+                    handledForefrontIds.add(requestId);
+                }
+                continue;
+            }
+            // In progress (fetched but not yet handled or reclaimed) — skip it, but remember that the
+            // queue is not truly empty.
+            if (this.inProgressRequestIds.has(requestId)) {
+                hasInProgressRequests = true;
+                continue;
+            }
+            if (items.length < limit) {
+                items.push(request);
+            }
+        }
+        this.forefrontRequestIds = this.forefrontRequestIds.filter((id) => !handledForefrontIds.has(id));
+        return {
+            items: items.sort((a, b) => a.orderNo - b.orderNo),
+            hasInProgressRequests: detectInProgressRequests ? hasInProgressRequests : undefined,
+        };
+    }
+    async fetchNextRequest() {
+        this.updateTimestamps(false);
+        await this.queueStateMutex.wait();
+        try {
+            const { items: [head], } = await this.listPendingHead(1);
+            if (!head) {
+                return undefined;
+            }
+            // Mark the request as in progress so it is not handed out again until it is handled or
+            // reclaimed. The request keeps its `orderNo` (and thus its forefront / normal ordering).
+            this.inProgressRequestIds.add(head.id);
+            return this._jsonToRequest(head.json) ?? undefined;
+        }
+        finally {
+            this.queueStateMutex.shift();
+        }
+    }
+    async addBatchOfRequests(requests, options = {}) {
+        batchRequestShapeWithoutId.parse(requests);
+        requestOptionsShape.parse(options);
+        // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so that the
+        // shared `requests` map, `forefrontRequestIds` array and request counts are not corrupted by a
+        // concurrent operation interleaving at one of the `await` points below.
+        await this.queueStateMutex.wait();
+        try {
+            const result = {
+                processedRequests: [],
+                unprocessedRequests: [],
+            };
+            for (const model of requests) {
+                const requestModel = this._createInternalRequest(model, options.forefront);
+                const existingRequestWithId = this.requests.get(requestModel.id);
+                if (existingRequestWithId) {
+                    result.processedRequests.push({
+                        requestId: existingRequestWithId.id,
+                        uniqueKey: existingRequestWithId.uniqueKey,
+                        wasAlreadyHandled: existingRequestWithId.orderNo === null,
+                        wasAlreadyPresent: true,
+                    });
+                    continue;
+                }
+                this.requests.set(requestModel.id, requestModel);
+                if (requestModel.orderNo) {
+                    this.pendingRequestCount += 1;
+                }
+                else {
+                    this.handledRequestCount += 1;
+                }
+                if (options.forefront) {
+                    this.forefrontRequestIds.push(requestModel.id);
+                }
+                result.processedRequests.push({
+                    requestId: requestModel.id,
+                    uniqueKey: requestModel.uniqueKey,
+                    // We return wasAlreadyHandled: false even though the request may
+                    // have been added as handled, because that's how API behaves.
+                    wasAlreadyHandled: false,
+                    wasAlreadyPresent: false,
+                });
+            }
+            this.updateTimestamps(true);
+            return result;
+        }
+        finally {
+            this.queueStateMutex.shift();
+        }
+    }
+    async getRequest(uniqueKey) {
+        s.string().parse(uniqueKey);
+        this.updateTimestamps(false);
+        const id = uniqueKeyToRequestId(uniqueKey);
+        const json = this.requests.get(id)?.json;
+        return this._jsonToRequest(json);
+    }
+    async markRequestAsHandled(request) {
+        requestShape.parse(request);
+        this.updateTimestamps(false);
+        // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
+        // `requests` map, `inProgressRequestIds` set and request counts stay consistent across the
+        // `await` points below.
+        await this.queueStateMutex.wait();
+        try {
+            const id = uniqueKeyToRequestId(request.uniqueKey);
+            const existingRequest = this.requests.get(id);
+            // The request must exist to be marked as handled. We intentionally do NOT require it to still
+            // be in progress: marking an already-released request handled must still succeed, otherwise
+            // the request could be handed out again and the queue would never finish.
+            if (!existingRequest) {
+                return undefined;
+            }
+            // A handled request has `orderNo === null`. Marking it again is an idempotent no-op.
+            const wasAlreadyHandled = existingRequest.orderNo === null;
+            const handledAt = request.handledAt ?? new Date().toISOString();
+            const requestModel = this._createInternalRequest({ ...request, handledAt }, false);
+            this.requests.set(id, requestModel);
+            // The request is no longer in progress for this client.
+            this.inProgressRequestIds.delete(id);
+            if (!wasAlreadyHandled) {
+                this.pendingRequestCount -= 1;
+                this.handledRequestCount += 1;
+            }
+            this.updateTimestamps(true);
+            return {
+                requestId: id,
+                wasAlreadyHandled,
+                wasAlreadyPresent: true,
+            };
+        }
+        finally {
+            this.queueStateMutex.shift();
+        }
+    }
+    async reclaimRequest(request, options = {}) {
+        requestShape.parse(request);
+        requestOptionsShape.parse(options);
+        this.updateTimestamps(false);
+        // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
+        // `requests` map, `forefrontRequestIds` array and `inProgressRequestIds` set stay consistent
+        // across the `await` points below.
+        await this.queueStateMutex.wait();
+        try {
+            const id = uniqueKeyToRequestId(request.uniqueKey);
+            const existingRequest = this.requests.get(id);
+            // The request must exist and not already be handled to be reclaimed. As with
+            // `markRequestAsHandled`, we do NOT require it to still be in progress — returning an
+            // already-released request to the queue (e.g. to honor a `forefront` reorder) must still
+            // work, rather than have the reclaim silently dropped.
+            if (!existingRequest || existingRequest.orderNo === null) {
+                return undefined;
+            }
+            // Reclaiming resets the `orderNo` to a fresh timestamp, restoring the request to the queue
+            // (at the front if `forefront`).
+            const requestModel = this._createInternalRequest(request, options.forefront);
+            this.requests.set(id, requestModel);
+            // The request is no longer in progress for this client.
+            this.inProgressRequestIds.delete(id);
+            if (options.forefront) {
+                this.forefrontRequestIds.push(id);
+            }
+            this.updateTimestamps(true);
+            return {
+                requestId: id,
+                wasAlreadyHandled: false,
+                wasAlreadyPresent: true,
+            };
+        }
+        finally {
+            this.queueStateMutex.shift();
+        }
+    }
+    async isEmpty() {
+        this.updateTimestamps(false);
+        // "Empty" means there is nothing left to fetch right now — i.e. the next `fetchNextRequest`
+        // would return `null`. Requests that are currently in progress are intentionally NOT counted
+        // here: they are not fetchable, so the queue is empty from a consumer's point of view. Whether
+        // those in-progress requests mean crawling is not yet done is a separate question, answered by
+        // `isFinished`.
+        //
+        // `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state mutex to avoid
+        // racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.
+        await this.queueStateMutex.wait();
+        try {
+            const { items } = await this.listPendingHead(1);
+            return items.length === 0;
+        }
+        finally {
+            this.queueStateMutex.shift();
+        }
+    }
+    async isFinished() {
+        this.updateTimestamps(false);
+        // The queue is finished only when there is nothing left to fetch AND nothing currently in
+        // progress. Counting in-progress requests is what allows a crawler with concurrency to keep
+        // waiting while it still holds the last requests, instead of finishing prematurely.
+        //
+        // Detecting in-progress requests requires a full scan, hence the `detectInProgressRequests`
+        // flag — unlike `fetchNextRequest`/`isEmpty`, which only need the head and can stop early.
+        //
+        // `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state mutex to avoid
+        // racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.
+        await this.queueStateMutex.wait();
+        try {
+            const { items, hasInProgressRequests } = await this.listPendingHead(1, true);
+            return items.length === 0 && !hasInProgressRequests;
+        }
+        finally {
+            this.queueStateMutex.shift();
+        }
+    }
+    /**
+     * Returns all pending (not yet handled, not currently in progress) requests in the queue, ordered
+     * the same way {@link fetchNextRequest} would hand them out. This does not mutate the queue,
+     * nothing is marked in progress.
+     */
+    async listItems() {
+        this.updateTimestamps(false);
+        // `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state
+        // mutex to avoid racing a concurrent mutator at its `await` points.
+        await this.queueStateMutex.wait();
+        try {
+            const { items } = await this.listPendingHead(Number.POSITIVE_INFINITY);
+            return items.map((request) => this._jsonToRequest(request.json));
+        }
+        finally {
+            this.queueStateMutex.shift();
+        }
+    }
+    toRequestQueueInfo() {
+        return {
+            accessedAt: this.accessedAt,
+            createdAt: this.createdAt,
+            handledRequestCount: this.handledRequestCount,
+            id: this.id,
+            modifiedAt: this.modifiedAt,
+            name: this.name,
+            pendingRequestCount: this.pendingRequestCount,
+            totalRequestCount: this.requests.size,
+        };
+    }
+    updateTimestamps(hasBeenModified) {
+        this.accessedAt = new Date();
+        if (hasBeenModified) {
+            this.modifiedAt = new Date();
+        }
+    }
+    _jsonToRequest(requestJson) {
+        if (!requestJson)
+            return undefined;
+        const request = JSON.parse(requestJson);
+        return purgeNullsFromObject(request);
+    }
+    _createInternalRequest(request, forefront) {
+        const orderNo = this._calculateOrderNo(request, forefront);
+        const id = uniqueKeyToRequestId(request.uniqueKey);
+        if (request.id && request.id !== id) {
+            throw new Error('Request ID does not match its uniqueKey.');
+        }
+        const json = JSON.stringify({ ...request, id });
+        return {
+            id,
+            json,
+            method: request.method,
+            orderNo,
+            retryCount: request.retryCount ?? 0,
+            uniqueKey: request.uniqueKey,
+            url: request.url,
+        };
+    }
+    _calculateOrderNo(request, forefront) {
+        if (request.handledAt)
+            return null;
+        const timestamp = Date.now();
+        return forefront ? -timestamp : timestamp;
+    }
+}
+//# sourceMappingURL=request-queue.js.map

package/memory-storage/resource-clients/request-queue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"request-queue.js","sourceRoot":"","sources":["../../../src/memory-storage/resource-clients/request-queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,CAAC,EAAE,MAAM,sBAAsB,CAAC;AAEzC,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,MAAM,YAAY,GAAG,CAAC;KACjB,MAAM,CAAC;IACJ,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IACd,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,gBAAgB,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;IAC9D,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACvC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;CAChE,CAAC;KACD,WAAW,EAAE,CAAC;AAEnB,MAAM,qBAAqB,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AAExD,MAAM,0BAA0B,GAAG,qBAAqB,CAAC,KAAK,EAAE,CAAC;AAEjE,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACpC,CAAC,CAAC;AAwBH,MAAM,OAAO,mBAAoB,SAAQ,UAAU;IAC/C,IAAI,CAAU;IACd;;;OAGG;IACH,QAAQ,CAAS;IACjB,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;IACxB,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;IACxB,mBAAmB,GAAG,CAAC,CAAC;IACxB,mBAAmB,GAAG,CAAC,CAAC;IACxB;;;;;;;OAOG;IACc,eAAe,GAAG,IAAI,UAAU,EAAE,CAAC;IAC5C,mBAAmB,GAAa,EAAE,CAAC;IAE3C;;;;;;;OAOG;IACc,oBAAoB,GAAG,IAAI,GAAG,EAAU,CAAC;IAEzC,QAAQ,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC9C,cAAc,CAAuB;IAEtD,YAAY,OAAmC;QAC3C,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,UAAU,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACzD,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,WAAW;QACb,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC,kBAAkB,EAAE,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,IAAI;QACN,kGAAkG;QAClG,mGAAmG;QACnG,0EAA0E;QAC1E,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAElC,IAAI,CAAC;YACD,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;YAE3G,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;gBACpB,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,wBAAwB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;gBACxF,UAAU,CAAC,mBAAmB,GAAG,CAAC,CAAC;gBACnC,sFAAsF;gBACtF,yFAAyF;gBACzF,2DAA2D;gBAC3D,UAAU,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAC5B,UAAU,CAAC,mBAAmB,GAAG,EAAE,CAAC;gBACpC,UAAU,CAAC,oBAAoB,CAAC,KAAK,EAAE,CAAC;YAC5C,CAAC;QACL,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QACjC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK;QACP,kGAAkG;QAClG,sFAAsF;QACtF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAElC,IAAI,CAAC;YACD,4BAA4B;YAC5B,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,mBAAmB,GAAG,EAAE,CAAC;YAC9B,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,CAAC;YAClC,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;YAC7B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;YAE7B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QACjC,CAAC;IACL,CAAC;IAEO,CAAC,kBAAkB;QACvB,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5D,MAAM,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACrC,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACK,KAAK,CAAC,eAAe,CACzB,KAAa,EACb,wBAAwB,GAAG,KAAK;QAEhC,MAAM,KAAK,GAAsB,EAAE,CAAC;QACpC,IAAI,qBAAqB,GAAG,KAAK,CAAC;QAElC,2GAA2G;QAC3G,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QACzC,uEAAuE;QACvE,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAU,CAAC;QAE9C,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC;YAChD,uFAAuF;YACvF,0FAA0F;YAC1F,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,IAAI,CAAC,CAAC,wBAAwB,IAAI,qBAAqB,CAAC,EAAE,CAAC;gBAChF,MAAM;YACV,CAAC;YAED,IAAI,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAChC,SAAS;YACb,CAAC;YAED,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAE9B,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;YAE9C,gGAAgG;YAChG,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC3B,IAAI,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC/C,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBACvC,CAAC;gBACD,SAAS;YACb,CAAC;YAED,0FAA0F;YAC1F,4BAA4B;YAC5B,IAAI,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC3C,qBAAqB,GAAG,IAAI,CAAC;gBAC7B,SAAS;YACb,CAAC;YAED,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;gBACvB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACL,CAAC;QAED,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAEjG,OAAO;YACH,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAQ,GAAG,CAAC,CAAC,OAAQ,CAAC;YACpD,qBAAqB,EAAE,wBAAwB,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS;SACtF,CAAC;IACN,CAAC;IAED,KAAK,CAAC,gBAAgB;QAClB,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE7B,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAElC,IAAI,CAAC;YACD,MAAM,EACF,KAAK,EAAE,CAAC,IAAI,CAAC,GAChB,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAElC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,uFAAuF;YACvF,yFAAyF;YACzF,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAEvC,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC;QACvD,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QACjC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,kBAAkB,CACpB,QAAiC,EACjC,UAAgD,EAAE;QAElD,0BAA0B,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3C,mBAAmB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,8FAA8F;QAC9F,+FAA+F;QAC/F,wEAAwE;QACxE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAElC,IAAI,CAAC;YACD,MAAM,MAAM,GAAmC;gBAC3C,iBAAiB,EAAE,EAAE;gBACrB,mBAAmB,EAAE,EAAE;aAC1B,CAAC;YAEF,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;gBAC3B,MAAM,YAAY,GAAG,IAAI,CAAC,sBAAsB,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;gBAE3E,MAAM,qBAAqB,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;gBAEjE,IAAI,qBAAqB,EAAE,CAAC;oBACxB,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC;wBAC1B,SAAS,EAAE,qBAAqB,CAAC,EAAE;wBACnC,SAAS,EAAE,qBAAqB,CAAC,SAAS;wBAC1C,iBAAiB,EAAE,qBAAqB,CAAC,OAAO,KAAK,IAAI;wBACzD,iBAAiB,EAAE,IAAI;qBAC1B,CAAC,CAAC;oBAEH,SAAS;gBACb,CAAC;gBAED,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;gBAEjD,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;oBACvB,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAC;gBAClC,CAAC;qBAAM,CAAC;oBACJ,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAC;gBAClC,CAAC;gBAED,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;oBACpB,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;gBACnD,CAAC;gBAED,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC;oBAC1B,SAAS,EAAE,YAAY,CAAC,EAAE;oBAC1B,SAAS,EAAE,YAAY,CAAC,SAAS;oBACjC,iEAAiE;oBACjE,8DAA8D;oBAC9D,iBAAiB,EAAE,KAAK;oBACxB,iBAAiB,EAAE,KAAK;iBAC3B,CAAC,CAAC;YACP,CAAC;YAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;YAE5B,OAAO,MAAM,CAAC;QAClB,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QACjC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,SAAiB;QAC9B,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC5B,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,EAAE,GAAG,oBAAoB,CAAC,SAAS,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC;QACzC,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,OAAoC;QAC3D,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE7B,gGAAgG;QAChG,2FAA2F;QAC3F,wBAAwB;QACxB,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAElC,IAAI,CAAC;YACD,MAAM,EAAE,GAAG,oBAAoB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAEnD,MAAM,eAAe,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAE9C,8FAA8F;YAC9F,4FAA4F;YAC5F,0EAA0E;YAC1E,IAAI,CAAC,eAAe,EAAE,CAAC;gBACnB,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,qFAAqF;YACrF,MAAM,iBAAiB,GAAG,eAAe,CAAC,OAAO,KAAK,IAAI,CAAC;YAE3D,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAChE,MAAM,YAAY,GAAG,IAAI,CAAC,sBAAsB,CAAC,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,EAAE,KAAK,CAAC,CAAC;YAEnF,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;YAEpC,wDAAwD;YACxD,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAErC,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBACrB,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAC;gBAC9B,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAC;YAClC,CAAC;YAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;YAE5B,OAAO;gBACH,SAAS,EAAE,EAAE;gBACb,iBAAiB;gBACjB,iBAAiB,EAAE,IAAI;aAC1B,CAAC;QACN,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QACjC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,cAAc,CAChB,OAAoC,EACpC,UAAgD,EAAE;QAElD,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5B,mBAAmB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE7B,gGAAgG;QAChG,6FAA6F;QAC7F,mCAAmC;QACnC,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAElC,IAAI,CAAC;YACD,MAAM,EAAE,GAAG,oBAAoB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAEnD,MAAM,eAAe,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAE9C,6EAA6E;YAC7E,sFAAsF;YACtF,yFAAyF;YACzF,uDAAuD;YACvD,IAAI,CAAC,eAAe,IAAI,eAAe,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBACvD,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,2FAA2F;YAC3F,iCAAiC;YACjC,MAAM,YAAY,GAAG,IAAI,CAAC,sBAAsB,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;YAE7E,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;YAEpC,wDAAwD;YACxD,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAErC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBACpB,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACtC,CAAC;YAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;YAE5B,OAAO;gBACH,SAAS,EAAE,EAAE;gBACb,iBAAiB,EAAE,KAAK;gBACxB,iBAAiB,EAAE,IAAI;aAC1B,CAAC;QACN,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QACjC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO;QACT,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE7B,4FAA4F;QAC5F,6FAA6F;QAC7F,+FAA+F;QAC/F,+FAA+F;QAC/F,gBAAgB;QAChB,EAAE;QACF,6GAA6G;QAC7G,iFAAiF;QACjF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAElC,IAAI,CAAC;YACD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAChD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC;QAC9B,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QACjC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU;QACZ,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE7B,0FAA0F;QAC1F,4FAA4F;QAC5F,oFAAoF;QACpF,EAAE;QACF,4FAA4F;QAC5F,2FAA2F;QAC3F,EAAE;QACF,6GAA6G;QAC7G,iFAAiF;QACjF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAElC,IAAI,CAAC;YACD,MAAM,EAAE,KAAK,EAAE,qBAAqB,EAAE,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;YAC7E,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC;QACxD,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QACjC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,SAAS;QACX,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAE7B,8FAA8F;QAC9F,oEAAoE;QACpE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAElC,IAAI,CAAC;YACD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACvE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAA8B,OAAO,CAAC,IAAI,CAAE,CAAC,CAAC;QACnG,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QACjC,CAAC;IACL,CAAC;IAED,kBAAkB;QACd,OAAO;YACH,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,iBAAiB,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;SACxC,CAAC;IACN,CAAC;IAEO,gBAAgB,CAAC,eAAwB;QAC7C,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QAE7B,IAAI,eAAe,EAAE,CAAC;YAClB,IAAI,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;QACjC,CAAC;IACL,CAAC;IAEO,cAAc,CAAI,WAAoB;QAC1C,IAAI,CAAC,WAAW;YAAE,OAAO,SAAS,CAAC;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACxC,OAAO,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;IAEO,sBAAsB,CAAC,OAA8B,EAAE,SAAmB;QAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC3D,MAAM,EAAE,GAAG,oBAAoB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEnD,IAAI,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QAChD,OAAO;YACH,EAAE;YACF,IAAI;YACJ,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO;YACP,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,CAAC;YACnC,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,GAAG,EAAE,OAAO,CAAC,GAAG;SACnB,CAAC;IACN,CAAC;IAEO,iBAAiB,CAAC,OAA8B,EAAE,SAAmB;QACzE,IAAI,OAAO,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAEnC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9C,CAAC;CACJ"}

package/memory-storage/utils.d.ts

@@ -0,0 +1,12 @@
+import { isBuffer, isStream, toBuffer } from '@crawlee/utils';
+/**
+ * Removes all properties with a null value
+ * from the provided object.
+ */
+export declare function purgeNullsFromObject<T>(object: T): T;
+/**
+ * Creates a standard request ID (same as Platform).
+ */
+export declare function uniqueKeyToRequestId(uniqueKey: string): string;
+export { isBuffer, isStream, toBuffer };
+//# sourceMappingURL=utils.d.ts.map

package/memory-storage/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/memory-storage/utils.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAI9D;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,CAQpD;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAO9D;AAED,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC"}

package/memory-storage/utils.js

@@ -0,0 +1,28 @@
+import { createHash } from 'node:crypto';
+import { isBuffer, isStream, toBuffer } from '@crawlee/utils';
+import { REQUEST_ID_LENGTH } from './consts.js';
+/**
+ * Removes all properties with a null value
+ * from the provided object.
+ */
+export function purgeNullsFromObject(object) {
+    if (object && typeof object === 'object' && !Array.isArray(object)) {
+        for (const [key, value] of Object.entries(object)) {
+            if (value === null)
+                Reflect.deleteProperty(object, key);
+        }
+    }
+    return object;
+}
+/**
+ * Creates a standard request ID (same as Platform).
+ */
+export function uniqueKeyToRequestId(uniqueKey) {
+    const str = createHash('sha256')
+        .update(uniqueKey)
+        .digest('base64')
+        .replace(/(\+|\/|=)/g, '');
+    return str.length > REQUEST_ID_LENGTH ? str.slice(0, REQUEST_ID_LENGTH) : str;
+}
+export { isBuffer, isStream, toBuffer };
+//# sourceMappingURL=utils.js.map

package/memory-storage/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/memory-storage/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAI,MAAS;IAC7C,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACjE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAChD,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAO,CAAC,cAAc,CAAC,MAAiC,EAAE,GAAG,CAAC,CAAC;QACvF,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAiB;IAClD,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC;SAC3B,MAAM,CAAC,SAAS,CAAC;SACjB,MAAM,CAAC,QAAQ,CAAC;SAChB,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAE/B,OAAO,GAAG,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAClF,CAAC;AAED,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.66",
+    "version": "4.0.0-beta.68",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,13 +53,16 @@
         "@apify/pseudo_url": "^2.0.59",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/memory-storage": "4.0.0-beta.66",
-        "@crawlee/types": "4.0.0-beta.66",
-        "@crawlee/utils": "4.0.0-beta.66",
+        "@crawlee/fs-storage": "4.0.0-beta.68",
+        "@crawlee/types": "4.0.0-beta.68",
+        "@crawlee/utils": "4.0.0-beta.68",
         "@sapphire/async-queue": "^1.5.5",
+        "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
+        "content-type": "^1.0.5",
         "csv-stringify": "^6.5.2",
         "json5": "^2.2.3",
+        "mime-types": "^3.0.1",
         "minimatch": "^10.0.1",
         "ow": "^2.0.0",
         "stream-json": "^1.9.1",
@@ -76,5 +79,5 @@
             }
         }
     },
-    "gitHead": "0890633b51caf16d93940be6f7da7881ad839992"
+    "gitHead": "0856e557d3eaa0ec8a5facc775bf4ba98dd6b020"
 }

package/request.d.ts

@@ -164,7 +164,7 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
      */
     pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
     /** @internal */
-    static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, }: ComputeUniqueKeyOptions): string;
+    static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, alwaysEnqueue, }: ComputeUniqueKeyOptions): string;
     /** @internal */
     static hashPayload(payload: BinaryLike): string;
 }
@@ -185,7 +185,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      * The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
      *
      * The `useExtendedUniqueKey` options determines whether method and payload are included in the `uniqueKey`,
-     * producing a `uniqueKey` in the following format: `METHOD(payloadHash):normalizedUrl`. This is useful
+     * producing a `uniqueKey` in the following format: `METHOD|payloadHash|normalizedUrl`. This is useful
      * when requests point to the same URL, but with different methods and payloads. For example: form submits.
      *
      * Pass an arbitrary non-empty text value to the `uniqueKey` property
@@ -233,6 +233,13 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      * @default false
      */
     useExtendedUniqueKey?: boolean;
+    /**
+     * If `true` then a random value is included in the `uniqueKey` computation, ensuring the request
+     * is always enqueued even if a request with the same URL (and method/payload) is already present
+     * in the queue. Cannot be used together with a custom `uniqueKey`.
+     * @default false
+     */
+    alwaysEnqueue?: boolean;
     /**
      * The `true` value indicates that the request will not be automatically retried on error.
      * @default false
@@ -291,6 +298,7 @@ interface ComputeUniqueKeyOptions {
     payload?: string | Buffer;
     keepUrlFragment?: boolean;
     useExtendedUniqueKey?: boolean;
+    alwaysEnqueue?: boolean;
 }
 export type Source = (Partial<RequestOptions> & {
     requestsFromUrl?: string;

package/request.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEtE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AA0BxD,oBAAY,YAAY;IACpB,WAAW,IAAA;IACX,UAAU,IAAA;IACV,SAAS,IAAA;IACT,eAAe,IAAA;IACf,IAAI,IAAA;IACJ,aAAa,IAAA;IACb,KAAK,IAAA;IACL,OAAO,IAAA;CACV;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,cAAM,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACzD,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAE3B,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,8FAA8F;IAC9F,OAAO,EAAE,OAAO,CAAC;IAEjB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,sEAAsE;IACtE,OAAO,CAAC,SAAS,CAA2B;IAE5C;;;;;OAKG;IACH,QAAQ,EAAE,QAAQ,CAAkB;IAEpC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;gBACS,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC;IA2H7C;;;OAGG;IACI,mBAAmB,IAAI,OAAO;IAQrC;;;;;;OAMG;IACH,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED;;;;;;OAMG;IACH,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,EAMhC;IAED;;;OAGG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;OAGG;IACH,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,EAG3B;IAED,kJAAkJ;IAClJ,IAAI,SAAS,IAAI,MAAM,GAAG,SAAS,CAElC;IAED,IAAI,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAGtC;IAED,oDAAoD;IACpD,IAAI,KAAK,IAAI,MAAM,GAAG,SAAS,CAE9B;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAElC;IAED,8HAA8H;IAC9H,IAAI,UAAU,IAAI,MAAM,GAAG,SAAS,CAEnC;IAED,8HAA8H;IAC9H,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAMvC;IAED,uDAAuD;IACvD,IAAI,KAAK,IAAI,YAAY,CAExB;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,EAM5B;IAED;;OAEG;IACH,IAAI,aAAa,IAAI,oBAAoB,GAAG,SAAS,CAEpD;IAED;;OAEG;IACH,IAAI,aAAa,CAAC,KAAK,EAAE,oBAAoB,GAAG,SAAS,EAMxD;IAED,OAAO,KAAK,eAAe,GAE1B;IAED,OAAO,KAAK,eAAe,QAM1B;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,GAAE,uBAA4B,GAAG,IAAI;IAgCtF,gBAAgB;IAChB,MAAM,CAAC,gBAAgB,CAAC,EACpB,GAAG,EACH,MAAc,EACd,OAAO,EACP,eAAuB,EACvB,oBAA4B,GAC/B,EAAE,uBAAuB;IAmB1B,gBAAgB;IAChB,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;CAGlD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACpE,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAE5D,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAEpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;;;;;;OAQG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,oBAAoB,CAAC;IAErC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,gBAAgB;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gBAAgB;IAChB,aAAa,CAAC,EAAE,IAAI,CAAC;IAErB,gBAAgB;IAChB,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,UAAU,uBAAuB;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,cAAc,CAAC;AAE/G,gBAAgB;AAChB,MAAM,WAAW,cAAc;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC"}
+{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEtE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AA2BxD,oBAAY,YAAY;IACpB,WAAW,IAAA;IACX,UAAU,IAAA;IACV,SAAS,IAAA;IACT,eAAe,IAAA;IACf,IAAI,IAAA;IACJ,aAAa,IAAA;IACb,KAAK,IAAA;IACL,OAAO,IAAA;CACV;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,cAAM,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACzD,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAE3B,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,8FAA8F;IAC9F,OAAO,EAAE,OAAO,CAAC;IAEjB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,sEAAsE;IACtE,OAAO,CAAC,SAAS,CAA2B;IAE5C;;;;;OAKG;IACH,QAAQ,EAAE,QAAQ,CAAkB;IAEpC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;gBACS,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC;IAuI7C;;;OAGG;IACI,mBAAmB,IAAI,OAAO;IAQrC;;;;;;OAMG;IACH,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED;;;;;;OAMG;IACH,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,EAMhC;IAED;;;OAGG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;OAGG;IACH,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,EAG3B;IAED,kJAAkJ;IAClJ,IAAI,SAAS,IAAI,MAAM,GAAG,SAAS,CAElC;IAED,IAAI,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAGtC;IAED,oDAAoD;IACpD,IAAI,KAAK,IAAI,MAAM,GAAG,SAAS,CAE9B;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAElC;IAED,8HAA8H;IAC9H,IAAI,UAAU,IAAI,MAAM,GAAG,SAAS,CAEnC;IAED,8HAA8H;IAC9H,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAMvC;IAED,uDAAuD;IACvD,IAAI,KAAK,IAAI,YAAY,CAExB;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,EAM5B;IAED;;OAEG;IACH,IAAI,aAAa,IAAI,oBAAoB,GAAG,SAAS,CAEpD;IAED;;OAEG;IACH,IAAI,aAAa,CAAC,KAAK,EAAE,oBAAoB,GAAG,SAAS,EAMxD;IAED,OAAO,KAAK,eAAe,GAE1B;IAED,OAAO,KAAK,eAAe,QAM1B;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,GAAE,uBAA4B,GAAG,IAAI;IAgCtF,gBAAgB;IAChB,MAAM,CAAC,gBAAgB,CAAC,EACpB,GAAG,EACH,MAAc,EACd,OAAO,EACP,eAAuB,EACvB,oBAA4B,EAC5B,aAAqB,GACxB,EAAE,uBAAuB;IA6B1B,gBAAgB;IAChB,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;CAGlD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACpE,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAE5D,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAEpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IAExB;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;;;;;;OAQG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,oBAAoB,CAAC;IAErC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,gBAAgB;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gBAAgB;IAChB,aAAa,CAAC,EAAE,IAAI,CAAC;IAErB,gBAAgB;IAChB,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,UAAU,uBAAuB;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,aAAa,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,cAAc,CAAC;AAE/G,gBAAgB;AAChB,MAAM,WAAW,cAAc;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC"}