apify 4.0.0-beta.19 → 4.0.0-beta.21

package/dist/apify_request_queue_shared_backend.d.ts

@@ -0,0 +1,44 @@
+import type { BatchAddRequestsResult, QueueOperationInfo, RequestQueueOperationOptions, RequestSchema, UpdateRequestSchema } from '@crawlee/types';
+import { ApifyRequestQueueBackend } from './apify_request_queue_backend.js';
+/**
+ * Request queue backend safe for multi-consumer scenarios on the Apify platform.
+ *
+ * Requests fetched via {@link fetchNextRequest} are locked server-side (`listAndLockHead`), so any
+ * number of clients — including other Actor runs — can process the same queue concurrently without
+ * handing out a request twice. The lock is held until the request is marked as handled or
+ * reclaimed, and its duration follows the consumer's expected processing time
+ * (see {@link setExpectedRequestProcessingTimeSecs}). This consistency costs roughly one extra API
+ * call per processed request compared to the single-consumer backend.
+ *
+ * @internal
+ */
+export declare class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
+    /** Ids of requests locked by this client and waiting to be handed out by `fetchNextRequest`. */
+    private readonly headIds;
+    /** Dedup records for requests known to exist on the platform, keyed by id. */
+    private readonly cachedRequestInfo;
+    /** Ids of requests currently being processed by this client. */
+    private readonly inProgressIds;
+    /** Whether the last head read reported any locked requests left in the queue (any client's). */
+    private queueHasLockedRequests?;
+    /** Set after a forefront insert — the next head read starts fresh so the insert is honored. */
+    private shouldCheckForefrontRequests;
+    /** Lock duration applied to fetched requests; raised via `setExpectedRequestProcessingTimeSecs`. */
+    private lockSecs;
+    /** Serializes head reads and reclaims — both reorder the shared head state. */
+    private readonly headLock;
+    setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+    addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
+    getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
+    fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;
+    markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | undefined>;
+    reclaimRequest(request: UpdateRequestSchema, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | undefined>;
+    isEmpty(): Promise<boolean>;
+    isFinished(): Promise<boolean>;
+    /** Must be called with the head lock held. */
+    private ensureHeadIsNonEmpty;
+    /** Must be called with the head lock held. */
+    private listAndLockHead;
+    private isKnownOrExists;
+    private cacheRequestInfo;
+}

package/dist/apify_request_queue_shared_backend.js

@@ -0,0 +1,211 @@
+import { LruCache } from '@apify/datastructures';
+import log from '@apify/log';
+import { ApifyRequestQueueBackend, AsyncLock } from './apify_request_queue_backend.js';
+/** Maximum number of request dedup records cached locally. */
+const MAX_CACHED_REQUESTS = 1_000_000;
+/** Default lock duration for requests fetched via `fetchNextRequest`. */
+const DEFAULT_REQUEST_LOCK_SECS = 3 * 60;
+/** How many head requests to lock per `listAndLockHead` round-trip. */
+const HEAD_LOCK_LIMIT = 25;
+/**
+ * Request queue backend safe for multi-consumer scenarios on the Apify platform.
+ *
+ * Requests fetched via {@link fetchNextRequest} are locked server-side (`listAndLockHead`), so any
+ * number of clients — including other Actor runs — can process the same queue concurrently without
+ * handing out a request twice. The lock is held until the request is marked as handled or
+ * reclaimed, and its duration follows the consumer's expected processing time
+ * (see {@link setExpectedRequestProcessingTimeSecs}). This consistency costs roughly one extra API
+ * call per processed request compared to the single-consumer backend.
+ *
+ * @internal
+ */
+export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
+    /** Ids of requests locked by this client and waiting to be handed out by `fetchNextRequest`. */
+    headIds = [];
+    /** Dedup records for requests known to exist on the platform, keyed by id. */
+    cachedRequestInfo = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
+    /** Ids of requests currently being processed by this client. */
+    inProgressIds = new Set();
+    /** Whether the last head read reported any locked requests left in the queue (any client's). */
+    queueHasLockedRequests;
+    /** Set after a forefront insert — the next head read starts fresh so the insert is honored. */
+    shouldCheckForefrontRequests = false;
+    /** Lock duration applied to fetched requests; raised via `setExpectedRequestProcessingTimeSecs`. */
+    lockSecs = DEFAULT_REQUEST_LOCK_SECS;
+    /** Serializes head reads and reclaims — both reorder the shared head state. */
+    headLock = new AsyncLock();
+    async setExpectedRequestProcessingTimeSecs(secs) {
+        // Only ever raise the lock duration — several consumers may share this client, and a
+        // short-lived one must not cut the reservation of a long-running one short.
+        this.lockSecs = Math.max(this.lockSecs, secs);
+    }
+    async addBatchOfRequests(requests, options = {}) {
+        const { forefront = false } = options;
+        // Skip requests this client already knows reached the platform — a platform write costs an
+        // API call and a paid write operation. Whether such a request has been handled in the
+        // meantime by another client is unknowable locally, so report its last known state.
+        const alreadyPresent = [];
+        const newRequests = [];
+        for (const request of requests) {
+            const id = this.requestIdFromUniqueKey(request.uniqueKey);
+            const cached = this.cachedRequestInfo.get(id);
+            if (cached) {
+                alreadyPresent.push({
+                    requestId: id,
+                    uniqueKey: request.uniqueKey,
+                    wasAlreadyPresent: true,
+                    wasAlreadyHandled: cached.wasAlreadyHandled || request.handledAt != null,
+                });
+            }
+            else {
+                newRequests.push(request);
+            }
+        }
+        let result = { processedRequests: [], unprocessedRequests: [] };
+        if (newRequests.length > 0) {
+            result = await this.sendBatch(newRequests, forefront);
+            for (const processed of result.processedRequests) {
+                this.cacheRequestInfo(processed.requestId, { wasAlreadyHandled: processed.wasAlreadyHandled });
+            }
+            // A forefront insert changes the head order — have the next head read re-fetch the
+            // front of the queue instead of draining the local buffer first.
+            if (forefront) {
+                this.shouldCheckForefrontRequests = true;
+            }
+        }
+        result.processedRequests.push(...alreadyPresent);
+        this.recordAddedRequests(result);
+        return result;
+    }
+    async getRequest(uniqueKey) {
+        // The queue is shared — another client may modify a request at any time, so always read
+        // through to the platform.
+        return this.getRequestById(this.requestIdFromUniqueKey(uniqueKey));
+    }
+    async fetchNextRequest() {
+        const id = await this.headLock.runExclusive(async () => {
+            await this.ensureHeadIsNonEmpty();
+            return this.headIds.shift();
+        });
+        if (!id)
+            return undefined;
+        // Head items carry only partial request data (no userData, payload or headers), so the
+        // full record has to be hydrated with a round-trip.
+        const request = await this.getRequestById(id);
+        if (!request) {
+            // The head read can briefly report a request the main table does not serve yet — leave
+            // it out of the local head; it will reappear in a later head read.
+            log.debug(`Request fetched from the queue head was not found (id: ${id}), will be retried later`);
+            return undefined;
+        }
+        if (request.handledAt) {
+            // Handled by another client in the meantime.
+            this.cacheRequestInfo(id, { wasAlreadyHandled: true });
+            return undefined;
+        }
+        this.inProgressIds.add(id);
+        return request;
+    }
+    async markRequestAsHandled(request) {
+        const id = this.requestIdFromUniqueKey(request.uniqueKey);
+        // Contract: marking a request that does not exist in the queue is a no-op — it must not be
+        // added as a side effect (the platform update endpoint would upsert it).
+        if (!(await this.isKnownOrExists(id))) {
+            this.inProgressIds.delete(id);
+            return undefined;
+        }
+        const handledAt = request.handledAt ?? new Date().toISOString();
+        const info = await this.updateRequestOnPlatform({ ...request, id, handledAt });
+        this.inProgressIds.delete(id);
+        this.cacheRequestInfo(id, { wasAlreadyHandled: true });
+        if (!info.wasAlreadyHandled) {
+            this.estimatedHandledRequestCount += 1;
+        }
+        return info;
+    }
+    async reclaimRequest(request, options = {}) {
+        const { forefront = false } = options;
+        const id = this.requestIdFromUniqueKey(request.uniqueKey);
+        // Same contract as `markRequestAsHandled` — never insert as a side effect.
+        if (!(await this.isKnownOrExists(id))) {
+            this.inProgressIds.delete(id);
+            return undefined;
+        }
+        return this.headLock.runExclusive(async () => {
+            const info = await this.updateRequestOnPlatform({ ...request, id, handledAt: undefined }, forefront);
+            // Release the server-side lock so the request becomes fetchable again immediately —
+            // by any consumer — rather than only after the lock expires.
+            try {
+                await this.client.deleteRequestLock(id, { forefront });
+            }
+            catch (err) {
+                log.debug(`Failed to delete the lock of a reclaimed request (id: ${id}): ${err.message}`);
+            }
+            this.inProgressIds.delete(id);
+            this.cacheRequestInfo(id, { wasAlreadyHandled: false });
+            if (forefront) {
+                this.shouldCheckForefrontRequests = true;
+            }
+            if (info.wasAlreadyHandled) {
+                this.estimatedHandledRequestCount -= 1;
+            }
+            return info;
+        });
+    }
+    async isEmpty() {
+        return this.headLock.runExclusive(async () => {
+            if (this.headIds.length > 0)
+                return false;
+            await this.listAndLockHead(1);
+            return this.headIds.length === 0;
+        });
+    }
+    async isFinished() {
+        return this.headLock.runExclusive(async () => {
+            if (this.headIds.length > 0)
+                return false;
+            // The head read also refreshes `queueHasLockedRequests`, so the order matters here.
+            await this.listAndLockHead(1);
+            return this.headIds.length === 0 && !this.queueHasLockedRequests;
+        });
+    }
+    /** Must be called with the head lock held. */
+    async ensureHeadIsNonEmpty() {
+        if (this.headIds.length > 1 && !this.shouldCheckForefrontRequests) {
+            return;
+        }
+        await this.listAndLockHead(HEAD_LOCK_LIMIT);
+    }
+    /** Must be called with the head lock held. */
+    async listAndLockHead(limit) {
+        // After a forefront insert the local buffer no longer starts at the true front of the
+        // queue — re-fetch the front and keep the already-locked leftovers for afterwards.
+        let leftoverIds = [];
+        if (this.shouldCheckForefrontRequests) {
+            leftoverIds = this.headIds.splice(0);
+            this.shouldCheckForefrontRequests = false;
+        }
+        const head = await this.client.listAndLockHead({ limit, lockSecs: this.lockSecs });
+        this.queueHasLockedRequests = head.queueHasLockedRequests;
+        for (const item of head.items) {
+            if (this.inProgressIds.has(item.id))
+                continue;
+            if (this.headIds.includes(item.id) || leftoverIds.includes(item.id))
+                continue;
+            this.cacheRequestInfo(item.id, { wasAlreadyHandled: false });
+            this.headIds.push(item.id);
+        }
+        this.headIds.push(...leftoverIds);
+    }
+    async isKnownOrExists(id) {
+        if (this.inProgressIds.has(id) || this.cachedRequestInfo.get(id)) {
+            return true;
+        }
+        return (await this.getRequestById(id)) !== undefined;
+    }
+    cacheRequestInfo(id, info) {
+        // `LruCache.add` does not overwrite existing entries, so remove first.
+        this.cachedRequestInfo.remove(id);
+        this.cachedRequestInfo.add(id, info);
+    }
+}

package/dist/apify_request_queue_single_backend.d.ts

@@ -0,0 +1,52 @@
+import type { BatchAddRequestsResult, QueueOperationInfo, RequestQueueOperationOptions, RequestSchema, UpdateRequestSchema } from '@crawlee/types';
+import { ApifyRequestQueueBackend } from './apify_request_queue_backend.js';
+/**
+ * Request queue backend optimized for single-consumer scenarios on the Apify platform.
+ *
+ * Minimizes API calls by keeping a local estimate of the queue head and a local cache of the
+ * requests this client added — a request fetched from the head is usually served straight from the
+ * cache, with no per-request round-trip and no server-side locking.
+ *
+ * ### Usage constraints
+ *
+ * - **Single consumer** — only one client may fetch and process requests from the queue at a time.
+ * - **Multiple producers allowed** — other clients may add requests concurrently, but their
+ *   forefront requests may not be prioritized immediately, as this client relies on a local head
+ *   estimate instead of frequent head fetching.
+ * - **Append-only queue** — other clients must not delete or modify existing requests, as such
+ *   changes are not reflected in the local cache. Marking requests as handled elsewhere is
+ *   tolerated, but may occasionally lead to a request being processed twice.
+ *
+ * If these constraints do not hold, use the shared backend (`requestQueueAccess: 'shared'`) instead.
+ *
+ * @internal
+ */
+export declare class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
+    /** Local estimate of the queue head — request ids in the order they should be fetched. */
+    private readonly headIds;
+    /** Unhandled full request objects added by (or fetched through) this client, keyed by id. */
+    private readonly cachedRequests;
+    /** Ids of requests known to be already handled — cheap dedup without caching full objects. */
+    private readonly handledIds;
+    /** Ids of requests currently being processed by this client. */
+    private readonly inProgressIds;
+    /** Memoized one-time prefetch of existing queue contents into the local caches. */
+    private initCachesPromise?;
+    addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
+    getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
+    fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;
+    markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | undefined>;
+    reclaimRequest(request: UpdateRequestSchema, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | undefined>;
+    isEmpty(): Promise<boolean>;
+    isFinished(): Promise<boolean>;
+    private ensureHeadIsNonEmpty;
+    private listHead;
+    /**
+     * One-time prefetch of the existing queue contents into the local caches, so that re-added
+     * requests of a resurrected run are deduplicated locally (one read API call for the whole
+     * cache) instead of on the platform (one write operation per request).
+     */
+    private initCaches;
+    private isKnownOrExists;
+    private cacheRequest;
+}

package/dist/apify_request_queue_single_backend.js

@@ -0,0 +1,256 @@
+import { LruCache } from '@apify/datastructures';
+import log from '@apify/log';
+import { ApifyRequestQueueBackend } from './apify_request_queue_backend.js';
+/** Maximum number of full request objects cached locally. */
+const MAX_CACHED_REQUESTS = 1_000_000;
+/** The maximum head items read count, limited by the API. */
+const MAX_HEAD_ITEMS = 1000;
+/** How many new head items to aim for per `listHead` round-trip. */
+const DESIRED_NEW_HEAD_ITEMS = 200;
+/** How many existing requests to prefetch into the local caches on the first add. */
+const INIT_CACHES_REQUEST_LIMIT = 10_000;
+/**
+ * Request queue backend optimized for single-consumer scenarios on the Apify platform.
+ *
+ * Minimizes API calls by keeping a local estimate of the queue head and a local cache of the
+ * requests this client added — a request fetched from the head is usually served straight from the
+ * cache, with no per-request round-trip and no server-side locking.
+ *
+ * ### Usage constraints
+ *
+ * - **Single consumer** — only one client may fetch and process requests from the queue at a time.
+ * - **Multiple producers allowed** — other clients may add requests concurrently, but their
+ *   forefront requests may not be prioritized immediately, as this client relies on a local head
+ *   estimate instead of frequent head fetching.
+ * - **Append-only queue** — other clients must not delete or modify existing requests, as such
+ *   changes are not reflected in the local cache. Marking requests as handled elsewhere is
+ *   tolerated, but may occasionally lead to a request being processed twice.
+ *
+ * If these constraints do not hold, use the shared backend (`requestQueueAccess: 'shared'`) instead.
+ *
+ * @internal
+ */
+export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
+    /** Local estimate of the queue head — request ids in the order they should be fetched. */
+    headIds = [];
+    /** Unhandled full request objects added by (or fetched through) this client, keyed by id. */
+    cachedRequests = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
+    /** Ids of requests known to be already handled — cheap dedup without caching full objects. */
+    handledIds = new Set();
+    /** Ids of requests currently being processed by this client. */
+    inProgressIds = new Set();
+    /** Memoized one-time prefetch of existing queue contents into the local caches. */
+    initCachesPromise;
+    async addBatchOfRequests(requests, options = {}) {
+        const { forefront = false } = options;
+        await (this.initCachesPromise ??= this.initCaches());
+        // Split the batch into requests we already know about (dedup them locally — a platform
+        // write costs an API call and a paid write operation) and genuinely new ones.
+        const alreadyPresent = [];
+        const newRequests = [];
+        for (const request of requests) {
+            const id = this.requestIdFromUniqueKey(request.uniqueKey);
+            if (this.handledIds.has(id)) {
+                alreadyPresent.push({
+                    requestId: id,
+                    uniqueKey: request.uniqueKey,
+                    wasAlreadyPresent: true,
+                    wasAlreadyHandled: true,
+                });
+            }
+            else if (this.cachedRequests.get(id)) {
+                alreadyPresent.push({
+                    requestId: id,
+                    uniqueKey: request.uniqueKey,
+                    wasAlreadyPresent: true,
+                    wasAlreadyHandled: request.handledAt != null,
+                });
+            }
+            else {
+                newRequests.push(request);
+            }
+        }
+        let result = { processedRequests: [], unprocessedRequests: [] };
+        if (newRequests.length > 0) {
+            result = await this.sendBatch(newRequests, forefront);
+            // Commit the accepted requests to the local caches and the head estimate. The platform
+            // response is authoritative — a request it reports as already handled (e.g. handled by
+            // a previous run of a resurrected Actor beyond the prefetch limit) must not re-enter
+            // the head.
+            const processedByKey = new Map(result.processedRequests.map((processed) => [processed.uniqueKey, processed]));
+            for (const request of newRequests) {
+                const processed = processedByKey.get(request.uniqueKey);
+                if (!processed)
+                    continue; // rejected by the platform, reported in `unprocessedRequests`
+                if (processed.wasAlreadyHandled) {
+                    this.handledIds.add(processed.requestId);
+                    continue;
+                }
+                this.cacheRequest({ ...request, id: processed.requestId });
+                if (forefront) {
+                    this.headIds.unshift(processed.requestId);
+                }
+                else {
+                    this.headIds.push(processed.requestId);
+                }
+            }
+        }
+        result.processedRequests.push(...alreadyPresent);
+        this.recordAddedRequests(result);
+        return result;
+    }
+    async getRequest(uniqueKey) {
+        const id = this.requestIdFromUniqueKey(uniqueKey);
+        const cached = this.cachedRequests.get(id);
+        if (cached)
+            return cached;
+        const request = await this.getRequestById(id);
+        if (!request)
+            return undefined;
+        // Requests already in progress are ones the client knows about — no caching needed.
+        if (!this.inProgressIds.has(id)) {
+            if (request.handledAt) {
+                this.handledIds.add(id);
+            }
+            else {
+                this.cacheRequest(request);
+            }
+        }
+        return request;
+    }
+    async fetchNextRequest() {
+        await this.ensureHeadIsNonEmpty();
+        while (this.headIds.length > 0) {
+            const id = this.headIds.shift();
+            if (this.inProgressIds.has(id) || this.handledIds.has(id)) {
+                continue;
+            }
+            this.inProgressIds.add(id);
+            // Requests added by this client are served straight from the cache; only requests
+            // discovered via `listHead` (added by another producer) need a round-trip.
+            const request = this.cachedRequests.get(id) ?? (await this.getRequestById(id));
+            if (!request) {
+                this.inProgressIds.delete(id);
+                continue;
+            }
+            if (request.handledAt) {
+                // Handled elsewhere in the meantime — skip it and remember the outcome.
+                this.inProgressIds.delete(id);
+                this.handledIds.add(id);
+                this.cachedRequests.remove(id);
+                continue;
+            }
+            return request;
+        }
+        return undefined;
+    }
+    async markRequestAsHandled(request) {
+        const id = this.requestIdFromUniqueKey(request.uniqueKey);
+        // Contract: marking a request that does not exist in the queue is a no-op — it must not be
+        // added as a side effect (the platform update endpoint would upsert it).
+        if (!(await this.isKnownOrExists(id))) {
+            this.inProgressIds.delete(id);
+            return undefined;
+        }
+        const handledAt = request.handledAt ?? new Date().toISOString();
+        const info = await this.updateRequestOnPlatform({ ...request, id, handledAt });
+        this.inProgressIds.delete(id);
+        this.handledIds.add(id);
+        this.cachedRequests.remove(id);
+        if (!info.wasAlreadyHandled) {
+            this.estimatedHandledRequestCount += 1;
+        }
+        return info;
+    }
+    async reclaimRequest(request, options = {}) {
+        const { forefront = false } = options;
+        const id = this.requestIdFromUniqueKey(request.uniqueKey);
+        // Same contract as `markRequestAsHandled` — never insert as a side effect.
+        if (!(await this.isKnownOrExists(id))) {
+            this.inProgressIds.delete(id);
+            return undefined;
+        }
+        // Reclaiming returns the request to the queue for reprocessing.
+        const reclaimed = { ...request, id, handledAt: undefined };
+        const info = await this.updateRequestOnPlatform(reclaimed, forefront);
+        this.inProgressIds.delete(id);
+        this.handledIds.delete(id);
+        this.cacheRequest(reclaimed);
+        // Return the id to the local head estimate right away — the platform head read can lag a
+        // few seconds behind the update, and `isFinished` must never report `true` while a
+        // reclaimed request is still waiting to be reprocessed.
+        if (!this.headIds.includes(id)) {
+            if (forefront) {
+                this.headIds.unshift(id);
+            }
+            else {
+                this.headIds.push(id);
+            }
+        }
+        if (info.wasAlreadyHandled) {
+            this.estimatedHandledRequestCount -= 1;
+        }
+        return info;
+    }
+    async isEmpty() {
+        await this.ensureHeadIsNonEmpty();
+        return this.headIds.length === 0;
+    }
+    async isFinished() {
+        return (await this.isEmpty()) && this.inProgressIds.size === 0;
+    }
+    async ensureHeadIsNonEmpty() {
+        if (this.headIds.length <= 1) {
+            await this.listHead();
+        }
+    }
+    async listHead() {
+        // The head read returns in-progress requests too, so fetch enough to find new ones.
+        const limit = Math.min(MAX_HEAD_ITEMS, DESIRED_NEW_HEAD_ITEMS + this.inProgressIds.size);
+        const head = await this.client.listHead({ limit });
+        for (const item of head.items) {
+            if (this.inProgressIds.has(item.id) || this.handledIds.has(item.id)) {
+                continue;
+            }
+            // `headIds` is nearly drained whenever this runs (see `ensureHeadIsNonEmpty`), so the
+            // linear dedup scan stays cheap.
+            if (!this.headIds.includes(item.id)) {
+                this.headIds.push(item.id);
+            }
+        }
+    }
+    /**
+     * One-time prefetch of the existing queue contents into the local caches, so that re-added
+     * requests of a resurrected run are deduplicated locally (one read API call for the whole
+     * cache) instead of on the platform (one write operation per request).
+     */
+    async initCaches() {
+        try {
+            const response = await this.client.listRequests({ limit: INIT_CACHES_REQUEST_LIMIT });
+            for (const request of response.items) {
+                if (request.handledAt) {
+                    this.handledIds.add(request.id);
+                }
+                else {
+                    this.cacheRequest(request);
+                }
+            }
+        }
+        catch (err) {
+            // The prefetch is a cost optimization, not a correctness requirement — deduplication
+            // falls back to the platform.
+            log.warning(`Failed to prefetch the request queue contents into the local cache: ${err.message}`);
+        }
+    }
+    async isKnownOrExists(id) {
+        if (this.inProgressIds.has(id) || this.handledIds.has(id) || this.cachedRequests.get(id)) {
+            return true;
+        }
+        return (await this.getRequestById(id)) !== undefined;
+    }
+    cacheRequest(request) {
+        // `LruCache.add` does not overwrite existing entries, so remove first.
+        this.cachedRequests.remove(request.id);
+        this.cachedRequests.add(request.id, request);
+    }
+}

package/dist/apify_storage_backend.d.ts

@@ -0,0 +1,109 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import type { DatasetBackend, KeyValueStoreBackend, RequestQueueBackend, StorageBackend, StorageIdentifier } from '@crawlee/types';
+import type { ApifyClient } from 'apify-client';
+import type { RequestQueueAccessMode } from './apify_request_queue_backend.js';
+import { type ChargeResult, type ChargingManager } from './charging.js';
+import type { Configuration } from './configuration.js';
+type StorageType = 'Dataset' | 'KeyValueStore' | 'RequestQueue';
+/** Marks a dataset backend whose underlying client charges for pushed items (pay-per-event). @internal */
+export declare const USES_PUSH_DATA_INTERCEPTION: unique symbol;
+/**
+ * Context of a single `Actor.pushData()` call, shared with the intercepted
+ * `pushItems()` calls so they can (1) know which event to charge and
+ * (2) aggregate the {@link ChargeResult} across the multiple `pushItems()`
+ * calls a single `pushData()` may trigger (Crawlee batches large pushes).
+ */
+export interface PpeAwarePushDataContext {
+    eventName: string | undefined;
+    chargeResult?: ChargeResult;
+}
+export declare const pushDataChargingContext: AsyncLocalStorage<PpeAwarePushDataContext>;
+export interface ApifyStorageBackendOptions {
+    /**
+     * SDK configuration providing the run's default storage ids and related environment values.
+     * Without it, opening storages requires an explicit id or name.
+     */
+    configuration?: Configuration;
+    /**
+     * Determines how request queues opened through this backend are consumed —
+     * `'single'` (default) assumes this is the queue's only consumer and skips request locking for
+     * fewer (paid) API calls; `'shared'` locks requests server-side so any number of concurrent
+     * consumers can process the same queue safely.
+     */
+    requestQueueAccess?: RequestQueueAccessMode;
+    /**
+     * Supplies the charging manager for pay-per-event runs, enabling the charging-aware default
+     * dataset client.
+     * @internal
+     */
+    getChargingManager?: () => ChargingManager;
+}
+/**
+ * Bridges `apify-client`'s synchronous resource accessors (`dataset(id)`,
+ * `keyValueStore(id)`, `requestQueue(id, options?)`) to crawlee v4's
+ * `StorageBackend` interface (async factory methods accepting an `id`,
+ * a `name`, or an `alias`).
+ *
+ * For the run's default dataset it transparently swaps in a charging-aware
+ * dataset client (pay-per-event on `Actor.pushData()`), provided a charging
+ * manager is supplied and a default-dataset-item price is configured.
+ *
+ * `Actor` wires this up automatically; construct it directly only to use Apify
+ * platform storage with crawlee's storage classes outside of `Actor` — e.g. to
+ * read another run's output with an explicit token:
+ *
+ * ```ts
+ * import { ApifyClient, ApifyStorageBackend, Dataset } from 'apify';
+ *
+ * const client = new ApifyClient({ token });
+ * const dataset = await Dataset.open(datasetId, { storageBackend: new ApifyStorageBackend(client) });
+ * const { items } = await dataset.getData();
+ * ```
+ */
+export declare class ApifyStorageBackend implements StorageBackend {
+    private readonly client;
+    private readonly config?;
+    private readonly requestQueueAccess;
+    private readonly getChargingManager?;
+    /** Unnamed storages created for aliases in this process, so an alias maps to one storage. */
+    private readonly aliasIdCache;
+    /** Fallback request queue client key when the run id is unavailable — one per backend. */
+    private fallbackClientKey?;
+    constructor(client: ApifyClient, options?: ApifyStorageBackendOptions);
+    /**
+     * Partitions crawlee's storage-instance cache by API base URL and token, so the same storage
+     * opened through two differently-authenticated backends is cached separately. The request
+     * queue access mode is deliberately not part of the key — opening the same queue in `single`
+     * and `shared` mode at once is not supported, and whichever backend opens it first wins.
+     */
+    getStorageBackendCacheKey(): string;
+    storageExists(id: string, type: StorageType): Promise<boolean>;
+    createDatasetBackend(options?: StorageIdentifier): Promise<DatasetBackend>;
+    createKeyValueStoreBackend(options?: StorageIdentifier): Promise<KeyValueStoreBackend>;
+    createRequestQueueBackend(options?: StorageIdentifier): Promise<RequestQueueBackend>;
+    /**
+     * A stable per-run client key makes the API's `hadMultipleClients` flag meaningful and lets a
+     * migrated or resurrected run re-acquire the request locks of its previous incarnation.
+     */
+    private requestQueueClientKey;
+    /**
+     * Returns a charging-aware dataset client when `id` is the run's default
+     * dataset and a default-dataset-item price is configured; otherwise
+     * `undefined` (caller uses the plain client).
+     */
+    private chargingDatasetClient;
+    /**
+     * Resolves a crawlee {@link StorageIdentifier} to a platform storage id.
+     *
+     * Aliases resolve to unnamed storages: the reserved `__default__` alias maps to the run's
+     * default storage, and other aliases to the storages declared in the Actor's schema (via the
+     * `ACTOR_STORAGES_JSON` environment variable, maintained by the platform). Outside the
+     * platform, an unnamed storage is created per alias instead (remembered for this process only).
+     */
+    private resolveId;
+    /** Looks an alias up in the Actor's schema storages (the `ACTOR_STORAGES_JSON` env var). */
+    private aliasFromActorStorages;
+    private resourceClient;
+    private collectionClient;
+}
+export {};