apify 4.0.0-beta.19 → 4.0.0-beta.21

package/dist/actor.d.ts

@@ -1,19 +1,33 @@
 import type { EventManager, EventTypeName, RecordOptions, UseStateOptions } from '@crawlee/core';
-import { Dataset, RequestQueue } from '@crawlee/core';
-import type { Awaitable, Dictionary, SetStatusMessageOptions, StorageClient } from '@crawlee/types';
+import { Dataset, KeyValueStore, RequestQueue } from '@crawlee/core';
+import type { Awaitable, Dictionary, StorageBackend } from '@crawlee/types';
 import type { ActorCallOptions, ActorStartOptions, ApifyClientOptions, RunAbortOptions, TaskCallOptions, Webhook, WebhookEventType } from 'apify-client';
 import { ActorRun as ClientActorRun, ApifyClient } from 'apify-client';
 import { type ACTOR_PERMISSION_LEVEL } from '@apify/consts';
+import type { RequestQueueAccessMode } from './apify_request_queue_backend.js';
 import type { ChargeOptions, ChargeResult } from './charging.js';
 import { ChargingManager } from './charging.js';
 import type { ConfigurationOptions } from './configuration.js';
 import { Configuration } from './configuration.js';
-import { KeyValueStore } from './key_value_store.js';
 import type { ProxyConfigurationOptions } from './proxy_configuration.js';
 import { ProxyConfiguration } from './proxy_configuration.js';
 import type { OpenStorageOptions, StorageIdentifier, StorageIdentifierWithoutAlias } from './storage.js';
 export interface InitOptions {
-    storage?: StorageClient;
+    storage?: StorageBackend;
+    /**
+     * Determines how request queues opened on the Apify platform are consumed.
+     *
+     * - `'single'` (default) assumes this run is the only consumer of its request queues. Requests
+     *   are not locked server-side and the queue head is estimated locally, which means fewer
+     *   (paid) API calls and better performance.
+     * - `'shared'` locks every fetched request server-side, so any number of concurrent consumers
+     *   (e.g. several Actor runs) can safely process the same queue, at the cost of roughly one
+     *   extra API call per processed request.
+     *
+     * Only applies on the Apify platform (or with `forceCloud`); local storage ignores it.
+     * @default 'single'
+     */
+    requestQueueAccess?: RequestQueueAccessMode;
     /**
      * Whether to automatically handle platform shutdown signals.
      * When enabled, `Actor.exit()` is called on `aborting` events and `Actor.reboot()` on `migrating` events.
@@ -56,6 +70,12 @@ export interface ExitOptions {
 }
 export interface MainOptions extends ExitOptions, InitOptions {
 }
+export interface SetStatusMessageOptions {
+    /** If `true`, the status message is treated as final and won't be overwritten by the platform. */
+    isStatusMessageTerminal?: boolean;
+    /** Log level used when the status message is also logged locally. Defaults to `INFO`. */
+    level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
+}
 /**
  * Parsed representation of the Apify environment variables.
  * This object is returned by the {@link Actor.getEnv} function.
@@ -304,7 +324,7 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
      * Configuration of this SDK instance (provided to its constructor). See {@link Configuration} for details.
      * @internal
      */
-    readonly config: Configuration;
+    readonly configuration: Configuration;
     /**
      * Default {@link ApifyClient} instance.
      * @internal
@@ -343,6 +363,8 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
      * @internal
      */
     purgedStorageAliases: Set<string>;
+    /** How Apify platform request queues are consumed; set from {@link InitOptions.requestQueueAccess}. */
+    private requestQueueAccess;
     constructor(options?: ActorOptions);
     /**
      * Runs the main user function that performs the job of the Actor
@@ -1361,13 +1383,14 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
     /** Default {@link ApifyClient} instance. */
     static get apifyClient(): ApifyClient;
     /** Default {@link Configuration} instance. */
-    static get config(): Configuration;
+    static get configuration(): Configuration;
     /** @internal */
     static getDefaultInstance(): Actor;
     private usesPushDataInterception;
     private pushDataViaInterceptedClient;
     private pushDataWithExplicitCharging;
     private _openStorage;
+    private createApifyStorageBackend;
     private _ensureActorInit;
     /**
      * Get time remaining from the Actor run timeout. Returns `undefined` if not on an Apify platform or the current

package/dist/actor.js

@@ -1,21 +1,20 @@
 import { createPrivateKey } from 'node:crypto';
-import { Dataset, purgeDefaultStorages, RequestQueue, serviceLocator } from '@crawlee/core';
-import { sleep, snakeCaseToCamelCase } from '@crawlee/utils';
+import { Dataset, KeyValueStore, purgeDefaultStorages, RequestQueue, serviceLocator } from '@crawlee/core';
+import { sleep } from '@crawlee/utils';
 import { ApifyClient } from 'apify-client';
 import { z } from 'zod';
 import { ACTOR_ENV_VARS, ACTOR_EVENT_NAMES, APIFY_ENV_VARS, INTEGER_ENV_VARS, } from '@apify/consts';
 import { decryptInputSecrets } from '@apify/input_secrets';
 import log from '@apify/log';
 import { addTimeoutToPromise } from '@apify/timeout';
-import { ApifyStorageClient, pushDataChargingContext, USES_PUSH_DATA_INTERCEPTION, } from './apify_storage_client.js';
+import { ApifyStorageBackend, pushDataChargingContext, USES_PUSH_DATA_INTERCEPTION, } from './apify_storage_backend.js';
 import { ChargingManager, pushDataAndCharge } from './charging.js';
 import { Configuration } from './configuration.js';
 import { getDefaultsFromInputSchema, noActorInputSchemaDefinedMarker, readInputSchema } from './input-schemas.js';
-import { KeyValueStore } from './key_value_store.js';
 import { PlatformEventManager } from './platform_event_manager.js';
 import { ProxyConfiguration } from './proxy_configuration.js';
 import { openStorage } from './storage.js';
-import { checkCrawleeVersion, getSystemInfo, isNonEmptyObject, printOutdatedSdkWarning, validate } from './utils.js';
+import { checkCrawleeVersion, getSystemInfo, isNonEmptyObject, printOutdatedSdkWarning, snakeCaseToCamelCase, validate, } from './utils.js';
 /**
  * Exit codes for the Actor process.
  * The error codes must be in the range 1-128, to avoid collision with signal exits
@@ -39,7 +38,7 @@ export class Actor {
      * Configuration of this SDK instance (provided to its constructor). See {@link Configuration} for details.
      * @internal
      */
-    config;
+    configuration;
     /**
      * Default {@link ApifyClient} instance.
      * @internal
@@ -78,6 +77,8 @@ export class Actor {
      * @internal
      */
     purgedStorageAliases = new Set();
+    /** How Apify platform request queues are consumed; set from {@link InitOptions.requestQueueAccess}. */
+    requestQueueAccess = 'single';
     constructor(options = {}) {
         const { configuration, ...configOptions } = options;
         if (configuration) {
@@ -91,18 +92,18 @@ export class Actor {
                 throw new Error('Actor `configuration` must be an Apify SDK Configuration (imported from `apify`), ' +
                     'not a crawlee Configuration, otherwise APIFY_*/ACTOR_* environment variables are not resolved.');
             }
-            this.config = configuration;
+            this.configuration = configuration;
         }
         else if (Object.keys(configOptions).length === 0) {
             // use default configuration object if nothing overridden (it fallbacks to env vars)
-            this.config = Configuration.getGlobalConfig();
+            this.configuration = Configuration.getGlobalConfiguration();
         }
         else {
-            this.config = new Configuration(configOptions);
+            this.configuration = new Configuration(configOptions);
         }
         this.apifyClient = this.newClient();
-        this.eventManager = new PlatformEventManager(this.config);
-        this.chargingManager = new ChargingManager(this.config, this.apifyClient);
+        this.eventManager = new PlatformEventManager(this.configuration);
+        this.chargingManager = new ChargingManager(this.configuration, this.apifyClient);
     }
     /**
      * Runs the main user function that performs the job of the Actor
@@ -206,13 +207,14 @@ export class Actor {
         // Register this Actor's config as the global one so crawlee storages and
         // the event manager resolve the same instance (`availableMemoryRatio` /
         // `disableBrowserSandbox` at-home defaults now live in `Configuration`).
-        serviceLocator.setConfiguration(this.config);
+        serviceLocator.setConfiguration(this.configuration);
+        this.requestQueueAccess = options.requestQueueAccess ?? 'single';
         if (this.isAtHome()) {
-            serviceLocator.setStorageClient(new ApifyStorageClient(this.apifyClient, this.config, () => this.chargingManager));
+            serviceLocator.setStorageBackend(this.createApifyStorageBackend());
             serviceLocator.setEventManager(this.eventManager);
         }
         else if (options.storage) {
-            serviceLocator.setStorageClient(options.storage);
+            serviceLocator.setStorageBackend(options.storage);
         }
         // Init the event manager the config uses
         await serviceLocator.getEventManager().init();
@@ -241,7 +243,7 @@ export class Actor {
             this.on(ACTOR_EVENT_NAMES.MIGRATING, this.gracefulShutdownHandlers.migrating);
         }
         await purgeDefaultStorages({
-            config: this.config,
+            configuration: this.configuration,
             onlyPurgeOnce: true,
         });
         log.debug(`Default storages purged`);
@@ -266,7 +268,7 @@ export class Actor {
         options.exitCode ??= EXIT_CODES.SUCCESS;
         options.timeoutSecs ??= 30;
         this._ensureActorInit('exit');
-        const client = serviceLocator.getStorageClient();
+        const client = serviceLocator.getStorageBackend();
         const events = serviceLocator.getEventManager();
         // Remove graceful shutdown handlers to prevent them from interfering with exit
         if (this.gracefulShutdownHandlers.aborting) {
@@ -476,8 +478,8 @@ export class Actor {
             log.warning('Actor.metamorph() is only supported when running on the Apify platform.');
             return;
         }
-        const { customAfterSleepMillis = this.config.metamorphAfterSleepMillis, ...metamorphOpts } = options;
-        const runId = this.config.actorRunId;
+        const { customAfterSleepMillis = this.configuration.metamorphAfterSleepMillis, ...metamorphOpts } = options;
+        const runId = this.configuration.actorRunId;
         await this.apifyClient.run(runId).metamorph(targetActorId, input, metamorphOpts);
         // Wait some time for container to be stopped.
         await sleep(customAfterSleepMillis);
@@ -513,10 +515,10 @@ export class Actor {
                 .listeners("migrating" /* EventType.MIGRATING */)
                 .map(async (x) => x({})),
         ]);
-        const runId = this.config.actorRunId;
+        const runId = this.configuration.actorRunId;
         await this.apifyClient.run(runId).reboot();
         // Wait some time for container to be stopped.
-        const { customAfterSleepMillis = this.config.metamorphAfterSleepMillis } = options;
+        const { customAfterSleepMillis = this.configuration.metamorphAfterSleepMillis } = options;
         await sleep(customAfterSleepMillis);
     }
     /**
@@ -550,7 +552,7 @@ export class Actor {
             log.warning('Actor.addWebhook() is only supported when running on the Apify platform. The webhook will not be invoked.');
             return undefined;
         }
-        const runId = this.config.actorRunId;
+        const runId = this.configuration.actorRunId;
         if (!runId) {
             throw new Error(`Environment variable ${ACTOR_ENV_VARS.RUN_ID} is not set!`);
         }
@@ -589,16 +591,10 @@ export class Actor {
                 log.info(loggedStatusMessage);
                 break;
         }
-        const client = serviceLocator.getStorageClient();
-        // just to be sure, this should be fast
-        await addTimeoutToPromise(async () => client.setStatusMessage(statusMessage, {
-            isStatusMessageTerminal,
-            level,
-        }), 1000, 'Setting status message timed out after 1s').catch((e) => log.warning(e.message));
-        const runId = this.config.actorRunId;
+        const runId = this.configuration.actorRunId;
         if (runId) {
             // just to be sure, this should be fast
-            const run = await addTimeoutToPromise(async () => this.apifyClient.run(runId).get(), 1000, 'Getting the current run timed out after 1s').catch((e) => log.warning(e.message));
+            const run = await addTimeoutToPromise(async () => this.apifyClient.run(runId).update({ statusMessage, isStatusMessageTerminal }), 1000, 'Setting status message timed out after 1s').catch((e) => log.warning(e.message));
             if (run) {
                 return run;
             }
@@ -769,8 +765,8 @@ export class Actor {
      */
     async getInput() {
         this._ensureActorInit('getInput');
-        const { inputSecretsPrivateKeyFile, inputSecretsPrivateKeyPassphrase } = this.config;
-        const rawInput = await this.getValue(this.config.inputKey);
+        const { inputSecretsPrivateKeyFile, inputSecretsPrivateKeyPassphrase } = this.configuration;
+        const rawInput = await this.getValue(this.configuration.inputKey);
         let input = rawInput;
         if (isNonEmptyObject(rawInput) && inputSecretsPrivateKeyFile && inputSecretsPrivateKeyPassphrase) {
             const privateKey = createPrivateKey({
@@ -838,8 +834,6 @@ export class Actor {
         validate(z.object({ forceCloud: z.boolean().optional() }).strict(), options);
         this._ensureActorInit('openRequestQueue');
         const queue = await this._openStorage(RequestQueue, queueIdOrName, options);
-        // eslint-disable-next-line dot-notation
-        queue['initialCount'] = (await queue.client.getMetadata())?.totalRequestCount ?? 0;
         return queue;
     }
     /**
@@ -891,7 +885,7 @@ export class Actor {
         if (dontUseApifyProxy && dontUseCustomProxies) {
             return undefined;
         }
-        const proxyConfiguration = new ProxyConfiguration(options, this.config);
+        const proxyConfiguration = new ProxyConfiguration(options, this.configuration);
         if (await proxyConfiguration.initialize({ checkAccess })) {
             return proxyConfiguration;
         }
@@ -981,12 +975,12 @@ export class Actor {
      * @ignore
      */
     newClient(options = {}) {
-        const { storageDir, ...storageClientOptions } = (this.config.storageClientOptions ?? {});
+        const { storageDir, ...storageClientOptions } = (this.configuration.storageClientOptions ?? {});
         const { apifyVersion, crawleeVersion } = getSystemInfo();
         return new ApifyClient({
-            baseUrl: this.config.apiBaseUrl,
-            publicBaseUrl: this.config.apiPublicBaseUrl,
-            token: this.config.token,
+            baseUrl: this.configuration.apiBaseUrl,
+            publicBaseUrl: this.configuration.apiPublicBaseUrl,
+            token: this.configuration.token,
             userAgentSuffix: [`SDK/${apifyVersion}`, `Crawlee/${crawleeVersion}`],
             ...storageClientOptions,
             ...options, // allow overriding the instance configuration
@@ -1011,7 +1005,7 @@ export class Actor {
     async useState(name, defaultValue = {}, options) {
         this._ensureActorInit('useState');
         const kvStore = await KeyValueStore.open(options?.keyValueStoreName, {
-            config: options?.config || Configuration.getGlobalConfig(),
+            configuration: options?.configuration || Configuration.getGlobalConfiguration(),
         });
         return kvStore.getAutoSavedValue(name || 'APIFY_GLOBAL_STATE', defaultValue);
     }
@@ -1572,8 +1566,8 @@ export class Actor {
         return Actor.getDefaultInstance().apifyClient;
     }
     /** Default {@link Configuration} instance. */
-    static get config() {
-        return Actor.getDefaultInstance().config;
+    static get configuration() {
+        return Actor.getDefaultInstance().configuration;
     }
     /** @internal */
     static getDefaultInstance() {
@@ -1581,7 +1575,7 @@ export class Actor {
         return this._instance;
     }
     usesPushDataInterception(dataset) {
-        return Boolean(dataset.client[USES_PUSH_DATA_INTERCEPTION]);
+        return Boolean(dataset.backend[USES_PUSH_DATA_INTERCEPTION]);
     }
     async pushDataViaInterceptedClient(dataset, item, eventName) {
         // PatchedDatasetClient will handle charging and item limiting.
@@ -1609,7 +1603,7 @@ export class Actor {
                 chargeableWithinLimit: {},
             };
         }
-        const isDefaultDataset = dataset.id === this.config.defaultDatasetId;
+        const isDefaultDataset = dataset.id === this.configuration.defaultDatasetId;
         return pushDataAndCharge({
             chargingManager: this.chargingManager,
             items,
@@ -1620,13 +1614,18 @@ export class Actor {
     }
     async _openStorage(storageClass, identifier, options = {}) {
         return openStorage(storageClass, identifier, {
-            config: this.config,
-            client: options.forceCloud
-                ? new ApifyStorageClient(this.apifyClient, this.config, () => this.chargingManager)
-                : undefined,
+            config: this.configuration,
+            backend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
             purgedStorageAliases: this.purgedStorageAliases,
         });
     }
+    createApifyStorageBackend() {
+        return new ApifyStorageBackend(this.apifyClient, {
+            configuration: this.configuration,
+            requestQueueAccess: this.requestQueueAccess,
+            getChargingManager: () => this.chargingManager,
+        });
+    }
     _ensureActorInit(methodCalled) {
         // If we already warned the user once, don't do it again to prevent spam
         if (this.warnedAboutMissingInitCall) {

package/dist/apify_dataset_backend.d.ts

@@ -0,0 +1,18 @@
+import type { DatasetBackend, DatasetBackendListOptions, DatasetInfo, Dictionary, PaginatedList } from '@crawlee/types';
+import type { DatasetClient } from 'apify-client';
+/**
+ * Implements crawlee v4's {@link DatasetBackend} interface on top of `apify-client`'s
+ * dataset API. A thin method-mapping wrapper — the interfaces differ only in naming
+ * (`getMetadata`/`get`, `drop`/`delete`, `pushData`/`pushItems`, `getData`/`listItems`).
+ *
+ * @internal
+ */
+export declare class ApifyDatasetBackend implements DatasetBackend {
+    private readonly client;
+    constructor(client: DatasetClient);
+    getMetadata(): Promise<DatasetInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    pushData(items: Dictionary[]): Promise<void>;
+    getData(options?: DatasetBackendListOptions): Promise<PaginatedList<Dictionary>>;
+}

package/dist/apify_dataset_backend.js

@@ -0,0 +1,33 @@
+/**
+ * Implements crawlee v4's {@link DatasetBackend} interface on top of `apify-client`'s
+ * dataset API. A thin method-mapping wrapper — the interfaces differ only in naming
+ * (`getMetadata`/`get`, `drop`/`delete`, `pushData`/`pushItems`, `getData`/`listItems`).
+ *
+ * @internal
+ */
+export class ApifyDatasetBackend {
+    client;
+    constructor(client) {
+        this.client = client;
+    }
+    async getMetadata() {
+        const metadata = await this.client.get();
+        if (!metadata) {
+            throw new Error('Dataset not found or has been deleted.');
+        }
+        return metadata;
+    }
+    async drop() {
+        await this.client.delete();
+    }
+    async purge() {
+        throw new Error('Purging a dataset is not supported on the Apify platform. ' +
+            'Use `drop()` to delete the dataset entirely, or open a new dataset instead.');
+    }
+    async pushData(items) {
+        await this.client.pushItems(items);
+    }
+    async getData(options) {
+        return await this.client.listItems(options);
+    }
+}

package/dist/apify_key_value_store_backend.d.ts

@@ -0,0 +1,23 @@
+import type { KeyValueStoreBackend, KeyValueStoreInfo, KeyValueStoreInputRecord, KeyValueStoreListKeysOptions, KeyValueStoreListKeysResult, KeyValueStoreRecord } from '@crawlee/types';
+import type { KeyValueStoreClient } from 'apify-client';
+/**
+ * Implements crawlee v4's {@link KeyValueStoreBackend} interface on top of `apify-client`'s
+ * key-value store API. Mostly a method-mapping wrapper (`getValue`/`getRecord`,
+ * `setValue`/`setRecord`, `drop`/`delete`, ...); the one semantic difference is that storage
+ * backends are byte transports, so records are read unparsed (see {@link getValue}).
+ *
+ * @internal
+ */
+export declare class ApifyKeyValueStoreBackend implements KeyValueStoreBackend {
+    private readonly client;
+    constructor(client: KeyValueStoreClient);
+    getMetadata(): Promise<KeyValueStoreInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    getValue(key: string): Promise<KeyValueStoreRecord | undefined>;
+    setValue(record: KeyValueStoreInputRecord): Promise<void>;
+    deleteValue(key: string): Promise<void>;
+    listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreListKeysResult>;
+    getPublicUrl(key: string): Promise<string | undefined>;
+    recordExists(key: string): Promise<boolean>;
+}

package/dist/apify_key_value_store_backend.js

@@ -0,0 +1,54 @@
+/**
+ * Implements crawlee v4's {@link KeyValueStoreBackend} interface on top of `apify-client`'s
+ * key-value store API. Mostly a method-mapping wrapper (`getValue`/`getRecord`,
+ * `setValue`/`setRecord`, `drop`/`delete`, ...); the one semantic difference is that storage
+ * backends are byte transports, so records are read unparsed (see {@link getValue}).
+ *
+ * @internal
+ */
+export class ApifyKeyValueStoreBackend {
+    client;
+    constructor(client) {
+        this.client = client;
+    }
+    async getMetadata() {
+        const metadata = await this.client.get();
+        if (!metadata) {
+            throw new Error('Key-value store not found or has been deleted.');
+        }
+        return metadata;
+    }
+    async drop() {
+        await this.client.delete();
+    }
+    async purge() {
+        throw new Error('Purging a key-value store is not supported on the Apify platform. ' +
+            'Use `drop()` to delete the store entirely, or open a new store instead.');
+    }
+    async getValue(key) {
+        // Storage backends are byte transports — the KeyValueStore frontend parses values
+        // according to their content type, so the record must be returned unparsed.
+        return this.client.getRecord(key, { buffer: true });
+    }
+    async setValue(record) {
+        await this.client.setRecord(record);
+    }
+    async deleteValue(key) {
+        await this.client.deleteRecord(key);
+    }
+    async listKeys(options) {
+        const result = await this.client.listKeys(options);
+        // The API does not report a content type for listed keys; crawlee's item shape
+        // requires the field, so it is left undefined via the cast.
+        return {
+            ...result,
+            items: result.items.map(({ key, size }) => ({ key, size })),
+        };
+    }
+    async getPublicUrl(key) {
+        return this.client.getRecordPublicUrl(key);
+    }
+    async recordExists(key) {
+        return this.client.recordExists(key);
+    }
+}

package/dist/apify_request_queue_backend.d.ts

@@ -0,0 +1,78 @@
+import type { BatchAddRequestsResult, QueueOperationInfo, RequestQueueBackend, RequestQueueInfo, RequestQueueOperationOptions, RequestSchema, UpdateRequestSchema } from '@crawlee/types';
+import type { RequestQueueClient as ApifyRequestQueueApiClient } from 'apify-client';
+/**
+ * Determines how an Apify platform request queue is consumed.
+ *
+ * - `'single'` — optimized for a single consumer. The client keeps a local estimate of the queue
+ *   head and never locks requests, which means fewer API calls, better performance and lower cost.
+ *   Multiple producers may still add requests concurrently, but only one client may *consume*
+ *   (fetch and process) them.
+ * - `'shared'` — safe for multiple concurrent consumers (e.g. several Actor runs processing one
+ *   queue). Requests are locked server-side while they are being processed, at the cost of more
+ *   API calls.
+ */
+export type RequestQueueAccessMode = 'single' | 'shared';
+/**
+ * Derives a request id from its unique key, exactly as the Apify platform does
+ * (`sha256(uniqueKey)` → base64 → strip `+`/`/`/`=` → first 15 chars). Lets us
+ * address a request by unique key without an extra round-trip.
+ */
+export declare function uniqueKeyToRequestId(uniqueKey: string): string;
+/**
+ * Common base of the Apify platform implementations of Crawlee v4's stateful, pull-based
+ * {@link RequestQueueBackend} interface, built on top of `apify-client`'s REST request-queue API.
+ *
+ * The mode-specific consumption logic lives in the subclasses:
+ * {@link ApifyRequestQueueSingleBackend} (single consumer, no locking) and
+ * {@link ApifyRequestQueueSharedBackend} (multiple consumers, server-side locking).
+ * Modeled on the Apify Python SDK's request-queue clients.
+ *
+ * @internal
+ */
+export declare abstract class ApifyRequestQueueBackend implements RequestQueueBackend {
+    protected readonly client: ApifyRequestQueueApiClient;
+    /**
+     * Local estimates of the queue counters, updated as this client adds/handles requests. The API
+     * counters can lag behind by a few seconds, so {@link getMetadata} reports whichever is higher.
+     */
+    protected estimatedTotalRequestCount: number;
+    protected estimatedHandledRequestCount: number;
+    constructor(client: ApifyRequestQueueApiClient);
+    abstract addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
+    abstract getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
+    abstract fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;
+    abstract markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | undefined>;
+    abstract reclaimRequest(request: UpdateRequestSchema, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | undefined>;
+    abstract isEmpty(): Promise<boolean>;
+    abstract isFinished(): Promise<boolean>;
+    setExpectedRequestProcessingTimeSecs(_secs: number): Promise<void>;
+    getMetadata(): Promise<RequestQueueInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    protected requestIdFromUniqueKey(uniqueKey: string): string;
+    /**
+     * Fetches the full request record by id.
+     *
+     * The apify-client return type understates the payload (the API returns the complete request
+     * record including `userData`, `payload`, `handledAt`, ...), hence the cast.
+     */
+    protected getRequestById(id: string): Promise<UpdateRequestSchema | undefined>;
+    /**
+     * Adds new requests to the platform queue. The API assigns ids itself, so any incoming id is
+     * stripped to pass its strict input validation. `apify-client` internally chunks the batch and
+     * retries transient failures.
+     */
+    protected sendBatch(requests: RequestSchema[], forefront?: boolean): Promise<BatchAddRequestsResult>;
+    /** Updates a request record on the platform and maps the result to crawlee's shape. */
+    protected updateRequestOnPlatform(request: UpdateRequestSchema, forefront?: boolean): Promise<QueueOperationInfo>;
+    /** Counts freshly added requests from an add-batch result into the local metadata estimates. */
+    protected recordAddedRequests(result: BatchAddRequestsResult): void;
+}
+/**
+ * A minimal FIFO mutex — serializes the async critical sections passed to {@link runExclusive}.
+ * @internal
+ */
+export declare class AsyncLock {
+    private tail;
+    runExclusive<T>(fn: () => Promise<T>): Promise<T>;
+}

package/dist/apify_request_queue_backend.js

@@ -0,0 +1,114 @@
+import { createHash } from 'node:crypto';
+/** Apify request IDs are the first 15 chars of a base64 SHA-256 of the unique key. */
+const REQUEST_ID_LENGTH = 15;
+/**
+ * Derives a request id from its unique key, exactly as the Apify platform does
+ * (`sha256(uniqueKey)` → base64 → strip `+`/`/`/`=` → first 15 chars). Lets us
+ * address a request by unique key without an extra round-trip.
+ */
+export function uniqueKeyToRequestId(uniqueKey) {
+    const hash = createHash('sha256').update(uniqueKey).digest('base64').replace(/[+/=]/g, '');
+    return hash.slice(0, REQUEST_ID_LENGTH);
+}
+/**
+ * Common base of the Apify platform implementations of Crawlee v4's stateful, pull-based
+ * {@link RequestQueueBackend} interface, built on top of `apify-client`'s REST request-queue API.
+ *
+ * The mode-specific consumption logic lives in the subclasses:
+ * {@link ApifyRequestQueueSingleBackend} (single consumer, no locking) and
+ * {@link ApifyRequestQueueSharedBackend} (multiple consumers, server-side locking).
+ * Modeled on the Apify Python SDK's request-queue clients.
+ *
+ * @internal
+ */
+export class ApifyRequestQueueBackend {
+    client;
+    /**
+     * Local estimates of the queue counters, updated as this client adds/handles requests. The API
+     * counters can lag behind by a few seconds, so {@link getMetadata} reports whichever is higher.
+     */
+    estimatedTotalRequestCount = 0;
+    estimatedHandledRequestCount = 0;
+    constructor(client) {
+        this.client = client;
+    }
+    async setExpectedRequestProcessingTimeSecs(_secs) {
+        // Only relevant for backends that reserve requests via locking; see the shared backend.
+    }
+    async getMetadata() {
+        const metadata = await this.client.get();
+        if (!metadata) {
+            throw new Error('Request queue not found or has been deleted.');
+        }
+        return {
+            id: metadata.id,
+            name: metadata.name,
+            createdAt: metadata.createdAt,
+            modifiedAt: metadata.modifiedAt,
+            accessedAt: metadata.accessedAt,
+            totalRequestCount: Math.max(metadata.totalRequestCount, this.estimatedTotalRequestCount),
+            handledRequestCount: Math.max(metadata.handledRequestCount, this.estimatedHandledRequestCount),
+            pendingRequestCount: metadata.pendingRequestCount,
+        };
+    }
+    async drop() {
+        await this.client.delete();
+    }
+    async purge() {
+        throw new Error('Purging a request queue is not supported on the Apify platform. ' +
+            'Use `drop()` to delete the queue entirely, or open a new queue instead.');
+    }
+    requestIdFromUniqueKey(uniqueKey) {
+        return uniqueKeyToRequestId(uniqueKey);
+    }
+    /**
+     * Fetches the full request record by id.
+     *
+     * The apify-client return type understates the payload (the API returns the complete request
+     * record including `userData`, `payload`, `handledAt`, ...), hence the cast.
+     */
+    async getRequestById(id) {
+        const request = await this.client.getRequest(id);
+        return request ?? undefined;
+    }
+    /**
+     * Adds new requests to the platform queue. The API assigns ids itself, so any incoming id is
+     * stripped to pass its strict input validation. `apify-client` internally chunks the batch and
+     * retries transient failures.
+     */
+    async sendBatch(requests, forefront) {
+        const apiRequests = requests.map((request) => {
+            const { id: _id, ...rest } = request;
+            return rest;
+        });
+        const result = await this.client.batchAddRequests(apiRequests, { forefront });
+        return result;
+    }
+    /** Updates a request record on the platform and maps the result to crawlee's shape. */
+    async updateRequestOnPlatform(request, forefront) {
+        const result = await this.client.updateRequest(request, { forefront });
+        return {
+            requestId: result.requestId,
+            wasAlreadyPresent: result.wasAlreadyPresent,
+            wasAlreadyHandled: result.wasAlreadyHandled,
+        };
+    }
+    /** Counts freshly added requests from an add-batch result into the local metadata estimates. */
+    recordAddedRequests(result) {
+        const newRequestCount = result.processedRequests.filter((request) => !request.wasAlreadyPresent && !request.wasAlreadyHandled).length;
+        this.estimatedTotalRequestCount += newRequestCount;
+    }
+}
+/**
+ * A minimal FIFO mutex — serializes the async critical sections passed to {@link runExclusive}.
+ * @internal
+ */
+export class AsyncLock {
+    tail = Promise.resolve();
+    async runExclusive(fn) {
+        const run = this.tail.then(fn);
+        // Keep the chain alive even when the critical section throws.
+        this.tail = run.catch(() => { });
+        return run;
+    }
+}