apify 4.0.0-beta.34 → 4.0.0-beta.36

package/dist/actor.d.ts

@@ -1381,22 +1381,6 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
      * @internal
      */
     static setDefaultInstance(instance?: Actor): void;
-    /**
-     * The backend the Actor installs: the caller's, the platform's, or crawlee's local default.
-     *
-     * Only the last two are wrapped for dataset-item charging. The platform counts an
-     * `apify-default-dataset-item` per item written to the run's default dataset through Apify
-     * storage, so a caller-supplied backend is billed nothing and must not be accounted for -
-     * charging for it would spend a budget nobody is consuming and trim the caller's items to fit
-     * it. crawlee's local default stands in for Apify storage, so it is wrapped to keep local
-     * pay-per-event testing faithful.
-     *
-     * Wrapping means owning the instance, which is why the local default is constructed here
-     * rather than left to be created lazily on first use.
-     */
-    private createStorageBackend;
-    /** Wrapped here rather than at the `init()` call site so that `forceCloud` storages are charged too. */
-    private createApifyStorageBackend;
     /**
      * crawlee's service locator is set-once, so a backend registered before `init()` wins and the
      * Actor cannot install its own over it. Replaces the resulting `ServiceConflictError` with the

package/dist/actor.js

@@ -15,6 +15,7 @@ import { Configuration } from './configuration.js';
 import { getDefaultsFromInputSchema, noActorInputSchemaDefinedMarker, readInputSchema } from './input-schemas.js';
 import { PlatformEventManager } from './platform_event_manager.js';
 import { ProxyConfiguration } from './proxy_configuration.js';
+import { SmartApifyStorageBackend } from './smart_apify_storage_backend.js';
 import { checkCrawleeVersion, getSystemInfo, isNonEmptyObject, printOutdatedSdkWarning, snakeCaseToCamelCase, } from './utils.js';
 /**
  * Exit codes for the Actor process.
@@ -78,6 +79,8 @@ export class Actor {
     #chargingManager;
     /** How Apify platform request queues are consumed; set from {@link InitOptions.requestQueueAccess}. */
     #requestQueueAccess = 'single';
+    /** Lazily built by the `#storageBackend` getter. */
+    #cachedStorageBackend;
     constructor(options = {}) {
         const { configuration, ...configOptions } = options;
         if (configuration) {
@@ -212,7 +215,7 @@ export class Actor {
             serviceLocator.setEventManager(this.eventManager);
         }
         if (!serviceLocator.getServicesIfSet().storageBackend) {
-            this.installStorageBackend(this.createStorageBackend(options.storage));
+            this.installStorageBackend(options.storage ?? this.#storageBackend);
         }
         else if (options.storage) {
             this.installStorageBackend(options.storage);
@@ -255,8 +258,7 @@ export class Actor {
         log.debug(`ChargingManager initialized`, this.#chargingManager.getPricingInfo());
         // Only knowable once the pricing is loaded. Reachable with a caller-supplied backend or one
         // registered with crawlee directly; the storages the Actor installs itself are wrapped.
-        if (this.#chargingManager.isPayPerEvent &&
-            !(serviceLocator.getStorageBackend() instanceof ChargingStorageBackend)) {
+        if (this.#chargingManager.isPayPerEvent && !this.#chargesDefaultDatasetItems()) {
             log.warning('Items pushed to the default dataset will not be charged for, because this run does not use Apify ' +
                 'storage - the platform only counts items it stores itself.');
         }
@@ -721,7 +723,9 @@ export class Actor {
         parseArgument(options, z.object({ forceCloud: z.boolean().optional() }).strict());
         this.ensureActorInit('openDataset');
         return Dataset.open(datasetIdOrName ?? null, {
-            storageBackend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
+            storageBackend: options.forceCloud
+                ? this.#storageBackend.getSuitableStorageBackend({ forceCloud: true })
+                : undefined,
         });
     }
     /**
@@ -871,7 +875,9 @@ export class Actor {
         parseArgument(options, z.object({ forceCloud: z.boolean().optional() }).strict());
         this.ensureActorInit('openKeyValueStore');
         return KeyValueStore.open(storeIdOrName ?? null, {
-            storageBackend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
+            storageBackend: options.forceCloud
+                ? this.#storageBackend.getSuitableStorageBackend({ forceCloud: true })
+                : undefined,
         });
     }
     /**
@@ -897,7 +903,9 @@ export class Actor {
         parseArgument(options, z.object({ forceCloud: z.boolean().optional() }).strict());
         this.ensureActorInit('openRequestQueue');
         return RequestQueue.open(queueIdOrName ?? null, {
-            storageBackend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
+            storageBackend: options.forceCloud
+                ? this.#storageBackend.getSuitableStorageBackend({ forceCloud: true })
+                : undefined,
         });
     }
     /**
@@ -1646,40 +1654,34 @@ export class Actor {
         Actor.#instance = instance;
     }
     /**
-     * The backend the Actor installs: the caller's, the platform's, or crawlee's local default.
+     * The backend the Actor installs unless the caller brings one: Apify platform storage on the
+     * platform, crawlee's local default outside of it.
      *
-     * Only the last two are wrapped for dataset-item charging. The platform counts an
-     * `apify-default-dataset-item` per item written to the run's default dataset through Apify
-     * storage, so a caller-supplied backend is billed nothing and must not be accounted for -
-     * charging for it would spend a budget nobody is consuming and trim the caller's items to fit
-     * it. crawlee's local default stands in for Apify storage, so it is wrapped to keep local
-     * pay-per-event testing faithful.
+     * Both sides are wrapped for dataset-item charging - crawlee's local default stands in for
+     * Apify storage, so local pay-per-event testing stays faithful. A caller-supplied backend
+     * (`Actor.init({ storage })`) is billed nothing and is installed unwrapped, in place of this.
      *
-     * Wrapping means owning the instance, which is why the local default is constructed here
-     * rather than left to be created lazily on first use.
+     * One instance per Actor: a second platform backend would mint unnamed storages of its own for
+     * aliases this one has already resolved.
      */
-    createStorageBackend(storage) {
-        if (storage) {
-            return storage;
-        }
-        if (this.isAtHome()) {
-            return this.createApifyStorageBackend();
-        }
-        return new ChargingStorageBackend(new ServiceLocator(this.configuration).getStorageBackend(), {
+    get #storageBackend() {
+        const charging = {
             configuration: this.configuration,
             getChargingManager: () => this.#chargingManager,
-        });
-    }
-    /** Wrapped here rather than at the `init()` call site so that `forceCloud` storages are charged too. */
-    createApifyStorageBackend() {
-        const backend = new ApifyStorageBackend(this.apifyClient, {
+        };
+        return (this.#cachedStorageBackend ??= new SmartApifyStorageBackend({
+            cloudStorageBackend: new ChargingStorageBackend(new ApifyStorageBackend(this.apifyClient, {
+                configuration: this.configuration,
+                requestQueueAccess: this.#requestQueueAccess,
+            }), charging),
+            localStorageBackend: new ChargingStorageBackend(new ServiceLocator(this.configuration).getStorageBackend(), charging),
             configuration: this.configuration,
-            requestQueueAccess: this.#requestQueueAccess,
-        });
-        return new ChargingStorageBackend(backend, {
-            configuration: this.configuration,
-            getChargingManager: () => this.#chargingManager,
-        });
+        }));
+    }
+    /** Whether items reaching the run's default dataset are counted for pay-per-event charging. */
+    #chargesDefaultDatasetItems() {
+        const installed = serviceLocator.getStorageBackend();
+        return installed instanceof SmartApifyStorageBackend || installed instanceof ChargingStorageBackend;
     }
     /**
      * crawlee's service locator is set-once, so a backend registered before `init()` wins and the

package/dist/apify_storage_backend.d.ts

@@ -2,7 +2,7 @@ import type { DatasetBackend, KeyValueStoreBackend, RequestQueueBackend, Storage
 import type { ApifyClient } from 'apify-client';
 import { type RequestQueueAccessMode } from './apify_request_queue_backend.js';
 import type { Configuration } from './configuration.js';
-type StorageType = 'Dataset' | 'KeyValueStore' | 'RequestQueue';
+export type StorageType = 'Dataset' | 'KeyValueStore' | 'RequestQueue';
 /** The reserved alias crawlee uses for the default (unnamed) storage. */
 export declare const DEFAULT_STORAGE_ALIAS = "__default__";
 export interface ApifyStorageBackendOptions {
@@ -88,4 +88,3 @@ export declare class ApifyStorageBackend implements StorageBackend {
     private resourceClient;
     private collectionClient;
 }
-export {};

package/dist/configuration.d.ts

@@ -9,9 +9,6 @@ export declare const apifyConfigFields: {
     persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
     storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
     containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean>>>;
-    defaultDatasetId: ConfigField<z.ZodDefault<z.ZodString>>;
-    defaultKeyValueStoreId: ConfigField<z.ZodDefault<z.ZodString>>;
-    defaultRequestQueueId: ConfigField<z.ZodDefault<z.ZodString>>;
     inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
     memoryMbytes: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
     availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
@@ -22,6 +19,9 @@ export declare const apifyConfigFields: {
     chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
     defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
     purgeOnStart: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
+    defaultDatasetId: ConfigField<z.ZodDefault<z.ZodString>>;
+    defaultKeyValueStoreId: ConfigField<z.ZodDefault<z.ZodString>>;
+    defaultRequestQueueId: ConfigField<z.ZodDefault<z.ZodString>>;
     metamorphAfterSleepMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
     actorEventsWsUrl: ConfigField<z.ZodOptional<z.ZodString>>;
     token: ConfigField<z.ZodOptional<z.ZodString>>;

package/dist/configuration.js

@@ -20,9 +20,6 @@ export const apifyConfigFields = {
     // take precedence; crawlee's own CRAWLEE_* var is reused as the fallback,
     // never re-typed). A schema is passed only where the SDK needs a different
     // default than crawlee's.
-    defaultDatasetId: withApifyEnv(crawleeConfigFields.defaultDatasetId, [ACTOR_ENV_VARS.DEFAULT_DATASET_ID, APIFY_ENV_VARS.DEFAULT_DATASET_ID], z.string().default(LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_DATASET_ID])),
-    defaultKeyValueStoreId: withApifyEnv(crawleeConfigFields.defaultKeyValueStoreId, [ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, APIFY_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID], z.string().default(LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID])),
-    defaultRequestQueueId: withApifyEnv(crawleeConfigFields.defaultRequestQueueId, [ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID, APIFY_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID], z.string().default(LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID])),
     inputKey: withApifyEnv(crawleeConfigFields.inputKey, [ACTOR_ENV_VARS.INPUT_KEY, APIFY_ENV_VARS.INPUT_KEY]),
     memoryMbytes: withApifyEnv(crawleeConfigFields.memoryMbytes, [
         ACTOR_ENV_VARS.MEMORY_MBYTES,
@@ -39,6 +36,20 @@ export const apifyConfigFields = {
     chromeExecutablePath: withApifyEnv(crawleeConfigFields.chromeExecutablePath, APIFY_ENV_VARS.CHROME_EXECUTABLE_PATH),
     defaultBrowserPath: withApifyEnv(crawleeConfigFields.defaultBrowserPath, 'APIFY_DEFAULT_BROWSER_PATH'),
     purgeOnStart: withApifyEnv(crawleeConfigFields.purgeOnStart, APIFY_ENV_VARS.PURGE_ON_START),
+    // Crawlee addresses the default storage by a reserved alias; on the platform
+    // each run gets real storage IDs, which the API calls need.
+    defaultDatasetId: field(z.string().default(LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_DATASET_ID]), [
+        ACTOR_ENV_VARS.DEFAULT_DATASET_ID,
+        APIFY_ENV_VARS.DEFAULT_DATASET_ID,
+    ]),
+    defaultKeyValueStoreId: field(z.string().default(LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID]), [
+        ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID,
+        APIFY_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID,
+    ]),
+    defaultRequestQueueId: field(z.string().default(LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID]), [
+        ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID,
+        APIFY_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID,
+    ]),
     // Apify-specific fields
     metamorphAfterSleepMillis: field(coerceNumber.default(300_000), APIFY_ENV_VARS.METAMORPH_AFTER_SLEEP_MILLIS),
     actorEventsWsUrl: field(z.string().optional(), [

package/dist/smart_apify_storage_backend.d.ts

@@ -0,0 +1,52 @@
+import type { DatasetBackend, KeyValueStoreBackend, RequestQueueBackend, StorageBackend, StorageIdentifier } from '@crawlee/types';
+import type { StorageType } from './apify_storage_backend.js';
+import type { Configuration } from './configuration.js';
+import type { OpenStorageOptions } from './storage.js';
+export interface SmartApifyStorageBackendOptions {
+    /** Used on the Apify platform, and locally for `forceCloud` storages. */
+    cloudStorageBackend: StorageBackend;
+    /** Used outside of the Apify platform, except for `forceCloud` storages. */
+    localStorageBackend: StorageBackend;
+    /** Supplies `isAtHome` and the token that cloud storage needs. */
+    configuration: Configuration;
+}
+/**
+ * Routes storages to Apify cloud storage or to a local one, depending on where the Actor runs.
+ *
+ * On the Apify platform (detected through the `APIFY_IS_AT_HOME` environment variable) storages are
+ * opened in the cloud; locally they are opened through `localStorageBackend`, unless `forceCloud`
+ * asks for the cloud one. `Actor` installs it; callers who want storages of their own pass a
+ * backend to `Actor.init({ storage })` instead.
+ *
+ * @internal
+ */
+export declare class SmartApifyStorageBackend implements StorageBackend {
+    #private;
+    constructor(options: SmartApifyStorageBackendOptions);
+    /**
+     * The backend a storage is opened through: the cloud one on the platform or with `forceCloud`,
+     * the local one otherwise.
+     *
+     * Exposed so that `forceCloud` storages can be opened through the very same cloud backend
+     * instance this one delegates to — two instances would resolve run-scoped aliases to two
+     * separate storages.
+     */
+    getSuitableStorageBackend(options?: OpenStorageOptions): StorageBackend;
+    get stats(): {
+        rateLimitErrors: number[];
+    } | undefined;
+    /** Repeats crawlee's own fallback, so that routing a backend does not change how storages are cached. */
+    getStorageBackendCacheKey(): string;
+    createDatasetBackend(options?: StorageIdentifier): Promise<DatasetBackend>;
+    createKeyValueStoreBackend(options?: StorageIdentifier): Promise<KeyValueStoreBackend>;
+    createRequestQueueBackend(options?: StorageIdentifier): Promise<RequestQueueBackend>;
+    /**
+     * Declared unconditionally, unlike on the backends this one delegates to: crawlee branches on
+     * the presence of the three methods below, and delegating to a backend that lacks one is
+     * indistinguishable from not having it — a `false` from `storageExists` means the same as its
+     * absence (crawlee falls through to a name lookup), and the other two have no return value.
+     */
+    storageExists(id: string, type: StorageType): Promise<boolean>;
+    purge(): Promise<void>;
+    teardown(): Promise<void>;
+}

package/dist/smart_apify_storage_backend.js

@@ -0,0 +1,73 @@
+/**
+ * Routes storages to Apify cloud storage or to a local one, depending on where the Actor runs.
+ *
+ * On the Apify platform (detected through the `APIFY_IS_AT_HOME` environment variable) storages are
+ * opened in the cloud; locally they are opened through `localStorageBackend`, unless `forceCloud`
+ * asks for the cloud one. `Actor` installs it; callers who want storages of their own pass a
+ * backend to `Actor.init({ storage })` instead.
+ *
+ * @internal
+ */
+export class SmartApifyStorageBackend {
+    #cloudStorageBackend;
+    #localStorageBackend;
+    #configuration;
+    constructor(options) {
+        this.#cloudStorageBackend = options.cloudStorageBackend;
+        this.#localStorageBackend = options.localStorageBackend;
+        this.#configuration = options.configuration;
+    }
+    /**
+     * The backend a storage is opened through: the cloud one on the platform or with `forceCloud`,
+     * the local one otherwise.
+     *
+     * Exposed so that `forceCloud` storages can be opened through the very same cloud backend
+     * instance this one delegates to — two instances would resolve run-scoped aliases to two
+     * separate storages.
+     */
+    getSuitableStorageBackend(options = {}) {
+        if (this.#configuration.isAtHome) {
+            return this.#cloudStorageBackend;
+        }
+        if (!options.forceCloud) {
+            return this.#localStorageBackend;
+        }
+        if (!this.#configuration.token) {
+            throw new Error('In order to use the Apify cloud storage from your computer, you need to provide an Apify token ' +
+                'using the APIFY_TOKEN environment variable.');
+        }
+        return this.#cloudStorageBackend;
+    }
+    get stats() {
+        return this.getSuitableStorageBackend().stats;
+    }
+    /** Repeats crawlee's own fallback, so that routing a backend does not change how storages are cached. */
+    getStorageBackendCacheKey() {
+        const backend = this.getSuitableStorageBackend();
+        return backend.getStorageBackendCacheKey?.() ?? backend.constructor.name;
+    }
+    async createDatasetBackend(options) {
+        return await this.getSuitableStorageBackend().createDatasetBackend(options);
+    }
+    async createKeyValueStoreBackend(options) {
+        return await this.getSuitableStorageBackend().createKeyValueStoreBackend(options);
+    }
+    async createRequestQueueBackend(options) {
+        return await this.getSuitableStorageBackend().createRequestQueueBackend(options);
+    }
+    /**
+     * Declared unconditionally, unlike on the backends this one delegates to: crawlee branches on
+     * the presence of the three methods below, and delegating to a backend that lacks one is
+     * indistinguishable from not having it — a `false` from `storageExists` means the same as its
+     * absence (crawlee falls through to a name lookup), and the other two have no return value.
+     */
+    async storageExists(id, type) {
+        return (await this.getSuitableStorageBackend().storageExists?.(id, type)) ?? false;
+    }
+    async purge() {
+        await this.getSuitableStorageBackend().purge?.();
+    }
+    async teardown() {
+        await this.getSuitableStorageBackend().teardown?.();
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "apify",
-  "version": "4.0.0-beta.34",
+  "version": "4.0.0-beta.36",
   "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
   "engines": {
     "node": ">=22.0.0"