apify 4.0.0-beta.19 → 4.0.0-beta.21

package/dist/apify_storage_backend.js

@@ -0,0 +1,249 @@
+/* eslint-disable max-classes-per-file */
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { createHash } from 'node:crypto';
+import { DatasetClient as ApifyDatasetClient } from 'apify-client';
+import { cryptoRandomObjectId } from '@apify/utilities';
+import { ApifyDatasetBackend } from './apify_dataset_backend.js';
+import { ApifyKeyValueStoreBackend } from './apify_key_value_store_backend.js';
+import { ApifyRequestQueueSharedBackend } from './apify_request_queue_shared_backend.js';
+import { ApifyRequestQueueSingleBackend } from './apify_request_queue_single_backend.js';
+import { DEFAULT_DATASET_ITEM_EVENT, mergeChargeResults, pushDataAndCharge, } from './charging.js';
+/** The reserved alias crawlee uses for the default (unnamed) storage. */
+const DEFAULT_STORAGE_ALIAS = '__default__';
+/** The maximum clientKey length accepted by the request queue API. */
+const MAX_CLIENT_KEY_LENGTH = 32;
+const DEFAULT_ID_CONFIG_KEY = {
+    Dataset: 'defaultDatasetId',
+    KeyValueStore: 'defaultKeyValueStoreId',
+    RequestQueue: 'defaultRequestQueueId',
+};
+const ACTOR_STORAGES_TYPE_KEY = {
+    Dataset: 'datasets',
+    KeyValueStore: 'keyValueStores',
+    RequestQueue: 'requestQueues',
+};
+/** Marks a dataset backend whose underlying client charges for pushed items (pay-per-event). @internal */
+export const USES_PUSH_DATA_INTERCEPTION = Symbol('apify:uses-push-data-interception');
+export const pushDataChargingContext = new AsyncLocalStorage();
+/**
+ * Default `DatasetClient` that charges for pushed items (pay-per-event). Used
+ * only for the run's default dataset when a `apify-default-dataset-item` price
+ * is configured; for everything else the plain `apify-client` dataset client is
+ * used.
+ */
+class PpeAwareDatasetClient extends ApifyDatasetClient {
+    getChargingManager;
+    constructor(options, getChargingManager) {
+        super(options);
+        this.getChargingManager = getChargingManager;
+    }
+    normalizeItems(items) {
+        if (typeof items === 'string') {
+            const parsed = JSON.parse(items);
+            return Array.isArray(parsed) ? parsed : [parsed];
+        }
+        if (Array.isArray(items)) {
+            return items.flatMap((item) => typeof item === 'string' ? JSON.parse(item) : item);
+        }
+        return [items];
+    }
+    async pushItems(items) {
+        const context = pushDataChargingContext.getStore();
+        // A single JSON string may encode multiple items (e.g. '[{...},{...}]'),
+        // which the charging logic would miscount — parse strings into arrays so
+        // each logical item is counted individually.
+        const normalizedItems = this.normalizeItems(items);
+        const result = await pushDataAndCharge({
+            chargingManager: this.getChargingManager(),
+            items: normalizedItems,
+            eventName: context?.eventName,
+            isDefaultDataset: true,
+            // stringify for faster validation in the Apify client
+            pushFn: async (limitedItems) => super.pushItems(JSON.stringify(limitedItems)),
+        });
+        if (!context)
+            return;
+        // One `Actor.pushData()` may map to several `pushItems()` calls — aggregate.
+        context.chargeResult =
+            context.chargeResult === undefined ? result : mergeChargeResults(context.chargeResult, result);
+    }
+}
+/**
+ * Bridges `apify-client`'s synchronous resource accessors (`dataset(id)`,
+ * `keyValueStore(id)`, `requestQueue(id, options?)`) to crawlee v4's
+ * `StorageBackend` interface (async factory methods accepting an `id`,
+ * a `name`, or an `alias`).
+ *
+ * For the run's default dataset it transparently swaps in a charging-aware
+ * dataset client (pay-per-event on `Actor.pushData()`), provided a charging
+ * manager is supplied and a default-dataset-item price is configured.
+ *
+ * `Actor` wires this up automatically; construct it directly only to use Apify
+ * platform storage with crawlee's storage classes outside of `Actor` — e.g. to
+ * read another run's output with an explicit token:
+ *
+ * ```ts
+ * import { ApifyClient, ApifyStorageBackend, Dataset } from 'apify';
+ *
+ * const client = new ApifyClient({ token });
+ * const dataset = await Dataset.open(datasetId, { storageBackend: new ApifyStorageBackend(client) });
+ * const { items } = await dataset.getData();
+ * ```
+ */
+export class ApifyStorageBackend {
+    client;
+    config;
+    requestQueueAccess;
+    getChargingManager;
+    /** Unnamed storages created for aliases in this process, so an alias maps to one storage. */
+    aliasIdCache = new Map();
+    /** Fallback request queue client key when the run id is unavailable — one per backend. */
+    fallbackClientKey;
+    constructor(client, options = {}) {
+        this.client = client;
+        this.config = options.configuration;
+        this.requestQueueAccess = options.requestQueueAccess ?? 'single';
+        this.getChargingManager = options.getChargingManager;
+    }
+    /**
+     * Partitions crawlee's storage-instance cache by API base URL and token, so the same storage
+     * opened through two differently-authenticated backends is cached separately. The request
+     * queue access mode is deliberately not part of the key — opening the same queue in `single`
+     * and `shared` mode at once is not supported, and whichever backend opens it first wins.
+     */
+    getStorageBackendCacheKey() {
+        const hash = createHash('sha256')
+            .update(`${this.client.publicBaseUrl}${this.client.token ?? ''}`)
+            .digest('hex')
+            .slice(0, 8);
+        return `ApifyStorageBackend:${hash}`;
+    }
+    async storageExists(id, type) {
+        // Lets `Dataset.open(idOrName)` and friends resolve a string to an id first (when one
+        // exists on the platform) and fall back to a name otherwise; without this, crawlee would
+        // treat every string as a name and silently create a new storage named like the passed id.
+        // Apify's `GET /v2/{kind}/{idOrName}` matches by either id or name;
+        // confirm it was an *id* match so crawlee can fall through to `{ name }`.
+        const info = await this.resourceClient(id, type).get();
+        return info?.id === id;
+    }
+    async createDatasetBackend(options) {
+        const id = await this.resolveId(options, 'Dataset');
+        const chargingClient = this.chargingDatasetClient(id);
+        const backend = new ApifyDatasetBackend(chargingClient ?? this.client.dataset(id));
+        if (chargingClient) {
+            // `Actor.pushData()` looks for this marker on the dataset's backend to know the
+            // pay-per-event charging happens inside the intercepted `pushItems()` calls.
+            Object.assign(backend, { [USES_PUSH_DATA_INTERCEPTION]: true });
+        }
+        return backend;
+    }
+    async createKeyValueStoreBackend(options) {
+        const id = await this.resolveId(options, 'KeyValueStore');
+        return new ApifyKeyValueStoreBackend(this.client.keyValueStore(id));
+    }
+    async createRequestQueueBackend(options) {
+        const id = await this.resolveId(options, 'RequestQueue');
+        const client = this.client.requestQueue(id, { clientKey: this.requestQueueClientKey() });
+        return this.requestQueueAccess === 'shared'
+            ? new ApifyRequestQueueSharedBackend(client)
+            : new ApifyRequestQueueSingleBackend(client);
+    }
+    /**
+     * A stable per-run client key makes the API's `hadMultipleClients` flag meaningful and lets a
+     * migrated or resurrected run re-acquire the request locks of its previous incarnation.
+     */
+    requestQueueClientKey() {
+        const key = this.config?.actorRunId ?? (this.fallbackClientKey ??= cryptoRandomObjectId(MAX_CLIENT_KEY_LENGTH));
+        return key.slice(0, MAX_CLIENT_KEY_LENGTH);
+    }
+    /**
+     * Returns a charging-aware dataset client when `id` is the run's default
+     * dataset and a default-dataset-item price is configured; otherwise
+     * `undefined` (caller uses the plain client).
+     */
+    chargingDatasetClient(id) {
+        const { getChargingManager } = this;
+        if (!getChargingManager)
+            return undefined;
+        if (id !== this.config?.defaultDatasetId)
+            return undefined;
+        const hasDefaultDatasetItemEvent = DEFAULT_DATASET_ITEM_EVENT in getChargingManager().getPricingInfo().perEventPrices;
+        if (!hasDefaultDatasetItemEvent)
+            return undefined;
+        return new PpeAwareDatasetClient({
+            id,
+            baseUrl: this.client.baseUrl,
+            publicBaseUrl: this.client.publicBaseUrl,
+            apifyClient: this.client,
+            httpClient: this.client.httpClient,
+        }, getChargingManager);
+    }
+    /**
+     * Resolves a crawlee {@link StorageIdentifier} to a platform storage id.
+     *
+     * Aliases resolve to unnamed storages: the reserved `__default__` alias maps to the run's
+     * default storage, and other aliases to the storages declared in the Actor's schema (via the
+     * `ACTOR_STORAGES_JSON` environment variable, maintained by the platform). Outside the
+     * platform, an unnamed storage is created per alias instead (remembered for this process only).
+     */
+    async resolveId(options, type) {
+        if (options?.id)
+            return options.id;
+        if (options?.name) {
+            return (await this.collectionClient(type).getOrCreate(options.name)).id;
+        }
+        const alias = (options && 'alias' in options && options.alias) || DEFAULT_STORAGE_ALIAS;
+        if (alias === DEFAULT_STORAGE_ALIAS) {
+            const defaultId = this.config?.[DEFAULT_ID_CONFIG_KEY[type]];
+            if (defaultId)
+                return defaultId;
+        }
+        else {
+            const declaredId = this.aliasFromActorStorages(alias, type);
+            if (declaredId)
+                return declaredId;
+            if (this.config?.isAtHome) {
+                throw new Error(`Storage alias "${alias}" cannot be resolved because it is not declared in the Actor's schema storages. ` +
+                    `Declare it in the Actor schema, or open the storage by name instead.`);
+            }
+        }
+        // No platform-provided id for this alias (e.g. cloud storage used locally via an API
+        // token) — create an unnamed storage for it, one per alias per process.
+        const cacheKey = `${type}:${alias}`;
+        const cachedId = this.aliasIdCache.get(cacheKey);
+        if (cachedId)
+            return cachedId;
+        const created = await this.collectionClient(type).getOrCreate();
+        this.aliasIdCache.set(cacheKey, created.id);
+        return created.id;
+    }
+    /** Looks an alias up in the Actor's schema storages (the `ACTOR_STORAGES_JSON` env var). */
+    aliasFromActorStorages(alias, type) {
+        const storagesJson = this.config?.actorStoragesJson;
+        if (!storagesJson)
+            return undefined;
+        let storages;
+        try {
+            storages = JSON.parse(storagesJson);
+        }
+        catch {
+            throw new Error(`Failed to parse ACTOR_STORAGES_JSON environment variable: ${storagesJson}`);
+        }
+        return storages[ACTOR_STORAGES_TYPE_KEY[type]]?.[alias];
+    }
+    resourceClient(id, type) {
+        if (type === 'Dataset')
+            return this.client.dataset(id);
+        if (type === 'KeyValueStore')
+            return this.client.keyValueStore(id);
+        return this.client.requestQueue(id);
+    }
+    collectionClient(type) {
+        if (type === 'Dataset')
+            return this.client.datasets();
+        if (type === 'KeyValueStore')
+            return this.client.keyValueStores();
+        return this.client.requestQueues();
+    }
+}

package/dist/configuration.d.ts

@@ -47,9 +47,11 @@ export declare const apifyConfigFields: {
     actorStoragesJson: ConfigField<z.ZodOptional<z.ZodString>>;
     storageClientOptions: ConfigField<z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
     maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
     systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
     logLevel: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<{} | null | undefined, unknown>, z.ZodEnum<typeof import("@apify/log").LogLevel>>>>;
     persistStorage: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+    storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
     containerized: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
 };
 export type ApifyConfigurationInput = FieldsInput<typeof apifyConfigFields>;
@@ -61,13 +63,13 @@ export interface Configuration extends ApifyResolvedConfigValues {
 /**
  * `Configuration` is a value object holding the SDK configuration. We can use it in two ways:
  *
- * 1. When using `Actor` class, we can get the instance configuration via `sdk.config`
+ * 1. When using `Actor` class, we can get the instance configuration via `sdk.configuration`
  *
  *    ```javascript
  *    import { Actor } from 'apify';
  *
  *    const sdk = new Actor({ token: '123' });
- *    console.log(sdk.config.token); // '123'
+ *    console.log(sdk.configuration.token); // '123'
  *    ```
  *
  * 2. To get the global configuration (singleton instance). It will respect the environment variables.
@@ -75,7 +77,7 @@ export interface Configuration extends ApifyResolvedConfigValues {
  *    ```javascript
  *    import { Configuration } from 'apify';
  *
- *    const config = Configuration.getGlobalConfig();
+ *    const config = Configuration.getGlobalConfiguration();
  *    console.log(config.headless);
  *    console.log(config.persistStateIntervalMillis);
  *    ```
@@ -139,5 +141,5 @@ export declare class Configuration extends CoreConfiguration {
      * what crawlee internals resolve against; this singleton is only the
      * fallback for code reaching for a configuration without an explicit one.
      */
-    static getGlobalConfig(): Configuration;
+    static getGlobalConfiguration(): Configuration;
 }

package/dist/configuration.js

@@ -89,13 +89,13 @@ export const apifyConfigFields = {
 /**
  * `Configuration` is a value object holding the SDK configuration. We can use it in two ways:
  *
- * 1. When using `Actor` class, we can get the instance configuration via `sdk.config`
+ * 1. When using `Actor` class, we can get the instance configuration via `sdk.configuration`
  *
  *    ```javascript
  *    import { Actor } from 'apify';
  *
  *    const sdk = new Actor({ token: '123' });
- *    console.log(sdk.config.token); // '123'
+ *    console.log(sdk.configuration.token); // '123'
  *    ```
  *
  * 2. To get the global configuration (singleton instance). It will respect the environment variables.
@@ -103,7 +103,7 @@ export const apifyConfigFields = {
  *    ```javascript
  *    import { Configuration } from 'apify';
  *
- *    const config = Configuration.getGlobalConfig();
+ *    const config = Configuration.getGlobalConfiguration();
  *    console.log(config.headless);
  *    console.log(config.persistStateIntervalMillis);
  *    ```
@@ -173,7 +173,7 @@ export class Configuration extends CoreConfiguration {
      * what crawlee internals resolve against; this singleton is only the
      * fallback for code reaching for a configuration without an explicit one.
      */
-    static getGlobalConfig() {
+    static getGlobalConfiguration() {
         Configuration.globalConfig ??= new Configuration();
         return Configuration.globalConfig;
     }

package/dist/index.d.ts

@@ -1,11 +1,12 @@
 export * from './actor.js';
-export { ApifyStorageClient } from './apify_storage_client.js';
+export { ApifyStorageBackend, type ApifyStorageBackendOptions } from './apify_storage_backend.js';
+export type { RequestQueueAccessMode } from './apify_request_queue_backend.js';
 export { ArgumentValidationError } from './utils.js';
 export type { OpenStorageOptions, StorageAlias, StorageId, StorageName, StorageIdentifier, StorageIdentifierWithoutAlias, } from './storage.js';
 export { ChargeOptions, ChargeResult, ActorPricingInfo, ChargingManager } from './charging.js';
 export * from './configuration.js';
 export * from './proxy_configuration.js';
 export * from './platform_event_manager.js';
-export * from './key_value_store.js';
-export { Dataset, DatasetDataOptions, DatasetIteratorOptions, DatasetConsumer, DatasetMapper, DatasetReducer, DatasetOptions, DatasetContent, RequestQueue, QueueOperationInfo, RequestQueueOperationOptions, RequestQueueOptions, KeyConsumer, KeyValueStoreOptions, RecordOptions, KeyValueStoreIteratorOptions, log, Log, LoggerOptions, LogLevel, Logger, LoggerJson, LoggerText, } from '@crawlee/core';
+export { Dataset, DatasetDataOptions, DatasetIteratorOptions, DatasetConsumer, DatasetMapper, DatasetReducer, DatasetOptions, DatasetContent, RequestQueue, RequestQueueOperationOptions, RequestQueueOptions, KeyValueStore, KeyConsumer, KeyValueStoreOptions, RecordOptions, KeyValueStoreIteratorOptions, log, Log, LoggerOptions, LogLevel, Logger, LoggerJson, LoggerText, } from '@crawlee/core';
+export type { QueueOperationInfo } from '@crawlee/types';
 export { ApifyClient, ApifyClientOptions } from 'apify-client';

package/dist/index.js

@@ -1,10 +1,9 @@
 export * from './actor.js';
-export { ApifyStorageClient } from './apify_storage_client.js';
+export { ApifyStorageBackend } from './apify_storage_backend.js';
 export { ArgumentValidationError } from './utils.js';
 export { ChargingManager } from './charging.js';
 export * from './configuration.js';
 export * from './proxy_configuration.js';
 export * from './platform_event_manager.js';
-export * from './key_value_store.js';
-export { Dataset, RequestQueue, log, Log, LogLevel, Logger, LoggerJson, LoggerText, } from '@crawlee/core';
+export { Dataset, RequestQueue, KeyValueStore, log, Log, LogLevel, Logger, LoggerJson, LoggerText, } from '@crawlee/core';
 export { ApifyClient } from 'apify-client';

package/dist/input-schemas.d.ts

@@ -1,4 +1,4 @@
-import type { Dictionary } from '@crawlee/utils';
+import type { Dictionary } from '@crawlee/types';
 /**
  * @ignore
  */

package/dist/platform_event_manager.d.ts

@@ -39,10 +39,10 @@ import { Configuration } from './configuration.js';
  *   you can achieve the same effect using `setInterval()` and listening for the `migrating` event.
  */
 export declare class PlatformEventManager extends EventManager {
-    readonly config: Configuration;
+    readonly configuration: Configuration;
     /** Websocket connection to Actor events. */
     private eventsWs?;
-    constructor(config?: Configuration);
+    constructor(configuration?: Configuration);
     /**
      * Initializes `Actor.events` event emitter by creating a connection to a websocket that provides them.
      * This is an internal function that is automatically called by `Actor.main()`.

package/dist/platform_event_manager.js

@@ -42,14 +42,14 @@ import { Configuration } from './configuration.js';
  *   you can achieve the same effect using `setInterval()` and listening for the `migrating` event.
  */
 export class PlatformEventManager extends EventManager {
-    config;
+    configuration;
     /** Websocket connection to Actor events. */
     eventsWs;
-    constructor(config = Configuration.getGlobalConfig()) {
+    constructor(configuration = Configuration.getGlobalConfiguration()) {
         super({
-            persistStateIntervalMillis: config.persistStateIntervalMillis,
+            persistStateIntervalMillis: configuration.persistStateIntervalMillis,
         });
-        this.config = config;
+        this.configuration = configuration;
     }
     /**
      * Initializes `Actor.events` event emitter by creating a connection to a websocket that provides them.
@@ -60,7 +60,7 @@ export class PlatformEventManager extends EventManager {
             return;
         }
         await super.init();
-        const eventsWsUrl = this.config.actorEventsWsUrl;
+        const eventsWsUrl = this.configuration.actorEventsWsUrl;
         // Locally there is no web socket to connect, so just print a log message.
         if (!eventsWsUrl) {
             this.log.debug(`Environment variable ${ACTOR_ENV_VARS.EVENTS_WEBSOCKET_URL} is not set, no events from Apify platform will be emitted.`);

package/dist/proxy_configuration.d.ts

@@ -151,7 +151,7 @@ export interface ProxyInfo extends CoreProxyInfo {
  * @category Scaling
  */
 export declare class ProxyConfiguration extends CoreProxyConfiguration {
-    readonly config: Configuration;
+    readonly configuration: Configuration;
     private groups;
     private countryCode?;
     private subdivisionCode?;
@@ -159,10 +159,11 @@ export declare class ProxyConfiguration extends CoreProxyConfiguration {
     private hostname;
     private port?;
     private usesApifyProxy?;
+    protected readonly log: import("@apify/log").Log;
     /**
      * @internal
      */
-    constructor(options?: ProxyConfigurationOptions, config?: Configuration);
+    constructor(options?: ProxyConfigurationOptions, configuration?: Configuration);
     /**
      * Loads proxy password if token is provided and checks access to Apify Proxy and provided proxy groups
      * if Apify Proxy configuration is used.
@@ -219,5 +220,10 @@ export declare class ProxyConfiguration extends CoreProxyConfiguration {
      * @internal
      */
     protected _throwCannotCombineCustomWithApify(): void;
+    /**
+     * Throws cannot combine custom proxies with custom generating function
+     * @internal
+     */
+    protected _throwCannotCombineCustomMethods(): void;
 }
 export {};

package/dist/proxy_configuration.js

@@ -4,6 +4,7 @@ import { json } from 'node:stream/consumers';
 import { ProxyConfiguration as CoreProxyConfiguration } from '@crawlee/core';
 import { z } from 'zod';
 import { APIFY_ENV_VARS, APIFY_PROXY_VALUE_REGEX } from '@apify/consts';
+import defaultLog from '@apify/log';
 import { cryptoRandomObjectId } from '@apify/utilities';
 import { Actor } from './actor.js';
 import { Configuration } from './configuration.js';
@@ -51,7 +52,7 @@ const SESSION_ID_LENGTH = 12;
  * @category Scaling
  */
 export class ProxyConfiguration extends CoreProxyConfiguration {
-    config;
+    configuration;
     groups;
     countryCode;
     subdivisionCode;
@@ -59,17 +60,18 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
     hostname;
     port;
     usesApifyProxy;
+    log = defaultLog.child({ prefix: 'ProxyConfiguration' });
     /**
      * @internal
      */
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
+    constructor(options = {}, configuration = Configuration.getGlobalConfiguration()) {
         const { proxyUrls, newUrlFunction, ...rest } = options;
         super({
             proxyUrls,
             newUrlFunction,
             ['validateRequired']: false,
         });
-        this.config = config;
+        this.configuration = configuration;
         validate(z
             .object({
             groups: z.array(z.string().regex(APIFY_PROXY_VALUE_REGEX)).optional(),
@@ -81,12 +83,12 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
             password: z.string().optional(),
         })
             .strict(), rest);
-        const { groups = [], apifyProxyGroups = [], countryCode, apifyProxyCountry, subdivisionCode, apifyProxySubdivision, password = config.proxyPassword, } = options;
+        const { groups = [], apifyProxyGroups = [], countryCode, apifyProxyCountry, subdivisionCode, apifyProxySubdivision, password = configuration.proxyPassword, } = options;
         const groupsToUse = groups.length ? groups : apifyProxyGroups;
         const countryCodeToUse = countryCode || apifyProxyCountry;
         const subdivisionCodeToUse = subdivisionCode || apifyProxySubdivision;
-        const hostname = config.proxyHostname;
-        const port = config.proxyPort;
+        const hostname = configuration.proxyHostname;
+        const port = configuration.proxyPort;
         // The Apify Proxy subdivision is expressed as part of the country
         // username parameter (`country-US_CA`), so a country is required.
         if (subdivisionCodeToUse && !countryCodeToUse) {
@@ -104,7 +106,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
         this.password = password;
         this.hostname = hostname;
         this.port = port;
-        this.usesApifyProxy = !this.proxyUrls && !this.newUrlFunction;
+        this.usesApifyProxy = !proxyUrls && !newUrlFunction;
         if (proxyUrls && proxyUrls.some((url) => url?.includes('apify.com'))) {
             this.log.warning('Some Apify proxy features may work incorrectly. Please consider setting up Apify properties instead of `proxyUrls`.\n' +
                 'See https://docs.apify.com/sdk/js/docs/concepts/proxy-management#apify-proxy-configuration');
@@ -174,7 +176,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
      * `proxyUrls`, the URLs are rotated round-robin.
      */
     async newUrl(options) {
-        if (this.newUrlFunction || this.proxyUrls) {
+        if (!this.usesApifyProxy) {
             return super.newUrl(options);
         }
         return this.composeDefaultUrl(cryptoRandomObjectId(SESSION_ID_LENGTH));
@@ -210,7 +212,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
      */
     // TODO: Make this private
     async _setPasswordIfToken() {
-        const { token } = this.config;
+        const { token } = this.configuration;
         if (!token)
             return;
         try {
@@ -241,6 +243,8 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
             return true;
         }
         const { connected, connectionError, isManInTheMiddle } = status;
+        // Declared `readonly false` on the base class; the status check is the one place that
+        // learns the actual value, so bypass the readonly marker.
         this.isManInTheMiddle = isManInTheMiddle;
         if (connected) {
             return true;
@@ -258,7 +262,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
      * Apify Proxy can be down for a second or a minute, but this should not crash processes.
      */
     async _fetchStatus() {
-        const { proxyStatusUrl } = this.config;
+        const { proxyStatusUrl } = this.configuration;
         const statusUrl = `${proxyStatusUrl}/?format=json`;
         const proxyUrl = await this.newUrl();
         // Without a proxy URL we can't perform the (proxied) status check.
@@ -319,4 +323,11 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
             '"options.groups", "options.apifyProxyGroups", "options.countryCode", "options.apifyProxyCountry", ' +
             '"options.subdivisionCode" or "options.apifyProxySubdivision".');
     }
+    /**
+     * Throws cannot combine custom proxies with custom generating function
+     * @internal
+     */
+    _throwCannotCombineCustomMethods() {
+        throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".');
+    }
 }

package/dist/storage.d.ts

@@ -1,5 +1,5 @@
-import type { Constructor, IStorage, StorageOpenOptions } from '@crawlee/core';
-import type { StorageClient } from '@crawlee/types';
+import type { IStorage, StorageOpenOptions } from '@crawlee/core';
+import type { Constructor, StorageBackend } from '@crawlee/types';
 import type { Configuration } from './configuration.js';
 export interface OpenStorageOptions {
     /**
@@ -47,7 +47,7 @@ export type StorageIdentifier = string | StorageAlias | StorageId | StorageName;
 export type StorageIdentifierWithoutAlias = string | StorageId | StorageName;
 export interface OpenStorageContext {
     config: Configuration;
-    client?: StorageClient;
+    backend?: StorageBackend;
     purgedStorageAliases: Set<string>;
 }
 /**

package/dist/storage.js

@@ -1,4 +1,4 @@
-import { ApifyStorageClient } from './apify_storage_client.js';
+import { ApifyStorageBackend } from './apify_storage_backend.js';
 const STORAGE_TYPE_KEYS = {
     Dataset: 'datasets',
     KeyValueStore: 'keyValueStores',
@@ -57,7 +57,7 @@ function resolveStorageIdentifier(storageType, identifier, config) {
  */
 export async function openStorage(storageClass, identifier, context) {
     const isAlias = identifier !== null && identifier !== undefined && typeof identifier === 'object' && 'alias' in identifier;
-    if (isAlias && !context.config.isAtHome && context.client instanceof ApifyStorageClient) {
+    if (isAlias && !context.config.isAtHome && context.backend instanceof ApifyStorageBackend) {
         throw new Error('The `alias` option is not allowed for Apify-based storages running outside of Apify');
     }
     const resolvedIdOrName = resolveStorageIdentifier(storageClass.name, identifier, context.config);
@@ -69,11 +69,11 @@ export async function openStorage(storageClass, identifier, context) {
         !context.purgedStorageAliases.has(identifier.alias)) {
         context.purgedStorageAliases.add(identifier.alias);
         const existingStorage = await storageClass.open(resolvedIdOrName ?? null, {
-            storageClient: context.client,
+            storageBackend: context.backend,
         });
         await existingStorage.drop();
     }
     return storageClass.open(resolvedIdOrName ?? null, {
-        storageClient: context.client,
+        storageBackend: context.backend,
     });
 }

package/dist/utils.d.ts

@@ -5,6 +5,11 @@ import type { z } from 'zod';
  * @internal
  */
 export declare function isNonEmptyObject(value: unknown): value is Record<string, unknown>;
+/**
+ * Converts a `SNAKE_CASE` string to `camelCase` (previously provided by `@crawlee/utils`).
+ * @internal
+ */
+export declare function snakeCaseToCamelCase(snakeCaseStr: string): string;
 /**
  * Error thrown when an argument fails validation (e.g. by `Actor.addWebhook()`
  * or the `ProxyConfiguration` constructor).

package/dist/utils.js

@@ -20,6 +20,17 @@ const require = createRequire(import.meta.url);
 export function isNonEmptyObject(value) {
     return typeof value === 'object' && value !== null && !Array.isArray(value) && Object.keys(value).length > 0;
 }
+/**
+ * Converts a `SNAKE_CASE` string to `camelCase` (previously provided by `@crawlee/utils`).
+ * @internal
+ */
+export function snakeCaseToCamelCase(snakeCaseStr) {
+    return snakeCaseStr
+        .toLowerCase()
+        .split('_')
+        .map((part, index) => (index > 0 ? part.charAt(0).toUpperCase() + part.slice(1) : part))
+        .join('');
+}
 /** Formats a zod issue path like `groups[0]` or `countryCode`. */
 function formatIssuePath(path) {
     let out = '';