apify 3.7.3-beta.9 → 4.0.0-beta.13

package/dist/apify_storage_client.d.ts

@@ -0,0 +1,54 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import type { CreateDatasetClientOptions, CreateKeyValueStoreClientOptions, CreateRequestQueueClientOptions, DatasetClient, KeyValueStoreClient, RequestQueueClient, StorageClient } from '@crawlee/types';
+import type { ApifyClient } from 'apify-client';
+import { type ChargeResult, type ChargingManager } from './charging.js';
+import type { Configuration } from './configuration.js';
+type StorageType = 'Dataset' | 'KeyValueStore' | 'RequestQueue';
+/** Marks a dataset client whose `pushItems` charges for pay-per-event. @internal */
+export declare const USES_PUSH_DATA_INTERCEPTION: unique symbol;
+/**
+ * Context of a single `Actor.pushData()` call, shared with the intercepted
+ * `pushItems()` calls so they can (1) know which event to charge and
+ * (2) aggregate the {@link ChargeResult} across the multiple `pushItems()`
+ * calls a single `pushData()` may trigger (Crawlee batches large pushes).
+ */
+export interface PpeAwarePushDataContext {
+    eventName: string | undefined;
+    chargeResult?: ChargeResult;
+}
+export declare const pushDataChargingContext: AsyncLocalStorage<PpeAwarePushDataContext>;
+/**
+ * Bridges `apify-client`'s synchronous resource accessors (`dataset(id)`,
+ * `keyValueStore(id)`, `requestQueue(id, options?)`) to crawlee v4's
+ * `StorageClient` interface (async factory methods accepting either an `id`
+ * or a `name`).
+ *
+ * For the run's default dataset it transparently swaps in a charging-aware
+ * dataset client (pay-per-event on `Actor.pushData()`), provided a charging
+ * manager is supplied and a default-dataset-item price is configured.
+ *
+ * `storageExists()` lets `Dataset.open(idOrName)` resolve a string to an id
+ * first (when one exists on the platform) and fall back to a name otherwise —
+ * otherwise crawlee's `resolveStorageIdentifier` treats every string as a name
+ * and the SDK would silently create a new storage named like the passed id.
+ */
+export declare class ApifyStorageClient implements StorageClient {
+    private readonly client;
+    private readonly config?;
+    private readonly getChargingManager?;
+    constructor(client: ApifyClient, config?: Configuration | undefined, getChargingManager?: (() => ChargingManager) | undefined);
+    storageExists(id: string, type: StorageType): Promise<boolean>;
+    createDatasetClient(options?: CreateDatasetClientOptions): Promise<DatasetClient>;
+    createKeyValueStoreClient(options?: CreateKeyValueStoreClientOptions): Promise<KeyValueStoreClient>;
+    createRequestQueueClient(options?: CreateRequestQueueClientOptions): Promise<RequestQueueClient>;
+    /**
+     * Returns a charging-aware dataset client when `id` is the run's default
+     * dataset and a default-dataset-item price is configured; otherwise
+     * `undefined` (caller uses the plain client).
+     */
+    private chargingDatasetClient;
+    private resolveId;
+    private resourceClient;
+    private collectionClient;
+}
+export {};

package/dist/apify_storage_client.js

@@ -0,0 +1,152 @@
+/* eslint-disable max-classes-per-file */
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { DatasetClient as ApifyDatasetClient } from 'apify-client';
+import { DEFAULT_DATASET_ITEM_EVENT, mergeChargeResults, pushDataAndCharge, } from './charging.js';
+const DEFAULT_ID_CONFIG_KEY = {
+    Dataset: 'defaultDatasetId',
+    KeyValueStore: 'defaultKeyValueStoreId',
+    RequestQueue: 'defaultRequestQueueId',
+};
+/** Marks a dataset client whose `pushItems` charges for pay-per-event. @internal */
+export const USES_PUSH_DATA_INTERCEPTION = Symbol('apify:uses-push-data-interception');
+export const pushDataChargingContext = new AsyncLocalStorage();
+/**
+ * Default `DatasetClient` that charges for pushed items (pay-per-event). Used
+ * only for the run's default dataset when a `apify-default-dataset-item` price
+ * is configured; for everything else the plain `apify-client` dataset client is
+ * used.
+ */
+class PpeAwareDatasetClient extends ApifyDatasetClient {
+    getChargingManager;
+    constructor(options, getChargingManager) {
+        super(options);
+        this.getChargingManager = getChargingManager;
+    }
+    normalizeItems(items) {
+        if (typeof items === 'string') {
+            const parsed = JSON.parse(items);
+            return Array.isArray(parsed) ? parsed : [parsed];
+        }
+        if (Array.isArray(items)) {
+            return items.flatMap((item) => typeof item === 'string' ? JSON.parse(item) : item);
+        }
+        return [items];
+    }
+    async pushItems(items) {
+        const context = pushDataChargingContext.getStore();
+        // A single JSON string may encode multiple items (e.g. '[{...},{...}]'),
+        // which the charging logic would miscount — parse strings into arrays so
+        // each logical item is counted individually.
+        const normalizedItems = this.normalizeItems(items);
+        const result = await pushDataAndCharge({
+            chargingManager: this.getChargingManager(),
+            items: normalizedItems,
+            eventName: context?.eventName,
+            isDefaultDataset: true,
+            // stringify for faster validation in the Apify client
+            pushFn: async (limitedItems) => super.pushItems(JSON.stringify(limitedItems)),
+        });
+        if (!context)
+            return;
+        // One `Actor.pushData()` may map to several `pushItems()` calls — aggregate.
+        context.chargeResult =
+            context.chargeResult === undefined ? result : mergeChargeResults(context.chargeResult, result);
+    }
+}
+/**
+ * Bridges `apify-client`'s synchronous resource accessors (`dataset(id)`,
+ * `keyValueStore(id)`, `requestQueue(id, options?)`) to crawlee v4's
+ * `StorageClient` interface (async factory methods accepting either an `id`
+ * or a `name`).
+ *
+ * For the run's default dataset it transparently swaps in a charging-aware
+ * dataset client (pay-per-event on `Actor.pushData()`), provided a charging
+ * manager is supplied and a default-dataset-item price is configured.
+ *
+ * `storageExists()` lets `Dataset.open(idOrName)` resolve a string to an id
+ * first (when one exists on the platform) and fall back to a name otherwise —
+ * otherwise crawlee's `resolveStorageIdentifier` treats every string as a name
+ * and the SDK would silently create a new storage named like the passed id.
+ */
+export class ApifyStorageClient {
+    client;
+    config;
+    getChargingManager;
+    constructor(client, config, getChargingManager) {
+        this.client = client;
+        this.config = config;
+        this.getChargingManager = getChargingManager;
+    }
+    async storageExists(id, type) {
+        // Apify's `GET /v2/{kind}/{idOrName}` matches by either id or name;
+        // confirm it was an *id* match so crawlee can fall through to `{ name }`.
+        const info = await this.resourceClient(id, type).get();
+        return info?.id === id;
+    }
+    async createDatasetClient(options) {
+        const id = await this.resolveId(options, 'Dataset');
+        const datasetClient = this.chargingDatasetClient(id) ?? this.client.dataset(id);
+        // apify-client's resource clients overlap with `@crawlee/types`' shapes
+        // but don't implement the v4-added members (`getMetadata`,
+        // `getRecordPublicUrl`), so cast through.
+        return datasetClient;
+    }
+    async createKeyValueStoreClient(options) {
+        const id = await this.resolveId(options, 'KeyValueStore');
+        return this.client.keyValueStore(id);
+    }
+    async createRequestQueueClient(options) {
+        const id = await this.resolveId(options, 'RequestQueue');
+        return this.client.requestQueue(id, options?.clientKey ? { clientKey: options.clientKey } : undefined);
+    }
+    /**
+     * Returns a charging-aware dataset client when `id` is the run's default
+     * dataset and a default-dataset-item price is configured; otherwise
+     * `undefined` (caller uses the plain client).
+     */
+    chargingDatasetClient(id) {
+        const { getChargingManager } = this;
+        if (!getChargingManager)
+            return undefined;
+        if (id !== this.config?.defaultDatasetId)
+            return undefined;
+        const hasDefaultDatasetItemEvent = DEFAULT_DATASET_ITEM_EVENT in getChargingManager().getPricingInfo().perEventPrices;
+        if (!hasDefaultDatasetItemEvent)
+            return undefined;
+        const datasetClient = new PpeAwareDatasetClient({
+            id,
+            baseUrl: this.client.baseUrl,
+            publicBaseUrl: this.client.publicBaseUrl,
+            apifyClient: this.client,
+            httpClient: this.client.httpClient,
+        }, getChargingManager);
+        Object.assign(datasetClient, {
+            [USES_PUSH_DATA_INTERCEPTION]: true,
+        });
+        return datasetClient;
+    }
+    async resolveId(options, type) {
+        if (options?.id)
+            return options.id;
+        if (options?.name) {
+            return (await this.collectionClient(type).getOrCreate(options.name)).id;
+        }
+        // No id/name (crawlee's `__default__` alias): use the default storage
+        // id from the run's environment. apify-client rejects an empty id.
+        return this.config?.[DEFAULT_ID_CONFIG_KEY[type]] ?? '';
+    }
+    resourceClient(id, type) {
+        if (type === 'Dataset')
+            return this.client.dataset(id);
+        if (type === 'KeyValueStore')
+            return this.client.keyValueStore(id);
+        return this.client.requestQueue(id);
+    }
+    collectionClient(type) {
+        if (type === 'Dataset')
+            return this.client.datasets();
+        if (type === 'KeyValueStore')
+            return this.client.keyValueStores();
+        return this.client.requestQueues();
+    }
+}

package/dist/charging.js

@@ -1,13 +1,7 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ChargingManager = exports.DEFAULT_DATASET_ITEM_EVENT = void 0;
-exports.mergeChargeResults = mergeChargeResults;
-exports.pushDataAndCharge = pushDataAndCharge;
-const tslib_1 = require("tslib");
-const core_1 = require("@crawlee/core");
-const log_1 = tslib_1.__importDefault(require("@apify/log"));
-exports.DEFAULT_DATASET_ITEM_EVENT = 'apify-default-dataset-item';
-function mergeChargeResults(a, b) {
+import { Dataset, KeyValueStore } from '@crawlee/core';
+import log from '@apify/log';
+export const DEFAULT_DATASET_ITEM_EVENT = 'apify-default-dataset-item';
+export function mergeChargeResults(a, b) {
     return {
         eventChargeLimitReached: a.eventChargeLimitReached || b.eventChargeLimitReached,
         chargedCount: a.chargedCount + b.chargedCount,
@@ -20,108 +14,39 @@ function mergeChargeResults(a, b) {
 /**
  * Handles pay-per-event charging.
  */
-class ChargingManager {
+export class ChargingManager {
+    configuration;
+    LOCAL_CHARGING_LOG_DATASET_NAME = 'charging_log';
+    PLATFORM_CHARGING_LOG_DATASET_ID_KEY = 'CHARGING_LOG_DATASET_ID';
+    maxTotalChargeUsd;
+    isAtHome;
+    actorRunId;
+    pricingModel;
+    purgeChargingLogDataset;
+    useChargingLogDataset;
+    notPpeWarningPrinted = false;
+    pricingInfo = {};
+    chargingState;
+    chargingLogDataset;
+    apifyClient;
     constructor(configuration, apifyClient) {
-        Object.defineProperty(this, "configuration", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: configuration
-        });
-        Object.defineProperty(this, "LOCAL_CHARGING_LOG_DATASET_NAME", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: 'charging_log'
-        });
-        Object.defineProperty(this, "PLATFORM_CHARGING_LOG_DATASET_ID_KEY", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: 'CHARGING_LOG_DATASET_ID'
-        });
-        Object.defineProperty(this, "maxTotalChargeUsd", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "isAtHome", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "actorRunId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "pricingModel", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "purgeChargingLogDataset", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "useChargingLogDataset", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "notPpeWarningPrinted", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: false
-        });
-        Object.defineProperty(this, "pricingInfo", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: {}
-        });
-        Object.defineProperty(this, "chargingState", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "chargingLogDataset", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "apifyClient", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.maxTotalChargeUsd = configuration.get('maxTotalChargeUsd') || Infinity; // convert `0` to `Infinity` in case the value is an empty string
-        this.isAtHome = configuration.get('isAtHome');
-        this.actorRunId = configuration.get('actorRunId');
-        this.purgeChargingLogDataset = configuration.get('purgeOnStart');
-        this.useChargingLogDataset = configuration.get('useChargingLogDataset');
+        this.configuration = configuration;
+        this.maxTotalChargeUsd = configuration.maxTotalChargeUsd || Infinity; // convert `0` to `Infinity` in case the value is an empty string
+        this.isAtHome = configuration.isAtHome;
+        this.actorRunId = configuration.actorRunId;
+        this.purgeChargingLogDataset = configuration.purgeOnStart;
+        this.useChargingLogDataset = configuration.useChargingLogDataset;
         this.apifyClient = apifyClient;
     }
     get isPayPerEvent() {
         return this.pricingModel === 'PAY_PER_EVENT';
     }
     async fetchPricingInfo() {
-        if (this.configuration.get('actorPricingInfo') && this.configuration.get('chargedEventCounts')) {
+        if (this.configuration.actorPricingInfo && this.configuration.chargedEventCounts) {
             return {
-                pricingInfo: JSON.parse(this.configuration.get('actorPricingInfo')),
-                chargedEventCounts: JSON.parse(this.configuration.get('chargedEventCounts')),
-                maxTotalChargeUsd: this.configuration.get('maxTotalChargeUsd') || Infinity,
+                pricingInfo: JSON.parse(this.configuration.actorPricingInfo),
+                chargedEventCounts: JSON.parse(this.configuration.chargedEventCounts),
+                maxTotalChargeUsd: this.configuration.maxTotalChargeUsd || Infinity,
             };
         }
         if (this.isAtHome) {
@@ -141,7 +66,7 @@ class ChargingManager {
         return {
             pricingInfo: undefined,
             chargedEventCounts: {},
-            maxTotalChargeUsd: this.configuration.get('maxTotalChargeUsd') || Infinity,
+            maxTotalChargeUsd: this.configuration.maxTotalChargeUsd || Infinity,
         };
     }
     /**
@@ -152,18 +77,18 @@ class ChargingManager {
         if (this.useChargingLogDataset && this.isAtHome) {
             throw new Error('Using the ACTOR_USE_CHARGING_LOG_DATASET environment variable is only supported in a local development environment');
         }
-        if (this.configuration.get('testPayPerEvent')) {
+        if (this.configuration.testPayPerEvent) {
             if (this.isAtHome) {
                 throw new Error('Using the ACTOR_TEST_PAY_PER_EVENT environment variable is only supported in a local development environment');
             }
         }
         // Retrieve pricing information
         const { pricingInfo, chargedEventCounts, maxTotalChargeUsd } = await this.fetchPricingInfo();
-        if (this.configuration.get('testPayPerEvent')) {
+        if (this.configuration.testPayPerEvent) {
             this.pricingModel = 'PAY_PER_EVENT';
         }
         else {
-            this.pricingModel ?? (this.pricingModel = pricingInfo?.pricingModel);
+            this.pricingModel ??= pricingInfo?.pricingModel;
         }
         // Load per-event pricing information
         if (pricingInfo?.pricingModel === 'PAY_PER_EVENT') {
@@ -188,18 +113,18 @@ class ChargingManager {
         // Set up charging log dataset
         if (this.isAtHome) {
             const datasetId = await this.ensureChargingLogDatasetOnPlatform();
-            this.chargingLogDataset = await core_1.Dataset.open(datasetId);
+            this.chargingLogDataset = await Dataset.open(datasetId);
         }
         else {
             if (this.purgeChargingLogDataset) {
-                const dataset = await core_1.Dataset.open(this.LOCAL_CHARGING_LOG_DATASET_NAME);
+                const dataset = await Dataset.open(this.LOCAL_CHARGING_LOG_DATASET_NAME);
                 await dataset.drop();
             }
-            this.chargingLogDataset = await core_1.Dataset.open(this.LOCAL_CHARGING_LOG_DATASET_NAME);
+            this.chargingLogDataset = await Dataset.open(this.LOCAL_CHARGING_LOG_DATASET_NAME);
         }
     }
     async ensureChargingLogDatasetOnPlatform() {
-        const defaultStore = await core_1.KeyValueStore.open();
+        const defaultStore = await KeyValueStore.open();
         const storedDatasetId = await defaultStore.getValue(this.PLATFORM_CHARGING_LOG_DATASET_ID_KEY);
         if (storedDatasetId !== null) {
             return storedDatasetId;
@@ -239,11 +164,10 @@ class ChargingManager {
      * @param options The name of the event to charge for and the number of events to be charged.
      */
     async charge({ eventName, count = 1 }) {
-        var _a;
         const calculateChargeableWithinLimit = () => Object.fromEntries(Object.keys(this.pricingInfo).map((name) => [name, this.calculateMaxEventChargeCountWithinLimit(name)]));
         if (!this.isPayPerEvent) {
             if (!this.notPpeWarningPrinted) {
-                log_1.default.warning('Ignored attempt to charge for an event - the Actor does not use the pay-per-event pricing');
+                log.warning('Ignored attempt to charge for an event - the Actor does not use the pay-per-event pricing');
                 this.notPpeWarningPrinted = true;
             }
             return {
@@ -280,10 +204,10 @@ class ChargingManager {
             price: this.isAtHome ? 0 : 1, // Use a nonzero price for local development so that the maximum budget can be reached
             title: `Unknown event '${eventName}'`,
         };
-        (_a = this.chargingState)[eventName] ?? (_a[eventName] = {
+        this.chargingState[eventName] ??= {
             chargeCount: 0,
             totalChargedAmount: 0,
-        });
+        };
         this.chargingState[eventName].chargeCount += chargedCount;
         this.chargingState[eventName].totalChargedAmount += chargedCount * pricingInfo.price;
         /* END OF CRITICAL SECTION */
@@ -296,7 +220,7 @@ class ChargingManager {
                 await this.apifyClient.run(this.actorRunId).charge({ eventName, count: chargedCount });
             }
             else {
-                log_1.default.warning(`Attempting to charge for an unknown event '${eventName}'`);
+                log.warning(`Attempting to charge for an unknown event '${eventName}'`);
             }
         }
         const timestamp = new Date().toISOString();
@@ -311,7 +235,7 @@ class ChargingManager {
         }
         if (chargedCount < count) {
             const subject = count === 1 ? 'instance' : 'instances';
-            log_1.default.info(`Charging ${count} ${subject} of '${eventName}' event would exceed maxTotalChargeUsd - only ${chargedCount} events were charged`);
+            log.info(`Charging ${count} ${subject} of '${eventName}' event would exceed maxTotalChargeUsd - only ${chargedCount} events were charged`);
         }
         return {
             eventChargeLimitReached: this.calculateMaxEventChargeCountWithinLimit(eventName) <= 0,
@@ -387,7 +311,7 @@ class ChargingManager {
             };
         }
         const itemPrice = ((eventName !== undefined ? this.calculateEventPrice(eventName) : undefined) ?? 0) +
-            ((isDefaultDataset ? this.calculateEventPrice(exports.DEFAULT_DATASET_ITEM_EVENT) : undefined) ?? 0);
+            ((isDefaultDataset ? this.calculateEventPrice(DEFAULT_DATASET_ITEM_EVENT) : undefined) ?? 0);
         const maxChargedCount = itemPrice > 0 ? this.calculateMaxChargesByPrice(itemPrice) : Infinity;
         const itemsToKeep = (() => {
             if (maxChargedCount >= itemsArray.length) {
@@ -408,7 +332,7 @@ class ChargingManager {
             eventsToCharge[eventName] = itemsToKeep;
         }
         if (isDefaultDataset && itemsToKeep > 0) {
-            eventsToCharge[exports.DEFAULT_DATASET_ITEM_EVENT] = itemsToKeep;
+            eventsToCharge[DEFAULT_DATASET_ITEM_EVENT] = itemsToKeep;
         }
         return {
             limitedItems: itemsToKeep >= itemsArray.length ? itemsArray : itemsArray.slice(0, itemsToKeep),
@@ -416,7 +340,6 @@ class ChargingManager {
         };
     }
 }
-exports.ChargingManager = ChargingManager;
 /**
  * Helper for PPE-aware pushing of data to the dataset.
  *
@@ -426,7 +349,7 @@ exports.ChargingManager = ChargingManager;
  *
  * @internal
  */
-async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultDataset, pushFn, }) {
+export async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultDataset, pushFn, }) {
     const { limitedItems, eventsToCharge } = chargingManager.calculatePushDataLimits({
         items,
         eventName,