apify 4.0.0-beta.19 → 4.0.0-beta.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "apify",
-  "version": "4.0.0-beta.19",
+  "version": "4.0.0-beta.21",
   "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
   "engines": {
     "node": ">=22.0.0"
@@ -54,13 +54,14 @@
   },
   "dependencies": {
     "@apify/consts": "^2.51.0",
+    "@apify/datastructures": "^2.0.3",
     "@apify/input_secrets": "^1.2.0",
     "@apify/log": "^2.4.3",
     "@apify/timeout": "^0.3.0",
     "@apify/utilities": "^2.13.0",
-    "@crawlee/core": "^4.0.0-beta.61",
-    "@crawlee/types": "^4.0.0-beta.61",
-    "@crawlee/utils": "^4.0.0-beta.61",
+    "@crawlee/core": "^4.0.0-beta.105",
+    "@crawlee/types": "^4.0.0-beta.105",
+    "@crawlee/utils": "^4.0.0-beta.105",
     "apify-client": "^2.23.4",
     "semver": "^7.5.4",
     "tslib": "^2.6.2",
@@ -71,7 +72,6 @@
     "@apify/oxlint-config": "^0.2.5",
     "@apify/tsconfig": "^0.1.2",
     "@commitlint/config-conventional": "^21.0.0",
-    "@crawlee/memory-storage": "^4.0.0-beta.61",
     "@playwright/browser-chromium": "^1.60.0",
     "@types/content-type": "^1.1.8",
     "@types/node": "^24.0.0",
@@ -79,7 +79,7 @@
     "@types/tough-cookie": "^4.0.5",
     "@types/ws": "^8.5.12",
     "commitlint": "^21.0.0",
-    "crawlee": "^4.0.0-beta.61",
+    "crawlee": "^4.0.0-beta.105",
     "globby": "^16.0.0",
     "husky": "^9.1.7",
     "lint-staged": "^17.0.0",

package/dist/apify_storage_client.d.ts

@@ -1,66 +0,0 @@
-import { AsyncLocalStorage } from 'node:async_hooks';
-import type { CreateDatasetClientOptions, CreateKeyValueStoreClientOptions, CreateRequestQueueClientOptions, DatasetClient, KeyValueStoreClient, RequestQueueClient, StorageClient } from '@crawlee/types';
-import type { ApifyClient } from 'apify-client';
-import { type ChargeResult, type ChargingManager } from './charging.js';
-import type { Configuration } from './configuration.js';
-type StorageType = 'Dataset' | 'KeyValueStore' | 'RequestQueue';
-/** Marks a dataset client whose `pushItems` charges for pay-per-event. @internal */
-export declare const USES_PUSH_DATA_INTERCEPTION: unique symbol;
-/**
- * Context of a single `Actor.pushData()` call, shared with the intercepted
- * `pushItems()` calls so they can (1) know which event to charge and
- * (2) aggregate the {@link ChargeResult} across the multiple `pushItems()`
- * calls a single `pushData()` may trigger (Crawlee batches large pushes).
- */
-export interface PpeAwarePushDataContext {
-    eventName: string | undefined;
-    chargeResult?: ChargeResult;
-}
-export declare const pushDataChargingContext: AsyncLocalStorage<PpeAwarePushDataContext>;
-/**
- * Bridges `apify-client`'s synchronous resource accessors (`dataset(id)`,
- * `keyValueStore(id)`, `requestQueue(id, options?)`) to crawlee v4's
- * `StorageClient` interface (async factory methods accepting either an `id`
- * or a `name`).
- *
- * For the run's default dataset it transparently swaps in a charging-aware
- * dataset client (pay-per-event on `Actor.pushData()`), provided a charging
- * manager is supplied and a default-dataset-item price is configured.
- *
- * `storageExists()` lets `Dataset.open(idOrName)` resolve a string to an id
- * first (when one exists on the platform) and fall back to a name otherwise —
- * otherwise crawlee's `resolveStorageIdentifier` treats every string as a name
- * and the SDK would silently create a new storage named like the passed id.
- *
- * `Actor` wires this up automatically; construct it directly only to use Apify
- * platform storage with crawlee's storage classes outside of `Actor` — e.g. to
- * read another run's output with an explicit token:
- *
- * ```ts
- * import { ApifyClient, ApifyStorageClient, Dataset } from 'apify';
- *
- * const client = new ApifyClient({ token });
- * const dataset = await Dataset.open(datasetId, { storageClient: new ApifyStorageClient(client) });
- * const { items } = await dataset.getData();
- * ```
- */
-export declare class ApifyStorageClient implements StorageClient {
-    private readonly client;
-    private readonly config?;
-    private readonly getChargingManager?;
-    constructor(client: ApifyClient, config?: Configuration | undefined, getChargingManager?: (() => ChargingManager) | undefined);
-    storageExists(id: string, type: StorageType): Promise<boolean>;
-    createDatasetClient(options?: CreateDatasetClientOptions): Promise<DatasetClient>;
-    createKeyValueStoreClient(options?: CreateKeyValueStoreClientOptions): Promise<KeyValueStoreClient>;
-    createRequestQueueClient(options?: CreateRequestQueueClientOptions): Promise<RequestQueueClient>;
-    /**
-     * Returns a charging-aware dataset client when `id` is the run's default
-     * dataset and a default-dataset-item price is configured; otherwise
-     * `undefined` (caller uses the plain client).
-     */
-    private chargingDatasetClient;
-    private resolveId;
-    private resourceClient;
-    private collectionClient;
-}
-export {};

package/dist/apify_storage_client.js

@@ -1,200 +0,0 @@
-/* eslint-disable max-classes-per-file */
-import { AsyncLocalStorage } from 'node:async_hooks';
-import { DatasetClient as ApifyDatasetClient } from 'apify-client';
-import { DEFAULT_DATASET_ITEM_EVENT, mergeChargeResults, pushDataAndCharge, } from './charging.js';
-const DEFAULT_ID_CONFIG_KEY = {
-    Dataset: 'defaultDatasetId',
-    KeyValueStore: 'defaultKeyValueStoreId',
-    RequestQueue: 'defaultRequestQueueId',
-};
-/** Marks a dataset client whose `pushItems` charges for pay-per-event. @internal */
-export const USES_PUSH_DATA_INTERCEPTION = Symbol('apify:uses-push-data-interception');
-export const pushDataChargingContext = new AsyncLocalStorage();
-/**
- * Default `DatasetClient` that charges for pushed items (pay-per-event). Used
- * only for the run's default dataset when a `apify-default-dataset-item` price
- * is configured; for everything else the plain `apify-client` dataset client is
- * used.
- */
-class PpeAwareDatasetClient extends ApifyDatasetClient {
-    getChargingManager;
-    constructor(options, getChargingManager) {
-        super(options);
-        this.getChargingManager = getChargingManager;
-    }
-    normalizeItems(items) {
-        if (typeof items === 'string') {
-            const parsed = JSON.parse(items);
-            return Array.isArray(parsed) ? parsed : [parsed];
-        }
-        if (Array.isArray(items)) {
-            return items.flatMap((item) => typeof item === 'string' ? JSON.parse(item) : item);
-        }
-        return [items];
-    }
-    async pushItems(items) {
-        const context = pushDataChargingContext.getStore();
-        // A single JSON string may encode multiple items (e.g. '[{...},{...}]'),
-        // which the charging logic would miscount — parse strings into arrays so
-        // each logical item is counted individually.
-        const normalizedItems = this.normalizeItems(items);
-        const result = await pushDataAndCharge({
-            chargingManager: this.getChargingManager(),
-            items: normalizedItems,
-            eventName: context?.eventName,
-            isDefaultDataset: true,
-            // stringify for faster validation in the Apify client
-            pushFn: async (limitedItems) => super.pushItems(JSON.stringify(limitedItems)),
-        });
-        if (!context)
-            return;
-        // One `Actor.pushData()` may map to several `pushItems()` calls — aggregate.
-        context.chargeResult =
-            context.chargeResult === undefined ? result : mergeChargeResults(context.chargeResult, result);
-    }
-}
-// crawlee v4's `StorageClient` sub-client interfaces use different method names
-// than `apify-client`'s resource clients (`getValue`/`getRecord`,
-// `pushData`/`pushItems`, `getData`/`listItems`, `getMetadata`/`get`,
-// `drop`/`delete`). `adapt` wraps a client in a name-remapping proxy: `renames`
-// aliases the differing methods and `overrides` replaces the few whose return
-// shape differs; everything else — identically-named methods and the
-// pay-per-event marker symbol — passes straight through.
-//
-// `purge()` has no apify-client equivalent and isn't needed on the platform
-// (a run's storages are already fresh), so it's a no-op.
-const noPurge = { purge: async () => { } };
-function adapt(client, renames, overrides = {}) {
-    return new Proxy(client, {
-        get(target, prop) {
-            if (typeof prop === 'string' && prop in overrides)
-                return overrides[prop];
-            const value = Reflect.get(target, (typeof prop === 'string' && renames[prop]) || prop, target);
-            return typeof value === 'function' ? value.bind(target) : value;
-        },
-    });
-}
-/**
- * Bridges `apify-client`'s synchronous resource accessors (`dataset(id)`,
- * `keyValueStore(id)`, `requestQueue(id, options?)`) to crawlee v4's
- * `StorageClient` interface (async factory methods accepting either an `id`
- * or a `name`).
- *
- * For the run's default dataset it transparently swaps in a charging-aware
- * dataset client (pay-per-event on `Actor.pushData()`), provided a charging
- * manager is supplied and a default-dataset-item price is configured.
- *
- * `storageExists()` lets `Dataset.open(idOrName)` resolve a string to an id
- * first (when one exists on the platform) and fall back to a name otherwise —
- * otherwise crawlee's `resolveStorageIdentifier` treats every string as a name
- * and the SDK would silently create a new storage named like the passed id.
- *
- * `Actor` wires this up automatically; construct it directly only to use Apify
- * platform storage with crawlee's storage classes outside of `Actor` — e.g. to
- * read another run's output with an explicit token:
- *
- * ```ts
- * import { ApifyClient, ApifyStorageClient, Dataset } from 'apify';
- *
- * const client = new ApifyClient({ token });
- * const dataset = await Dataset.open(datasetId, { storageClient: new ApifyStorageClient(client) });
- * const { items } = await dataset.getData();
- * ```
- */
-export class ApifyStorageClient {
-    client;
-    config;
-    getChargingManager;
-    constructor(client, config, getChargingManager) {
-        this.client = client;
-        this.config = config;
-        this.getChargingManager = getChargingManager;
-    }
-    async storageExists(id, type) {
-        // Apify's `GET /v2/{kind}/{idOrName}` matches by either id or name;
-        // confirm it was an *id* match so crawlee can fall through to `{ name }`.
-        const info = await this.resourceClient(id, type).get();
-        return info?.id === id;
-    }
-    async createDatasetClient(options) {
-        const id = await this.resolveId(options, 'Dataset');
-        const client = this.chargingDatasetClient(id) ?? this.client.dataset(id);
-        return adapt(client, {
-            getMetadata: 'get',
-            drop: 'delete',
-            pushData: 'pushItems',
-            getData: 'listItems',
-        }, noPurge);
-    }
-    async createKeyValueStoreClient(options) {
-        const id = await this.resolveId(options, 'KeyValueStore');
-        const client = this.client.keyValueStore(id);
-        return adapt(client, {
-            getMetadata: 'get',
-            getValue: 'getRecord',
-            setValue: 'setRecord',
-            deleteValue: 'deleteRecord',
-            drop: 'delete',
-            getPublicUrl: 'getRecordPublicUrl',
-        }, {
-            ...noPurge,
-            // crawlee expects an array; apify-client returns `{ items }`.
-            listKeys: async (opts) => (await client.listKeys(opts)).items,
-        });
-    }
-    async createRequestQueueClient(options) {
-        const id = await this.resolveId(options, 'RequestQueue');
-        const client = this.client.requestQueue(id, options?.clientKey ? { clientKey: options.clientKey } : undefined);
-        return adapt(client, { getMetadata: 'get', drop: 'delete' }, noPurge);
-    }
-    /**
-     * Returns a charging-aware dataset client when `id` is the run's default
-     * dataset and a default-dataset-item price is configured; otherwise
-     * `undefined` (caller uses the plain client).
-     */
-    chargingDatasetClient(id) {
-        const { getChargingManager } = this;
-        if (!getChargingManager)
-            return undefined;
-        if (id !== this.config?.defaultDatasetId)
-            return undefined;
-        const hasDefaultDatasetItemEvent = DEFAULT_DATASET_ITEM_EVENT in getChargingManager().getPricingInfo().perEventPrices;
-        if (!hasDefaultDatasetItemEvent)
-            return undefined;
-        const datasetClient = new PpeAwareDatasetClient({
-            id,
-            baseUrl: this.client.baseUrl,
-            publicBaseUrl: this.client.publicBaseUrl,
-            apifyClient: this.client,
-            httpClient: this.client.httpClient,
-        }, getChargingManager);
-        Object.assign(datasetClient, {
-            [USES_PUSH_DATA_INTERCEPTION]: true,
-        });
-        return datasetClient;
-    }
-    async resolveId(options, type) {
-        if (options?.id)
-            return options.id;
-        if (options?.name) {
-            return (await this.collectionClient(type).getOrCreate(options.name)).id;
-        }
-        // No id/name (crawlee's `__default__` alias): use the default storage
-        // id from the run's environment. apify-client rejects an empty id.
-        return this.config?.[DEFAULT_ID_CONFIG_KEY[type]] ?? '';
-    }
-    resourceClient(id, type) {
-        if (type === 'Dataset')
-            return this.client.dataset(id);
-        if (type === 'KeyValueStore')
-            return this.client.keyValueStore(id);
-        return this.client.requestQueue(id);
-    }
-    collectionClient(type) {
-        if (type === 'Dataset')
-            return this.client.datasets();
-        if (type === 'KeyValueStore')
-            return this.client.keyValueStores();
-        return this.client.requestQueues();
-    }
-}

package/dist/key_value_store.d.ts

@@ -1,21 +0,0 @@
-import type { StorageOpenOptions } from '@crawlee/core';
-import { KeyValueStore as CoreKeyValueStore } from '@crawlee/core';
-/**
- * @inheritDoc
- */
-export declare class KeyValueStore extends CoreKeyValueStore {
-    /**
-     * Returns a URL for the given key that may be used to publicly
-     * access the value in the remote key-value store.
-     *
-     * On the Apify platform the URL is signed with the store's
-     * `urlSigningSecretKey` so that anyone with the URL can read the record
-     * without authentication. Locally we delegate to crawlee's default
-     * implementation (which produces a `file://` URL or returns `undefined`).
-     */
-    getPublicUrl(key: string): Promise<string | undefined>;
-    /**
-     * @inheritDoc
-     */
-    static open(storeIdOrName?: string | null, options?: StorageOpenOptions): Promise<KeyValueStore>;
-}

package/dist/key_value_store.js

@@ -1,41 +0,0 @@
-import { KeyValueStore as CoreKeyValueStore } from '@crawlee/core';
-import { KeyValueStoreClient as RemoteKeyValueStoreClient } from 'apify-client';
-import { createHmacSignature } from '@apify/utilities';
-/**
- * @inheritDoc
- */
-export class KeyValueStore extends CoreKeyValueStore {
-    /**
-     * Returns a URL for the given key that may be used to publicly
-     * access the value in the remote key-value store.
-     *
-     * On the Apify platform the URL is signed with the store's
-     * `urlSigningSecretKey` so that anyone with the URL can read the record
-     * without authentication. Locally we delegate to crawlee's default
-     * implementation (which produces a `file://` URL or returns `undefined`).
-     */
-    async getPublicUrl(key) {
-        const config = this.config;
-        // Detect a remote (Apify) store by its client type rather than by
-        // `isAtHome`, so that a `forceCloud` store opened locally still gets a
-        // signed Apify URL (matching the platform behaviour). `client` is
-        // `private` on `CoreKeyValueStore`, so bypass the visibility check.
-        const { client } = this;
-        const isLocalStore = !(client instanceof RemoteKeyValueStoreClient);
-        if (isLocalStore) {
-            return super.getPublicUrl(key);
-        }
-        const publicUrl = new URL(`${config.apiPublicBaseUrl}/v2/key-value-stores/${this.id}/records/${key}`);
-        const metadata = (await client.getMetadata());
-        if (metadata?.urlSigningSecretKey) {
-            publicUrl.searchParams.append('signature', createHmacSignature(metadata.urlSigningSecretKey, key));
-        }
-        return publicUrl.toString();
-    }
-    /**
-     * @inheritDoc
-     */
-    static async open(storeIdOrName, options = {}) {
-        return super.open(storeIdOrName, options);
-    }
-}