@crawlee/core 4.0.0-beta.7 → 4.0.0-beta.70

package/storages/dataset.d.ts

@@ -1,27 +1,21 @@
-import type { DatasetClient, DatasetInfo, Dictionary, StorageClient } from '@crawlee/types';
+import type { DatasetBackend, DatasetInfo, Dictionary } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
-import { type Log } from '../log.js';
+import type { CrawleeLogger } from '../log.js';
 import type { Awaitable } from '../typedefs.js';
-import type { StorageManagerOptions } from './storage_manager.js';
+import type { DatasetStats } from './storage_stats.js';
+import type { StorageIdentifier } from './storage_instance_manager.js';
+import type { StorageOpenOptions } from './utils.js';
 /** @internal */
 export declare const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
 /**
- * Accepts a JSON serializable object as an input, validates its serializability,
- * and validates its serialized size against limitBytes. Optionally accepts its index
- * in an array to provide better error messages. Returns serialized object.
- * @ignore
- */
-export declare function checkAndSerialize<T>(item: T, limitBytes: number, index?: number): string;
-/**
- * Takes an array of JSONs (payloads) as input and produces an array of JSON strings
- * where each string is a JSON array of payloads with a maximum size of limitBytes per one
- * JSON array. Fits as many payloads as possible into a single JSON array and then moves
- * on to the next, preserving item order.
+ * Validates that the given value is a plain JSON-serializable object
+ * (not an array, not a primitive, not circular).
  *
- * The function assumes that none of the items is larger than limitBytes and does not validate.
+ * @param item The value to validate.
+ * @param index Optional index for error messages when validating inside an array.
  * @ignore
  */
-export declare function chunkBySize(items: string[], limitBytes: number): string[];
+export declare function assertJsonSerializable<T>(item: T, index?: number): void;
 export interface DatasetDataOptions {
     /**
      * Number of array elements that should be skipped at the start.
@@ -67,6 +61,11 @@ export interface DatasetDataOptions {
     skipEmpty?: boolean;
 }
 export interface DatasetExportOptions extends Omit<DatasetDataOptions, 'offset' | 'limit'> {
+    /**
+     * If true, includes all unique keys from all dataset items in the CSV export header.
+     * If omitted or false, only keys from the first item are used.
+     */
+    collectAllKeys?: boolean;
 }
 export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset' | 'limit' | 'clean' | 'skipHidden' | 'skipEmpty'> {
     /** @internal */
@@ -86,8 +85,8 @@ export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset
     format?: string;
 }
 export interface DatasetExportToOptions extends DatasetExportOptions {
-    fromDataset?: string;
-    toKVS?: string;
+    fromDataset?: string | StorageIdentifier;
+    toKVS?: string | StorageIdentifier;
 }
 /**
  * The `Dataset` class represents a store for structured data where each object stored has the same attributes,
@@ -144,12 +143,18 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
     readonly config: Configuration;
     id: string;
     name?: string;
-    client: DatasetClient<Data>;
-    log: Log;
+    backend: DatasetBackend<Data>;
+    log: CrawleeLogger;
+    private readonly statsTracker;
     /**
      * @internal
      */
     constructor(options: DatasetOptions, config?: Configuration);
+    /**
+     * Backend-independent usage counters tracked for this dataset (read / write operations issued to
+     * the underlying storage backend). Counted per backend call.
+     */
+    get stats(): DatasetStats;
     /**
      * Stores an object or an array of objects to the dataset.
      * The function returns a promise that resolves when the operation finishes.
@@ -158,21 +163,8 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
      * otherwise the crawler process might finish before the data is stored!
      *
-     * The size of the data is limited by the receiving API and therefore `pushData()` will only
-     * allow objects whose JSON representation is smaller than 9MB. When an array is passed,
-     * none of the included objects
-     * may be larger than 9MB, but the array itself may be of any size.
-     *
-     * The function internally
-     * chunks the array into separate items and pushes them sequentially.
-     * The chunking process is stable (keeps order of data), but it does not provide a transaction
-     * safety mechanism. Therefore, in the event of an uploading error (after several automatic retries),
-     * the function's Promise will reject and the dataset will be left in a state where some of
-     * the items have already been saved to the dataset while other items from the source array were not.
-     * To overcome this limitation, the developer may, for example, read the last item saved in the dataset
-     * and re-attempt the save of the data from this item onwards to prevent duplicates.
      * @param data Object or array of objects containing data to be stored in the default dataset.
-     *   The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
+     *   The objects must be serializable to JSON.
      */
     pushData(data: Data | Data[]): Promise<void>;
     /**
@@ -223,26 +215,21 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
     /**
      * Returns an object containing general information about the dataset.
      *
-     * The function returns the same object as the Apify API Client's
-     * [getDataset](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-datasets-getDataset)
-     * function, which in turn calls the
-     * [Get dataset](https://apify.com/docs/api/v2#/reference/datasets/dataset/get-dataset)
-     * API endpoint.
-     *
      * **Example:**
      * ```
      * {
      *   id: "WkzbQMuFYuamGv3YF",
      *   name: "my-dataset",
-     *   userId: "wRsJZtadYvn4mBZmm",
      *   createdAt: new Date("2015-12-12T07:34:14.202Z"),
      *   modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
      *   accessedAt: new Date("2015-12-14T08:36:13.202Z"),
      *   itemCount: 14,
      * }
      * ```
+     *
+     * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
-    getInfo(): Promise<DatasetInfo | undefined>;
+    getInfo(): Promise<DatasetInfo>;
     /**
      * Iterates over dataset items, yielding each in turn to an `iteratee` function.
      * Each invocation of `iteratee` is called with two arguments: `(item, index)`.
@@ -326,6 +313,69 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * @param [options] An object containing extra options for `reduce()`
      */
     reduce<T>(iteratee: DatasetReducer<T, Data>, memo: T, options?: DatasetIteratorOptions): Promise<T>;
+    private fetchEntryPages;
+    private fetchPages;
+    /**
+     * Returns dataset items.
+     *
+     * When awaited (`await dataset.values()`), returns all items as a flat `Data[]` array.
+     * When used as an async iterable (`for await...of`), iterates over all items across pages
+     * without loading everything into memory at once.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const dataset = await Dataset.open('my-results');
+     *
+     * // Iterate over all items (memory-efficient for large datasets)
+     * for await (const item of dataset.values()) {
+     *   console.log(item);
+     * }
+     *
+     * // Or fetch all items at once
+     * const items = await dataset.values();
+     * console.log(items);
+     * ```
+     *
+     * @param options Options for the iteration.
+     */
+    values(options?: DatasetIteratorOptions): AsyncIterable<Data> & Promise<Data[]>;
+    /**
+     * Returns dataset entries (index-value pairs).
+     *
+     * When awaited (`await dataset.entries()`), returns all entries as a flat `[index, item][]` array.
+     * When used as an async iterable (`for await...of`), iterates over all entries across pages
+     * without loading everything into memory at once.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const dataset = await Dataset.open('my-results');
+     *
+     * // Iterate over all entries
+     * for await (const [index, item] of dataset.entries()) {
+     *   console.log(`Item at ${index}: ${JSON.stringify(item)}`);
+     * }
+     *
+     * // Or fetch all at once
+     * const entries = await dataset.entries();
+     * console.log(entries);
+     * ```
+     *
+     * @param options Options for the iteration.
+     */
+    entries(options?: DatasetIteratorOptions): AsyncIterable<[number, Data]> & Promise<[number, Data][]>;
+    /**
+     * Default async iterator for the dataset, iterating over items.
+     * Allows using the dataset directly in a `for await...of` loop.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const dataset = await Dataset.open('my-results');
+     * for await (const item of dataset) {
+     *   console.log(item);
+     * }
+     * ```
+     */
+    [Symbol.asyncIterator](): AsyncGenerator<Data, void, undefined>;
     /**
      * Removes the dataset either from the Apify cloud storage or from the local directory,
      * depending on the mode of operation.
@@ -340,12 +390,13 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      *
      * For more details and code examples, see the {@link Dataset} class.
      *
-     * @param [datasetIdOrName]
-     *   ID or name of the dataset to be opened. If `null` or `undefined`,
-     *   the function returns the default dataset associated with the crawler run.
+     * @param [identifier]
+     *   ID or name of the dataset to be opened. If a string is provided, it will first be
+     *   looked up as an ID; if no such storage exists, it will be treated as a name.
+     *   If `null` or `undefined`, the function returns the default dataset associated with the crawler run.
      * @param [options] Storage manager options.
      */
-    static open<Data extends Dictionary = Dictionary>(datasetIdOrName?: string | null, options?: StorageManagerOptions): Promise<Dataset<Data>>;
+    static open<Data extends Dictionary = Dictionary>(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<Dataset<Data>>;
     /**
      * Stores an object or an array of objects to the default {@link Dataset} of the current crawler run.
      *
@@ -411,7 +462,7 @@ export interface DatasetReducer<T, Data> {
 export interface DatasetOptions {
     id: string;
     name?: string;
-    client: StorageClient;
+    backend: DatasetBackend;
 }
 export interface DatasetContent<Data> {
     /** Total count of entries in the dataset. */

package/storages/dataset.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAM5F,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,KAAK,GAAG,EAAO,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAIlE,gBAAgB;AAChB,eAAO,MAAM,+BAA+B,QAAQ,CAAC;AAIrD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAsBxF;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CA8BzE;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,oBAAqB,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,CAAC;CAAG;AAE7F,MAAM,WAAW,sBACb,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,YAAY,GAAG,WAAW,CAAC;IAC3F,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,gBAAgB;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB,gBAAgB;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,sBAAuB,SAAQ,oBAAoB;IAChE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,qBAAa,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IAWjD,QAAQ,CAAC,MAAM;IAVnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IAC5B,GAAG,EAAE,GAAG,CAAoC;IAE5C;;OAEG;gBAEC,OAAO,EAAE,cAAc,EACd,MAAM,gBAAkC;IAOrD;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBlD;;OAEG;IACG,OAAO,CAAC,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAgB9E;;;OAGG;IACG,MAAM,CAAC,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAyBjE;;;;;;OAMG;IACG,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IA0BpG;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC;IAIrF;;;;;OAKG;IACG,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC;IAIpF;;;;;OAKG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAOvE;;;;;OAKG;WACU,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAOtE;;;;;;;;;;;;;;;;;;;;;OAqBG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;IAMjD;;;;;;;;;;;;;;;;;;;OAmBG;IACG,OAAO,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,OAAO,GAAE,sBAA2B,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqB9G;;;;;;;;OAQG;IACG,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAalG;;;;;;;;;;;;;;;;OAgBG;IACG,MAAM,CAAC,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;IAE7E;;;;;;;;;;;;;;;;;;OAkBG;IACG,MAAM,CACR,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,EACpC,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,sBAAsB,GAChC,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;IAE5B;;;;;;;;;;;;OAYG;IACG,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,CAAC,CAAC;IAyBzG;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAClD,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,EAC/B,OAAO,GAAE,qBAA0B,GACpC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAsBzB;;;;;;;;;;;;;;;;;;;;;;;OAuBG;WACU,QAAQ,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAK/F;;OAEG;WACU,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EACrD,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;CAInC;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,IAAI;IACjC;;;OAGG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,EAAE,CAAC;IAClC;;;;OAIG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,CAAC,EAAE,IAAI;IACnC;;;;OAIG;IACH,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CACtD;AAED,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,cAAc,CAAC,IAAI;IAChC,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,iDAAiD;IACjD,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB"}
+{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,UAAU,EAAiB,MAAM,gBAAgB,CAAC;AAI7F,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAIrD,gBAAgB;AAChB,eAAO,MAAM,+BAA+B,QAAQ,CAAC;AAErD;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAcvE;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,oBAAqB,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,CAAC;IACtF;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,sBAAuB,SAAQ,IAAI,CAChD,kBAAkB,EAClB,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,YAAY,GAAG,WAAW,CAC5D;IACG,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,gBAAgB;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB,gBAAgB;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,sBAAuB,SAAQ,oBAAoB;IAChE,WAAW,CAAC,EAAE,MAAM,GAAG,iBAAiB,CAAC;IACzC,KAAK,CAAC,EAAE,MAAM,GAAG,iBAAiB,CAAC;CACtC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,qBAAa,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IAgBjD,QAAQ,CAAC,MAAM;IAfnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9B,GAAG,EAAE,aAAa,CAAC;IAEnB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAG1B;IAEH;;OAEG;gBAEC,OAAO,EAAE,cAAc,EACd,MAAM,gBAAkC;IAQrD;;;OAGG;IACH,IAAI,KAAK,IAAI,YAAY,CAExB;IAED;;;;;;;;;;OAUG;IACG,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAelD;;OAEG;IACG,OAAO,CAAC,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAiB9E;;;OAGG;IACG,MAAM,CAAC,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAYjE;;;;;;OAMG;IACG,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAiCpG;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC;IAIrF;;;;;OAKG;IACG,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC;IAIpF;;;;;OAKG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAOvE;;;;;OAKG;WACU,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAOtE;;;;;;;;;;;;;;;;OAgBG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IAMrC;;;;;;;;;;;;;;;;;;;OAmBG;IACG,OAAO,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,OAAO,GAAE,sBAA2B,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqB9G;;;;;;;;OAQG;IACG,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAalG;;;;;;;;;;;;;;;;OAgBG;IACG,MAAM,CAAC,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;IAE7E;;;;;;;;;;;;;;;;;;OAkBG;IACG,MAAM,CACR,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,EACpC,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,sBAAsB,GAChC,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;IAE5B;;;;;;;;;;;;OAYG;IACG,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,CAAC,CAAC;YAyB1F,eAAe;YAUf,UAAU;IAsBzB;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,MAAM,CAAC,OAAO,GAAE,sBAA2B,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IASnF;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,OAAO,CAAC,OAAO,GAAE,sBAA2B,GAAG,aAAa,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;IASxG;;;;;;;;;;;OAWG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC;IAItE;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B;;;;;;;;;;;;;;OAcG;WACU,IAAI,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAClD,UAAU,CAAC,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,EAC9C,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IA0BzB;;;;;;;;;;;;;;;;;;;;;;;OAuBG;WACU,QAAQ,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAK/F;;OAEG;WACU,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EACrD,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;CAInC;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,IAAI;IACjC;;;OAGG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,EAAE,CAAC;IAClC;;;;OAIG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,CAAC,EAAE,IAAI;IACnC;;;;OAIG;IACH,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CACtD;AAED,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,cAAc,CAAC;CAC3B;AAED,MAAM,WAAW,cAAc,CAAC,IAAI;IAChC,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,iDAAiD;IACjD,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB"}

package/storages/dataset.js

@@ -1,80 +1,35 @@
 import { stringify } from 'csv-stringify/sync';
 import ow from 'ow';
-import { MAX_PAYLOAD_SIZE_BYTES } from '@apify/consts';
 import { Configuration } from '../configuration.js';
-import { log } from '../log.js';
+import { serviceLocator } from '../service_locator.js';
 import { checkStorageAccess } from './access_checking.js';
 import { KeyValueStore } from './key_value_store.js';
-import { StorageManager } from './storage_manager.js';
-import { purgeDefaultStorages } from './utils.js';
+import { StorageStatsTracker } from './storage_stats.js';
+import { resolveStorageIdentifier } from './storage_instance_manager.js';
+import { createDualIterable, purgeDefaultStorages } from './utils.js';
 /** @internal */
 export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
-const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
 /**
- * Accepts a JSON serializable object as an input, validates its serializability,
- * and validates its serialized size against limitBytes. Optionally accepts its index
- * in an array to provide better error messages. Returns serialized object.
+ * Validates that the given value is a plain JSON-serializable object
+ * (not an array, not a primitive, not circular).
+ *
+ * @param item The value to validate.
+ * @param index Optional index for error messages when validating inside an array.
  * @ignore
  */
-export function checkAndSerialize(item, limitBytes, index) {
+export function assertJsonSerializable(item, index) {
     const s = typeof index === 'number' ? ` at index ${index} ` : ' ';
     const isItemObject = item && typeof item === 'object' && !Array.isArray(item);
     if (!isItemObject) {
         throw new Error(`Data item${s}is not an object. You can push only objects into a dataset.`);
     }
-    let payload;
     try {
-        payload = JSON.stringify(item);
+        JSON.stringify(item);
     }
     catch (e) {
         const err = e;
         throw new Error(`Data item${s}is not serializable to JSON.\nCause: ${err.message}`);
     }
-    const bytes = Buffer.byteLength(payload);
-    if (bytes > limitBytes) {
-        throw new Error(`Data item${s}is too large (size: ${bytes} bytes, limit: ${limitBytes} bytes)`);
-    }
-    return payload;
-}
-/**
- * Takes an array of JSONs (payloads) as input and produces an array of JSON strings
- * where each string is a JSON array of payloads with a maximum size of limitBytes per one
- * JSON array. Fits as many payloads as possible into a single JSON array and then moves
- * on to the next, preserving item order.
- *
- * The function assumes that none of the items is larger than limitBytes and does not validate.
- * @ignore
- */
-export function chunkBySize(items, limitBytes) {
-    if (!items.length)
-        return [];
-    if (items.length === 1)
-        return items;
-    // Split payloads into buckets of valid size.
-    let lastChunkBytes = 2; // Add 2 bytes for [] wrapper.
-    const chunks = [];
-    for (const payload of items) {
-        const bytes = Buffer.byteLength(payload);
-        if (bytes <= limitBytes && bytes + 2 > limitBytes) {
-            // Handle cases where wrapping with [] would fail, but solo object is fine.
-            chunks.push(payload);
-            lastChunkBytes = bytes;
-        }
-        else if (lastChunkBytes + bytes <= limitBytes) {
-            // ensure array
-            if (!Array.isArray(chunks[chunks.length - 1])) {
-                chunks.push([]);
-            }
-            chunks[chunks.length - 1].push(payload);
-            lastChunkBytes += bytes + 1; // Add 1 byte for ',' separator.
-        }
-        else {
-            chunks.push([payload]);
-            lastChunkBytes = bytes + 2; // Add 2 bytes for [] wrapper.
-        }
-    }
-    // Stringify array chunks.
-    return chunks.map((chunk) => (typeof chunk === 'string' ? chunk : `[${chunk.join(',')}]`));
 }
 /**
  * The `Dataset` class represents a store for structured data where each object stored has the same attributes,
@@ -131,8 +86,12 @@ export class Dataset {
     config;
     id;
     name;
-    client;
-    log = log.child({ prefix: 'Dataset' });
+    backend;
+    log;
+    statsTracker = new StorageStatsTracker({
+        readCount: 0,
+        writeCount: 0,
+    });
     /**
      * @internal
      */
@@ -140,7 +99,15 @@ export class Dataset {
         this.config = config;
         this.id = options.id;
         this.name = options.name;
-        this.client = options.client.dataset(this.id);
+        this.backend = options.backend;
+        this.log = serviceLocator.getLogger().child({ prefix: 'Dataset' });
+    }
+    /**
+     * Backend-independent usage counters tracked for this dataset (read / write operations issued to
+     * the underlying storage backend). Counted per backend call.
+     */
+    get stats() {
+        return this.statsTracker.current;
     }
     /**
      * Stores an object or an array of objects to the dataset.
@@ -150,40 +117,19 @@ export class Dataset {
      * **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
      * otherwise the crawler process might finish before the data is stored!
      *
-     * The size of the data is limited by the receiving API and therefore `pushData()` will only
-     * allow objects whose JSON representation is smaller than 9MB. When an array is passed,
-     * none of the included objects
-     * may be larger than 9MB, but the array itself may be of any size.
-     *
-     * The function internally
-     * chunks the array into separate items and pushes them sequentially.
-     * The chunking process is stable (keeps order of data), but it does not provide a transaction
-     * safety mechanism. Therefore, in the event of an uploading error (after several automatic retries),
-     * the function's Promise will reject and the dataset will be left in a state where some of
-     * the items have already been saved to the dataset while other items from the source array were not.
-     * To overcome this limitation, the developer may, for example, read the last item saved in the dataset
-     * and re-attempt the save of the data from this item onwards to prevent duplicates.
      * @param data Object or array of objects containing data to be stored in the default dataset.
-     *   The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
+     *   The objects must be serializable to JSON.
      */
     async pushData(data) {
         checkStorageAccess();
         ow(data, 'data', ow.object);
-        const dispatch = async (payload) => this.client.pushItems(payload);
-        const limit = MAX_PAYLOAD_SIZE_BYTES - Math.ceil(MAX_PAYLOAD_SIZE_BYTES * SAFETY_BUFFER_PERCENT);
-        // Handle singular Objects
-        if (!Array.isArray(data)) {
-            const payload = checkAndSerialize(data, limit);
-            await dispatch(payload);
-            return;
-        }
-        // Handle Arrays
-        const payloads = data.map((item, index) => checkAndSerialize(item, limit, index));
-        const chunks = chunkBySize(payloads, limit);
-        // Invoke client in series to preserve order of data
-        for (const chunk of chunks) {
-            await dispatch(chunk);
+        // Normalize to array and validate each item
+        const items = Array.isArray(data) ? data : [data];
+        for (let i = 0; i < items.length; i++) {
+            assertJsonSerializable(items[i], i);
         }
+        this.statsTracker.add('writeCount');
+        await this.backend.pushData(items);
     }
     /**
      * Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
@@ -191,7 +137,8 @@ export class Dataset {
     async getData(options = {}) {
         checkStorageAccess();
         try {
-            return await this.client.listItems(options);
+            this.statsTracker.add('readCount');
+            return await this.backend.getData(options);
         }
         catch (e) {
             const error = e;
@@ -208,18 +155,9 @@ export class Dataset {
     async export(options = {}) {
         checkStorageAccess();
         const items = [];
-        const fetchNextChunk = async (offset = 0) => {
-            const limit = 1000;
-            const value = await this.client.listItems({ offset, limit, ...options });
-            if (value.count === 0) {
-                return;
-            }
-            items.push(...value.items);
-            if (value.total > offset + value.count) {
-                await fetchNextChunk(offset + value.count);
-            }
-        };
-        await fetchNextChunk();
+        for await (const page of this.fetchPages(options)) {
+            items.push(...page.items);
+        }
         return items;
     }
     /**
@@ -233,7 +171,14 @@ export class Dataset {
         const kvStore = await KeyValueStore.open(options?.toKVS ?? null, { config: this.config });
         const items = await this.export(options);
         if (contentType === 'text/csv') {
-            const keys = Object.keys(items[0]);
+            // To handle empty dataset exports gracefully.
+            if (items.length === 0) {
+                await kvStore.setValue(key, '', { contentType });
+                return items;
+            }
+            const keys = options?.collectAllKeys
+                ? Array.from(new Set(items.flatMap(Object.keys)))
+                : Object.keys(items[0]);
             const value = stringify([
                 keys,
                 ...items.map((item) => {
@@ -248,7 +193,6 @@ export class Dataset {
             return items;
         }
         throw new Error(`Unsupported content type: ${contentType}`);
-        return items;
     }
     /**
      * Save entire default dataset's contents into one JSON file within a key-value store.
@@ -293,28 +237,23 @@ export class Dataset {
     /**
      * Returns an object containing general information about the dataset.
      *
-     * The function returns the same object as the Apify API Client's
-     * [getDataset](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-datasets-getDataset)
-     * function, which in turn calls the
-     * [Get dataset](https://apify.com/docs/api/v2#/reference/datasets/dataset/get-dataset)
-     * API endpoint.
-     *
      * **Example:**
      * ```
      * {
      *   id: "WkzbQMuFYuamGv3YF",
      *   name: "my-dataset",
-     *   userId: "wRsJZtadYvn4mBZmm",
      *   createdAt: new Date("2015-12-12T07:34:14.202Z"),
      *   modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
      *   accessedAt: new Date("2015-12-14T08:36:13.202Z"),
      *   itemCount: 14,
      * }
      * ```
+     *
+     * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
     async getInfo() {
         checkStorageAccess();
-        return this.client.get();
+        return this.backend.getMetadata();
     }
     /**
      * Iterates over dataset items, yielding each in turn to an `iteratee` function.
@@ -388,15 +327,115 @@ export class Dataset {
         await this.forEach(wrappedFunc, options);
         return currentMemo;
     }
+    async *fetchEntryPages(options) {
+        let index = options.offset ?? 0;
+        for await (const page of this.fetchPages(options)) {
+            yield {
+                ...page,
+                items: page.items.map((item) => [index++, item]),
+            };
+        }
+    }
+    async *fetchPages(options, pageSize = DATASET_ITERATORS_DEFAULT_LIMIT) {
+        let offset = options.offset ?? 0;
+        const totalLimit = options.limit;
+        let yielded = 0;
+        while (true) {
+            const fetchLimit = totalLimit !== undefined ? Math.min(pageSize, totalLimit - yielded) : pageSize;
+            if (fetchLimit <= 0)
+                break;
+            this.statsTracker.add('readCount');
+            const page = await this.backend.getData({ ...options, offset, limit: fetchLimit });
+            yield page;
+            yielded += page.items.length;
+            if (page.items.length < fetchLimit || offset + page.items.length >= page.total)
+                break;
+            offset += page.items.length;
+        }
+    }
+    /**
+     * Returns dataset items.
+     *
+     * When awaited (`await dataset.values()`), returns all items as a flat `Data[]` array.
+     * When used as an async iterable (`for await...of`), iterates over all items across pages
+     * without loading everything into memory at once.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const dataset = await Dataset.open('my-results');
+     *
+     * // Iterate over all items (memory-efficient for large datasets)
+     * for await (const item of dataset.values()) {
+     *   console.log(item);
+     * }
+     *
+     * // Or fetch all items at once
+     * const items = await dataset.values();
+     * console.log(items);
+     * ```
+     *
+     * @param options Options for the iteration.
+     */
+    values(options = {}) {
+        checkStorageAccess();
+        return createDualIterable({
+            createPages: () => this.fetchPages(options),
+            extractItems: (page) => page.items,
+        });
+    }
+    /**
+     * Returns dataset entries (index-value pairs).
+     *
+     * When awaited (`await dataset.entries()`), returns all entries as a flat `[index, item][]` array.
+     * When used as an async iterable (`for await...of`), iterates over all entries across pages
+     * without loading everything into memory at once.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const dataset = await Dataset.open('my-results');
+     *
+     * // Iterate over all entries
+     * for await (const [index, item] of dataset.entries()) {
+     *   console.log(`Item at ${index}: ${JSON.stringify(item)}`);
+     * }
+     *
+     * // Or fetch all at once
+     * const entries = await dataset.entries();
+     * console.log(entries);
+     * ```
+     *
+     * @param options Options for the iteration.
+     */
+    entries(options = {}) {
+        checkStorageAccess();
+        return createDualIterable({
+            createPages: () => this.fetchEntryPages(options),
+            extractItems: (page) => page.items,
+        });
+    }
+    /**
+     * Default async iterator for the dataset, iterating over items.
+     * Allows using the dataset directly in a `for await...of` loop.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const dataset = await Dataset.open('my-results');
+     * for await (const item of dataset) {
+     *   console.log(item);
+     * }
+     * ```
+     */
+    async *[Symbol.asyncIterator]() {
+        yield* this.values();
+    }
     /**
      * Removes the dataset either from the Apify cloud storage or from the local directory,
      * depending on the mode of operation.
      */
     async drop() {
         checkStorageAccess();
-        await this.client.delete();
-        const manager = StorageManager.getManager(Dataset, this.config);
-        manager.closeStorage(this);
+        await this.backend.drop();
+        serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
     /**
      * Opens a dataset and returns a promise resolving to an instance of the {@link Dataset} class.
@@ -407,23 +446,27 @@ export class Dataset {
      *
      * For more details and code examples, see the {@link Dataset} class.
      *
-     * @param [datasetIdOrName]
-     *   ID or name of the dataset to be opened. If `null` or `undefined`,
-     *   the function returns the default dataset associated with the crawler run.
+     * @param [identifier]
+     *   ID or name of the dataset to be opened. If a string is provided, it will first be
+     *   looked up as an ID; if no such storage exists, it will be treated as a name.
+     *   If `null` or `undefined`, the function returns the default dataset associated with the crawler run.
      * @param [options] Storage manager options.
      */
-    static async open(datasetIdOrName, options = {}) {
+    static async open(identifier, options = {}) {
         checkStorageAccess();
-        ow(datasetIdOrName, ow.optional.string);
         ow(options, ow.object.exactShape({
             config: ow.optional.object.instanceOf(Configuration),
-            storageClient: ow.optional.object,
+            storageBackend: ow.optional.object,
         }));
         options.config ??= Configuration.getGlobalConfig();
-        options.storageClient ??= options.config.getStorageClient();
-        await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
-        const manager = StorageManager.getManager(this, options.config);
-        return manager.openStorage(datasetIdOrName, options.storageClient);
+        const storageBackend = options.storageBackend ?? serviceLocator.getStorageBackend();
+        await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, config: options.config });
+        const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'Dataset');
+        return serviceLocator.getStorageInstanceManager().openStorage(this, {
+            ...resolved,
+            backendOpener: () => storageBackend.createDatasetBackend(resolved),
+            backendCacheKey: storageBackend.getStorageBackendCacheKey?.() ?? storageBackend.constructor.name,
+        });
     }
     /**
      * Stores an object or an array of objects to the default {@link Dataset} of the current crawler run.