@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.130

package/storages/dataset.d.ts

@@ -1,27 +1,21 @@
-import type { DatasetClient, DatasetInfo, Dictionary, StorageClient } from '@crawlee/types';
+import type { Awaitable, DatasetBackend, DatasetInfo, Dictionary } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
-import { type Log } from '../log.js';
-import type { Awaitable } from '../typedefs.js';
-import type { StorageManagerOptions } from './storage_manager.js';
+import type { CrawleeLogger } from '../log.js';
+import type { JournalEntry } from './transaction.js';
+import type { DatasetStats } from './storage_stats.js';
+import type { StorageOpenOptions } from './utils.js';
+import type { StorageIdentifier } from './storage_instance_manager.js';
 /** @internal */
 export declare const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
 /**
- * Accepts a JSON serializable object as an input, validates its serializability,
- * and validates its serialized size against limitBytes. Optionally accepts its index
- * in an array to provide better error messages. Returns serialized object.
- * @ignore
- */
-export declare function checkAndSerialize<T>(item: T, limitBytes: number, index?: number): string;
-/**
- * Takes an array of JSONs (payloads) as input and produces an array of JSON strings
- * where each string is a JSON array of payloads with a maximum size of limitBytes per one
- * JSON array. Fits as many payloads as possible into a single JSON array and then moves
- * on to the next, preserving item order.
+ * Validates that the given value is a plain JSON-serializable object
+ * (not an array, not a primitive, not circular).
  *
- * The function assumes that none of the items is larger than limitBytes and does not validate.
+ * @param item The value to validate.
+ * @param index Optional index for error messages when validating inside an array.
  * @ignore
  */
-export declare function chunkBySize(items: string[], limitBytes: number): string[];
+export declare function assertJsonSerializable<T>(item: T, index?: number): void;
 export interface DatasetDataOptions {
     /**
      * Number of array elements that should be skipped at the start.
@@ -67,6 +61,11 @@ export interface DatasetDataOptions {
     skipEmpty?: boolean;
 }
 export interface DatasetExportOptions extends Omit<DatasetDataOptions, 'offset' | 'limit'> {
+    /**
+     * If true, includes all unique keys from all dataset items in the CSV export header.
+     * If omitted or false, only keys from the first item are used.
+     */
+    collectAllKeys?: boolean;
 }
 export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset' | 'limit' | 'clean' | 'skipHidden' | 'skipEmpty'> {
     /** @internal */
@@ -86,8 +85,8 @@ export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset
     format?: string;
 }
 export interface DatasetExportToOptions extends DatasetExportOptions {
-    fromDataset?: string;
-    toKVS?: string;
+    fromDataset?: string | StorageIdentifier;
+    toKVS?: string | StorageIdentifier;
 }
 /**
  * The `Dataset` class represents a store for structured data where each object stored has the same attributes,
@@ -141,15 +140,21 @@ export interface DatasetExportToOptions extends DatasetExportOptions {
  * @category Result Stores
  */
 export declare class Dataset<Data extends Dictionary = Dictionary> {
-    readonly config: Configuration;
+    #private;
+    readonly configuration: Configuration;
     id: string;
     name?: string;
-    client: DatasetClient<Data>;
-    log: Log;
+    backend: DatasetBackend<Data>;
+    log: CrawleeLogger;
     /**
      * @internal
      */
-    constructor(options: DatasetOptions, config?: Configuration);
+    constructor(options: DatasetOptions, configuration?: Configuration);
+    /**
+     * Backend-independent usage counters tracked for this dataset (read / write operations issued to
+     * the underlying storage backend). Counted per backend call.
+     */
+    get stats(): DatasetStats;
     /**
      * Stores an object or an array of objects to the dataset.
      * The function returns a promise that resolves when the operation finishes.
@@ -158,27 +163,24 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
      * otherwise the crawler process might finish before the data is stored!
      *
-     * The size of the data is limited by the receiving API and therefore `pushData()` will only
-     * allow objects whose JSON representation is smaller than 9MB. When an array is passed,
-     * none of the included objects
-     * may be larger than 9MB, but the array itself may be of any size.
-     *
-     * The function internally
-     * chunks the array into separate items and pushes them sequentially.
-     * The chunking process is stable (keeps order of data), but it does not provide a transaction
-     * safety mechanism. Therefore, in the event of an uploading error (after several automatic retries),
-     * the function's Promise will reject and the dataset will be left in a state where some of
-     * the items have already been saved to the dataset while other items from the source array were not.
-     * To overcome this limitation, the developer may, for example, read the last item saved in the dataset
-     * and re-attempt the save of the data from this item onwards to prevent duplicates.
      * @param data Object or array of objects containing data to be stored in the default dataset.
-     *   The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
+     *   The objects must be serializable to JSON.
      */
     pushData(data: Data | Data[]): Promise<void>;
     /**
      * Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
      */
     getData(options?: DatasetDataOptions): Promise<DatasetContent<Data>>;
+    /**
+     * The single transaction-aware page read all dataset read paths go through — both `getData()` and
+     * the private `fetchPages()`. Returns the real page concatenated with the current transaction's
+     * buffered items, with `offset` / `limit` / `desc` windowing applied across the concatenation.
+     */
+    private readPage;
+    /** The active transaction's buffered writes to this dataset, derived from its journal. */
+    private bufferedJournalEntries;
+    /** @internal */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
      * Returns all the data from the dataset. This will iterate through the whole dataset
      * via the `listItems()` client method, which gives you only paginated results.
@@ -223,26 +225,21 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
     /**
      * Returns an object containing general information about the dataset.
      *
-     * The function returns the same object as the Apify API Client's
-     * [getDataset](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-datasets-getDataset)
-     * function, which in turn calls the
-     * [Get dataset](https://apify.com/docs/api/v2#/reference/datasets/dataset/get-dataset)
-     * API endpoint.
-     *
      * **Example:**
      * ```
      * {
      *   id: "WkzbQMuFYuamGv3YF",
      *   name: "my-dataset",
-     *   userId: "wRsJZtadYvn4mBZmm",
      *   createdAt: new Date("2015-12-12T07:34:14.202Z"),
      *   modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
      *   accessedAt: new Date("2015-12-14T08:36:13.202Z"),
      *   itemCount: 14,
      * }
      * ```
+     *
+     * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
-    getInfo(): Promise<DatasetInfo | undefined>;
+    getInfo(): Promise<DatasetInfo>;
     /**
      * Iterates over dataset items, yielding each in turn to an `iteratee` function.
      * Each invocation of `iteratee` is called with two arguments: `(item, index)`.
@@ -308,7 +305,7 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * If the dataset is empty, reduce will return undefined.
      *
      * @param iteratee
-     * @param memo Unset parameter, neccesary to be able to pass options
+     * @param memo Unset parameter, necessary to be able to pass options
      * @param [options] An object containing extra options for `reduce()`
      */
     reduce(iteratee: DatasetReducer<Data, Data>, memo: undefined, options: DatasetIteratorOptions): Promise<Data | undefined>;
@@ -326,6 +323,69 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * @param [options] An object containing extra options for `reduce()`
      */
     reduce<T>(iteratee: DatasetReducer<T, Data>, memo: T, options?: DatasetIteratorOptions): Promise<T>;
+    private fetchEntryPages;
+    private fetchPages;
+    /**
+     * Returns dataset items.
+     *
+     * When awaited (`await dataset.values()`), returns all items as a flat `Data[]` array.
+     * When used as an async iterable (`for await...of`), iterates over all items across pages
+     * without loading everything into memory at once.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const dataset = await Dataset.open('my-results');
+     *
+     * // Iterate over all items (memory-efficient for large datasets)
+     * for await (const item of dataset.values()) {
+     *   console.log(item);
+     * }
+     *
+     * // Or fetch all items at once
+     * const items = await dataset.values();
+     * console.log(items);
+     * ```
+     *
+     * @param options Options for the iteration.
+     */
+    values(options?: DatasetIteratorOptions): AsyncIterable<Data> & Promise<Data[]>;
+    /**
+     * Returns dataset entries (index-value pairs).
+     *
+     * When awaited (`await dataset.entries()`), returns all entries as a flat `[index, item][]` array.
+     * When used as an async iterable (`for await...of`), iterates over all entries across pages
+     * without loading everything into memory at once.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const dataset = await Dataset.open('my-results');
+     *
+     * // Iterate over all entries
+     * for await (const [index, item] of dataset.entries()) {
+     *   console.log(`Item at ${index}: ${JSON.stringify(item)}`);
+     * }
+     *
+     * // Or fetch all at once
+     * const entries = await dataset.entries();
+     * console.log(entries);
+     * ```
+     *
+     * @param options Options for the iteration.
+     */
+    entries(options?: DatasetIteratorOptions): AsyncIterable<[number, Data]> & Promise<[number, Data][]>;
+    /**
+     * Default async iterator for the dataset, iterating over items.
+     * Allows using the dataset directly in a `for await...of` loop.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const dataset = await Dataset.open('my-results');
+     * for await (const item of dataset) {
+     *   console.log(item);
+     * }
+     * ```
+     */
+    [Symbol.asyncIterator](): AsyncGenerator<Data, void, undefined>;
     /**
      * Removes the dataset either from the Apify cloud storage or from the local directory,
      * depending on the mode of operation.
@@ -340,12 +400,13 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      *
      * For more details and code examples, see the {@link Dataset} class.
      *
-     * @param [datasetIdOrName]
-     *   ID or name of the dataset to be opened. If `null` or `undefined`,
-     *   the function returns the default dataset associated with the crawler run.
+     * @param [identifier]
+     *   ID or name of the dataset to be opened. If a string is provided, it will first be
+     *   looked up as an ID; if no such storage exists, it will be treated as a name.
+     *   If `null` or `undefined`, the function returns the default dataset associated with the crawler run.
      * @param [options] Storage manager options.
      */
-    static open<Data extends Dictionary = Dictionary>(datasetIdOrName?: string | null, options?: StorageManagerOptions): Promise<Dataset<Data>>;
+    static open<Data extends Dictionary = Dictionary>(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<Dataset<Data>>;
     /**
      * Stores an object or an array of objects to the default {@link Dataset} of the current crawler run.
      *
@@ -409,9 +470,9 @@ export interface DatasetReducer<T, Data> {
     (memo: T, item: Data, index: number): Awaitable<T>;
 }
 export interface DatasetOptions {
-    id: string;
-    name?: string;
-    client: StorageClient;
+    /** Resolved metadata for the dataset, as returned by the backend's `getMetadata()`. */
+    metadata: DatasetInfo;
+    backend: DatasetBackend;
 }
 export interface DatasetContent<Data> {
     /** Total count of entries in the dataset. */
@@ -427,4 +488,3 @@ export interface DatasetContent<Data> {
     /** Should the results be in descending order. */
     desc?: boolean;
 }
-//# sourceMappingURL=dataset.d.ts.map