@crawlee/core 4.0.0-beta.2 → 4.0.0-beta.200

package/storages/key_value_store.d.ts

@@ -1,15 +1,9 @@
-import type { Dictionary, StorageClient } from '@crawlee/types';
+import type { Awaitable, Dictionary, KeyValueStoreBackend, KeyValueStoreInfo } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
-import type { Awaitable } from '../typedefs.js';
-import type { StorageManagerOptions } from './storage_manager.js';
-/**
- * Helper function to possibly stringify value if options.contentType is not set.
- *
- * @ignore
- */
-export declare const maybeStringify: <T>(value: T, options: {
-    contentType?: string;
-}) => T;
+import type { JournalEntry } from './transaction.js';
+import type { KeyValueStoreStats } from './storage_stats.js';
+import type { StorageOpenOptions } from './utils.js';
+import type { StorageIdentifier } from './storage_instance_manager.js';
 /**
  * The `KeyValueStore` class represents a key-value store, a simple data storage that is used
  * for saving and reading data records or files. Each data record is
@@ -26,8 +20,6 @@ export declare const maybeStringify: <T>(value: T, options: {
  * To access the default key-value store directly, you can use the
  * {@link KeyValueStore.getValue} and {@link KeyValueStore.setValue} convenience functions.
  *
- * To access the input, you can also use the {@link KeyValueStore.getInput} convenience function.
- *
  * `KeyValueStore` stores its data on a local disk.
  *
  * If the `CRAWLEE_STORAGE_DIR` environment variable is set, the data is stored in
@@ -35,15 +27,14 @@ export declare const maybeStringify: <T>(value: T, options: {
  * ```
  * {CRAWLEE_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}
  * ```
- * Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,
- * unless you override it by setting the `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
+ * Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`.
  * The `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.
  *
  * **Example usage:**
  *
  * ```javascript
  * // Get crawler input from the default key-value store.
- * const input = await KeyValueStore.getInput();
+ * const input = await KeyValueStore.getValue('INPUT');
  * // Get some value from the default key-value store.
  * const otherValue = await KeyValueStore.getValue('my-key');
  *
@@ -67,18 +58,20 @@ export declare const maybeStringify: <T>(value: T, options: {
  * @category Result Stores
  */
 export declare class KeyValueStore {
-    readonly config: Configuration;
+    #private;
+    readonly configuration: Configuration;
     readonly id: string;
     readonly name?: string;
-    readonly storageObject?: Record<string, unknown>;
-    private readonly client;
-    private persistStateEventStarted;
-    /** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
-    private readonly cache;
+    readonly backend: KeyValueStoreBackend;
     /**
      * @internal
      */
-    constructor(options: KeyValueStoreOptions, config?: Configuration);
+    constructor(options: KeyValueStoreOptions, configuration?: Configuration);
+    /**
+     * Backend-independent usage counters tracked for this key-value store (read / write / delete /
+     * list operations issued to the underlying storage backend). Counted per backend call.
+     */
+    get stats(): KeyValueStoreStats;
     /**
      * Gets a value from the key-value store.
      *
@@ -143,6 +136,48 @@ export declare class KeyValueStore {
      *   on the MIME content type of the record, or the default value if the key is missing from the store.
      */
     getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
+    /**
+     * The active transaction's last buffered write per key for this store, derived from its journal.
+     * An entry with a `null` value is a tombstone (an in-transaction deletion).
+     */
+    private bufferedJournalEntries;
+    /**
+     * The single transaction-aware record read shared by `getValue`, `getRecord`, `recordExists` and the
+     * listing paths: buffered key → serialized through the standard codec (same fidelity as a real
+     * round-trip); tombstoned key → `null`; otherwise the backend.
+     *
+     * The per-key buffered lookup requires the whole journal to be reduced to a last-write-per-key map,
+     * which is O(journal). Single-record callers let it default (rebuilt per call); the listing paths,
+     * which read many keys, pass a map built once so the read stays O(1) per key instead of O(journal).
+     */
+    private readRecord;
+    /**
+     * Reads a record from the key-value store without parsing the value.
+     *
+     * Use this when you need the raw bytes and the content type — for example, to run your own
+     * parser (`simdjson`, a custom XML library, etc.) or to forward the bytes verbatim.
+     *
+     * There is no symmetric `setRecord` method, because {@link KeyValueStore.setValue} already
+     * passes a `Buffer` (or `string` / `Stream`) through unchanged when an explicit `contentType`
+     * is provided. To write pre-serialized bytes, call
+     * `setValue(key, buffer, { contentType: 'application/json; charset=utf-8' })`.
+     *
+     * Returns `null` if the record does not exist.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const store = await KeyValueStore.open();
+     * const record = await store.getRecord('huge.json');
+     * if (record) {
+     *     const data = simdjson.parse(record.value);
+     * }
+     * ```
+     *
+     * @param key
+     *   Unique key of the record. It can be at most 256 characters long and only consist
+     *   of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
+     */
+    getRecord(key: string): Promise<KeyValueStoreRawRecord | null>;
     /**
      * Tests whether a record with the given key exists in the key-value store without retrieving its value.
      *
@@ -152,6 +187,8 @@ export declare class KeyValueStore {
     recordExists(key: string): Promise<boolean>;
     getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
     private ensurePersistStateEvent;
+    private fetchKeyValuePages;
+    private fetchKeyPages;
     /**
      * Saves or deletes a record in the key-value store.
      * The function returns a promise that resolves once the record has been saved or deleted.
@@ -196,18 +233,25 @@ export declare class KeyValueStore {
      * @param [options] Record options.
      */
     setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
+    /** @internal */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
      * Removes the key-value store either from the Apify cloud storage or from the local directory,
      * depending on the mode of operation.
      */
     drop(): Promise<void>;
+    /**
+     * Removes all records from the store but keeps the store itself, along with its
+     * {@link KeyValueStore.id|`id`} and {@link KeyValueStore.name|`name`}.
+     */
+    purge(): Promise<void>;
     /** @internal */
     clearCache(): void;
     /**
      * Iterates over key-value store keys, yielding each in turn to an `iteratee` function.
      * Each invocation of `iteratee` is called with three arguments: `(key, index, info)`, where `key`
      * is the record key, `index` is a zero-based index of the key in the current iteration
-     * (regardless of `options.exclusiveStartKey`) and `info` is an object that contains a single property `size`
+     * and `info` is an object that contains a single property `size`
      * indicating size of the record in bytes.
      *
      * If the `iteratee` function returns a Promise then it is awaited before the next call.
@@ -225,11 +269,101 @@ export declare class KeyValueStore {
      * @param [options] All `forEachKey()` parameters.
      */
     forEachKey(iteratee: KeyConsumer, options?: KeyValueStoreIteratorOptions): Promise<void>;
-    private _forEachKey;
+    /**
+     * Returns key-value store keys.
+     *
+     * When awaited (`await store.keys()`), returns all keys as a flat `string[]` array.
+     * When used as an async iterable (`for await...of`), iterates over all keys across pages
+     * without loading everything into memory at once.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const keyValueStore = await KeyValueStore.open();
+     *
+     * // Iterate over all keys (memory-efficient for large stores)
+     * for await (const key of keyValueStore.keys()) {
+     *   console.log(key);
+     * }
+     *
+     * // Or fetch all keys at once
+     * const allKeys = await keyValueStore.keys();
+     * console.log(allKeys);
+     * ```
+     *
+     * @param options Options for the iteration.
+     */
+    keys(options?: KeyValueStoreIteratorOptions): AsyncIterable<string> & Promise<string[]>;
+    /**
+     * Returns key-value store values.
+     *
+     * When awaited (`await store.values()`), returns all values as a flat `T[]` array.
+     * When used as an async iterable (`for await...of`), iterates over all values across pages
+     * without loading everything into memory at once.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const keyValueStore = await KeyValueStore.open();
+     *
+     * // Iterate over all values (memory-efficient for large stores)
+     * for await (const value of keyValueStore.values()) {
+     *   console.log(value);
+     * }
+     *
+     * // Or fetch all values at once
+     * const allValues = await keyValueStore.values();
+     * console.log(allValues);
+     * ```
+     *
+     * @param options Options for the iteration.
+     */
+    values<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<T> & Promise<T[]>;
+    /**
+     * Returns key-value store entries (key-value pairs).
+     *
+     * When awaited (`await store.entries()`), returns all entries as a flat `[key, value][]` array.
+     * When used as an async iterable (`for await...of`), iterates over all entries across pages
+     * without loading everything into memory at once.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const keyValueStore = await KeyValueStore.open();
+     *
+     * // Iterate over all entries (memory-efficient for large stores)
+     * for await (const [key, value] of keyValueStore.entries()) {
+     *   console.log(`${key}: ${value}`);
+     * }
+     *
+     * // Or fetch all entries at once
+     * const allEntries = await keyValueStore.entries();
+     * console.log(allEntries);
+     * ```
+     *
+     * @param options Options for the iteration.
+     */
+    entries<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<[string, T]> & Promise<[string, T][]>;
+    /**
+     * Default async iterator for the key-value store, iterating over entries (key-value pairs).
+     * Allows using the store directly in a `for await...of` loop.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const keyValueStore = await KeyValueStore.open();
+     * for await (const [key, value] of keyValueStore) {
+     *   console.log(`${key}: ${value}`);
+     * }
+     * ```
+     */
+    [Symbol.asyncIterator]<T = unknown>(): AsyncGenerator<[string, T], void, undefined>;
     /**
      * Returns a file URL for the given key.
+     *
+     * The URL is derived from the key, so it is also returned for a record that does not exist (yet) —
+     * including one written earlier in an uncommitted storage transaction. Returns `undefined` only when
+     * the storage has no file URLs at all (e.g. the in-memory storage).
+     *
+     * @param key The key of the record to generate the public URL for.
      */
-    getPublicUrl(key: string): string;
+    getPublicUrl(key: string): Promise<string | undefined>;
     /**
      * Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
      *
@@ -239,12 +373,13 @@ export declare class KeyValueStore {
      *
      * For more details and code examples, see the {@link KeyValueStore} class.
      *
-     * @param [storeIdOrName]
-     *   ID or name of the key-value store to be opened. If `null` or `undefined`,
-     *   the function returns the default key-value store associated with the crawler run.
+     * @param [identifier]
+     *   ID or name of the key-value store to be opened. If a string is provided, it will first be
+     *   looked up as an ID; if no such storage exists, it will be treated as a name.
+     *   If `null` or `undefined`, the function returns the default key-value store associated with the crawler run.
      * @param [options] Storage manager options.
      */
-    static open(storeIdOrName?: string | null, options?: StorageManagerOptions): Promise<KeyValueStore>;
+    static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<KeyValueStore>;
     /**
      * Gets a value from the default {@link KeyValueStore} associated with the current crawler run.
      *
@@ -303,6 +438,19 @@ export declare class KeyValueStore {
      * @ignore
      */
     static getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
+    /**
+     * Reads a record from the default {@link KeyValueStore} associated with the current crawler run
+     * without parsing the value.
+     *
+     * This is just a convenient shortcut for {@link KeyValueStore.getRecord}. Returns `null` if the
+     * record does not exist.
+     *
+     * @param key
+     *   Unique key of the record. It can be at most 256 characters long and only consist
+     *   of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
+     * @ignore
+     */
+    static getRecord(key: string): Promise<KeyValueStoreRawRecord | null>;
     /**
      * Tests whether a record with the given key exists in the default {@link KeyValueStore} associated with the current crawler run.
      * @param key The queried record key.
@@ -342,26 +490,6 @@ export declare class KeyValueStore {
      * @ignore
      */
     static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
-    /**
-     * Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
-     * By default, it will try to find root input files (either extension-less, `.json` or `.txt`),
-     * or alternatively read the input from the default {@link KeyValueStore}.
-     *
-     * Note that the `getInput()` function does not cache the value read from the key-value store.
-     * If you need to use the input multiple times in your crawler,
-     * it is far more efficient to read it once and store it locally.
-     *
-     * For more information, see {@link KeyValueStore.open}
-     * and {@link KeyValueStore.getValue}.
-     *
-     * @returns
-     *   Returns a promise that resolves to an object, string
-     *   or [`Buffer`](https://nodejs.org/api/buffer.html), depending
-     *   on the MIME content type of the record, or `null`
-     *   if the record is missing.
-     * @ignore
-     */
-    static getInput<T = Dictionary | string | Buffer>(): Promise<T | null>;
 }
 /**
  * User-function used in the  {@link KeyValueStore.forEachKey} method.
@@ -377,30 +505,29 @@ export interface KeyConsumer {
         size: number;
     }): Awaitable<void>;
 }
+/** @internal */
 export interface KeyValueStoreOptions {
-    id: string;
-    name?: string;
-    client: StorageClient;
-    storageObject?: Record<string, unknown>;
+    /** Resolved metadata for the key-value store, as returned by the backend's `getMetadata()`. */
+    metadata: KeyValueStoreInfo;
+    backend: KeyValueStoreBackend;
+}
+/**
+ * A raw, unparsed key-value store record as returned by {@link KeyValueStore.getRecord}: the
+ * verbatim bytes plus the content type, with parsing left to the caller.
+ */
+export interface KeyValueStoreRawRecord {
+    value: Buffer | ArrayBuffer;
+    contentType: string | null;
 }
 export interface RecordOptions {
     /**
      * Specifies a custom MIME content type of the record.
      */
     contentType?: string;
-    /**
-     * Specifies a custom timeout for the `set-record` API call, in seconds.
-     */
-    timeoutSecs?: number;
-    /**
-     * If set to `true`, the `set-record` API call will not be retried if it times out.
-     */
-    doNotRetryTimeouts?: boolean;
 }
 export interface KeyValueStoreIteratorOptions {
     /**
-     * All keys up to this one (including) are skipped from the result.
+     * If set, only keys that start with this prefix are returned.
      */
-    exclusiveStartKey?: string;
+    prefix?: string;
 }
-//# sourceMappingURL=key_value_store.d.ts.map