@crawlee/core 4.0.0-beta.2 → 4.0.0-beta.200

package/storages/key_value_store.js

@@ -1,42 +1,31 @@
-import { readFile } from 'node:fs/promises';
-import { join } from 'node:path';
-import JSON5 from 'json5';
-import ow, { ArgumentError } from 'ow';
+import { z } from 'zod';
 import { KEY_VALUE_STORE_KEY_REGEX } from '@apify/consts';
-import log from '@apify/log';
-import { jsonStringifyExtended } from '@apify/utilities';
+import { tryCancel } from '@apify/timeout';
 import { Configuration } from '../configuration.js';
-import { checkStorageAccess } from './access_checking.js';
-import { StorageManager } from './storage_manager.js';
-import { purgeDefaultStorages } from './utils.js';
-/**
- * Helper function to possibly stringify value if options.contentType is not set.
- *
- * @ignore
- */
-export const maybeStringify = (value, options) => {
-    // If contentType is missing, value will be stringified to JSON
-    if (options.contentType === null || options.contentType === undefined) {
-        options.contentType = 'application/json; charset=utf-8';
-        try {
-            // Format JSON to simplify debugging, the overheads with compression is negligible
-            value = jsonStringifyExtended(value, null, 2);
-        }
-        catch (e) {
-            const error = e;
-            // Give more meaningful error message
-            if (error.message?.includes('Invalid string length')) {
-                error.message = 'Object is too large';
-            }
-            throw new Error(`The "value" parameter cannot be stringified to JSON: ${error.message}`);
-        }
-        if (value === undefined) {
-            throw new Error('The "value" parameter was stringified to JSON and returned undefined. ' +
-                "Make sure you're not trying to stringify an undefined value.");
-        }
-    }
-    return value;
-};
+import { serviceLocator } from '../service_locator.js';
+import { parseArgument, schemas, validators } from '../validators.js';
+import { activeStorageTransaction, operationRejectedInTransaction, rejectOperationInTransaction, snapshotValue, withDirectStorageAccess, } from './transaction.js';
+import { parseValue, serializeValue } from './key_value_store_codec.js';
+import { StorageStatsTracker } from './storage_stats.js';
+import { resolveStorageIdentifier } from './storage_instance_manager.js';
+import { createDualIterable, purgeDefaultStorages } from './utils.js';
+import { isBuffer, isStream } from '../byte_utils.js';
+/** @internal */
+const KVS_KEYS_DEFAULT_LIMIT = 1000;
+const keySchema = z.string().nonempty();
+const setValueKeySchema = z.string().nonempty().regex(KEY_VALUE_STORE_KEY_REGEX, {
+    message: `The "key" argument must be at most 256 characters long and only contain the following characters: a-zA-Z0-9!-_.'()`,
+});
+const recordOptionsSchema = z.strictObject({
+    contentType: z.string().nonempty().optional(),
+});
+const iteratorOptionsSchema = z.strictObject({
+    prefix: z.string().optional(),
+});
+const openOptionsSchema = z.strictObject({
+    configuration: z.instanceof(Configuration).optional(),
+    storageBackend: validators.storageBackend.optional(),
+});
 /**
  * The `KeyValueStore` class represents a key-value store, a simple data storage that is used
  * for saving and reading data records or files. Each data record is
@@ -53,8 +42,6 @@ export const maybeStringify = (value, options) => {
  * To access the default key-value store directly, you can use the
  * {@link KeyValueStore.getValue} and {@link KeyValueStore.setValue} convenience functions.
  *
- * To access the input, you can also use the {@link KeyValueStore.getInput} convenience function.
- *
  * `KeyValueStore` stores its data on a local disk.
  *
  * If the `CRAWLEE_STORAGE_DIR` environment variable is set, the data is stored in
@@ -62,15 +49,14 @@ export const maybeStringify = (value, options) => {
  * ```
  * {CRAWLEE_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}
  * ```
- * Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,
- * unless you override it by setting the `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
+ * Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`.
  * The `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.
  *
  * **Example usage:**
  *
  * ```javascript
  * // Get crawler input from the default key-value store.
- * const input = await KeyValueStore.getInput();
+ * const input = await KeyValueStore.getValue('INPUT');
  * // Get some value from the default key-value store.
  * const otherValue = await KeyValueStore.getValue('my-key');
  *
@@ -94,23 +80,34 @@ export const maybeStringify = (value, options) => {
  * @category Result Stores
  */
 export class KeyValueStore {
-    config;
+    configuration;
     id;
     name;
-    storageObject;
-    client;
-    persistStateEventStarted = false;
+    backend;
+    #persistStateEventStarted = false;
     /** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
-    cache = new Map();
+    #cache = new Map();
+    #statsTracker = new StorageStatsTracker({
+        readCount: 0,
+        writeCount: 0,
+        deleteCount: 0,
+        listCount: 0,
+    });
     /**
      * @internal
      */
-    constructor(options, config = Configuration.getGlobalConfig()) {
-        this.config = config;
-        this.id = options.id;
-        this.name = options.name;
-        this.storageObject = options.storageObject;
-        this.client = options.client.keyValueStore(this.id);
+    constructor(options, configuration = Configuration.getGlobalConfiguration()) {
+        this.configuration = configuration;
+        this.id = options.metadata.id;
+        this.name = options.metadata.name;
+        this.backend = options.backend;
+    }
+    /**
+     * Backend-independent usage counters tracked for this key-value store (read / write / delete /
+     * list operations issued to the underlying storage backend). Counted per backend call.
+     */
+    get stats() {
+        return this.#statsTracker.current;
     }
     /**
      * Gets a value from the key-value store.
@@ -145,10 +142,93 @@ export class KeyValueStore {
      *   on the MIME content type of the record, or `null` if the key is missing from the store.
      */
     async getValue(key, defaultValue) {
-        checkStorageAccess();
-        ow(key, ow.string.nonEmpty);
-        const record = await this.client.getRecord(key);
-        return record?.value ?? defaultValue ?? null;
+        tryCancel();
+        parseArgument(key, keySchema);
+        const record = await this.readRecord(key);
+        // A missing record falls back to the default; a record that parses to a falsy value (including
+        // a stored literal `null`) is returned verbatim, so callers can tell "stored null" from "absent".
+        if (!record) {
+            return defaultValue ?? null;
+        }
+        // Storage backends are byte transports — the value is raw bytes; the frontend parses it here.
+        return parseValue(record.value, record.contentType ?? null);
+    }
+    /**
+     * The active transaction's last buffered write per key for this store, derived from its journal.
+     * An entry with a `null` value is a tombstone (an in-transaction deletion).
+     */
+    bufferedJournalEntries() {
+        const transaction = activeStorageTransaction();
+        if (!transaction)
+            return undefined;
+        const lastWritePerKey = new Map();
+        for (const entry of transaction.journal) {
+            if (entry.type === 'keyValueStore' && entry.participant === this) {
+                lastWritePerKey.set(entry.key, entry);
+            }
+        }
+        return lastWritePerKey;
+    }
+    /**
+     * The single transaction-aware record read shared by `getValue`, `getRecord`, `recordExists` and the
+     * listing paths: buffered key → serialized through the standard codec (same fidelity as a real
+     * round-trip); tombstoned key → `null`; otherwise the backend.
+     *
+     * The per-key buffered lookup requires the whole journal to be reduced to a last-write-per-key map,
+     * which is O(journal). Single-record callers let it default (rebuilt per call); the listing paths,
+     * which read many keys, pass a map built once so the read stays O(1) per key instead of O(journal).
+     */
+    async readRecord(key, buffered = this.bufferedJournalEntries()) {
+        const entry = buffered?.get(key);
+        if (entry) {
+            if (entry.value === null) {
+                return null;
+            }
+            const serialized = serializeValue(entry.value, entry.options?.contentType);
+            return {
+                value: normalizeSerializedValue(serialized.value),
+                contentType: serialized.contentType ?? null,
+            };
+        }
+        this.#statsTracker.add('readCount');
+        const record = await this.backend.getValue(key);
+        if (!record)
+            return null;
+        return {
+            value: record.value,
+            contentType: record.contentType ?? null,
+        };
+    }
+    /**
+     * Reads a record from the key-value store without parsing the value.
+     *
+     * Use this when you need the raw bytes and the content type — for example, to run your own
+     * parser (`simdjson`, a custom XML library, etc.) or to forward the bytes verbatim.
+     *
+     * There is no symmetric `setRecord` method, because {@link KeyValueStore.setValue} already
+     * passes a `Buffer` (or `string` / `Stream`) through unchanged when an explicit `contentType`
+     * is provided. To write pre-serialized bytes, call
+     * `setValue(key, buffer, { contentType: 'application/json; charset=utf-8' })`.
+     *
+     * Returns `null` if the record does not exist.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const store = await KeyValueStore.open();
+     * const record = await store.getRecord('huge.json');
+     * if (record) {
+     *     const data = simdjson.parse(record.value);
+     * }
+     * ```
+     *
+     * @param key
+     *   Unique key of the record. It can be at most 256 characters long and only consist
+     *   of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
+     */
+    async getRecord(key) {
+        tryCancel();
+        parseArgument(key, keySchema);
+        return this.readRecord(key);
     }
     /**
      * Tests whether a record with the given key exists in the key-value store without retrieving its value.
@@ -157,45 +237,96 @@ export class KeyValueStore {
      * @returns `true` if the record exists, `false` if it does not.
      */
     async recordExists(key) {
-        checkStorageAccess();
-        ow(key, ow.string.nonEmpty);
-        return this.client.recordExists(key);
+        tryCancel();
+        parseArgument(key, keySchema);
+        const entry = this.bufferedJournalEntries()?.get(key);
+        if (entry) {
+            return entry.value !== null;
+        }
+        return this.backend.recordExists(key);
     }
     async getAutoSavedValue(key, defaultValue = {}) {
-        checkStorageAccess();
-        if (this.cache.has(key)) {
-            return this.cache.get(key);
+        tryCancel();
+        if (this.#cache.has(key)) {
+            return this.#cache.get(key);
         }
-        const value = await this.getValue(key, defaultValue);
+        // Auto-saved state is deliberately *not* transactional. The direct read bypasses any active
+        // transaction - a buffered value seeded into this shared cache would survive a rollback forever.
+        const value = await withDirectStorageAccess(async () => this.getValue(key, defaultValue));
         // The await above could have run in parallel with another call to this function. If the other call finished more quickly,
         // the value will in cache at this point, and returning the new fetched value would introduce two different instances of
         // the auto-saved object, and only the latter one would be persisted.
         // Therefore we re-check the cache here, and if such race condition happened, we drop the fetched value and return the cached one.
-        if (this.cache.has(key)) {
-            return this.cache.get(key);
+        if (this.#cache.has(key)) {
+            return this.#cache.get(key);
         }
-        this.cache.set(key, value);
+        this.#cache.set(key, value);
         this.ensurePersistStateEvent();
         return value;
     }
     ensurePersistStateEvent() {
-        if (this.persistStateEventStarted) {
+        if (this.#persistStateEventStarted) {
             return;
         }
-        // use half the interval of `persistState` to avoid race conditions
-        const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
-        const timeoutSecs = persistStateIntervalMillis / 2_000;
-        this.config.getEventManager().on('persistState', async () => {
+        serviceLocator.getEventManager().on('persistState', async () => {
             const promises = [];
-            for (const [key, value] of this.cache) {
-                promises.push(this.setValue(key, value, {
-                    timeoutSecs,
-                    doNotRetryTimeouts: true,
-                }).catch((error) => log.warning(`Failed to persist the state value to ${key}`, { error })));
+            for (const [key, value] of this.#cache) {
+                promises.push(this.setValue(key, value).catch((error) => serviceLocator.getLogger().warning(`Failed to persist the state value to ${key}`, { error })));
             }
             await Promise.all(promises);
         });
-        this.persistStateEventStarted = true;
+        this.#persistStateEventStarted = true;
+    }
+    async *fetchKeyValuePages(options, mapRecord) {
+        // Reduce the journal once for the whole iteration, not once per key inside `readRecord`.
+        const buffered = this.bufferedJournalEntries();
+        for await (const page of this.fetchKeyPages(options, buffered)) {
+            const results = [];
+            for (const item of page) {
+                // The shared transaction-aware read, so a key that exists only in the transaction resolves
+                // here instead of being dropped (`values()` would disagree with `keys()` on length).
+                const record = await this.readRecord(item.key, buffered);
+                if (record) {
+                    const parsed = parseValue(record.value, record.contentType ?? null);
+                    results.push(mapRecord(item.key, parsed));
+                }
+            }
+            yield results;
+        }
+    }
+    async *fetchKeyPages(options, buffered = this.bufferedJournalEntries(), limit = KVS_KEYS_DEFAULT_LIMIT) {
+        // Buffered keys are emitted first, then the real pages with any buffered (or tombstoned) key
+        // skipped - a merge-join is not an option, since `listKeys` promises no sort order.
+        const shadowedKeys = new Set();
+        if (buffered) {
+            const bufferedItems = [];
+            for (const [key, entry] of buffered) {
+                shadowedKeys.add(key);
+                if (entry.value === null)
+                    continue;
+                if (options.prefix !== undefined && !key.startsWith(options.prefix))
+                    continue;
+                bufferedItems.push(bufferedKeyItemData(key, entry));
+            }
+            if (bufferedItems.length > 0) {
+                bufferedItems.sort((a, b) => (a.key < b.key ? -1 : 1));
+                yield bufferedItems;
+            }
+        }
+        let exclusiveStartKey;
+        while (true) {
+            this.#statsTracker.add('listCount');
+            const { items, isTruncated, nextExclusiveStartKey } = await this.backend.listKeys({
+                ...options,
+                exclusiveStartKey,
+                limit,
+            });
+            yield shadowedKeys.size > 0 ? items.filter((item) => !shadowedKeys.has(item.key)) : items;
+            if (!isTruncated)
+                break;
+            // Paginate from the raw backend cursor - it may reject a key it did not hand out.
+            exclusiveStartKey = nextExclusiveStartKey;
+        }
     }
     /**
      * Saves or deletes a record in the key-value store.
@@ -241,73 +372,110 @@ export class KeyValueStore {
      * @param [options] Record options.
      */
     async setValue(key, value, options = {}) {
-        checkStorageAccess();
-        ow(key, 'key', ow.string.nonEmpty);
-        ow(key, ow.string.validate((k) => ({
-            validator: ow.isValid(k, ow.string.matches(KEY_VALUE_STORE_KEY_REGEX)),
-            message: `The "key" argument "${key}" must be at most 256 characters long and only contain the following characters: a-zA-Z0-9!-_.'()`,
-        })));
-        if (options.contentType &&
-            !(ow.isValid(value, ow.any(ow.string, ow.uint8Array)) ||
-                (ow.isValid(value, ow.object) && typeof value.pipe === 'function'))) {
-            throw new ArgumentError('The "value" parameter must be a String, Buffer or Stream when "options.contentType" is specified.', this.setValue);
+        const transaction = activeStorageTransaction();
+        parseArgument(key, setValueKeySchema);
+        if (options.contentType && !(typeof value === 'string' || isBuffer(value) || isStream(value))) {
+            throw new Error('The "value" parameter must be a String, Buffer, ArrayBuffer, TypedArray, or Stream when "options.contentType" is specified.');
+        }
+        // The parse result is a fresh copy, so we never update what user passed.
+        const optionsCopy = parseArgument(options, recordOptionsSchema);
+        // The whole transaction branch sits *above* the auto-saved cache update below, so a buffered
+        // write touches nothing outside the journal. That cache is shared, process-lifetime frontend
+        // state, so mutating it here would survive a rollback and later be persisted by `persistState`.
+        // The commit replay re-enters this method with no active transaction and updates it then.
+        if (transaction) {
+            if (isStream(value)) {
+                // A stream cannot serve both a read-your-own-writes read and the commit replay. The
+                // transaction is known-active here, so throw directly rather than via the conditional guard.
+                throw operationRejectedInTransaction(`KeyValueStore.setValue() with a stream value (key "${key}")`, 'a stream can only be consumed once, so it cannot be buffered until commit.');
+            }
+            // Validation only, result discarded: the journal snapshot (`structuredClone`) accepts values
+            // JSON cannot, which would otherwise only throw at a later read or at commit.
+            if (value !== null) {
+                serializeValue(value, optionsCopy.contentType);
+            }
+            // One snapshot serves both the reads and the commit replay; `null` is a tombstone.
+            transaction.recordJournalEntry({
+                type: 'keyValueStore',
+                participant: this,
+                storageId: this.id,
+                key,
+                value: value === null ? null : snapshotValue(value),
+                options: optionsCopy,
+            });
+            return;
         }
-        ow(options, ow.object.exactShape({
-            contentType: ow.optional.string.nonEmpty,
-            timeoutSecs: ow.optional.number,
-            doNotRetryTimeouts: ow.optional.boolean,
-        }));
-        // Make copy of options, don't update what user passed.
-        const optionsCopy = { ...options };
         // If we try to set the value of a cached state to a different reference, we need to update the cache accordingly.
-        const cachedValue = this.cache.get(key);
+        const cachedValue = this.#cache.get(key);
         if (cachedValue && cachedValue !== value) {
             if (value === null) {
                 // Cached state can be only object, so a propagation of `null` means removing all its properties.
-                Object.keys(cachedValue).forEach((k) => this.cache.delete(k));
+                Object.keys(cachedValue).forEach((k) => this.#cache.delete(k));
             }
             else if (typeof value === 'object') {
                 // We need to remove the keys that are no longer present in the new value.
                 Object.keys(cachedValue)
                     .filter((k) => !(k in value))
-                    .forEach((k) => this.cache.delete(k));
+                    .forEach((k) => this.#cache.delete(k));
                 // And update the existing ones + add new ones.
                 Object.assign(cachedValue, value);
             }
         }
         // In this case delete the record.
-        if (value === null)
-            return this.client.deleteRecord(key);
-        value = maybeStringify(value, optionsCopy);
-        return this.client.setRecord({
+        if (value === null) {
+            this.#statsTracker.add('deleteCount');
+            return this.backend.deleteValue(key);
+        }
+        const serialized = serializeValue(value, optionsCopy.contentType);
+        this.#statsTracker.add('writeCount');
+        return this.backend.setValue({
             key,
-            value,
-            contentType: optionsCopy.contentType,
-        }, {
-            timeoutSecs: optionsCopy.timeoutSecs,
-            doNotRetryTimeouts: optionsCopy.doNotRetryTimeouts,
+            value: serialized.value,
+            contentType: serialized.contentType,
         });
     }
+    /** @internal */
+    async commitJournalEntries(entries) {
+        // One `setValue` per key, last write wins - idempotent under retry.
+        const lastWritePerKey = new Map();
+        for (const entry of entries) {
+            if (entry.type === 'keyValueStore') {
+                lastWritePerKey.set(entry.key, { value: entry.value, options: entry.options });
+            }
+        }
+        for (const [key, { value, options }] of lastWritePerKey) {
+            await this.setValue(key, value, options);
+        }
+    }
     /**
      * Removes the key-value store either from the Apify cloud storage or from the local directory,
      * depending on the mode of operation.
      */
     async drop() {
-        checkStorageAccess();
-        await this.client.delete();
-        const manager = StorageManager.getManager(KeyValueStore, this.config);
-        manager.closeStorage(this);
+        rejectOperationInTransaction('KeyValueStore.drop()');
+        await this.backend.drop();
+        serviceLocator.getStorageInstanceManager().removeFromCache(this);
+    }
+    /**
+     * Removes all records from the store but keeps the store itself, along with its
+     * {@link KeyValueStore.id|`id`} and {@link KeyValueStore.name|`name`}.
+     */
+    async purge() {
+        rejectOperationInTransaction('KeyValueStore.purge()');
+        await this.backend.purge();
+        // The auto-saved values this cache holds are no longer in the store.
+        this.#cache.clear();
     }
     /** @internal */
     clearCache() {
-        checkStorageAccess();
-        this.cache.clear();
+        rejectOperationInTransaction('KeyValueStore.clearCache()');
+        this.#cache.clear();
     }
     /**
      * Iterates over key-value store keys, yielding each in turn to an `iteratee` function.
      * Each invocation of `iteratee` is called with three arguments: `(key, index, info)`, where `key`
      * is the record key, `index` is a zero-based index of the key in the current iteration
-     * (regardless of `options.exclusiveStartKey`) and `info` is an object that contains a single property `size`
+     * and `info` is an object that contains a single property `size`
      * indicating size of the record in bytes.
      *
      * If the `iteratee` function returns a Promise then it is awaited before the next call.
@@ -325,30 +493,132 @@ export class KeyValueStore {
      * @param [options] All `forEachKey()` parameters.
      */
     async forEachKey(iteratee, options = {}) {
-        checkStorageAccess();
-        return this._forEachKey(iteratee, options);
-    }
-    async _forEachKey(iteratee, options = {}, index = 0) {
-        const { exclusiveStartKey } = options;
-        ow(iteratee, ow.function);
-        ow(options, ow.object.exactShape({
-            exclusiveStartKey: ow.optional.string,
-        }));
-        const response = await this.client.listKeys({ exclusiveStartKey });
-        const { nextExclusiveStartKey, isTruncated, items } = response;
-        for (const item of items) {
-            await iteratee(item.key, index++, { size: item.size });
+        tryCancel();
+        parseArgument(iteratee, schemas.anyFunction);
+        const parsedOptions = parseArgument(options, iteratorOptionsSchema);
+        let index = 0;
+        for await (const page of this.fetchKeyPages(parsedOptions)) {
+            for (const item of page) {
+                await iteratee(item.key, index++, { size: item.size });
+            }
         }
-        return isTruncated
-            ? this._forEachKey(iteratee, { exclusiveStartKey: nextExclusiveStartKey }, index)
-            : undefined; // [].forEach() returns undefined.
+    }
+    /**
+     * Returns key-value store keys.
+     *
+     * When awaited (`await store.keys()`), returns all keys as a flat `string[]` array.
+     * When used as an async iterable (`for await...of`), iterates over all keys across pages
+     * without loading everything into memory at once.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const keyValueStore = await KeyValueStore.open();
+     *
+     * // Iterate over all keys (memory-efficient for large stores)
+     * for await (const key of keyValueStore.keys()) {
+     *   console.log(key);
+     * }
+     *
+     * // Or fetch all keys at once
+     * const allKeys = await keyValueStore.keys();
+     * console.log(allKeys);
+     * ```
+     *
+     * @param options Options for the iteration.
+     */
+    keys(options = {}) {
+        tryCancel();
+        return createDualIterable({
+            createPages: () => this.fetchKeyPages(options),
+            extractItems: (page) => page.map((item) => item.key),
+        });
+    }
+    /**
+     * Returns key-value store values.
+     *
+     * When awaited (`await store.values()`), returns all values as a flat `T[]` array.
+     * When used as an async iterable (`for await...of`), iterates over all values across pages
+     * without loading everything into memory at once.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const keyValueStore = await KeyValueStore.open();
+     *
+     * // Iterate over all values (memory-efficient for large stores)
+     * for await (const value of keyValueStore.values()) {
+     *   console.log(value);
+     * }
+     *
+     * // Or fetch all values at once
+     * const allValues = await keyValueStore.values();
+     * console.log(allValues);
+     * ```
+     *
+     * @param options Options for the iteration.
+     */
+    values(options = {}) {
+        tryCancel();
+        return createDualIterable({
+            createPages: () => this.fetchKeyValuePages(options, (_key, value) => value),
+            extractItems: (page) => page,
+        });
+    }
+    /**
+     * Returns key-value store entries (key-value pairs).
+     *
+     * When awaited (`await store.entries()`), returns all entries as a flat `[key, value][]` array.
+     * When used as an async iterable (`for await...of`), iterates over all entries across pages
+     * without loading everything into memory at once.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const keyValueStore = await KeyValueStore.open();
+     *
+     * // Iterate over all entries (memory-efficient for large stores)
+     * for await (const [key, value] of keyValueStore.entries()) {
+     *   console.log(`${key}: ${value}`);
+     * }
+     *
+     * // Or fetch all entries at once
+     * const allEntries = await keyValueStore.entries();
+     * console.log(allEntries);
+     * ```
+     *
+     * @param options Options for the iteration.
+     */
+    entries(options = {}) {
+        tryCancel();
+        return createDualIterable({
+            createPages: () => this.fetchKeyValuePages(options, (key, value) => [key, value]),
+            extractItems: (page) => page,
+        });
+    }
+    /**
+     * Default async iterator for the key-value store, iterating over entries (key-value pairs).
+     * Allows using the store directly in a `for await...of` loop.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const keyValueStore = await KeyValueStore.open();
+     * for await (const [key, value] of keyValueStore) {
+     *   console.log(`${key}: ${value}`);
+     * }
+     * ```
+     */
+    async *[Symbol.asyncIterator]() {
+        yield* this.entries();
     }
     /**
      * Returns a file URL for the given key.
+     *
+     * The URL is derived from the key, so it is also returned for a record that does not exist (yet) —
+     * including one written earlier in an uncommitted storage transaction. Returns `undefined` only when
+     * the storage has no file URLs at all (e.g. the in-memory storage).
+     *
+     * @param key The key of the record to generate the public URL for.
      */
-    getPublicUrl(key) {
-        const name = this.name ?? this.config.get('defaultKeyValueStoreId');
-        return `file://${process.cwd()}/storage/key_value_stores/${name}/${key}`;
+    async getPublicUrl(key) {
+        return this.backend.getPublicUrl(key);
     }
     /**
      * Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
@@ -359,23 +629,24 @@ export class KeyValueStore {
      *
      * For more details and code examples, see the {@link KeyValueStore} class.
      *
-     * @param [storeIdOrName]
-     *   ID or name of the key-value store to be opened. If `null` or `undefined`,
-     *   the function returns the default key-value store associated with the crawler run.
+     * @param [identifier]
+     *   ID or name of the key-value store to be opened. If a string is provided, it will first be
+     *   looked up as an ID; if no such storage exists, it will be treated as a name.
+     *   If `null` or `undefined`, the function returns the default key-value store associated with the crawler run.
      * @param [options] Storage manager options.
      */
-    static async open(storeIdOrName, options = {}) {
-        checkStorageAccess();
-        ow(storeIdOrName, ow.optional.any(ow.string, ow.null));
-        ow(options, ow.object.exactShape({
-            config: ow.optional.object.instanceOf(Configuration),
-            storageClient: ow.optional.object,
-        }));
-        options.config ??= Configuration.getGlobalConfig();
-        options.storageClient ??= options.config.getStorageClient();
-        await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
-        const manager = StorageManager.getManager(this, options.config);
-        return manager.openStorage(storeIdOrName, options.storageClient);
+    static async open(identifier, options = {}) {
+        tryCancel();
+        const parsedOptions = parseArgument(options, openOptionsSchema);
+        const configuration = parsedOptions.configuration ?? Configuration.getGlobalConfiguration();
+        const storageBackend = parsedOptions.storageBackend ?? serviceLocator.getStorageBackend();
+        await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration });
+        const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'KeyValueStore');
+        return serviceLocator.getStorageInstanceManager().openStorage(this, {
+            ...resolved,
+            backendOpener: () => storageBackend.createKeyValueStoreBackend(resolved),
+            backendCacheKey: storageBackend.getStorageBackendCacheKey?.() ?? storageBackend.constructor.name,
+        });
     }
     /**
      * Gets a value from the default {@link KeyValueStore} associated with the current crawler run.
@@ -410,6 +681,22 @@ export class KeyValueStore {
         const store = await this.open();
         return store.getValue(key, defaultValue);
     }
+    /**
+     * Reads a record from the default {@link KeyValueStore} associated with the current crawler run
+     * without parsing the value.
+     *
+     * This is just a convenient shortcut for {@link KeyValueStore.getRecord}. Returns `null` if the
+     * record does not exist.
+     *
+     * @param key
+     *   Unique key of the record. It can be at most 256 characters long and only consist
+     *   of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
+     * @ignore
+     */
+    static async getRecord(key) {
+        const store = await this.open();
+        return store.getRecord(key);
+    }
     /**
      * Tests whether a record with the given key exists in the default {@link KeyValueStore} associated with the current crawler run.
      * @param key The queried record key.
@@ -458,50 +745,23 @@ export class KeyValueStore {
         const store = await this.open();
         return store.setValue(key, value, options);
     }
-    /**
-     * Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
-     * By default, it will try to find root input files (either extension-less, `.json` or `.txt`),
-     * or alternatively read the input from the default {@link KeyValueStore}.
-     *
-     * Note that the `getInput()` function does not cache the value read from the key-value store.
-     * If you need to use the input multiple times in your crawler,
-     * it is far more efficient to read it once and store it locally.
-     *
-     * For more information, see {@link KeyValueStore.open}
-     * and {@link KeyValueStore.getValue}.
-     *
-     * @returns
-     *   Returns a promise that resolves to an object, string
-     *   or [`Buffer`](https://nodejs.org/api/buffer.html), depending
-     *   on the MIME content type of the record, or `null`
-     *   if the record is missing.
-     * @ignore
-     */
-    static async getInput() {
-        const store = await this.open();
-        const inputKey = store.config.get('inputKey');
-        const cwd = process.cwd();
-        const possibleExtensions = ['', '.json', '.txt'];
-        // Attempt to read input from root file instead of key-value store
-        for (const extension of possibleExtensions) {
-            const inputFile = join(cwd, `${inputKey}${extension}`);
-            let input;
-            // Try getting the file from the file system
-            try {
-                input = await readFile(inputFile);
-            }
-            catch {
-                continue;
-            }
-            // Attempt to parse as JSON, or return the input as is otherwise
-            try {
-                return JSON5.parse(input.toString());
-            }
-            catch {
-                return input;
-            }
-        }
-        return store.getValue(inputKey);
+}
+/** Normalizes a codec-serialized value into the `Buffer | ArrayBuffer` shape raw record reads promise. */
+function normalizeSerializedValue(value) {
+    if (typeof value === 'string') {
+        return Buffer.from(value);
+    }
+    if (ArrayBuffer.isView(value)) {
+        return Buffer.isBuffer(value) ? value : Buffer.from(value.buffer, value.byteOffset, value.byteLength);
     }
+    return value;
+}
+/** Computes the key listing item (serialized byte size and content type) of a buffered entry. */
+function bufferedKeyItemData(key, entry) {
+    const serialized = serializeValue(entry.value, entry.options?.contentType);
+    return {
+        key,
+        size: normalizeSerializedValue(serialized.value).byteLength,
+        contentType: serialized.contentType,
+    };
 }
-//# sourceMappingURL=key_value_store.js.map