@crawlee/core 4.0.0-beta.23 → 4.0.0-beta.230

package/storages/key-value-store-codec.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Canonical write path for key-value store records.
+ *
+ * When a content type is provided, the value passes through unchanged — it is the caller's
+ * responsibility to supply a String/Buffer/Stream (the frontend validates this).
+ *
+ * When no content type is provided, it is inferred from the value's shape:
+ * - Buffer / typed array / ArrayBuffer / stream → `application/octet-stream` (passthrough)
+ * - `string` → `text/plain; charset=utf-8` (passthrough)
+ * - anything else → `application/json; charset=utf-8` (serialized via `jsonStringifyExtended`)
+ *
+ * Does NOT drain streams — that is storage mechanics and stays in the storage backend.
+ *
+ * Backend-independent.
+ */
+export declare function serializeValue(value: unknown, contentType?: string): {
+    value: Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
+    contentType: string;
+};
+/**
+ * Parses a Buffer or ArrayBuffer using the provided content type header.
+ *
+ * - application/json is returned as a parsed object.
+ * - application/*xml and text/* are returned as strings.
+ * - everything else is returned as original body.
+ *
+ * If the header includes a charset, the body will be stringified only
+ * if the charset represents a known encoding to Node.js or Browser.
+ *
+ * Backend-independent — this is the canonical read path for the {@link KeyValueStore} frontend.
+ */
+export declare function parseValue(body: Buffer | ArrayBuffer | string, contentTypeHeader: string | null): string | Buffer | ArrayBuffer | Record<string, unknown>;

package/storages/key-value-store-codec.js

@@ -0,0 +1,108 @@
+import { isTypeValid, parse as parseContentType } from 'content-type';
+import JSON5 from 'json5';
+import { jsonStringifyExtended } from '@apify/utilities';
+import { isBuffer, isStream } from '../byte-utils.js';
+const CONTENT_TYPE_JSON = 'application/json';
+const STRINGIFIABLE_CONTENT_TYPE_RXS = [new RegExp(`^${CONTENT_TYPE_JSON}$`, 'i'), /^application\/.*xml$/i, /^text\//i];
+/**
+ * Canonical write path for key-value store records.
+ *
+ * When a content type is provided, the value passes through unchanged — it is the caller's
+ * responsibility to supply a String/Buffer/Stream (the frontend validates this).
+ *
+ * When no content type is provided, it is inferred from the value's shape:
+ * - Buffer / typed array / ArrayBuffer / stream → `application/octet-stream` (passthrough)
+ * - `string` → `text/plain; charset=utf-8` (passthrough)
+ * - anything else → `application/json; charset=utf-8` (serialized via `jsonStringifyExtended`)
+ *
+ * Does NOT drain streams — that is storage mechanics and stays in the storage backend.
+ *
+ * Backend-independent.
+ */
+export function serializeValue(value, contentType) {
+    if (contentType !== null && contentType !== undefined) {
+        return { value: value, contentType };
+    }
+    if (isStream(value) || isBuffer(value)) {
+        return {
+            value,
+            contentType: 'application/octet-stream',
+        };
+    }
+    if (typeof value === 'string') {
+        return { value, contentType: 'text/plain; charset=utf-8' };
+    }
+    let serialized;
+    try {
+        // Format JSON to simplify debugging, the overheads with compression is negligible
+        serialized = jsonStringifyExtended(value, null, 2);
+    }
+    catch (e) {
+        const error = e;
+        // Give more meaningful error message
+        if (error.message?.includes('Invalid string length')) {
+            error.message = 'Object is too large';
+        }
+        throw new Error(`The "value" parameter cannot be stringified to JSON: ${error.message}`);
+    }
+    if (serialized === undefined) {
+        throw new Error('The "value" parameter was stringified to JSON and returned undefined. ' +
+            "Make sure you're not trying to stringify an undefined value.");
+    }
+    return { value: serialized, contentType: 'application/json; charset=utf-8' };
+}
+/**
+ * Parses a Buffer or ArrayBuffer using the provided content type header.
+ *
+ * - application/json is returned as a parsed object.
+ * - application/*xml and text/* are returned as strings.
+ * - everything else is returned as original body.
+ *
+ * If the header includes a charset, the body will be stringified only
+ * if the charset represents a known encoding to Node.js or Browser.
+ *
+ * Backend-independent — this is the canonical read path for the {@link KeyValueStore} frontend.
+ */
+export function parseValue(body, contentTypeHeader) {
+    // No content type at all → we have no basis for interpretation; hand back the raw value.
+    if (contentTypeHeader === null)
+        return body;
+    const result = parseContentType(contentTypeHeader);
+    // Unparseable header → keep the original value rather than a mangled string.
+    if (!isTypeValid(result.type))
+        return body;
+    const contentType = result.type;
+    const charset = result.parameters.charset;
+    // If we can't successfully interpret it, we return the original value rather than mangling it.
+    if (!areDataStringifiable(contentType, charset))
+        return body;
+    // Decode raw bytes using the resolved charset. An already-decoded string passes through (callers
+    // may hand us one directly), avoiding a needless re-encode round-trip.
+    const dataString = typeof body === 'string' ? body : isomorphicBufferToString(body, charset);
+    return contentType === CONTENT_TYPE_JSON ? JSON5.parse(dataString) : dataString;
+}
+function isomorphicBufferToString(buffer, encoding) {
+    if (buffer.constructor.name !== ArrayBuffer.name) {
+        return buffer.toString(encoding);
+    }
+    // In Node, wrap the ArrayBuffer in a Buffer so the resolved charset is honored (the caller already
+    // checked it via `Buffer.isEncoding`). Only the browser, which lacks Buffer, is limited to UTF-8.
+    if (typeof Buffer !== 'undefined') {
+        return Buffer.from(buffer).toString(encoding);
+    }
+    const decoder = new TextDecoder(encoding);
+    return decoder.decode(new Uint8Array(buffer));
+}
+function isCharsetStringifiable(charset) {
+    if (!charset)
+        return true; // hope that it's utf-8
+    return Buffer.isEncoding(charset);
+}
+function isContentTypeStringifiable(contentType) {
+    if (!contentType)
+        return false; // keep buffer
+    return STRINGIFIABLE_CONTENT_TYPE_RXS.some((rx) => rx.test(contentType));
+}
+function areDataStringifiable(contentType, charset) {
+    return isContentTypeStringifiable(contentType) && isCharsetStringifiable(charset);
+}

package/storages/{key_value_store.d.ts → key-value-store.d.ts}

@@ -1,15 +1,9 @@
-import type { Dictionary, StorageClient } from '@crawlee/types';
+import type { Awaitable, Dictionary, KeyValueStoreBackend, KeyValueStoreInfo } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
-import type { Awaitable } from '../typedefs.js';
-import type { StorageManagerOptions } from './storage_manager.js';
-/**
- * Helper function to possibly stringify value if options.contentType is not set.
- *
- * @ignore
- */
-export declare const maybeStringify: <T>(value: T, options: {
-    contentType?: string;
-}) => T;
+import type { JournalEntry } from './transaction.js';
+import type { KeyValueStoreStats } from './storage-stats.js';
+import type { StorageOpenOptions } from './utils.js';
+import type { StorageIdentifier } from './storage-instance-manager.js';
 /**
  * The `KeyValueStore` class represents a key-value store, a simple data storage that is used
  * for saving and reading data records or files. Each data record is
@@ -26,8 +20,6 @@ export declare const maybeStringify: <T>(value: T, options: {
  * To access the default key-value store directly, you can use the
  * {@link KeyValueStore.getValue} and {@link KeyValueStore.setValue} convenience functions.
  *
- * To access the input, you can also use the {@link KeyValueStore.getInput} convenience function.
- *
  * `KeyValueStore` stores its data on a local disk.
  *
  * If the `CRAWLEE_STORAGE_DIR` environment variable is set, the data is stored in
@@ -35,15 +27,14 @@ export declare const maybeStringify: <T>(value: T, options: {
  * ```
  * {CRAWLEE_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}
  * ```
- * Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,
- * unless you override it by setting the `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
+ * Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`.
  * The `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.
  *
  * **Example usage:**
  *
  * ```javascript
  * // Get crawler input from the default key-value store.
- * const input = await KeyValueStore.getInput();
+ * const input = await KeyValueStore.getValue('INPUT');
  * // Get some value from the default key-value store.
  * const otherValue = await KeyValueStore.getValue('my-key');
  *
@@ -67,18 +58,20 @@ export declare const maybeStringify: <T>(value: T, options: {
  * @category Result Stores
  */
 export declare class KeyValueStore {
-    readonly config: Configuration;
+    #private;
+    readonly configuration: Configuration;
     readonly id: string;
     readonly name?: string;
-    readonly storageObject?: Record<string, unknown>;
-    private readonly client;
-    private persistStateEventStarted;
-    /** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
-    private readonly cache;
+    readonly backend: KeyValueStoreBackend;
     /**
      * @internal
      */
-    constructor(options: KeyValueStoreOptions, config?: Configuration);
+    constructor(options: KeyValueStoreOptions, configuration?: Configuration);
+    /**
+     * Backend-independent usage counters tracked for this key-value store (read / write / delete /
+     * list operations issued to the underlying storage backend). Counted per backend call.
+     */
+    get stats(): KeyValueStoreStats;
     /**
      * Gets a value from the key-value store.
      *
@@ -143,6 +136,33 @@ export declare class KeyValueStore {
      *   on the MIME content type of the record, or the default value if the key is missing from the store.
      */
     getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
+    /**
+     * Reads a record from the key-value store without parsing the value.
+     *
+     * Use this when you need the raw bytes and the content type — for example, to run your own
+     * parser (`simdjson`, a custom XML library, etc.) or to forward the bytes verbatim.
+     *
+     * There is no symmetric `setRecord` method, because {@link KeyValueStore.setValue} already
+     * passes a `Buffer` (or `string` / `Stream`) through unchanged when an explicit `contentType`
+     * is provided. To write pre-serialized bytes, call
+     * `setValue(key, buffer, { contentType: 'application/json; charset=utf-8' })`.
+     *
+     * Returns `null` if the record does not exist.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const store = await KeyValueStore.open();
+     * const record = await store.getRecord('huge.json');
+     * if (record) {
+     *     const data = simdjson.parse(record.value);
+     * }
+     * ```
+     *
+     * @param key
+     *   Unique key of the record. It can be at most 256 characters long and only consist
+     *   of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
+     */
+    getRecord(key: string): Promise<KeyValueStoreRawRecord | null>;
     /**
      * Tests whether a record with the given key exists in the key-value store without retrieving its value.
      *
@@ -151,7 +171,6 @@ export declare class KeyValueStore {
      */
     recordExists(key: string): Promise<boolean>;
     getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
-    private ensurePersistStateEvent;
     /**
      * Saves or deletes a record in the key-value store.
      * The function returns a promise that resolves once the record has been saved or deleted.
@@ -196,18 +215,25 @@ export declare class KeyValueStore {
      * @param [options] Record options.
      */
     setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
+    /** @internal */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
      * Removes the key-value store either from the Apify cloud storage or from the local directory,
      * depending on the mode of operation.
      */
     drop(): Promise<void>;
+    /**
+     * Removes all records from the store but keeps the store itself, along with its
+     * {@link KeyValueStore.id|`id`} and {@link KeyValueStore.name|`name`}.
+     */
+    purge(): Promise<void>;
     /** @internal */
     clearCache(): void;
     /**
      * Iterates over key-value store keys, yielding each in turn to an `iteratee` function.
      * Each invocation of `iteratee` is called with three arguments: `(key, index, info)`, where `key`
      * is the record key, `index` is a zero-based index of the key in the current iteration
-     * (regardless of `options.exclusiveStartKey`) and `info` is an object that contains a single property `size`
+     * and `info` is an object that contains a single property `size`
      * indicating size of the record in bytes.
      *
      * If the `iteratee` function returns a Promise then it is awaited before the next call.
@@ -225,11 +251,97 @@ export declare class KeyValueStore {
      * @param [options] All `forEachKey()` parameters.
      */
     forEachKey(iteratee: KeyConsumer, options?: KeyValueStoreIteratorOptions): Promise<void>;
-    private _forEachKey;
+    /**
+     * Returns key-value store keys.
+     *
+     * When awaited (`await store.keys()`), returns all keys as a flat `string[]` array.
+     * When used as an async iterable (`for await...of`), iterates over all keys across pages
+     * without loading everything into memory at once.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const keyValueStore = await KeyValueStore.open();
+     *
+     * // Iterate over all keys (memory-efficient for large stores)
+     * for await (const key of keyValueStore.keys()) {
+     *   console.log(key);
+     * }
+     *
+     * // Or fetch all keys at once
+     * const allKeys = await keyValueStore.keys();
+     * console.log(allKeys);
+     * ```
+     *
+     * @param options Options for the iteration.
+     */
+    keys(options?: KeyValueStoreIteratorOptions): AsyncIterable<string> & Promise<string[]>;
+    /**
+     * Returns key-value store values.
+     *
+     * When awaited (`await store.values()`), returns all values as a flat `T[]` array.
+     * When used as an async iterable (`for await...of`), iterates over all values across pages
+     * without loading everything into memory at once.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const keyValueStore = await KeyValueStore.open();
+     *
+     * // Iterate over all values (memory-efficient for large stores)
+     * for await (const value of keyValueStore.values()) {
+     *   console.log(value);
+     * }
+     *
+     * // Or fetch all values at once
+     * const allValues = await keyValueStore.values();
+     * console.log(allValues);
+     * ```
+     *
+     * @param options Options for the iteration.
+     */
+    values<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<T> & Promise<T[]>;
+    /**
+     * Returns key-value store entries (key-value pairs).
+     *
+     * When awaited (`await store.entries()`), returns all entries as a flat `[key, value][]` array.
+     * When used as an async iterable (`for await...of`), iterates over all entries across pages
+     * without loading everything into memory at once.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const keyValueStore = await KeyValueStore.open();
+     *
+     * // Iterate over all entries (memory-efficient for large stores)
+     * for await (const [key, value] of keyValueStore.entries()) {
+     *   console.log(`${key}: ${value}`);
+     * }
+     *
+     * // Or fetch all entries at once
+     * const allEntries = await keyValueStore.entries();
+     * console.log(allEntries);
+     * ```
+     *
+     * @param options Options for the iteration.
+     */
+    entries<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<[string, T]> & Promise<[string, T][]>;
+    /**
+     * Default async iterator for the key-value store, iterating over entries (key-value pairs).
+     * Allows using the store directly in a `for await...of` loop.
+     *
+     * **Example usage:**
+     * ```javascript
+     * const keyValueStore = await KeyValueStore.open();
+     * for await (const [key, value] of keyValueStore) {
+     *   console.log(`${key}: ${value}`);
+     * }
+     * ```
+     */
+    [Symbol.asyncIterator]<T = unknown>(): AsyncGenerator<[string, T], void, undefined>;
     /**
      * Returns a file URL for the given key.
      *
-     * If the record does not exist or has no associated file path (i.e., it is not stored as a file), returns `undefined`.
+     * The URL is derived from the key, so it is also returned for a record that does not exist (yet) —
+     * including one written earlier in an uncommitted storage transaction. Returns `undefined` only when
+     * the storage has no file URLs at all (e.g. the in-memory storage).
      *
      * @param key The key of the record to generate the public URL for.
      */
@@ -243,12 +355,13 @@ export declare class KeyValueStore {
      *
      * For more details and code examples, see the {@link KeyValueStore} class.
      *
-     * @param [storeIdOrName]
-     *   ID or name of the key-value store to be opened. If `null` or `undefined`,
-     *   the function returns the default key-value store associated with the crawler run.
+     * @param [identifier]
+     *   ID or name of the key-value store to be opened. If a string is provided, it will first be
+     *   looked up as an ID; if no such storage exists, it will be treated as a name.
+     *   If `null` or `undefined`, the function returns the default key-value store associated with the crawler run.
      * @param [options] Storage manager options.
      */
-    static open(storeIdOrName?: string | null, options?: StorageManagerOptions): Promise<KeyValueStore>;
+    static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<KeyValueStore>;
     /**
      * Gets a value from the default {@link KeyValueStore} associated with the current crawler run.
      *
@@ -307,6 +420,19 @@ export declare class KeyValueStore {
      * @ignore
      */
     static getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
+    /**
+     * Reads a record from the default {@link KeyValueStore} associated with the current crawler run
+     * without parsing the value.
+     *
+     * This is just a convenient shortcut for {@link KeyValueStore.getRecord}. Returns `null` if the
+     * record does not exist.
+     *
+     * @param key
+     *   Unique key of the record. It can be at most 256 characters long and only consist
+     *   of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
+     * @ignore
+     */
+    static getRecord(key: string): Promise<KeyValueStoreRawRecord | null>;
     /**
      * Tests whether a record with the given key exists in the default {@link KeyValueStore} associated with the current crawler run.
      * @param key The queried record key.
@@ -346,26 +472,6 @@ export declare class KeyValueStore {
      * @ignore
      */
     static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
-    /**
-     * Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
-     * By default, it will try to find root input files (either extension-less, `.json` or `.txt`),
-     * or alternatively read the input from the default {@link KeyValueStore}.
-     *
-     * Note that the `getInput()` function does not cache the value read from the key-value store.
-     * If you need to use the input multiple times in your crawler,
-     * it is far more efficient to read it once and store it locally.
-     *
-     * For more information, see {@link KeyValueStore.open}
-     * and {@link KeyValueStore.getValue}.
-     *
-     * @returns
-     *   Returns a promise that resolves to an object, string
-     *   or [`Buffer`](https://nodejs.org/api/buffer.html), depending
-     *   on the MIME content type of the record, or `null`
-     *   if the record is missing.
-     * @ignore
-     */
-    static getInput<T = Dictionary | string | Buffer>(): Promise<T | null>;
 }
 /**
  * User-function used in the  {@link KeyValueStore.forEachKey} method.
@@ -381,38 +487,29 @@ export interface KeyConsumer {
         size: number;
     }): Awaitable<void>;
 }
+/** @internal */
 export interface KeyValueStoreOptions {
-    id: string;
-    name?: string;
-    client: StorageClient;
-    storageObject?: Record<string, unknown>;
+    /** Resolved metadata for the key-value store, as returned by the backend's `getMetadata()`. */
+    metadata: KeyValueStoreInfo;
+    backend: KeyValueStoreBackend;
+}
+/**
+ * A raw, unparsed key-value store record as returned by {@link KeyValueStore.getRecord}: the
+ * verbatim bytes plus the content type, with parsing left to the caller.
+ */
+export interface KeyValueStoreRawRecord {
+    value: Buffer | ArrayBuffer;
+    contentType: string | null;
 }
 export interface RecordOptions {
     /**
      * Specifies a custom MIME content type of the record.
      */
     contentType?: string;
-    /**
-     * Specifies a custom timeout for the `set-record` API call, in seconds.
-     */
-    timeoutSecs?: number;
-    /**
-     * If set to `true`, the `set-record` API call will not be retried if it times out.
-     */
-    doNotRetryTimeouts?: boolean;
 }
 export interface KeyValueStoreIteratorOptions {
-    /**
-     * All keys up to this one (including) are skipped from the result.
-     */
-    exclusiveStartKey?: string;
     /**
      * If set, only keys that start with this prefix are returned.
      */
     prefix?: string;
-    /**
-     * Collection name to use for listing keys.
-     */
-    collection?: string;
 }
-//# sourceMappingURL=key_value_store.d.ts.map