@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.0

package/storages/key_value_store.js

@@ -1,8 +1,10 @@
-import ow, { ArgumentError } from 'ow';
+import { z } from 'zod';
 import { KEY_VALUE_STORE_KEY_REGEX } from '@apify/consts';
+import { tryCancel } from '@apify/timeout';
 import { Configuration } from '../configuration.js';
 import { serviceLocator } from '../service_locator.js';
-import { checkStorageAccess } from './access_checking.js';
+import { parseArgument, schemas, validators } from '../validators.js';
+import { activeStorageTransaction, operationRejectedInTransaction, rejectOperationInTransaction, snapshotValue, withDirectStorageAccess, } from './transaction.js';
 import { parseValue, serializeValue } from './key_value_store_codec.js';
 import { StorageStatsTracker } from './storage_stats.js';
 import { resolveStorageIdentifier } from './storage_instance_manager.js';
@@ -10,6 +12,20 @@ import { createDualIterable, purgeDefaultStorages } from './utils.js';
 import { isBuffer, isStream } from '../byte_utils.js';
 /** @internal */
 const KVS_KEYS_DEFAULT_LIMIT = 1000;
+const keySchema = z.string().nonempty();
+const setValueKeySchema = z.string().nonempty().regex(KEY_VALUE_STORE_KEY_REGEX, {
+    message: `The "key" argument must be at most 256 characters long and only contain the following characters: a-zA-Z0-9!-_.'()`,
+});
+const recordOptionsSchema = z.strictObject({
+    contentType: z.string().nonempty().optional(),
+});
+const iteratorOptionsSchema = z.strictObject({
+    prefix: z.string().optional(),
+});
+const openOptionsSchema = z.strictObject({
+    configuration: z.instanceof(Configuration).optional(),
+    storageBackend: validators.storageBackend.optional(),
+});
 /**
  * The `KeyValueStore` class represents a key-value store, a simple data storage that is used
  * for saving and reading data records or files. Each data record is
@@ -70,11 +86,12 @@ export class KeyValueStore {
     configuration;
     id;
     name;
+    // kept as TS-private: key_value_store tests spy on the backend directly
     backend;
-    persistStateEventStarted = false;
+    #persistStateEventStarted = false;
     /** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
-    cache = new Map();
-    statsTracker = new StorageStatsTracker({
+    #cache = new Map();
+    #statsTracker = new StorageStatsTracker({
         readCount: 0,
         writeCount: 0,
         deleteCount: 0,
@@ -94,7 +111,7 @@ export class KeyValueStore {
      * list operations issued to the underlying storage backend). Counted per backend call.
      */
     get stats() {
-        return this.statsTracker.current;
+        return this.#statsTracker.current;
     }
     /**
      * Gets a value from the key-value store.
@@ -129,10 +146,9 @@ export class KeyValueStore {
      *   on the MIME content type of the record, or `null` if the key is missing from the store.
      */
     async getValue(key, defaultValue) {
-        checkStorageAccess();
-        ow(key, ow.string.nonEmpty);
-        this.statsTracker.add('readCount');
-        const record = await this.backend.getValue(key);
+        tryCancel();
+        parseArgument(key, keySchema);
+        const record = await this.readRecord(key);
         // A missing record falls back to the default; a record that parses to a falsy value (including
         // a stored literal `null`) is returned verbatim, so callers can tell "stored null" from "absent".
         if (!record) {
@@ -141,6 +157,52 @@ export class KeyValueStore {
         // Storage backends are byte transports — the value is raw bytes; the frontend parses it here.
         return parseValue(record.value, record.contentType ?? null);
     }
+    /**
+     * The active transaction's last buffered write per key for this store, derived from its journal.
+     * An entry with a `null` value is a tombstone (an in-transaction deletion).
+     */
+    bufferedJournalEntries() {
+        const transaction = activeStorageTransaction();
+        if (!transaction)
+            return undefined;
+        const lastWritePerKey = new Map();
+        for (const entry of transaction.journal) {
+            if (entry.type === 'keyValueStore' && entry.participant === this) {
+                lastWritePerKey.set(entry.key, entry);
+            }
+        }
+        return lastWritePerKey;
+    }
+    /**
+     * The single transaction-aware record read shared by `getValue`, `getRecord`, `recordExists` and the
+     * listing paths: buffered key → serialized through the standard codec (same fidelity as a real
+     * round-trip); tombstoned key → `null`; otherwise the backend.
+     *
+     * The per-key buffered lookup requires the whole journal to be reduced to a last-write-per-key map,
+     * which is O(journal). Single-record callers let it default (rebuilt per call); the listing paths,
+     * which read many keys, pass a map built once so the read stays O(1) per key instead of O(journal).
+     */
+    async readRecord(key, buffered = this.bufferedJournalEntries()) {
+        const entry = buffered?.get(key);
+        if (entry) {
+            if (entry.value === null) {
+                return null;
+            }
+            const serialized = serializeValue(entry.value, entry.options?.contentType);
+            return {
+                value: normalizeSerializedValue(serialized.value),
+                contentType: serialized.contentType ?? null,
+            };
+        }
+        this.#statsTracker.add('readCount');
+        const record = await this.backend.getValue(key);
+        if (!record)
+            return null;
+        return {
+            value: record.value,
+            contentType: record.contentType ?? null,
+        };
+    }
     /**
      * Reads a record from the key-value store without parsing the value.
      *
@@ -168,16 +230,9 @@ export class KeyValueStore {
      *   of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
      */
     async getRecord(key) {
-        checkStorageAccess();
-        ow(key, ow.string.nonEmpty);
-        this.statsTracker.add('readCount');
-        const record = await this.backend.getValue(key);
-        if (!record)
-            return null;
-        return {
-            value: record.value,
-            contentType: record.contentType ?? null,
-        };
+        tryCancel();
+        parseArgument(key, keySchema);
+        return this.readRecord(key);
     }
     /**
      * Tests whether a record with the given key exists in the key-value store without retrieving its value.
@@ -186,46 +241,55 @@ export class KeyValueStore {
      * @returns `true` if the record exists, `false` if it does not.
      */
     async recordExists(key) {
-        checkStorageAccess();
-        ow(key, ow.string.nonEmpty);
+        tryCancel();
+        parseArgument(key, keySchema);
+        const entry = this.bufferedJournalEntries()?.get(key);
+        if (entry) {
+            return entry.value !== null;
+        }
         return this.backend.recordExists(key);
     }
     async getAutoSavedValue(key, defaultValue = {}) {
-        checkStorageAccess();
-        if (this.cache.has(key)) {
-            return this.cache.get(key);
+        tryCancel();
+        if (this.#cache.has(key)) {
+            return this.#cache.get(key);
         }
-        const value = await this.getValue(key, defaultValue);
+        // Auto-saved state is deliberately *not* transactional. The direct read bypasses any active
+        // transaction - a buffered value seeded into this shared cache would survive a rollback forever.
+        const value = await withDirectStorageAccess(async () => this.getValue(key, defaultValue));
         // The await above could have run in parallel with another call to this function. If the other call finished more quickly,
         // the value will in cache at this point, and returning the new fetched value would introduce two different instances of
         // the auto-saved object, and only the latter one would be persisted.
         // Therefore we re-check the cache here, and if such race condition happened, we drop the fetched value and return the cached one.
-        if (this.cache.has(key)) {
-            return this.cache.get(key);
+        if (this.#cache.has(key)) {
+            return this.#cache.get(key);
         }
-        this.cache.set(key, value);
+        this.#cache.set(key, value);
         this.ensurePersistStateEvent();
         return value;
     }
     ensurePersistStateEvent() {
-        if (this.persistStateEventStarted) {
+        if (this.#persistStateEventStarted) {
             return;
         }
         serviceLocator.getEventManager().on('persistState', async () => {
             const promises = [];
-            for (const [key, value] of this.cache) {
+            for (const [key, value] of this.#cache) {
                 promises.push(this.setValue(key, value).catch((error) => serviceLocator.getLogger().warning(`Failed to persist the state value to ${key}`, { error })));
             }
             await Promise.all(promises);
         });
-        this.persistStateEventStarted = true;
+        this.#persistStateEventStarted = true;
     }
     async *fetchKeyValuePages(options, mapRecord) {
-        for await (const page of this.fetchKeyPages(options)) {
+        // Reduce the journal once for the whole iteration, not once per key inside `readRecord`.
+        const buffered = this.bufferedJournalEntries();
+        for await (const page of this.fetchKeyPages(options, buffered)) {
             const results = [];
             for (const item of page) {
-                this.statsTracker.add('readCount');
-                const record = await this.backend.getValue(item.key);
+                // The shared transaction-aware read, so a key that exists only in the transaction resolves
+                // here instead of being dropped (`values()` would disagree with `keys()` on length).
+                const record = await this.readRecord(item.key, buffered);
                 if (record) {
                     const parsed = parseValue(record.value, record.contentType ?? null);
                     results.push(mapRecord(item.key, parsed));
@@ -234,18 +298,37 @@ export class KeyValueStore {
             yield results;
         }
     }
-    async *fetchKeyPages(options, limit = KVS_KEYS_DEFAULT_LIMIT) {
+    async *fetchKeyPages(options, buffered = this.bufferedJournalEntries(), limit = KVS_KEYS_DEFAULT_LIMIT) {
+        // Buffered keys are emitted first, then the real pages with any buffered (or tombstoned) key
+        // skipped - a merge-join is not an option, since `listKeys` promises no sort order.
+        const shadowedKeys = new Set();
+        if (buffered) {
+            const bufferedItems = [];
+            for (const [key, entry] of buffered) {
+                shadowedKeys.add(key);
+                if (entry.value === null)
+                    continue;
+                if (options.prefix !== undefined && !key.startsWith(options.prefix))
+                    continue;
+                bufferedItems.push(bufferedKeyItemData(key, entry));
+            }
+            if (bufferedItems.length > 0) {
+                bufferedItems.sort((a, b) => (a.key < b.key ? -1 : 1));
+                yield bufferedItems;
+            }
+        }
         let exclusiveStartKey;
         while (true) {
-            this.statsTracker.add('listCount');
+            this.#statsTracker.add('listCount');
             const { items, isTruncated, nextExclusiveStartKey } = await this.backend.listKeys({
                 ...options,
                 exclusiveStartKey,
                 limit,
             });
-            yield items;
+            yield shadowedKeys.size > 0 ? items.filter((item) => !shadowedKeys.has(item.key)) : items;
             if (!isTruncated)
                 break;
+            // Paginate from the raw backend cursor - it may reject a key it did not hand out.
             exclusiveStartKey = nextExclusiveStartKey;
         }
     }
@@ -293,62 +376,94 @@ export class KeyValueStore {
      * @param [options] Record options.
      */
     async setValue(key, value, options = {}) {
-        checkStorageAccess();
-        ow(key, 'key', ow.string.nonEmpty);
-        ow(key, ow.string.validate((k) => ({
-            validator: ow.isValid(k, ow.string.matches(KEY_VALUE_STORE_KEY_REGEX)),
-            message: `The "key" argument "${key}" must be at most 256 characters long and only contain the following characters: a-zA-Z0-9!-_.'()`,
-        })));
+        const transaction = activeStorageTransaction();
+        parseArgument(key, setValueKeySchema);
         if (options.contentType && !(typeof value === 'string' || isBuffer(value) || isStream(value))) {
-            throw new ArgumentError('The "value" parameter must be a String, Buffer, ArrayBuffer, TypedArray, or Stream when "options.contentType" is specified.', this.setValue);
+            throw new Error('The "value" parameter must be a String, Buffer, ArrayBuffer, TypedArray, or Stream when "options.contentType" is specified.');
+        }
+        // The parse result is a fresh copy, so we never update what user passed.
+        const optionsCopy = parseArgument(options, recordOptionsSchema);
+        // The whole transaction branch sits *above* the auto-saved cache update below, so a buffered
+        // write touches nothing outside the journal. That cache is shared, process-lifetime frontend
+        // state, so mutating it here would survive a rollback and later be persisted by `persistState`.
+        // The commit replay re-enters this method with no active transaction and updates it then.
+        if (transaction) {
+            if (isStream(value)) {
+                // A stream cannot serve both a read-your-own-writes read and the commit replay. The
+                // transaction is known-active here, so throw directly rather than via the conditional guard.
+                throw operationRejectedInTransaction(`KeyValueStore.setValue() with a stream value (key "${key}")`, 'a stream can only be consumed once, so it cannot be buffered until commit.');
+            }
+            // Validation only, result discarded: the journal snapshot (`structuredClone`) accepts values
+            // JSON cannot, which would otherwise only throw at a later read or at commit.
+            if (value !== null) {
+                serializeValue(value, optionsCopy.contentType);
+            }
+            // One snapshot serves both the reads and the commit replay; `null` is a tombstone.
+            transaction.recordJournalEntry({
+                type: 'keyValueStore',
+                participant: this,
+                storageId: this.id,
+                key,
+                value: value === null ? null : snapshotValue(value),
+                options: optionsCopy,
+            });
+            return;
         }
-        ow(options, ow.object.exactShape({
-            contentType: ow.optional.string.nonEmpty,
-        }));
-        // Make copy of options, don't update what user passed.
-        const optionsCopy = { ...options };
         // If we try to set the value of a cached state to a different reference, we need to update the cache accordingly.
-        const cachedValue = this.cache.get(key);
+        const cachedValue = this.#cache.get(key);
         if (cachedValue && cachedValue !== value) {
             if (value === null) {
                 // Cached state can be only object, so a propagation of `null` means removing all its properties.
-                Object.keys(cachedValue).forEach((k) => this.cache.delete(k));
+                Object.keys(cachedValue).forEach((k) => this.#cache.delete(k));
             }
             else if (typeof value === 'object') {
                 // We need to remove the keys that are no longer present in the new value.
                 Object.keys(cachedValue)
                     .filter((k) => !(k in value))
-                    .forEach((k) => this.cache.delete(k));
+                    .forEach((k) => this.#cache.delete(k));
                 // And update the existing ones + add new ones.
                 Object.assign(cachedValue, value);
             }
         }
         // In this case delete the record.
         if (value === null) {
-            this.statsTracker.add('deleteCount');
+            this.#statsTracker.add('deleteCount');
             return this.backend.deleteValue(key);
         }
         const serialized = serializeValue(value, optionsCopy.contentType);
-        this.statsTracker.add('writeCount');
+        this.#statsTracker.add('writeCount');
         return this.backend.setValue({
             key,
             value: serialized.value,
             contentType: serialized.contentType,
         });
     }
+    /** @internal */
+    async commitJournalEntries(entries) {
+        // One `setValue` per key, last write wins - idempotent under retry.
+        const lastWritePerKey = new Map();
+        for (const entry of entries) {
+            if (entry.type === 'keyValueStore') {
+                lastWritePerKey.set(entry.key, { value: entry.value, options: entry.options });
+            }
+        }
+        for (const [key, { value, options }] of lastWritePerKey) {
+            await this.setValue(key, value, options);
+        }
+    }
     /**
      * Removes the key-value store either from the Apify cloud storage or from the local directory,
      * depending on the mode of operation.
      */
     async drop() {
-        checkStorageAccess();
+        rejectOperationInTransaction('KeyValueStore.drop()');
         await this.backend.drop();
         serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
     /** @internal */
     clearCache() {
-        checkStorageAccess();
-        this.cache.clear();
+        rejectOperationInTransaction('KeyValueStore.clearCache()');
+        this.#cache.clear();
     }
     /**
      * Iterates over key-value store keys, yielding each in turn to an `iteratee` function.
@@ -372,13 +487,11 @@ export class KeyValueStore {
      * @param [options] All `forEachKey()` parameters.
      */
     async forEachKey(iteratee, options = {}) {
-        checkStorageAccess();
-        ow(iteratee, ow.function);
-        ow(options, ow.object.exactShape({
-            prefix: ow.optional.string,
-        }));
+        tryCancel();
+        parseArgument(iteratee, schemas.anyFunction);
+        const parsedOptions = parseArgument(options, iteratorOptionsSchema);
         let index = 0;
-        for await (const page of this.fetchKeyPages(options)) {
+        for await (const page of this.fetchKeyPages(parsedOptions)) {
             for (const item of page) {
                 await iteratee(item.key, index++, { size: item.size });
             }
@@ -408,7 +521,7 @@ export class KeyValueStore {
      * @param options Options for the iteration.
      */
     keys(options = {}) {
-        checkStorageAccess();
+        tryCancel();
         return createDualIterable({
             createPages: () => this.fetchKeyPages(options),
             extractItems: (page) => page.map((item) => item.key),
@@ -438,7 +551,7 @@ export class KeyValueStore {
      * @param options Options for the iteration.
      */
     values(options = {}) {
-        checkStorageAccess();
+        tryCancel();
         return createDualIterable({
             createPages: () => this.fetchKeyValuePages(options, (_key, value) => value),
             extractItems: (page) => page,
@@ -468,7 +581,7 @@ export class KeyValueStore {
      * @param options Options for the iteration.
      */
     entries(options = {}) {
-        checkStorageAccess();
+        tryCancel();
         return createDualIterable({
             createPages: () => this.fetchKeyValuePages(options, (key, value) => [key, value]),
             extractItems: (page) => page,
@@ -515,14 +628,11 @@ export class KeyValueStore {
      * @param [options] Storage manager options.
      */
     static async open(identifier, options = {}) {
-        checkStorageAccess();
-        ow(options, ow.object.exactShape({
-            configuration: ow.optional.object.instanceOf(Configuration),
-            storageBackend: ow.optional.object,
-        }));
-        options.configuration ??= Configuration.getGlobalConfiguration();
-        const storageBackend = options.storageBackend ?? serviceLocator.getStorageBackend();
-        await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration: options.configuration });
+        tryCancel();
+        const parsedOptions = parseArgument(options, openOptionsSchema);
+        const configuration = parsedOptions.configuration ?? Configuration.getGlobalConfiguration();
+        const storageBackend = parsedOptions.storageBackend ?? serviceLocator.getStorageBackend();
+        await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration });
         const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'KeyValueStore');
         return serviceLocator.getStorageInstanceManager().openStorage(this, {
             ...resolved,
@@ -652,3 +762,22 @@ export class KeyValueStore {
         return store.getValue(store.configuration.inputKey);
     }
 }
+/** Normalizes a codec-serialized value into the `Buffer | ArrayBuffer` shape raw record reads promise. */
+function normalizeSerializedValue(value) {
+    if (typeof value === 'string') {
+        return Buffer.from(value);
+    }
+    if (ArrayBuffer.isView(value)) {
+        return Buffer.isBuffer(value) ? value : Buffer.from(value.buffer, value.byteOffset, value.byteLength);
+    }
+    return value;
+}
+/** Computes the key listing item (serialized byte size and content type) of a buffered entry. */
+function bufferedKeyItemData(key, entry) {
+    const serialized = serializeValue(entry.value, entry.options?.contentType);
+    return {
+        key,
+        size: normalizeSerializedValue(serialized.value).byteLength,
+        contentType: serialized.contentType,
+    };
+}

package/storages/request_dedup_cache.d.ts

@@ -12,9 +12,8 @@
  * @internal
  */
 export declare class RequestDeduplicationCache {
+    #private;
     private readonly size;
-    private keys;
-    private ids;
     constructor(size?: number);
     get(cacheKey: string): string | null;
     add(cacheKey: string, requestId: string): void;

package/storages/request_dedup_cache.js

@@ -13,26 +13,26 @@
  */
 export class RequestDeduplicationCache {
     size;
-    keys;
-    ids;
+    #keys;
+    #ids;
     // The slot count is the same for every queue, so it's a fixed default rather than a per-consumer option.
     constructor(size = 1_000_000) {
         this.size = size;
-        this.keys = new Array(size);
-        this.ids = new Array(size);
+        this.#keys = new Array(size);
+        this.#ids = new Array(size);
     }
     get(cacheKey) {
         const index = this.indexOf(cacheKey);
-        return this.keys[index] === cacheKey ? this.ids[index] : null;
+        return this.#keys[index] === cacheKey ? this.#ids[index] : null;
     }
     add(cacheKey, requestId) {
         const index = this.indexOf(cacheKey);
-        this.keys[index] = cacheKey;
-        this.ids[index] = requestId;
+        this.#keys[index] = cacheKey;
+        this.#ids[index] = requestId;
     }
     clear() {
-        this.keys = new Array(this.size);
-        this.ids = new Array(this.size);
+        this.#keys = new Array(this.size);
+        this.#ids = new Array(this.size);
     }
     // A cheap FNV-1a hash of the cache key — avoids pulling in a dedicated hashing dependency.
     indexOf(cacheKey) {

package/storages/request_list.d.ts

@@ -1,4 +1,5 @@
-import type { BaseHttpClient, Dictionary } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/http-client';
+import type { Dictionary } from '@crawlee/types';
 import type { Configuration } from '../configuration.js';
 import type { IProxyConfiguration } from '../proxy_configuration.js';
 import { Request, type RequestOptions, type Source } from '../request.js';
@@ -216,28 +217,18 @@ export interface RequestListOptions {
  * @category Sources
  */
 export declare class RequestList implements IRequestLoader {
-    private log;
+    #private;
     /**
      * Array of all requests from all sources, in the order as they appeared in sources.
      * All requests in the array have distinct uniqueKey!
      * @internal
      */
     readonly requests: (Request | RequestOptions)[];
-    /** Index to the next item in requests array to fetch. All previous requests are either handled or in progress. */
-    private nextIndex;
-    /** Dictionary, key is Request.uniqueKey, value is corresponding index in the requests array. */
-    private uniqueKeyToIndex;
     /**
      * Set of `uniqueKey`s of requests that were returned by fetchNextRequest().
      * @internal
      */
     inProgress: Set<string>;
-    /**
-     * `uniqueKey`s of requests that were in progress when the state was last persisted and thus need to be
-     * re-crawled after a restart. They are served before advancing through the rest of the sources.
-     * @internal
-     */
-    private requestsToRetry;
     /**
      * Starts as true because until we handle the first request, the list is effectively persisted by doing nothing.
      * @internal
@@ -248,17 +239,7 @@ export declare class RequestList implements IRequestLoader {
      * @internal
      */
     areRequestsPersisted: boolean;
-    private isLoading;
-    private isInitialized;
-    private persistStateKey?;
-    private persistRequestsKey?;
-    private initialState?;
-    private store?;
-    private keepDuplicateUrls;
     private sources;
-    private sourcesFunction?;
-    private proxyConfiguration?;
-    private httpClient?;
     /**
      * To create new instance of `RequestList` we need to use `RequestList.open()` factory method.
      * @param options All `RequestList` configuration options
@@ -437,7 +418,7 @@ export declare class RequestList implements IRequestLoader {
     /**
      * @internal wraps public utility for mocking purposes
      */
-    private _downloadListOfUrls;
+    private downloadListOfUrls;
 }
 /**
  * Represents state of a {@link RequestList}. It can be used to resume a {@link RequestList} which has been previously processed.