@crawlee/core 4.0.0-beta.103 → 4.0.0-beta.105

package/storages/key_value_store.js

@@ -1,8 +1,9 @@
 import ow, { ArgumentError } from 'ow';
 import { KEY_VALUE_STORE_KEY_REGEX } from '@apify/consts';
+import { tryCancel } from '@apify/timeout';
 import { Configuration } from '../configuration.js';
 import { serviceLocator } from '../service_locator.js';
-import { checkStorageAccess } from './access_checking.js';
+import { activeStorageTransaction, operationRejectedInTransaction, rejectOperationInTransaction, snapshotValue, withDirectStorageAccess, } from './transaction.js';
 import { parseValue, serializeValue } from './key_value_store_codec.js';
 import { StorageStatsTracker } from './storage_stats.js';
 import { resolveStorageIdentifier } from './storage_instance_manager.js';
@@ -129,10 +130,9 @@ export class KeyValueStore {
      *   on the MIME content type of the record, or `null` if the key is missing from the store.
      */
     async getValue(key, defaultValue) {
-        checkStorageAccess();
+        tryCancel();
         ow(key, ow.string.nonEmpty);
-        this.statsTracker.add('readCount');
-        const record = await this.backend.getValue(key);
+        const record = await this.readRecord(key);
         // A missing record falls back to the default; a record that parses to a falsy value (including
         // a stored literal `null`) is returned verbatim, so callers can tell "stored null" from "absent".
         if (!record) {
@@ -141,6 +141,52 @@ export class KeyValueStore {
         // Storage backends are byte transports — the value is raw bytes; the frontend parses it here.
         return parseValue(record.value, record.contentType ?? null);
     }
+    /**
+     * The active transaction's last buffered write per key for this store, derived from its journal.
+     * An entry with a `null` value is a tombstone (an in-transaction deletion).
+     */
+    bufferedJournalEntries() {
+        const transaction = activeStorageTransaction();
+        if (!transaction)
+            return undefined;
+        const lastWritePerKey = new Map();
+        for (const entry of transaction.journal) {
+            if (entry.type === 'keyValueStore' && entry.participant === this) {
+                lastWritePerKey.set(entry.key, entry);
+            }
+        }
+        return lastWritePerKey;
+    }
+    /**
+     * The single transaction-aware record read shared by `getValue`, `getRecord`, `recordExists` and the
+     * listing paths: buffered key → serialized through the standard codec (same fidelity as a real
+     * round-trip); tombstoned key → `null`; otherwise the backend.
+     *
+     * The per-key buffered lookup requires the whole journal to be reduced to a last-write-per-key map,
+     * which is O(journal). Single-record callers let it default (rebuilt per call); the listing paths,
+     * which read many keys, pass a map built once so the read stays O(1) per key instead of O(journal).
+     */
+    async readRecord(key, buffered = this.bufferedJournalEntries()) {
+        const entry = buffered?.get(key);
+        if (entry) {
+            if (entry.value === null) {
+                return null;
+            }
+            const serialized = serializeValue(entry.value, entry.options?.contentType);
+            return {
+                value: normalizeSerializedValue(serialized.value),
+                contentType: serialized.contentType ?? null,
+            };
+        }
+        this.statsTracker.add('readCount');
+        const record = await this.backend.getValue(key);
+        if (!record)
+            return null;
+        return {
+            value: record.value,
+            contentType: record.contentType ?? null,
+        };
+    }
     /**
      * Reads a record from the key-value store without parsing the value.
      *
@@ -168,16 +214,9 @@ export class KeyValueStore {
      *   of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
      */
     async getRecord(key) {
-        checkStorageAccess();
+        tryCancel();
         ow(key, ow.string.nonEmpty);
-        this.statsTracker.add('readCount');
-        const record = await this.backend.getValue(key);
-        if (!record)
-            return null;
-        return {
-            value: record.value,
-            contentType: record.contentType ?? null,
-        };
+        return this.readRecord(key);
     }
     /**
      * Tests whether a record with the given key exists in the key-value store without retrieving its value.
@@ -186,16 +225,22 @@ export class KeyValueStore {
      * @returns `true` if the record exists, `false` if it does not.
      */
     async recordExists(key) {
-        checkStorageAccess();
+        tryCancel();
         ow(key, ow.string.nonEmpty);
+        const entry = this.bufferedJournalEntries()?.get(key);
+        if (entry) {
+            return entry.value !== null;
+        }
         return this.backend.recordExists(key);
     }
     async getAutoSavedValue(key, defaultValue = {}) {
-        checkStorageAccess();
+        tryCancel();
         if (this.cache.has(key)) {
             return this.cache.get(key);
         }
-        const value = await this.getValue(key, defaultValue);
+        // Auto-saved state is deliberately *not* transactional. The direct read bypasses any active
+        // transaction - a buffered value seeded into this shared cache would survive a rollback forever.
+        const value = await withDirectStorageAccess(async () => this.getValue(key, defaultValue));
         // The await above could have run in parallel with another call to this function. If the other call finished more quickly,
         // the value will in cache at this point, and returning the new fetched value would introduce two different instances of
         // the auto-saved object, and only the latter one would be persisted.
@@ -221,11 +266,14 @@ export class KeyValueStore {
         this.persistStateEventStarted = true;
     }
     async *fetchKeyValuePages(options, mapRecord) {
-        for await (const page of this.fetchKeyPages(options)) {
+        // Reduce the journal once for the whole iteration, not once per key inside `readRecord`.
+        const buffered = this.bufferedJournalEntries();
+        for await (const page of this.fetchKeyPages(options, buffered)) {
             const results = [];
             for (const item of page) {
-                this.statsTracker.add('readCount');
-                const record = await this.backend.getValue(item.key);
+                // The shared transaction-aware read, so a key that exists only in the transaction resolves
+                // here instead of being dropped (`values()` would disagree with `keys()` on length).
+                const record = await this.readRecord(item.key, buffered);
                 if (record) {
                     const parsed = parseValue(record.value, record.contentType ?? null);
                     results.push(mapRecord(item.key, parsed));
@@ -234,7 +282,25 @@ export class KeyValueStore {
             yield results;
         }
     }
-    async *fetchKeyPages(options, limit = KVS_KEYS_DEFAULT_LIMIT) {
+    async *fetchKeyPages(options, buffered = this.bufferedJournalEntries(), limit = KVS_KEYS_DEFAULT_LIMIT) {
+        // Buffered keys are emitted first, then the real pages with any buffered (or tombstoned) key
+        // skipped - a merge-join is not an option, since `listKeys` promises no sort order.
+        const shadowedKeys = new Set();
+        if (buffered) {
+            const bufferedItems = [];
+            for (const [key, entry] of buffered) {
+                shadowedKeys.add(key);
+                if (entry.value === null)
+                    continue;
+                if (options.prefix !== undefined && !key.startsWith(options.prefix))
+                    continue;
+                bufferedItems.push(bufferedKeyItemData(key, entry));
+            }
+            if (bufferedItems.length > 0) {
+                bufferedItems.sort((a, b) => (a.key < b.key ? -1 : 1));
+                yield bufferedItems;
+            }
+        }
         let exclusiveStartKey;
         while (true) {
             this.statsTracker.add('listCount');
@@ -243,9 +309,10 @@ export class KeyValueStore {
                 exclusiveStartKey,
                 limit,
             });
-            yield items;
+            yield shadowedKeys.size > 0 ? items.filter((item) => !shadowedKeys.has(item.key)) : items;
             if (!isTruncated)
                 break;
+            // Paginate from the raw backend cursor - it may reject a key it did not hand out.
             exclusiveStartKey = nextExclusiveStartKey;
         }
     }
@@ -293,7 +360,7 @@ export class KeyValueStore {
      * @param [options] Record options.
      */
     async setValue(key, value, options = {}) {
-        checkStorageAccess();
+        const transaction = activeStorageTransaction();
         ow(key, 'key', ow.string.nonEmpty);
         ow(key, ow.string.validate((k) => ({
             validator: ow.isValid(k, ow.string.matches(KEY_VALUE_STORE_KEY_REGEX)),
@@ -307,6 +374,32 @@ export class KeyValueStore {
         }));
         // Make copy of options, don't update what user passed.
         const optionsCopy = { ...options };
+        // The whole transaction branch sits *above* the auto-saved cache update below, so a buffered
+        // write touches nothing outside the journal. That cache is shared, process-lifetime frontend
+        // state, so mutating it here would survive a rollback and later be persisted by `persistState`.
+        // The commit replay re-enters this method with no active transaction and updates it then.
+        if (transaction) {
+            if (isStream(value)) {
+                // A stream cannot serve both a read-your-own-writes read and the commit replay. The
+                // transaction is known-active here, so throw directly rather than via the conditional guard.
+                throw operationRejectedInTransaction(`KeyValueStore.setValue() with a stream value (key "${key}")`, 'a stream can only be consumed once, so it cannot be buffered until commit.');
+            }
+            // Validation only, result discarded: the journal snapshot (`structuredClone`) accepts values
+            // JSON cannot, which would otherwise only throw at a later read or at commit.
+            if (value !== null) {
+                serializeValue(value, optionsCopy.contentType);
+            }
+            // One snapshot serves both the reads and the commit replay; `null` is a tombstone.
+            transaction.recordJournalEntry({
+                type: 'keyValueStore',
+                participant: this,
+                storageId: this.id,
+                key,
+                value: value === null ? null : snapshotValue(value),
+                options: optionsCopy,
+            });
+            return;
+        }
         // If we try to set the value of a cached state to a different reference, we need to update the cache accordingly.
         const cachedValue = this.cache.get(key);
         if (cachedValue && cachedValue !== value) {
@@ -336,18 +429,31 @@ export class KeyValueStore {
             contentType: serialized.contentType,
         });
     }
+    /** @internal */
+    async commitJournalEntries(entries) {
+        // One `setValue` per key, last write wins - idempotent under retry.
+        const lastWritePerKey = new Map();
+        for (const entry of entries) {
+            if (entry.type === 'keyValueStore') {
+                lastWritePerKey.set(entry.key, { value: entry.value, options: entry.options });
+            }
+        }
+        for (const [key, { value, options }] of lastWritePerKey) {
+            await this.setValue(key, value, options);
+        }
+    }
     /**
      * Removes the key-value store either from the Apify cloud storage or from the local directory,
      * depending on the mode of operation.
      */
     async drop() {
-        checkStorageAccess();
+        rejectOperationInTransaction('KeyValueStore.drop()');
         await this.backend.drop();
         serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
     /** @internal */
     clearCache() {
-        checkStorageAccess();
+        rejectOperationInTransaction('KeyValueStore.clearCache()');
         this.cache.clear();
     }
     /**
@@ -372,7 +478,7 @@ export class KeyValueStore {
      * @param [options] All `forEachKey()` parameters.
      */
     async forEachKey(iteratee, options = {}) {
-        checkStorageAccess();
+        tryCancel();
         ow(iteratee, ow.function);
         ow(options, ow.object.exactShape({
             prefix: ow.optional.string,
@@ -408,7 +514,7 @@ export class KeyValueStore {
      * @param options Options for the iteration.
      */
     keys(options = {}) {
-        checkStorageAccess();
+        tryCancel();
         return createDualIterable({
             createPages: () => this.fetchKeyPages(options),
             extractItems: (page) => page.map((item) => item.key),
@@ -438,7 +544,7 @@ export class KeyValueStore {
      * @param options Options for the iteration.
      */
     values(options = {}) {
-        checkStorageAccess();
+        tryCancel();
         return createDualIterable({
             createPages: () => this.fetchKeyValuePages(options, (_key, value) => value),
             extractItems: (page) => page,
@@ -468,7 +574,7 @@ export class KeyValueStore {
      * @param options Options for the iteration.
      */
     entries(options = {}) {
-        checkStorageAccess();
+        tryCancel();
         return createDualIterable({
             createPages: () => this.fetchKeyValuePages(options, (key, value) => [key, value]),
             extractItems: (page) => page,
@@ -515,7 +621,7 @@ export class KeyValueStore {
      * @param [options] Storage manager options.
      */
     static async open(identifier, options = {}) {
-        checkStorageAccess();
+        tryCancel();
         ow(options, ow.object.exactShape({
             configuration: ow.optional.object.instanceOf(Configuration),
             storageBackend: ow.optional.object,
@@ -652,3 +758,22 @@ export class KeyValueStore {
         return store.getValue(store.configuration.inputKey);
     }
 }
+/** Normalizes a codec-serialized value into the `Buffer | ArrayBuffer` shape raw record reads promise. */
+function normalizeSerializedValue(value) {
+    if (typeof value === 'string') {
+        return Buffer.from(value);
+    }
+    if (ArrayBuffer.isView(value)) {
+        return Buffer.isBuffer(value) ? value : Buffer.from(value.buffer, value.byteOffset, value.byteLength);
+    }
+    return value;
+}
+/** Computes the key listing item (serialized byte size and content type) of a buffered entry. */
+function bufferedKeyItemData(key, entry) {
+    const serialized = serializeValue(entry.value, entry.options?.contentType);
+    return {
+        key,
+        size: normalizeSerializedValue(serialized.value).byteLength,
+        contentType: serialized.contentType,
+    };
+}

package/storages/request_queue.d.ts

@@ -4,6 +4,7 @@ import type { CrawleeLogger } from '../log.js';
 import type { IProxyConfiguration } from '../proxy_configuration.js';
 import type { Source } from '../request.js';
 import { Request } from '../request.js';
+import type { JournalEntry } from './transaction.js';
 import type { IRequestManager, RequestsLike } from './request_manager.js';
 import type { RequestQueueStats } from './storage_stats.js';
 import type { IStorage, StorageIdentifier } from './storage_instance_manager.js';
@@ -102,6 +103,24 @@ export declare class RequestQueue implements IStorage, IRequestManager {
      * @param [options] Request queue operation options.
      */
     addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    /**
+     * Journals an addition for introspection only; these entries are never replayed. A no-op unless the
+     * transaction is open, so detached and outliving writers stay out of the journal.
+     */
+    private recordRequestJournalEntry;
+    /**
+     * The requests buffered by the given transaction for this queue, keyed by `uniqueKey` — a dedup
+     * index derived from the transaction journal.
+     */
+    private bufferedRequests;
+    /**
+     * Adds a request under the `deferred` policy: journaled now, really added by the commit replay.
+     * A new request's `requestId` is the local `uniqueKey` hash and is **provisional** — never write it
+     * to `request.id` or the dedup caches. Dedup is cheapest-first: buffer, caches, then a backend probe.
+     */
+    private addRequestDeferred;
+    /** @internal */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
      * Adds requests to the queue in batches of 25. This method will wait till all the requests are added
      * to the queue before resolving. You should prefer using `queue.addRequestsBatched()` or `crawler.addRequests()`