@crawlee/core 4.0.0-beta.103 → 4.0.0-beta.105

package/storages/request_queue.js

@@ -2,12 +2,13 @@ import { inspect } from 'node:util';
 import { downloadListOfUrls, isAsyncIterable, isIterable, sleep } from '@crawlee/utils';
 import ow from 'ow';
 import { LruCache } from '@apify/datastructures';
+import { tryCancel } from '@apify/timeout';
 import { Configuration } from '../configuration.js';
 import { getObjectType } from '../debug.js';
 import { chunkedAsyncIterable, peekableAsyncIterable } from '../iterables.js';
 import { Request } from '../request.js';
 import { serviceLocator } from '../service_locator.js';
-import { checkStorageAccess } from './access_checking.js';
+import { activeStorageTransaction, rejectOperationInTransaction, withDirectStorageAccess } from './transaction.js';
 import { StorageStatsTracker } from './storage_stats.js';
 import { resolveStorageIdentifier } from './storage_instance_manager.js';
 import { getRequestId, purgeDefaultStorages } from './utils.js';
@@ -134,7 +135,7 @@ export class RequestQueue {
      * @param [options] Request queue operation options.
      */
     async addRequest(requestLike, options = {}) {
-        checkStorageAccess();
+        const transaction = activeStorageTransaction();
         ow(requestLike, ow.object);
         ow(options, ow.object.exactShape({
             forefront: ow.optional.boolean,
@@ -150,10 +151,14 @@ export class RequestQueue {
             id: ow.undefined,
         }));
         const request = requestLike instanceof Request ? requestLike : new Request(requestLike);
+        if (transaction?.policy.requestQueue === 'deferred') {
+            return this.addRequestDeferred(transaction, request, forefront);
+        }
         const cacheKey = getRequestId(request.uniqueKey);
         const cachedInfo = this.requestCache.get(cacheKey);
         if (cachedInfo) {
             request.id = cachedInfo.id;
+            this.recordRequestJournalEntry(transaction, [request], forefront, true);
             return {
                 wasAlreadyPresent: true,
                 // We may assume that if request is in local cache then also the information if the
@@ -166,6 +171,7 @@ export class RequestQueue {
         }
         this.statsTracker.add('writeCount');
         const { processedRequests } = await this.backend.addBatchOfRequests([request], { forefront });
+        this.recordRequestJournalEntry(transaction, [request], forefront, true);
         const queueOperationInfo = {
             ...processedRequests[0],
             uniqueKey: request.uniqueKey,
@@ -175,6 +181,151 @@ export class RequestQueue {
         this.requestSeenCache.add(cacheKey, request.id);
         return queueOperationInfo;
     }
+    /**
+     * Journals an addition for introspection only; these entries are never replayed. A no-op unless the
+     * transaction is open, so detached and outliving writers stay out of the journal.
+     */
+    recordRequestJournalEntry(transaction, requests, forefront, writeThrough) {
+        if (!transaction?.isActive || requests.length === 0)
+            return;
+        transaction.recordJournalEntry({
+            type: 'requestQueue',
+            participant: this,
+            requests: requests.map((request) => ({
+                url: request.url,
+                uniqueKey: request.uniqueKey,
+                label: request.label,
+            })),
+            forefront,
+            writeThrough,
+        });
+    }
+    /**
+     * The requests buffered by the given transaction for this queue, keyed by `uniqueKey` — a dedup
+     * index derived from the transaction journal.
+     */
+    bufferedRequests(transaction) {
+        const buffered = new Map();
+        // Only `deferred` records snapshots, so scanning the journal under `writeThrough` never finds any.
+        if (transaction.policy.requestQueue !== 'deferred')
+            return buffered;
+        for (const entry of transaction.journal) {
+            if (entry.type !== 'requestQueue' || entry.participant !== this)
+                continue;
+            for (const request of entry.requests) {
+                if (request.snapshot !== undefined)
+                    buffered.set(request.uniqueKey, request.snapshot);
+            }
+        }
+        return buffered;
+    }
+    /**
+     * Adds a request under the `deferred` policy: journaled now, really added by the commit replay.
+     * A new request's `requestId` is the local `uniqueKey` hash and is **provisional** — never write it
+     * to `request.id` or the dedup caches. Dedup is cheapest-first: buffer, caches, then a backend probe.
+     */
+    async addRequestDeferred(transaction, request, forefront, buffered = this.bufferedRequests(transaction)) {
+        // This transaction's own buffered adds; the shared caches never see them (provisional ids).
+        if (buffered.has(request.uniqueKey)) {
+            this.recordRequestJournalEntry(transaction, [request], forefront, false);
+            return {
+                wasAlreadyPresent: true,
+                wasAlreadyHandled: false,
+                requestId: getRequestId(request.uniqueKey),
+                uniqueKey: request.uniqueKey,
+                forefront,
+            };
+        }
+        // The caches hold real backend ids. Only *writing* provisional ids to them would be wrong;
+        // reading saves a probe. Same lookup as the write-through path.
+        const cacheKey = getRequestId(request.uniqueKey);
+        const cachedInfo = this.requestCache.get(cacheKey);
+        const knownRequestId = cachedInfo?.id ?? this.requestSeenCache.get(cacheKey);
+        if (knownRequestId) {
+            this.recordRequestJournalEntry(transaction, [request], forefront, false);
+            return {
+                wasAlreadyPresent: true,
+                // The dedup cache doesn't track the handled state; only the full record does.
+                wasAlreadyHandled: cachedInfo?.isHandled ?? false,
+                requestId: knownRequestId,
+                uniqueKey: request.uniqueKey,
+                forefront,
+            };
+        }
+        // The caches are bounded, so a miss is not proof of absence - probe for an accurate answer.
+        const existing = await this.backend.getRequest(request.uniqueKey);
+        if (existing) {
+            this.recordRequestJournalEntry(transaction, [request], forefront, false);
+            return {
+                wasAlreadyPresent: true,
+                wasAlreadyHandled: existing.handledAt != null,
+                requestId: existing.id,
+                uniqueKey: request.uniqueKey,
+                forefront,
+            };
+        }
+        // The entry below *is* the write, so a transaction closed during the probe must not receive it -
+        // pass through instead, per the closed-transaction rule. Under `deferred` that can land an
+        // addition a rollback would have discarded; dedup bounds that cost, silent loss is unbounded.
+        if (!transaction.isActive) {
+            return await this.addRequest(request, { forefront });
+        }
+        const snapshot = JSON.parse(JSON.stringify(request));
+        // Strip-list, not allow-list: every user-facing field flows through, including ones added to
+        // `Request` in the future. The exceptions are `id` and `handledAt`, the two backend-owned
+        // lifecycle fields.
+        delete snapshot.id;
+        delete snapshot.handledAt;
+        transaction.recordJournalEntry({
+            type: 'requestQueue',
+            participant: this,
+            requests: [{ url: request.url, uniqueKey: request.uniqueKey, label: request.label, snapshot }],
+            forefront,
+            writeThrough: false,
+        });
+        buffered.set(request.uniqueKey, snapshot);
+        return {
+            wasAlreadyPresent: false,
+            wasAlreadyHandled: false,
+            requestId: getRequestId(request.uniqueKey),
+            uniqueKey: request.uniqueKey,
+            forefront,
+        };
+    }
+    /** @internal */
+    async commitJournalEntries(entries) {
+        // Replay through `backend.addBatchOfRequests`, *not* the batched frontend wrapper - the wrapper
+        // resolves after the first chunk and sleeps between the rest, neither of which commit may
+        // inherit. One call per `forefront` flag; the order of forefront additions is arbitrary anyway.
+        for (const forefront of [false, true]) {
+            const requests = entries.flatMap((entry) => entry.type === 'requestQueue' && entry.forefront === forefront
+                ? // Requests without a snapshot were deduplicated or written through; nothing to replay.
+                    entry.requests
+                        .filter((journaled) => journaled.snapshot !== undefined)
+                        .map((journaled) => new Request(journaled.snapshot))
+                : []);
+            if (requests.length === 0)
+                continue;
+            this.statsTracker.add('writeCount');
+            const { processedRequests, unprocessedRequests } = await this.backend.addBatchOfRequests(requests, {
+                forefront,
+            });
+            // Only now, with the real backend-assigned ids, may the shared dedup caches be populated.
+            for (const processed of processedRequests) {
+                const cacheKey = getRequestId(processed.uniqueKey);
+                this.cacheRequest(cacheKey, { ...processed, forefront });
+                this.requestSeenCache.add(cacheKey, processed.requestId);
+            }
+            if (unprocessedRequests.length > 0) {
+                // Warn and skip, rather than retry or fail. `unprocessedRequests` is what remains after
+                // the backend's own transient-error handling - a semantic rejection that retrying here
+                // would only re-poke. And failing the commit would let one malformed request hold the
+                // whole transaction hostage.
+                this.log.warning('Some requests were rejected by the request queue while committing a storage transaction and will be skipped. ' +
+                    "This usually means the request data is malformed (e.g. an invalid 'userData' shape).", { unprocessedRequests });
+            }
+        }
+    }
     /**
      * Adds requests to the queue in batches of 25. This method will wait till all the requests are added
      * to the queue before resolving. You should prefer using `queue.addRequestsBatched()` or `crawler.addRequests()`
@@ -190,7 +341,7 @@ export class RequestQueue {
      * @param [options] Request queue operation options.
      */
     async addRequests(requestsLike, options = {}) {
-        checkStorageAccess();
+        const transaction = activeStorageTransaction();
         ow(requestsLike, ow.object
             .is((value) => isIterable(value) || isAsyncIterable(value))
             .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
@@ -225,6 +376,14 @@ export class RequestQueue {
                 requests.push(requestLike instanceof Request ? requestLike : new Request(requestLike));
             }
         }
+        if (transaction?.policy.requestQueue === 'deferred') {
+            const buffered = this.bufferedRequests(transaction);
+            for (const request of requests) {
+                results.processedRequests.push(await this.addRequestDeferred(transaction, request, forefront, buffered));
+            }
+            return results;
+        }
+        this.recordRequestJournalEntry(transaction, requests, forefront, true);
         const requestsToAdd = new Map();
         for (const request of requests) {
             const cacheKey = getCachedRequestId(request.uniqueKey);
@@ -276,7 +435,8 @@ export class RequestQueue {
      * @param options Options for the request queue
      */
     async addRequestsBatched(requests, options = {}) {
-        checkStorageAccess();
+        const transaction = activeStorageTransaction();
+        const deferred = transaction?.policy.requestQueue === 'deferred';
         ow(requests, ow.object
             .is((value) => isIterable(value) || isAsyncIterable(value))
             .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
@@ -313,7 +473,9 @@ export class RequestQueue {
                 }
             }
         }
-        const { batchSize = 1000, waitBetweenBatchesMillis = 1000, maxNewRequests = undefined } = options;
+        const { batchSize = 1000, maxNewRequests = undefined } = options;
+        // Under `deferred` no chunk performs backend I/O, so pacing them would only stall the handler.
+        const waitBetweenBatchesMillis = deferred ? 0 : (options.waitBetweenBatchesMillis ?? 1000);
         let remainingBudget = maxNewRequests ?? Infinity;
         const requestsOverLimit = [];
         // If there's a limit on the number of added requests, do not send batches bigger than the limit
@@ -373,21 +535,38 @@ export class RequestQueue {
         if ((await chunksIterator.peek()) === undefined) {
             return buildResult(addedRequests, Promise.resolve([]), requestIterator);
         }
-        // eslint-disable-next-line no-async-promise-executor
-        const promise = new Promise(async (resolve) => {
+        const processRemainingChunks = async () => {
             const finalAddedRequests = [];
             for await (const requestChunk of chunks) {
                 finalAddedRequests.push(...(await processChunk(requestChunk, false)));
                 await sleep(waitBetweenBatchesMillis);
             }
-            resolve(finalAddedRequests);
+            return finalAddedRequests;
+        };
+        // maxNewRequests needs all batches to report skipped requests accurately; `deferred` needs them
+        // too - a writer that finishes after commit would have nowhere to put its journal entries.
+        const awaitsRemainingChunks = options.waitForAllRequestsToBeAdded || maxNewRequests !== undefined || deferred;
+        // eslint-disable-next-line no-async-promise-executor
+        const promise = new Promise(async (resolve) => {
+            if (awaitsRemainingChunks) {
+                // Awaited below, i.e. still within the caller's transaction scope, so the additions are
+                // journaled like the initial chunk - introspection must not depend on where the chunk
+                // boundary happened to fall.
+                resolve(await processRemainingChunks());
+            }
+            else {
+                // Nobody awaits this writer, so it outlives the transaction scope it inherits and must
+                // not record into a transaction that may already be closed. It writes directly - its
+                // write-through additions were never going to be rolled back anyway - which means the
+                // requests it adds are not journaled. See `StorageTransactionView.enqueuedUrls`.
+                resolve(await withDirectStorageAccess(processRemainingChunks));
+            }
         });
         this.inProgressRequestBatchCount += 1;
         void promise.finally(() => {
             this.inProgressRequestBatchCount -= 1;
         });
-        // When maxNewRequests is set, we must wait for all batches so we can accurately report skipped requests.
-        if (options.waitForAllRequestsToBeAdded || maxNewRequests !== undefined) {
+        if (awaitsRemainingChunks) {
             addedRequests.push(...(await promise));
         }
         return buildResult(addedRequests, promise, requestIterator);
@@ -399,8 +578,13 @@ export class RequestQueue {
      * @returns Returns the request object, or `null` if it was not found.
      */
     async getRequest(uniqueKey) {
-        checkStorageAccess();
+        const transaction = activeStorageTransaction();
         ow(uniqueKey, ow.string);
+        // Requests buffered by the active transaction (under the `deferred` write policy) are visible to it.
+        const buffered = transaction && this.bufferedRequests(transaction).get(uniqueKey);
+        if (buffered) {
+            return new Request(buffered);
+        }
         const requestOptions = await this.backend.getRequest(uniqueKey);
         if (!requestOptions)
             return null;
@@ -424,7 +608,7 @@ export class RequestQueue {
      *   Returns the request object or `null` if there are no more pending requests.
      */
     async fetchNextRequest() {
-        checkStorageAccess();
+        rejectOperationInTransaction('RequestQueue.fetchNextRequest()', 'it is part of the crawler request-processing bookkeeping, which a transaction must not affect.');
         if (this.queuePausedForMigration) {
             return null;
         }
@@ -441,7 +625,7 @@ export class RequestQueue {
      * Handled requests will never again be returned by the `fetchNextRequest` function.
      */
     async markRequestAsHandled(request) {
-        checkStorageAccess();
+        rejectOperationInTransaction('RequestQueue.markRequestAsHandled()', 'it is part of the crawler request-processing bookkeeping, which a transaction must not affect.');
         ow(request, ow.object.partialShape({
             id: ow.string,
             uniqueKey: ow.string,
@@ -474,7 +658,7 @@ export class RequestQueue {
      * For example, this lets you store the number of retries or error messages for the request.
      */
     async reclaimRequest(request, options = {}) {
-        checkStorageAccess();
+        rejectOperationInTransaction('RequestQueue.reclaimRequest()', 'it is part of the crawler request-processing bookkeeping, which a transaction must not affect.');
         ow(request, ow.object.partialShape({
             id: ow.string,
             uniqueKey: ow.string,
@@ -508,7 +692,11 @@ export class RequestQueue {
      * {@link RequestQueue.isFinished}.
      */
     async isEmpty() {
-        checkStorageAccess();
+        const transaction = activeStorageTransaction();
+        // Requests buffered by the active transaction count as pending from its point of view.
+        if (transaction && this.bufferedRequests(transaction).size > 0) {
+            return false;
+        }
         return this.backend.isEmpty();
     }
     /**
@@ -520,11 +708,15 @@ export class RequestQueue {
      * a false negative, but it shall never return a false positive.
      */
     async isFinished() {
-        checkStorageAccess();
+        const transaction = activeStorageTransaction();
         // We are not finished if we're still adding new requests in the background.
         if (this.inProgressRequestBatchCount > 0) {
             return false;
         }
+        // Requests buffered by the active transaction count as pending from its point of view.
+        if (transaction && this.bufferedRequests(transaction).size > 0) {
+            return false;
+        }
         return this.backend.isFinished();
     }
     /**
@@ -564,7 +756,7 @@ export class RequestQueue {
      * depending on the mode of operation.
      */
     async drop() {
-        checkStorageAccess();
+        rejectOperationInTransaction('RequestQueue.drop()');
         await this.backend.drop();
         serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
@@ -573,7 +765,7 @@ export class RequestQueue {
      * so it can be reused (e.g. across multiple `crawler.run()` calls).
      */
     async purge() {
-        checkStorageAccess();
+        rejectOperationInTransaction('RequestQueue.purge()');
         await this.backend.purge();
         // Reset in-memory bookkeeping so the queue behaves as if freshly opened.
         this.requestCache.clear();
@@ -630,8 +822,17 @@ export class RequestQueue {
      * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
     async getInfo() {
-        checkStorageAccess();
-        return this.backend.getMetadata();
+        const transaction = activeStorageTransaction();
+        const metadata = await this.backend.getMetadata();
+        const bufferedCount = transaction ? this.bufferedRequests(transaction).size : 0;
+        if (bufferedCount > 0) {
+            return {
+                ...metadata,
+                totalRequestCount: metadata.totalRequestCount + bufferedCount,
+                pendingRequestCount: metadata.pendingRequestCount + bufferedCount,
+            };
+        }
+        return metadata;
     }
     /**
      * Fetches URLs from requestsFromUrl and returns them in format of list of requests
@@ -700,7 +901,7 @@ export class RequestQueue {
      * @param [options] Open Request Queue options.
      */
     static async open(identifier, options = {}) {
-        checkStorageAccess();
+        tryCancel();
         ow(options, ow.object.exactShape({
             configuration: ow.optional.object.instanceOf(Configuration),
             storageBackend: ow.optional.object,

package/storages/transaction.d.ts

@@ -0,0 +1,254 @@
+import type { Awaitable, Dictionary } from '@crawlee/types';
+import type { RecordOptions } from './key_value_store.js';
+/**
+ * Governs whether writes of a given storage type performed inside a {@link StorageTransaction} are
+ * applied immediately (`writeThrough`) or recorded and replayed on commit (`deferred`).
+ */
+export type StorageWriteMode = 'deferred' | 'writeThrough';
+/**
+ * Per-storage-type write policy of a {@link StorageTransaction}. Datasets and key-value stores are
+ * always `deferred` and not configurable — deferring is the only safe mode for non-idempotent writes,
+ * and {@link withDirectStorageAccess} covers one-off immediate writes.
+ */
+export interface StorageWritePolicy {
+    /**
+     * Write mode for request queue additions. Note that this is a *write policy* for the queue, not the
+     * queue instance itself (which is the top-level `requestQueue` crawler option).
+     *
+     * - `writeThrough` (default): requests are added immediately and are **not** rolled back with the
+     *   transaction. This is safe (additions are deduplicated by `uniqueKey`, so a retry is idempotent)
+     *   and keeps new requests visible to the crawler while the handler still runs.
+     * - `deferred`: requests are only added when the transaction commits — strict all-or-nothing
+     *   semantics, at the cost of the crawler not seeing them until the handler finishes.
+     */
+    requestQueue: StorageWriteMode;
+}
+export type StorageTransactionState = 'open' | 'committing' | 'committed' | 'failed' | 'rolledBack';
+/**
+ * A storage frontend that can record operations in a transaction journal.
+ * @internal
+ */
+export interface TransactionParticipant {
+    /**
+     * Replay the given buffered journal entries (all recorded by this participant) into the real storage
+     * backend. Called during commit, with the transaction already in the `committing` state, so the
+     * replayed operations pass through.
+     */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
+}
+/**
+ * A single dataset write (`pushData`) recorded in a transaction journal.
+ */
+export interface DatasetJournalEntry {
+    type: 'dataset';
+    /** @internal **/
+    participant: TransactionParticipant;
+    storageId: string;
+    /** The pushed items, captured by `structuredClone` at write time. */
+    items: Dictionary[];
+    recordedAt: Date;
+}
+/**
+ * A single key-value store write (`setValue`) recorded in a transaction journal.
+ */
+export interface KeyValueStoreJournalEntry {
+    type: 'keyValueStore';
+    /** @internal **/
+    participant: TransactionParticipant;
+    storageId: string;
+    key: string;
+    /** The original, pre-serialization value captured by `structuredClone`; `null` denotes a deletion. */
+    value: unknown;
+    options?: RecordOptions;
+}
+/**
+ * A request recorded in a transaction journal.
+ */
+export interface JournaledRequest {
+    url: string;
+    uniqueKey: string;
+    label?: string;
+    /**
+     * A full JSON snapshot of the request for the commit replay. Only present for buffered additions —
+     * deduplicated and write-through ones are journaled for introspection only.
+     */
+    snapshot?: Dictionary;
+}
+/**
+ * A batch of request queue additions recorded in a transaction journal.
+ */
+export interface RequestQueueJournalEntry {
+    type: 'requestQueue';
+    /** @internal **/
+    participant: TransactionParticipant;
+    requests: JournaledRequest[];
+    forefront: boolean;
+    /** Write-through entries were applied immediately; they are never replayed. */
+    writeThrough: boolean;
+}
+export type JournalEntry = DatasetJournalEntry | KeyValueStoreJournalEntry | RequestQueueJournalEntry;
+/**
+ * A read-only view of a {@link StorageTransaction}: only the journal-backed introspection accessors,
+ * without the lifecycle methods. The accessors are synchronous and expose the original pre-serialization
+ * values. They cover every write recorded while the transaction was open, under either write policy —
+ * with the one exception noted on {@link StorageTransactionView.enqueuedUrls|`enqueuedUrls`}. A view
+ * is valid until the transaction is disposed.
+ */
+export interface StorageTransactionView {
+    readonly state: StorageTransactionState;
+    /** Items pushed to datasets during the transaction, in push order. */
+    readonly datasetItems: {
+        item: Dictionary;
+        datasetId: string;
+    }[];
+    /**
+     * URLs enqueued to request queues during the transaction, under either write policy. Recorded as
+     * requested, so duplicate, already-present and backend-rejected URLs are included.
+     *
+     * One gap: unless a caller of `addRequestsBatched()` waits for every chunk
+     * (`waitForAllRequestsToBeAdded` or `maxNewRequests`, both of which {@link enqueueLinks} sets when
+     * a crawl limit applies), the chunks after the first are added by a background writer that outlives
+     * the transaction and is not recorded here.
+     */
+    readonly enqueuedUrls: {
+        url: string;
+        label?: string;
+    }[];
+    /** Key-value store changes made during the transaction, keyed by store id, last write per key. */
+    readonly keyValueStoreChanges: Record<string, Record<string, {
+        changedValue: unknown;
+        options?: RecordOptions;
+    }>>;
+}
+export interface StorageTransactionOptions {
+    /** Overrides of the per-storage-type write policy. See {@link StorageWritePolicy}. */
+    policy?: Partial<StorageWritePolicy>;
+    /**
+     * How long a commit may take before it fails, in milliseconds. There is no automatic retry — the
+     * replay of dataset items is not idempotent.
+     * @default 300000
+     */
+    commitTimeoutMillis?: number;
+}
+/**
+ * A storage transaction scoped to a request's lifecycle. Writes made through the storage frontends
+ * ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}) while the transaction is active
+ * are recorded rather than applied; on {@link StorageTransaction.commit|`commit()`} they are replayed
+ * into real storage, on {@link StorageTransaction.rollback|`rollback()`} they are dropped. Reads consult
+ * the recorded writes first, so a handler sees its own writes.
+ *
+ * Create one with {@link createStorageTransaction} (explicit commit/rollback) or
+ * {@link withStorageTransaction} (scoped sugar). Crawlers open one automatically around every request
+ * handler unless `transactionalStorage: false` is set.
+ */
+export declare class StorageTransaction implements StorageTransactionView {
+    /** The ordered, append-only journal — the source of truth for commit, introspection and reads. */
+    readonly journal: JournalEntry[];
+    /** Per-storage-type write policy. */
+    readonly policy: StorageWritePolicy;
+    private readonly commitTimeoutMillis;
+    private _state;
+    private disposed;
+    /** @internal */
+    constructor(options?: StorageTransactionOptions);
+    get state(): StorageTransactionState;
+    /**
+     * `true` only while `state === 'open'`. This is the single predicate every storage operation
+     * consults — operations performed after the transaction is closed pass through to the real backend.
+     */
+    get isActive(): boolean;
+    /** Runs `callback` with this transaction installed in the async context. */
+    run<T>(callback: () => Awaitable<T>): Promise<T>;
+    /**
+     * Records a write operation in the journal.
+     * @internal
+     */
+    recordJournalEntry(entry: JournalEntry): void;
+    /**
+     * Replays the journaled writes into real storage. A no-op unless the transaction is `open`.
+     *
+     * The transaction transitions to `committing` *before* anything is flushed, so a commit that throws
+     * partway lands in `failed` (never back in `open`) and subsequent storage operations pass through
+     * rather than recording into a dead transaction. Delivery is at-least-once — a commit that fails
+     * partway may have applied some of the writes already.
+     */
+    commit(): Promise<void>;
+    private flush;
+    /**
+     * Discards the journaled writes. A no-op unless the transaction is `open` — in particular, calling it
+     * after a successful `commit()` (which the crawler's error handling can legitimately do) does nothing
+     * and never throws.
+     */
+    rollback(): void;
+    /**
+     * Releases the journal and the write-time snapshots it holds. Must be called for *every* terminal
+     * state, `failed` included. Idempotent, never throws, and does not change `state`. Any
+     * {@link StorageTransactionView} of this transaction is only valid until this is called.
+     */
+    dispose(): void;
+    get datasetItems(): {
+        item: Dictionary;
+        datasetId: string;
+    }[];
+    get enqueuedUrls(): {
+        url: string;
+        label?: string;
+    }[];
+    get keyValueStoreChanges(): Record<string, Record<string, {
+        changedValue: unknown;
+        options?: RecordOptions;
+    }>>;
+}
+/**
+ * Opens a {@link StorageTransaction} without running anything yet. The caller owns the outcome:
+ * `run()`, then `commit()` or `rollback()`, and always `dispose()` when done. For the common
+ * open-run-commit flow, prefer {@link withStorageTransaction}.
+ */
+export declare function createStorageTransaction(options?: StorageTransactionOptions): StorageTransaction;
+/**
+ * Runs `callback` inside a new {@link StorageTransaction}: storage writes made in the callback are
+ * committed when it returns and rolled back when it throws. If a transaction is already active in the
+ * current async context, it is reused and its outcome is left to its owner (and `options` are ignored)
+ * — there are no nested transaction semantics.
+ */
+export declare function withStorageTransaction<T>(callback: (transaction: StorageTransaction) => Awaitable<T>, options?: StorageTransactionOptions): Promise<T>;
+/**
+ * Runs `callback` outside of any storage transaction — the per-call-site escape hatch. Storage operations
+ * made inside it hit the real backend directly, are not rolled back, and operations that a transaction
+ * rejects (`drop`, stream-valued `setValue`, request queue internals, ...) are permitted.
+ */
+export declare function withDirectStorageAccess<T>(callback: () => Awaitable<T>): Promise<T>;
+/**
+ * The per-operation hook consulted by every storage frontend method: performs the cancellation check
+ * that aborts storage operations when the request handler times out, and returns the active storage
+ * transaction. Returns `undefined` when there is no transaction in the async context *or* when it is no
+ * longer open — operations on a closed transaction deliberately pass through to the real backend.
+ * @internal
+ */
+export declare function activeStorageTransaction(): StorageTransaction | undefined;
+/**
+ * Returns the transaction installed in the current async context, regardless of its state. Used by the
+ * crawler to drive the outcome of the transaction it opened.
+ * @internal
+ */
+export declare function currentStorageTransaction(): StorageTransaction | undefined;
+/**
+ * Captures a value at write time, so that later mutations of the caller's object affect neither the
+ * read-your-own-writes reads nor the commit replay. `structuredClone` for fidelity (`Date`, `Map`, `Set`,
+ * typed arrays, `undefined`); values it cannot handle fall back to the JSON round-trip the storage
+ * backends perform anyway.
+ * @internal
+ */
+export declare function snapshotValue<T>(value: T): T;
+/**
+ * The guard for operations that cannot be performed inside a storage transaction: throws when one is
+ * active, and performs the per-operation cancellation check either way.
+ * @internal
+ */
+export declare function rejectOperationInTransaction(operation: string, reason?: string): void;
+/**
+ * Builds the "operation not allowed in a transaction" error, for a call site that has already
+ * established a transaction is active and so wants to `throw` unconditionally.
+ * @internal
+ */
+export declare function operationRejectedInTransaction(operation: string, reason?: string): Error;