@crawlee/core 4.0.0-rc.0 → 4.0.0-rc.1

package/storages/request_queue.js

@@ -38,19 +38,20 @@ const addRequestsBatchedOptionsSchema = z.strictObject({
     waitBetweenBatchesMillis: schemas.anyNumber.default(1000),
     maxNewRequests: schemas.anyNumber.optional(),
 });
-const newRequestLikeSchema = z.looseObject({
+// Compiled: these run once per request.
+const newRequestLikeSchema = z.compile(z.looseObject({
     url: z.string(),
     id: z.undefined().optional(),
-});
-const handledRequestSchema = z.looseObject({
+}));
+const handledRequestSchema = z.compile(z.looseObject({
     id: z.string(),
     uniqueKey: z.string(),
     handledAt: z.string().optional(),
-});
-const reclaimedRequestSchema = z.looseObject({
+}));
+const reclaimedRequestSchema = z.compile(z.looseObject({
     id: z.string(),
     uniqueKey: z.string(),
-});
+}));
 const uniqueKeySchema = z.string();
 const openOptionsSchema = z.strictObject({
     configuration: z.instanceof(Configuration).optional(),
@@ -98,8 +99,7 @@ export class RequestQueue {
     backend;
     #proxyConfiguration;
     log;
-    // kept as TS-private: request_queue tests read this cache directly
-    requestCache;
+    #requestCache;
     /**
      * Remembers the `requestId` of every request already submitted to the client — including background
      * batches that `requestCache` skips — so overlapping URL sets aren't re-submitted.
@@ -107,8 +107,7 @@ export class RequestQueue {
      */
     #requestSeenCache;
     #queuePausedForMigration = false;
-    // kept as TS-private: packages/core/test request-queue tests write this counter directly
-    inProgressRequestBatchCount = 0;
+    #inProgressRequestBatchCount = 0;
     /**
      * The largest expected request-processing time (in seconds) seen so far via
      * {@link setExpectedRequestProcessingTimeSecs}. Used to ensure that value is only ever raised, never
@@ -137,7 +136,7 @@ export class RequestQueue {
         this.#events = serviceLocator.getEventManager();
         this.backend = options.backend;
         this.#proxyConfiguration = options.proxyConfiguration;
-        this.requestCache = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
+        this.#requestCache = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
         this.#requestSeenCache = new RequestDeduplicationCache();
         this.log = serviceLocator.getLogger().child({ prefix: `RequestQueue(${this.id}, ${this.name ?? 'no-name'})` });
         this.#events.on(EventType.MIGRATING, async () => {
@@ -181,20 +180,20 @@ export class RequestQueue {
         parseArgument(requestLike, schemas.anyObject);
         const { forefront } = parseArgument(options, operationOptionsSchema);
         if ('requestsFromUrl' in requestLike) {
-            const requests = await this.fetchRequestsFromUrl(requestLike);
-            const processedRequests = await this.addFetchedRequests(requestLike, requests, options);
+            const requests = await this.#fetchRequestsFromUrl(requestLike);
+            const processedRequests = await this.#addFetchedRequests(requestLike, requests, options);
             return { ...processedRequests[0], forefront };
         }
         parseArgument(requestLike, newRequestLikeSchema);
         const request = requestLike instanceof Request ? requestLike : new Request(requestLike);
         if (transaction?.policy.requestQueue === 'deferred') {
-            return this.addRequestDeferred(transaction, request, forefront);
+            return this.#addRequestDeferred(transaction, request, forefront);
         }
         const cacheKey = getRequestId(request.uniqueKey);
-        const cachedInfo = this.requestCache.get(cacheKey);
+        const cachedInfo = this.#requestCache.get(cacheKey);
         if (cachedInfo) {
             request.id = cachedInfo.id;
-            this.recordRequestJournalEntry(transaction, [request], forefront, true);
+            this.#recordRequestJournalEntry(transaction, [request], forefront, true);
             return {
                 wasAlreadyPresent: true,
                 // We may assume that if request is in local cache then also the information if the
@@ -207,13 +206,13 @@ export class RequestQueue {
         }
         this.#statsTracker.add('writeCount');
         const { processedRequests } = await this.backend.addBatchOfRequests([request], { forefront });
-        this.recordRequestJournalEntry(transaction, [request], forefront, true);
+        this.#recordRequestJournalEntry(transaction, [request], forefront, true);
         const queueOperationInfo = {
             ...processedRequests[0],
             uniqueKey: request.uniqueKey,
             forefront,
         };
-        this.cacheRequest(cacheKey, queueOperationInfo);
+        this.#cacheRequest(cacheKey, queueOperationInfo);
         this.#requestSeenCache.add(cacheKey, request.id);
         return queueOperationInfo;
     }
@@ -221,7 +220,7 @@ export class RequestQueue {
      * Journals an addition for introspection only; these entries are never replayed. A no-op unless the
      * transaction is open, so detached and outliving writers stay out of the journal.
      */
-    recordRequestJournalEntry(transaction, requests, forefront, writeThrough) {
+    #recordRequestJournalEntry(transaction, requests, forefront, writeThrough) {
         if (!transaction?.isActive || requests.length === 0)
             return;
         transaction.recordJournalEntry({
@@ -240,7 +239,7 @@ export class RequestQueue {
      * The requests buffered by the given transaction for this queue, keyed by `uniqueKey` — a dedup
      * index derived from the transaction journal.
      */
-    bufferedRequests(transaction) {
+    #bufferedRequests(transaction) {
         const buffered = new Map();
         // Only `deferred` records snapshots, so scanning the journal under `writeThrough` never finds any.
         if (transaction.policy.requestQueue !== 'deferred')
@@ -260,10 +259,10 @@ export class RequestQueue {
      * A new request's `requestId` is the local `uniqueKey` hash and is **provisional** — never write it
      * to `request.id` or the dedup caches. Dedup is cheapest-first: buffer, caches, then a backend probe.
      */
-    async addRequestDeferred(transaction, request, forefront, buffered = this.bufferedRequests(transaction)) {
+    async #addRequestDeferred(transaction, request, forefront, buffered = this.#bufferedRequests(transaction)) {
         // This transaction's own buffered adds; the shared caches never see them (provisional ids).
         if (buffered.has(request.uniqueKey)) {
-            this.recordRequestJournalEntry(transaction, [request], forefront, false);
+            this.#recordRequestJournalEntry(transaction, [request], forefront, false);
             return {
                 wasAlreadyPresent: true,
                 wasAlreadyHandled: false,
@@ -275,10 +274,10 @@ export class RequestQueue {
         // The caches hold real backend ids. Only *writing* provisional ids to them would be wrong;
         // reading saves a probe. Same lookup as the write-through path.
         const cacheKey = getRequestId(request.uniqueKey);
-        const cachedInfo = this.requestCache.get(cacheKey);
+        const cachedInfo = this.#requestCache.get(cacheKey);
         const knownRequestId = cachedInfo?.id ?? this.#requestSeenCache.get(cacheKey);
         if (knownRequestId) {
-            this.recordRequestJournalEntry(transaction, [request], forefront, false);
+            this.#recordRequestJournalEntry(transaction, [request], forefront, false);
             return {
                 wasAlreadyPresent: true,
                 // The dedup cache doesn't track the handled state; only the full record does.
@@ -291,7 +290,7 @@ export class RequestQueue {
         // The caches are bounded, so a miss is not proof of absence - probe for an accurate answer.
         const existing = await this.backend.getRequest(request.uniqueKey);
         if (existing) {
-            this.recordRequestJournalEntry(transaction, [request], forefront, false);
+            this.#recordRequestJournalEntry(transaction, [request], forefront, false);
             return {
                 wasAlreadyPresent: true,
                 wasAlreadyHandled: existing.handledAt != null,
@@ -338,7 +337,7 @@ export class RequestQueue {
                 ? // Requests without a snapshot were deduplicated or written through; nothing to replay.
                     entry.requests
                         .filter((journaled) => journaled.snapshot !== undefined)
-                        .map((journaled) => new Request(journaled.snapshot))
+                        .map((journaled) => Request.fromSchema(journaled.snapshot))
                 : []);
             if (requests.length === 0)
                 continue;
@@ -349,7 +348,7 @@ export class RequestQueue {
             // Only now, with the real backend-assigned ids, may the shared dedup caches be populated.
             for (const processed of processedRequests) {
                 const cacheKey = getRequestId(processed.uniqueKey);
-                this.cacheRequest(cacheKey, { ...processed, forefront });
+                this.#cacheRequest(cacheKey, { ...processed, forefront });
                 this.#requestSeenCache.add(cacheKey, processed.requestId);
             }
             if (unprocessedRequests.length > 0) {
@@ -399,26 +398,26 @@ export class RequestQueue {
                 requests.push(new Request({ url: requestLike }));
             }
             else if ('requestsFromUrl' in requestLike) {
-                const fetchedRequests = await this.fetchRequestsFromUrl(requestLike);
-                await this.addFetchedRequests(requestLike, fetchedRequests, options);
+                const fetchedRequests = await this.#fetchRequestsFromUrl(requestLike);
+                await this.#addFetchedRequests(requestLike, fetchedRequests, options);
             }
             else {
                 requests.push(requestLike instanceof Request ? requestLike : new Request(requestLike));
             }
         }
         if (transaction?.policy.requestQueue === 'deferred') {
-            const buffered = this.bufferedRequests(transaction);
+            const buffered = this.#bufferedRequests(transaction);
             for (const request of requests) {
-                results.processedRequests.push(await this.addRequestDeferred(transaction, request, forefront, buffered));
+                results.processedRequests.push(await this.#addRequestDeferred(transaction, request, forefront, buffered));
             }
             return results;
         }
-        this.recordRequestJournalEntry(transaction, requests, forefront, true);
+        this.#recordRequestJournalEntry(transaction, requests, forefront, true);
         const requestsToAdd = new Map();
         for (const request of requests) {
             const cacheKey = getCachedRequestId(request.uniqueKey);
             // Prefer the full `requestCache` record; fall back to the dedup cache for background batches it skips.
-            const cachedInfo = this.requestCache.get(cacheKey);
+            const cachedInfo = this.#requestCache.get(cacheKey);
             const knownRequestId = cachedInfo?.id ?? this.#requestSeenCache.get(cacheKey);
             if (knownRequestId) {
                 request.id = knownRequestId;
@@ -448,7 +447,7 @@ export class RequestQueue {
             results.processedRequests.push(newRequest);
             const cacheKey = getCachedRequestId(newRequest.uniqueKey);
             if (cache) {
-                this.cacheRequest(cacheKey, { ...newRequest, forefront });
+                this.#cacheRequest(cacheKey, { ...newRequest, forefront });
             }
             // Unlike `requestCache`, populate this on every batch (including background ones).
             this.#requestSeenCache.add(cacheKey, newRequest.requestId);
@@ -475,7 +474,7 @@ export class RequestQueue {
                     if (opts.url !== undefined && typeof opts.url !== 'string') {
                         throw new Error(`Request options are not valid, the 'url' property is not a string. Input: ${inspect(opts)}`);
                     }
-                    if (opts.id !== undefined) {
+                    if ('id' in opts && opts.id !== undefined) {
                         throw new Error(`Request options are not valid, the 'id' property must not be present. Input: ${inspect(opts)}`);
                     }
                     if (opts.requestsFromUrl !== undefined &&
@@ -517,9 +516,9 @@ export class RequestQueue {
                 return processedRequests;
             },
             trackBackgroundBatches: (batches) => {
-                this.inProgressRequestBatchCount += 1;
+                this.#inProgressRequestBatchCount += 1;
                 void batches.finally(() => {
-                    this.inProgressRequestBatchCount -= 1;
+                    this.#inProgressRequestBatchCount -= 1;
                 });
             },
         });
@@ -534,14 +533,14 @@ export class RequestQueue {
         const transaction = activeStorageTransaction();
         parseArgument(uniqueKey, uniqueKeySchema);
         // Requests buffered by the active transaction (under the `deferred` write policy) are visible to it.
-        const buffered = transaction && this.bufferedRequests(transaction).get(uniqueKey);
+        const buffered = transaction && this.#bufferedRequests(transaction).get(uniqueKey);
         if (buffered) {
-            return new Request(buffered);
+            return Request.fromSchema(buffered);
         }
-        const requestOptions = await this.backend.getRequest(uniqueKey);
-        if (!requestOptions)
+        const schema = await this.backend.getRequest(uniqueKey);
+        if (!schema)
             return null;
-        return new Request(requestOptions);
+        return Request.fromSchema(schema);
     }
     /**
      * Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
@@ -555,7 +554,7 @@ export class RequestQueue {
      * Note that the `null` return value doesn't mean the queue processing finished,
      * it means there are currently no pending requests.
      * To check whether all requests in queue were finished,
-     * use {@link RequestQueue.isFinished} instead.
+     * use {@link RequestQueue.checkReadiness} instead.
      *
      * @returns
      *   Returns the request object or `null` if there are no more pending requests.
@@ -566,10 +565,10 @@ export class RequestQueue {
             return null;
         }
         this.#statsTracker.add('headItemReadCount');
-        const requestOptions = await this.backend.fetchNextRequest();
-        if (!requestOptions)
+        const schema = await this.backend.fetchNextRequest();
+        if (!schema)
             return null;
-        return new Request(requestOptions);
+        return Request.fromSchema(schema);
     }
     /**
      * Marks a request that was previously returned by the
@@ -580,7 +579,7 @@ export class RequestQueue {
     async markRequestAsHandled(request) {
         rejectOperationInTransaction('RequestQueue.markRequestAsHandled()', 'it is part of the crawler request-processing bookkeeping, which a transaction must not affect.');
         parseArgument(request, handledRequestSchema);
-        const forefront = this.requestCache.get(getRequestId(request.uniqueKey))?.forefront ?? false;
+        const forefront = this.#requestCache.get(getRequestId(request.uniqueKey))?.forefront ?? false;
         const handledAt = request.handledAt ?? new Date().toISOString();
         this.#statsTracker.add('writeCount');
         const processedRequest = await this.backend.markRequestAsHandled({
@@ -597,7 +596,7 @@ export class RequestQueue {
             uniqueKey: request.uniqueKey,
             forefront,
         };
-        this.cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
+        this.#cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
         return queueOperationInfo;
     }
     /**
@@ -623,46 +622,41 @@ export class RequestQueue {
             uniqueKey: request.uniqueKey,
             forefront,
         };
-        this.cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
+        this.#cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
         return queueOperationInfo;
     }
     /**
-     * Resolves to `true` if the next call to {@link RequestQueue.fetchNextRequest} would return
-     * `null`, i.e. there are no pending requests to fetch right now. Otherwise it resolves to `false`.
-     *
-     * Note that even if the queue is empty, there might be some requests currently being processed
-     * (fetched but not yet handled or reclaimed). An empty queue therefore does not mean crawling is
-     * finished — those in-progress requests may still be reclaimed, and background tasks may still be
-     * adding more requests. To check whether all activity in the queue has finished, use
-     * {@link RequestQueue.isFinished}.
+     * A queue hands requests out as fast as they are asked for; pacing is a job for a manager wrapped around it,
+     * such as {@link ThrottlingRequestManager}.
+     * @inheritdoc
      */
-    async isEmpty() {
-        const transaction = activeStorageTransaction();
-        // Requests buffered by the active transaction count as pending from its point of view.
-        if (transaction && this.bufferedRequests(transaction).size > 0) {
-            return false;
-        }
-        return this.backend.isEmpty();
+    recordPacingSignal(_signal) {
+        return false;
     }
     /**
-     * Resolves to `true` if all requests were already handled and there are no more left — including no
-     * requests currently in progress (fetched but not yet handled or reclaimed, including requests
-     * locked by other clients sharing the same queue) and no background add operations still in flight.
+     * Reports whether the queue has a request to hand over, is waiting on one, or is done.
      *
-     * Due to the nature of distributed storage used by the queue, the function may occasionally return
-     * a false negative, but it shall never return a false positive.
+     * `waiting` means requests are in progress (fetched but not yet handled or reclaimed, possibly by another
+     * client sharing the queue) or a background add is still landing; neither has a clock, so no `readyAt`.
+     *
+     * Due to the nature of distributed storage used by the queue, `finished` may occasionally arrive a probe or
+     * two late, but it is never reported early.
      */
-    async isFinished() {
+    async checkReadiness() {
         const transaction = activeStorageTransaction();
-        // We are not finished if we're still adding new requests in the background.
-        if (this.inProgressRequestBatchCount > 0) {
-            return false;
-        }
         // Requests buffered by the active transaction count as pending from its point of view.
-        if (transaction && this.bufferedRequests(transaction).size > 0) {
-            return false;
+        if (transaction && this.#bufferedRequests(transaction).size > 0) {
+            return { status: 'ready' };
+        }
+        // Something fetchable outranks everything below, so this is the only backend call a probe needs.
+        if (!(await this.backend.isEmpty())) {
+            return { status: 'ready' };
         }
-        return this.backend.isFinished();
+        // We are not finished if we're still adding new requests in the background.
+        if (this.#inProgressRequestBatchCount > 0) {
+            return { status: 'waiting' };
+        }
+        return (await this.backend.isFinished()) ? { status: 'finished' } : { status: 'waiting' };
     }
     /**
      * Tells the queue how long a consumer expects to hold a fetched request before marking it handled
@@ -681,18 +675,28 @@ export class RequestQueue {
         this.#expectedRequestProcessingSecs = secs;
         await this.backend.setExpectedRequestProcessingTimeSecs?.(secs);
     }
+    /**
+     * @inheritdoc
+     * Unlike {@link RequestQueue.setExpectedRequestProcessingTimeSecs}, which sizes every future lock,
+     * this only touches the one request it is given.
+     */
+    async extendRequestProcessingTimeSecs(request, secs) {
+        if (!request.id) {
+            return false;
+        }
+        return (await this.backend.extendRequestProcessingTimeSecs?.(request.id, secs)) ?? false;
+    }
     /**
      * Caches information about request to beware of unneeded addRequest() calls.
      */
-    cacheRequest(cacheKey, queueOperationInfo) {
+    #cacheRequest(cacheKey, queueOperationInfo) {
         // Remove the previous entry, as otherwise our cache will never update 👀
-        this.requestCache.remove(cacheKey);
-        this.requestCache.add(cacheKey, {
+        this.#requestCache.remove(cacheKey);
+        this.#requestCache.add(cacheKey, {
             id: queueOperationInfo.requestId,
             isHandled: queueOperationInfo.wasAlreadyHandled,
             uniqueKey: queueOperationInfo.uniqueKey,
             hydrated: null,
-            lockExpiresAt: null,
             forefront: queueOperationInfo.forefront,
         });
     }
@@ -713,9 +717,9 @@ export class RequestQueue {
         rejectOperationInTransaction('RequestQueue.purge()');
         await this.backend.purge();
         // Reset in-memory bookkeeping so the queue behaves as if freshly opened.
-        this.requestCache.clear();
+        this.#requestCache.clear();
         this.#requestSeenCache.clear();
-        this.inProgressRequestBatchCount = 0;
+        this.#inProgressRequestBatchCount = 0;
         // Reset the expected-processing-time high-water mark too, otherwise the monotonic-raise guard
         // in `setExpectedRequestProcessingTimeSecs` would let a value raised in an earlier run leak into a
         // later one and silently swallow a lower hint (the queue is meant to be reusable across runs).
@@ -769,7 +773,7 @@ export class RequestQueue {
     async getInfo() {
         const transaction = activeStorageTransaction();
         const metadata = await this.backend.getMetadata();
-        const bufferedCount = transaction ? this.bufferedRequests(transaction).size : 0;
+        const bufferedCount = transaction ? this.#bufferedRequests(transaction).size : 0;
         if (bufferedCount > 0) {
             return {
                 ...metadata,
@@ -782,12 +786,12 @@ export class RequestQueue {
     /**
      * Fetches URLs from requestsFromUrl and returns them in format of list of requests
      */
-    async fetchRequestsFromUrl(source) {
+    async #fetchRequestsFromUrl(source) {
         const { requestsFromUrl, regex, ...sharedOpts } = source;
         // Download remote resource and parse URLs.
         let urlsArr;
         try {
-            urlsArr = await this.downloadListOfUrls({
+            urlsArr = await this.#downloadListOfUrls({
                 url: requestsFromUrl,
                 urlRegExp: regex,
                 proxyUrl: (await this.#proxyConfiguration?.newProxyInfo())?.url,
@@ -806,7 +810,7 @@ export class RequestQueue {
     /**
      * Adds all fetched requests from a URL from a remote resource.
      */
-    async addFetchedRequests(source, fetchedRequests, options) {
+    async #addFetchedRequests(source, fetchedRequests, options) {
         const { requestsFromUrl, regex } = source;
         const { addedRequests } = await this.addRequestsBatched(fetchedRequests, options);
         this.log.info('Fetched and loaded Requests from a remote resource.', {
@@ -822,7 +826,7 @@ export class RequestQueue {
     /**
      * @internal wraps public utility for mocking purposes
      */
-    async downloadListOfUrls(options) {
+    async #downloadListOfUrls(options) {
         return downloadListOfUrls({
             ...options,
             httpClient: this.#httpClient,

package/storages/storage_instance_manager.d.ts

@@ -7,7 +7,6 @@ export interface IStorage {
     id: string;
     name?: string;
 }
-type Hashable = string;
 /**
  * Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
  *
@@ -35,7 +34,7 @@ export declare class StorageInstanceManager {
      */
     openStorage<TStorage extends IStorage>(cls: Constructor<TStorage>, { id, name, alias, backendOpener, backendCacheKey, }: (ExplicitStorageIdentifier | DefaultStorageIdentifier) & {
         backendOpener: () => Promise<DatasetBackend | KeyValueStoreBackend | RequestQueueBackend>;
-        backendCacheKey: Hashable;
+        backendCacheKey: string;
     }): Promise<TStorage>;
     /**
      * Remove a storage instance from the cache (called from `storage.drop()`).

package/storages/storage_instance_manager.js

@@ -32,7 +32,7 @@ class StorageCache {
         return undefined;
     }
     /** Write a single entry into a given tier. */
-    setInMap(tier, cls, key, instance, backendCacheKey) {
+    #setInMap(tier, cls, key, instance, backendCacheKey) {
         if (!tier.has(cls))
             tier.set(cls, new Map());
         const keyMap = tier.get(cls);
@@ -45,14 +45,14 @@ class StorageCache {
      */
     set(cls, instance, backendCacheKey, alias) {
         // Always cache by id.
-        this.setInMap(this.byId, cls, instance.id, instance, backendCacheKey);
+        this.#setInMap(this.byId, cls, instance.id, instance, backendCacheKey);
         // Cache by name — only for named storages.
         if (instance.name) {
-            this.setInMap(this.byName, cls, instance.name, instance, backendCacheKey);
+            this.#setInMap(this.byName, cls, instance.name, instance, backendCacheKey);
         }
         // Cache by alias — only for unnamed storages opened via alias.
         if (alias !== undefined) {
-            this.setInMap(this.byAlias, cls, alias, instance, backendCacheKey);
+            this.#setInMap(this.byAlias, cls, alias, instance, backendCacheKey);
         }
     }
     removeFromCache(instance) {

package/storages/transaction.d.ts

@@ -1,4 +1,4 @@
-import type { Awaitable, Dictionary } from '@crawlee/types';
+import type { Awaitable, Dictionary, RequestSchema } from '@crawlee/types';
 import type { RecordOptions } from './key_value_store.js';
 /**
  * Governs whether writes of a given storage type performed inside a {@link StorageTransaction} are
@@ -38,6 +38,7 @@ export interface TransactionParticipant {
 }
 /**
  * A single dataset write (`pushData`) recorded in a transaction journal.
+ * @internal
  */
 export interface DatasetJournalEntry {
     type: 'dataset';
@@ -50,6 +51,7 @@ export interface DatasetJournalEntry {
 }
 /**
  * A single key-value store write (`setValue`) recorded in a transaction journal.
+ * @internal
  */
 export interface KeyValueStoreJournalEntry {
     type: 'keyValueStore';
@@ -63,6 +65,7 @@ export interface KeyValueStoreJournalEntry {
 }
 /**
  * A request recorded in a transaction journal.
+ * @internal
  */
 export interface JournaledRequest {
     url: string;
@@ -72,10 +75,11 @@ export interface JournaledRequest {
      * A full JSON snapshot of the request for the commit replay. Only present for buffered additions —
      * deduplicated and write-through ones are journaled for introspection only.
      */
-    snapshot?: Dictionary;
+    snapshot?: RequestSchema;
 }
 /**
  * A batch of request queue additions recorded in a transaction journal.
+ * @internal
  */
 export interface RequestQueueJournalEntry {
     type: 'requestQueue';
@@ -86,6 +90,7 @@ export interface RequestQueueJournalEntry {
     /** Write-through entries were applied immediately; they are never replayed. */
     writeThrough: boolean;
 }
+/** @internal */
 export type JournalEntry = DatasetJournalEntry | KeyValueStoreJournalEntry | RequestQueueJournalEntry;
 /**
  * A read-only view of a {@link StorageTransaction}: only the journal-backed introspection accessors,
@@ -143,12 +148,18 @@ export interface StorageTransactionOptions {
  */
 export declare class StorageTransaction implements StorageTransactionView {
     #private;
-    /** The ordered, append-only journal — the source of truth for commit, introspection and reads. */
+    /**
+     * The ordered, append-only journal — the source of truth for commit, introspection and reads.
+     * @internal
+     */
     readonly journal: JournalEntry[];
-    /** Per-storage-type write policy. */
-    readonly policy: StorageWritePolicy;
     /** @internal */
     constructor(options?: StorageTransactionOptions);
+    /**
+     * Per-storage-type write policy.
+     * @internal
+     */
+    get policy(): StorageWritePolicy;
     get state(): StorageTransactionState;
     /**
      * `true` only while `state === 'open'`. This is the single predicate every storage operation
@@ -162,6 +173,13 @@ export declare class StorageTransaction implements StorageTransactionView {
      * @internal
      */
     recordJournalEntry(entry: JournalEntry): void;
+    /**
+     * Registers a callback to run once a commit has been attempted, whether it succeeded (`error` is
+     * `undefined`) or failed - the only point at which a deferred write can be reacted to where it was
+     * made. Callbacks run in registration order; the first to throw propagates, the rest do not run, and
+     * on a failed commit its error replaces the commit error. Not run on rollback: nothing was written.
+     */
+    afterCommit(callback: (error?: Error) => Awaitable<void>): void;
     /**
      * Replays the journaled writes into real storage. A no-op unless the transaction is `open`.
      *
@@ -171,7 +189,6 @@ export declare class StorageTransaction implements StorageTransactionView {
      * partway may have applied some of the writes already.
      */
     commit(): Promise<void>;
-    private flush;
     /**
      * Discards the journaled writes. A no-op unless the transaction is `open` — in particular, calling it
      * after a successful `commit()` (which the crawler's error handling can legitimately do) does nothing
@@ -179,9 +196,10 @@ export declare class StorageTransaction implements StorageTransactionView {
      */
     rollback(): void;
     /**
-     * Releases the journal and the write-time snapshots it holds. Must be called for *every* terminal
-     * state, `failed` included. Idempotent, never throws, and does not change `state`. Any
-     * {@link StorageTransactionView} of this transaction is only valid until this is called.
+     * Releases the journal, the write-time snapshots it holds and any registered commit callbacks. Must
+     * be called for *every* terminal state, `failed` included. Idempotent, never throws, and does not
+     * change `state`. Any {@link StorageTransactionView} of this transaction is only valid until this
+     * is called.
      */
     dispose(): void;
     get datasetItems(): {