@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.0

package/storages/request_queue.js

@@ -1,13 +1,17 @@
 import { inspect } from 'node:util';
-import { downloadListOfUrls, isAsyncIterable, isIterable, sleep } from '@crawlee/utils';
-import ow from 'ow';
+import { isAsyncIterable, isIterable } from '@crawlee/utils/internal';
+import { downloadListOfUrls } from '@crawlee/utils';
+import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
+import { tryCancel } from '@apify/timeout';
 import { Configuration } from '../configuration.js';
 import { getObjectType } from '../debug.js';
-import { chunkedAsyncIterable, peekableAsyncIterable } from '../iterables.js';
+import { EventType } from '../events/event_manager.js';
 import { Request } from '../request.js';
 import { serviceLocator } from '../service_locator.js';
-import { checkStorageAccess } from './access_checking.js';
+import { parseArgument, schemas, validators } from '../validators.js';
+import { activeStorageTransaction, rejectOperationInTransaction } from './transaction.js';
+import { drainRequestBatches } from './batched_adds.js';
 import { StorageStatsTracker } from './storage_stats.js';
 import { resolveStorageIdentifier } from './storage_instance_manager.js';
 import { getRequestId, purgeDefaultStorages } from './utils.js';
@@ -17,6 +21,43 @@ import { RequestDeduplicationCache } from './request_dedup_cache.js';
  * @internal
  */
 const MAX_CACHED_REQUESTS = 2_000_000;
+const iterableSchema = z.custom((value) => isIterable(value) || isAsyncIterable(value), {
+    error: (issue) => `Expected an iterable or async iterable, got ${getObjectType(issue.input)}`,
+});
+const operationOptionsSchema = z.strictObject({
+    forefront: z.boolean().default(false),
+});
+const addRequestsOptionsSchema = z.strictObject({
+    forefront: z.boolean().default(false),
+    cache: z.boolean().default(true),
+});
+const addRequestsBatchedOptionsSchema = z.strictObject({
+    forefront: z.boolean().optional(),
+    waitForAllRequestsToBeAdded: z.boolean().default(false),
+    batchSize: schemas.anyNumber.default(1000),
+    waitBetweenBatchesMillis: schemas.anyNumber.default(1000),
+    maxNewRequests: schemas.anyNumber.optional(),
+});
+const newRequestLikeSchema = z.looseObject({
+    url: z.string(),
+    id: z.undefined().optional(),
+});
+const handledRequestSchema = z.looseObject({
+    id: z.string(),
+    uniqueKey: z.string(),
+    handledAt: z.string().optional(),
+});
+const reclaimedRequestSchema = z.looseObject({
+    id: z.string(),
+    uniqueKey: z.string(),
+});
+const uniqueKeySchema = z.string();
+const openOptionsSchema = z.strictObject({
+    configuration: z.instanceof(Configuration).optional(),
+    storageBackend: validators.storageBackend.optional(),
+    proxyConfiguration: validators.proxyConfiguration.optional(),
+    httpClient: schemas.httpClient.optional(),
+});
 /**
  * Represents a queue of URLs to crawl, which is used for deep crawling of websites
  * where you start with several URLs and then recursively
@@ -55,26 +96,28 @@ export class RequestQueue {
     id;
     name;
     backend;
-    proxyConfiguration;
+    #proxyConfiguration;
     log;
+    // kept as TS-private: request_queue tests read this cache directly
     requestCache;
     /**
      * Remembers the `requestId` of every request already submitted to the client — including background
      * batches that `requestCache` skips — so overlapping URL sets aren't re-submitted.
      * See {@link RequestDeduplicationCache} for why this is a separate, cheaper cache.
      */
-    requestSeenCache;
-    queuePausedForMigration = false;
+    #requestSeenCache;
+    #queuePausedForMigration = false;
+    // kept as TS-private: packages/core/test request-queue tests write this counter directly
     inProgressRequestBatchCount = 0;
     /**
      * The largest expected request-processing time (in seconds) seen so far via
      * {@link setExpectedRequestProcessingTimeSecs}. Used to ensure that value is only ever raised, never
      * lowered, before being forwarded to the storage backend.
      */
-    expectedRequestProcessingSecs = 0;
-    httpClient;
-    events;
-    statsTracker = new StorageStatsTracker({
+    #expectedRequestProcessingSecs = 0;
+    #httpClient;
+    #events;
+    #statsTracker = new StorageStatsTracker({
         writeCount: 0,
         headItemReadCount: 0,
     });
@@ -83,7 +126,7 @@ export class RequestQueue {
      * queue-head reads issued to the underlying storage backend). Counted per backend call.
      */
     get stats() {
-        return this.statsTracker.current;
+        return this.#statsTracker.current;
     }
     /**
      * @internal
@@ -91,14 +134,14 @@ export class RequestQueue {
     constructor(options) {
         this.id = options.metadata.id;
         this.name = options.metadata.name;
-        this.events = serviceLocator.getEventManager();
+        this.#events = serviceLocator.getEventManager();
         this.backend = options.backend;
-        this.proxyConfiguration = options.proxyConfiguration;
+        this.#proxyConfiguration = options.proxyConfiguration;
         this.requestCache = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
-        this.requestSeenCache = new RequestDeduplicationCache();
+        this.#requestSeenCache = new RequestDeduplicationCache();
         this.log = serviceLocator.getLogger().child({ prefix: `RequestQueue(${this.id}, ${this.name ?? 'no-name'})` });
-        this.events.on("migrating" /* EventType.MIGRATING */, async () => {
-            this.queuePausedForMigration = true;
+        this.#events.on(EventType.MIGRATING, async () => {
+            this.#queuePausedForMigration = true;
         });
     }
     /**
@@ -134,26 +177,24 @@ export class RequestQueue {
      * @param [options] Request queue operation options.
      */
     async addRequest(requestLike, options = {}) {
-        checkStorageAccess();
-        ow(requestLike, ow.object);
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-        }));
-        const { forefront = false } = options;
+        const transaction = activeStorageTransaction();
+        parseArgument(requestLike, schemas.anyObject);
+        const { forefront } = parseArgument(options, operationOptionsSchema);
         if ('requestsFromUrl' in requestLike) {
             const requests = await this.fetchRequestsFromUrl(requestLike);
             const processedRequests = await this.addFetchedRequests(requestLike, requests, options);
             return { ...processedRequests[0], forefront };
         }
-        ow(requestLike, ow.object.partialShape({
-            url: ow.string,
-            id: ow.undefined,
-        }));
+        parseArgument(requestLike, newRequestLikeSchema);
         const request = requestLike instanceof Request ? requestLike : new Request(requestLike);
+        if (transaction?.policy.requestQueue === 'deferred') {
+            return this.addRequestDeferred(transaction, request, forefront);
+        }
         const cacheKey = getRequestId(request.uniqueKey);
         const cachedInfo = this.requestCache.get(cacheKey);
         if (cachedInfo) {
             request.id = cachedInfo.id;
+            this.recordRequestJournalEntry(transaction, [request], forefront, true);
             return {
                 wasAlreadyPresent: true,
                 // We may assume that if request is in local cache then also the information if the
@@ -164,17 +205,163 @@ export class RequestQueue {
                 forefront,
             };
         }
-        this.statsTracker.add('writeCount');
+        this.#statsTracker.add('writeCount');
         const { processedRequests } = await this.backend.addBatchOfRequests([request], { forefront });
+        this.recordRequestJournalEntry(transaction, [request], forefront, true);
         const queueOperationInfo = {
             ...processedRequests[0],
             uniqueKey: request.uniqueKey,
             forefront,
         };
         this.cacheRequest(cacheKey, queueOperationInfo);
-        this.requestSeenCache.add(cacheKey, request.id);
+        this.#requestSeenCache.add(cacheKey, request.id);
         return queueOperationInfo;
     }
+    /**
+     * Journals an addition for introspection only; these entries are never replayed. A no-op unless the
+     * transaction is open, so detached and outliving writers stay out of the journal.
+     */
+    recordRequestJournalEntry(transaction, requests, forefront, writeThrough) {
+        if (!transaction?.isActive || requests.length === 0)
+            return;
+        transaction.recordJournalEntry({
+            type: 'requestQueue',
+            participant: this,
+            requests: requests.map((request) => ({
+                url: request.url,
+                uniqueKey: request.uniqueKey,
+                label: request.label,
+            })),
+            forefront,
+            writeThrough,
+        });
+    }
+    /**
+     * The requests buffered by the given transaction for this queue, keyed by `uniqueKey` — a dedup
+     * index derived from the transaction journal.
+     */
+    bufferedRequests(transaction) {
+        const buffered = new Map();
+        // Only `deferred` records snapshots, so scanning the journal under `writeThrough` never finds any.
+        if (transaction.policy.requestQueue !== 'deferred')
+            return buffered;
+        for (const entry of transaction.journal) {
+            if (entry.type !== 'requestQueue' || entry.participant !== this)
+                continue;
+            for (const request of entry.requests) {
+                if (request.snapshot !== undefined)
+                    buffered.set(request.uniqueKey, request.snapshot);
+            }
+        }
+        return buffered;
+    }
+    /**
+     * Adds a request under the `deferred` policy: journaled now, really added by the commit replay.
+     * A new request's `requestId` is the local `uniqueKey` hash and is **provisional** — never write it
+     * to `request.id` or the dedup caches. Dedup is cheapest-first: buffer, caches, then a backend probe.
+     */
+    async addRequestDeferred(transaction, request, forefront, buffered = this.bufferedRequests(transaction)) {
+        // This transaction's own buffered adds; the shared caches never see them (provisional ids).
+        if (buffered.has(request.uniqueKey)) {
+            this.recordRequestJournalEntry(transaction, [request], forefront, false);
+            return {
+                wasAlreadyPresent: true,
+                wasAlreadyHandled: false,
+                requestId: getRequestId(request.uniqueKey),
+                uniqueKey: request.uniqueKey,
+                forefront,
+            };
+        }
+        // The caches hold real backend ids. Only *writing* provisional ids to them would be wrong;
+        // reading saves a probe. Same lookup as the write-through path.
+        const cacheKey = getRequestId(request.uniqueKey);
+        const cachedInfo = this.requestCache.get(cacheKey);
+        const knownRequestId = cachedInfo?.id ?? this.#requestSeenCache.get(cacheKey);
+        if (knownRequestId) {
+            this.recordRequestJournalEntry(transaction, [request], forefront, false);
+            return {
+                wasAlreadyPresent: true,
+                // The dedup cache doesn't track the handled state; only the full record does.
+                wasAlreadyHandled: cachedInfo?.isHandled ?? false,
+                requestId: knownRequestId,
+                uniqueKey: request.uniqueKey,
+                forefront,
+            };
+        }
+        // The caches are bounded, so a miss is not proof of absence - probe for an accurate answer.
+        const existing = await this.backend.getRequest(request.uniqueKey);
+        if (existing) {
+            this.recordRequestJournalEntry(transaction, [request], forefront, false);
+            return {
+                wasAlreadyPresent: true,
+                wasAlreadyHandled: existing.handledAt != null,
+                requestId: existing.id,
+                uniqueKey: request.uniqueKey,
+                forefront,
+            };
+        }
+        // The entry below *is* the write, so a transaction closed during the probe must not receive it -
+        // pass through instead, per the closed-transaction rule. Under `deferred` that can land an
+        // addition a rollback would have discarded; dedup bounds that cost, silent loss is unbounded.
+        if (!transaction.isActive) {
+            return await this.addRequest(request, { forefront });
+        }
+        const snapshot = JSON.parse(JSON.stringify(request));
+        // Strip-list, not allow-list: every user-facing field flows through, including ones added to
+        // `Request` in the future. The exceptions are `id` and `handledAt`, the two backend-owned
+        // lifecycle fields.
+        delete snapshot.id;
+        delete snapshot.handledAt;
+        transaction.recordJournalEntry({
+            type: 'requestQueue',
+            participant: this,
+            requests: [{ url: request.url, uniqueKey: request.uniqueKey, label: request.label, snapshot }],
+            forefront,
+            writeThrough: false,
+        });
+        buffered.set(request.uniqueKey, snapshot);
+        return {
+            wasAlreadyPresent: false,
+            wasAlreadyHandled: false,
+            requestId: getRequestId(request.uniqueKey),
+            uniqueKey: request.uniqueKey,
+            forefront,
+        };
+    }
+    /** @internal */
+    async commitJournalEntries(entries) {
+        // Replay through `backend.addBatchOfRequests`, *not* the batched frontend wrapper - the wrapper
+        // resolves after the first chunk and sleeps between the rest, neither of which commit may
+        // inherit. One call per `forefront` flag; the order of forefront additions is arbitrary anyway.
+        for (const forefront of [false, true]) {
+            const requests = entries.flatMap((entry) => entry.type === 'requestQueue' && entry.forefront === forefront
+                ? // Requests without a snapshot were deduplicated or written through; nothing to replay.
+                    entry.requests
+                        .filter((journaled) => journaled.snapshot !== undefined)
+                        .map((journaled) => new Request(journaled.snapshot))
+                : []);
+            if (requests.length === 0)
+                continue;
+            this.#statsTracker.add('writeCount');
+            const { processedRequests, unprocessedRequests } = await this.backend.addBatchOfRequests(requests, {
+                forefront,
+            });
+            // Only now, with the real backend-assigned ids, may the shared dedup caches be populated.
+            for (const processed of processedRequests) {
+                const cacheKey = getRequestId(processed.uniqueKey);
+                this.cacheRequest(cacheKey, { ...processed, forefront });
+                this.#requestSeenCache.add(cacheKey, processed.requestId);
+            }
+            if (unprocessedRequests.length > 0) {
+                // Warn and skip, rather than retry or fail. `unprocessedRequests` is what remains after
+                // the backend's own transient-error handling - a semantic rejection that retrying here
+                // would only re-poke. And failing the commit would let one malformed request hold the
+                // whole transaction hostage.
+                this.log.warning('Some requests were rejected by the request queue while committing a storage transaction and will be skipped. ' +
+                    "This usually means the request data is malformed (e.g. an invalid 'userData' shape).", { unprocessedRequests });
+            }
+        }
+    }
     /**
      * Adds requests to the queue in batches of 25. This method will wait till all the requests are added
      * to the queue before resolving. You should prefer using `queue.addRequestsBatched()` or `crawler.addRequests()`
@@ -190,15 +377,9 @@ export class RequestQueue {
      * @param [options] Request queue operation options.
      */
     async addRequests(requestsLike, options = {}) {
-        checkStorageAccess();
-        ow(requestsLike, ow.object
-            .is((value) => isIterable(value) || isAsyncIterable(value))
-            .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-            cache: ow.optional.boolean,
-        }));
-        const { forefront = false, cache = true } = options;
+        const transaction = activeStorageTransaction();
+        parseArgument(requestsLike, iterableSchema);
+        const { forefront, cache } = parseArgument(options, addRequestsOptionsSchema);
         const uniqueKeyToCacheKey = new Map();
         const getCachedRequestId = (uniqueKey) => {
             const cached = uniqueKeyToCacheKey.get(uniqueKey);
@@ -225,12 +406,20 @@ export class RequestQueue {
                 requests.push(requestLike instanceof Request ? requestLike : new Request(requestLike));
             }
         }
+        if (transaction?.policy.requestQueue === 'deferred') {
+            const buffered = this.bufferedRequests(transaction);
+            for (const request of requests) {
+                results.processedRequests.push(await this.addRequestDeferred(transaction, request, forefront, buffered));
+            }
+            return results;
+        }
+        this.recordRequestJournalEntry(transaction, requests, forefront, true);
         const requestsToAdd = new Map();
         for (const request of requests) {
             const cacheKey = getCachedRequestId(request.uniqueKey);
             // Prefer the full `requestCache` record; fall back to the dedup cache for background batches it skips.
             const cachedInfo = this.requestCache.get(cacheKey);
-            const knownRequestId = cachedInfo?.id ?? this.requestSeenCache.get(cacheKey);
+            const knownRequestId = cachedInfo?.id ?? this.#requestSeenCache.get(cacheKey);
             if (knownRequestId) {
                 request.id = knownRequestId;
                 results.processedRequests.push({
@@ -249,7 +438,7 @@ export class RequestQueue {
         if (!requestsToAdd.size) {
             return results;
         }
-        this.statsTracker.add('writeCount');
+        this.#statsTracker.add('writeCount');
         const apiResults = await this.backend.addBatchOfRequests([...requestsToAdd.values()], { forefront });
         // Report unprocessed requests
         results.unprocessedRequests = apiResults.unprocessedRequests;
@@ -262,7 +451,7 @@ export class RequestQueue {
                 this.cacheRequest(cacheKey, { ...newRequest, forefront });
             }
             // Unlike `requestCache`, populate this on every batch (including background ones).
-            this.requestSeenCache.add(cacheKey, newRequest.requestId);
+            this.#requestSeenCache.add(cacheKey, newRequest.requestId);
         }
         return results;
     }
@@ -276,17 +465,8 @@ export class RequestQueue {
      * @param options Options for the request queue
      */
     async addRequestsBatched(requests, options = {}) {
-        checkStorageAccess();
-        ow(requests, ow.object
-            .is((value) => isIterable(value) || isAsyncIterable(value))
-            .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-            waitForAllRequestsToBeAdded: ow.optional.boolean,
-            batchSize: ow.optional.number,
-            waitBetweenBatchesMillis: ow.optional.number,
-            maxNewRequests: ow.optional.number,
-        }));
+        parseArgument(requests, iterableSchema);
+        const { forefront, waitForAllRequestsToBeAdded, batchSize, waitBetweenBatchesMillis, maxNewRequests } = parseArgument(options, addRequestsBatchedOptionsSchema);
         const addRequest = this.addRequest.bind(this);
         async function* generateRequests() {
             for await (const opts of requests) {
@@ -305,7 +485,7 @@ export class RequestQueue {
                 }
                 if (opts && typeof opts === 'object' && 'requestsFromUrl' in opts) {
                     // Handle URL lists right away
-                    await addRequest(opts, { forefront: options.forefront });
+                    await addRequest(opts, { forefront });
                 }
                 else {
                     // Yield valid requests
@@ -313,84 +493,36 @@ export class RequestQueue {
                 }
             }
         }
-        const { batchSize = 1000, waitBetweenBatchesMillis = 1000, maxNewRequests = undefined } = options;
-        let remainingBudget = maxNewRequests ?? Infinity;
-        const requestsOverLimit = [];
-        // If there's a limit on the number of added requests, do not send batches bigger than the limit
-        const effectiveChunkSize = maxNewRequests !== undefined ? () => Math.min(batchSize, remainingBudget) : batchSize;
-        // Hold onto the underlying iterator so we can drain leftovers from it in buildResult
-        const requestIterator = generateRequests();
-        const chunks = peekableAsyncIterable(chunkedAsyncIterable(requestIterator, effectiveChunkSize));
-        const chunksIterator = chunks[Symbol.asyncIterator]();
-        /**
-         * Process a chunk: send it to the queue, then update the remaining budget if maxNewRequests is active.
-         *
-         * Requests the backend reports as unprocessed are warned about and skipped rather than retried:
-         * `unprocessedRequests` is what remains after the backend's own transient-error handling - a
-         * semantic rejection (e.g. a malformed `userData` shape) that re-sending would only re-poke.
-         * Retrying transient failures is the storage backend's job, not the frontend's.
-         */
-        const processChunk = async (chunk, cache = true) => {
-            const { processedRequests, unprocessedRequests } = await this.addRequests(chunk, {
-                forefront: options.forefront,
-                cache,
-            });
-            if (unprocessedRequests.length > 0) {
-                this.log.warning('Some requests were rejected by the request queue and will be skipped. ' +
-                    "This usually means the request data is malformed (e.g. an invalid 'userData' shape).", { unprocessedRequests });
-            }
-            if (maxNewRequests !== undefined) {
-                remainingBudget -= processedRequests.filter((r) => !r.wasAlreadyPresent).length;
-            }
-            return processedRequests;
-        };
-        /**
-         * Build the final result. When maxNewRequests is set, drains any remaining items
-         * from the underlying request iterator into requestsOverLimit.
-         *
-         * We accept the iterator explicitly (rather than closing over it) to make it obvious
-         * that this is the *same* iterator that `chunkedAsyncIterable` has been consuming —
-         * so only unconsumed items are drained. We drain `requestIterator` (not `chunks`)
-         * because `chunkedAsyncIterable` stops yielding when the budget-based chunk size
-         * drops to 0, leaving unconsumed items in the underlying iterator.
-         */
-        const buildResult = async (addedRequests, waitForAllRequestsToBeAdded, unconsumedIterator) => {
-            if (maxNewRequests !== undefined) {
-                for await (const request of unconsumedIterator) {
-                    requestsOverLimit.push(request);
+        return drainRequestBatches({
+            items: generateRequests(),
+            batchSize,
+            waitBetweenBatchesMillis,
+            waitForAllRequestsToBeAdded,
+            maxNewRequests,
+            /**
+             * Requests the backend reports as unprocessed are warned about and skipped rather than retried:
+             * `unprocessedRequests` is what remains after the backend's own transient-error handling - a
+             * semantic rejection (e.g. a malformed `userData` shape) that re-sending would only re-poke.
+             * Retrying transient failures is the storage backend's job, not the frontend's.
+             */
+            processChunk: async (chunk, isInitial) => {
+                const { processedRequests, unprocessedRequests } = await this.addRequests(chunk, {
+                    forefront,
+                    cache: isInitial,
+                });
+                if (unprocessedRequests.length > 0) {
+                    this.log.warning('Some requests were rejected by the request queue and will be skipped. ' +
+                        "This usually means the request data is malformed (e.g. an invalid 'userData' shape).", { unprocessedRequests });
                 }
-            }
-            return { addedRequests, waitForAllRequestsToBeAdded, requestsOverLimit };
-        };
-        // Add initial batch to process right away
-        const initialChunk = await chunksIterator.peek();
-        if (initialChunk === undefined) {
-            return buildResult([], Promise.resolve([]), requestIterator);
-        }
-        const addedRequests = await processChunk(initialChunk);
-        await chunksIterator.next();
-        // If we have no more requests to add (either exhausted or budget hit), return immediately
-        if ((await chunksIterator.peek()) === undefined) {
-            return buildResult(addedRequests, Promise.resolve([]), requestIterator);
-        }
-        // eslint-disable-next-line no-async-promise-executor
-        const promise = new Promise(async (resolve) => {
-            const finalAddedRequests = [];
-            for await (const requestChunk of chunks) {
-                finalAddedRequests.push(...(await processChunk(requestChunk, false)));
-                await sleep(waitBetweenBatchesMillis);
-            }
-            resolve(finalAddedRequests);
-        });
-        this.inProgressRequestBatchCount += 1;
-        void promise.finally(() => {
-            this.inProgressRequestBatchCount -= 1;
+                return processedRequests;
+            },
+            trackBackgroundBatches: (batches) => {
+                this.inProgressRequestBatchCount += 1;
+                void batches.finally(() => {
+                    this.inProgressRequestBatchCount -= 1;
+                });
+            },
         });
-        // When maxNewRequests is set, we must wait for all batches so we can accurately report skipped requests.
-        if (options.waitForAllRequestsToBeAdded || maxNewRequests !== undefined) {
-            addedRequests.push(...(await promise));
-        }
-        return buildResult(addedRequests, promise, requestIterator);
     }
     /**
      * Gets the request from the queue specified by its `uniqueKey`.
@@ -399,8 +531,13 @@ export class RequestQueue {
      * @returns Returns the request object, or `null` if it was not found.
      */
     async getRequest(uniqueKey) {
-        checkStorageAccess();
-        ow(uniqueKey, ow.string);
+        const transaction = activeStorageTransaction();
+        parseArgument(uniqueKey, uniqueKeySchema);
+        // Requests buffered by the active transaction (under the `deferred` write policy) are visible to it.
+        const buffered = transaction && this.bufferedRequests(transaction).get(uniqueKey);
+        if (buffered) {
+            return new Request(buffered);
+        }
         const requestOptions = await this.backend.getRequest(uniqueKey);
         if (!requestOptions)
             return null;
@@ -424,11 +561,11 @@ export class RequestQueue {
      *   Returns the request object or `null` if there are no more pending requests.
      */
     async fetchNextRequest() {
-        checkStorageAccess();
-        if (this.queuePausedForMigration) {
+        rejectOperationInTransaction('RequestQueue.fetchNextRequest()', 'it is part of the crawler request-processing bookkeeping, which a transaction must not affect.');
+        if (this.#queuePausedForMigration) {
             return null;
         }
-        this.statsTracker.add('headItemReadCount');
+        this.#statsTracker.add('headItemReadCount');
         const requestOptions = await this.backend.fetchNextRequest();
         if (!requestOptions)
             return null;
@@ -441,15 +578,11 @@ export class RequestQueue {
      * Handled requests will never again be returned by the `fetchNextRequest` function.
      */
     async markRequestAsHandled(request) {
-        checkStorageAccess();
-        ow(request, ow.object.partialShape({
-            id: ow.string,
-            uniqueKey: ow.string,
-            handledAt: ow.optional.string,
-        }));
+        rejectOperationInTransaction('RequestQueue.markRequestAsHandled()', 'it is part of the crawler request-processing bookkeeping, which a transaction must not affect.');
+        parseArgument(request, handledRequestSchema);
         const forefront = this.requestCache.get(getRequestId(request.uniqueKey))?.forefront ?? false;
         const handledAt = request.handledAt ?? new Date().toISOString();
-        this.statsTracker.add('writeCount');
+        this.#statsTracker.add('writeCount');
         const processedRequest = await this.backend.markRequestAsHandled({
             ...request,
             handledAt,
@@ -474,17 +607,13 @@ export class RequestQueue {
      * For example, this lets you store the number of retries or error messages for the request.
      */
     async reclaimRequest(request, options = {}) {
-        checkStorageAccess();
-        ow(request, ow.object.partialShape({
-            id: ow.string,
-            uniqueKey: ow.string,
-        }));
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-        }));
-        const { forefront = false } = options;
-        this.statsTracker.add('writeCount');
-        const processedRequest = await this.backend.reclaimRequest(request, { forefront });
+        rejectOperationInTransaction('RequestQueue.reclaimRequest()', 'it is part of the crawler request-processing bookkeeping, which a transaction must not affect.');
+        parseArgument(request, reclaimedRequestSchema);
+        const { forefront } = parseArgument(options, operationOptionsSchema);
+        this.#statsTracker.add('writeCount');
+        const processedRequest = await this.backend.reclaimRequest(request, {
+            forefront,
+        });
         // The request was not in progress — nothing to reclaim.
         if (!processedRequest) {
             return null;
@@ -508,7 +637,11 @@ export class RequestQueue {
      * {@link RequestQueue.isFinished}.
      */
     async isEmpty() {
-        checkStorageAccess();
+        const transaction = activeStorageTransaction();
+        // Requests buffered by the active transaction count as pending from its point of view.
+        if (transaction && this.bufferedRequests(transaction).size > 0) {
+            return false;
+        }
         return this.backend.isEmpty();
     }
     /**
@@ -520,11 +653,15 @@ export class RequestQueue {
      * a false negative, but it shall never return a false positive.
      */
     async isFinished() {
-        checkStorageAccess();
+        const transaction = activeStorageTransaction();
         // We are not finished if we're still adding new requests in the background.
         if (this.inProgressRequestBatchCount > 0) {
             return false;
         }
+        // Requests buffered by the active transaction count as pending from its point of view.
+        if (transaction && this.bufferedRequests(transaction).size > 0) {
+            return false;
+        }
         return this.backend.isFinished();
     }
     /**
@@ -538,10 +675,10 @@ export class RequestQueue {
      * short the reservation of a long-lived one and have its in-flight request stolen.
      */
     async setExpectedRequestProcessingTimeSecs(secs) {
-        if (secs <= this.expectedRequestProcessingSecs) {
+        if (secs <= this.#expectedRequestProcessingSecs) {
             return;
         }
-        this.expectedRequestProcessingSecs = secs;
+        this.#expectedRequestProcessingSecs = secs;
         await this.backend.setExpectedRequestProcessingTimeSecs?.(secs);
     }
     /**
@@ -564,7 +701,7 @@ export class RequestQueue {
      * depending on the mode of operation.
      */
     async drop() {
-        checkStorageAccess();
+        rejectOperationInTransaction('RequestQueue.drop()');
         await this.backend.drop();
         serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
@@ -573,16 +710,16 @@ export class RequestQueue {
      * so it can be reused (e.g. across multiple `crawler.run()` calls).
      */
     async purge() {
-        checkStorageAccess();
+        rejectOperationInTransaction('RequestQueue.purge()');
         await this.backend.purge();
         // Reset in-memory bookkeeping so the queue behaves as if freshly opened.
         this.requestCache.clear();
-        this.requestSeenCache.clear();
+        this.#requestSeenCache.clear();
         this.inProgressRequestBatchCount = 0;
         // Reset the expected-processing-time high-water mark too, otherwise the monotonic-raise guard
         // in `setExpectedRequestProcessingTimeSecs` would let a value raised in an earlier run leak into a
         // later one and silently swallow a lower hint (the queue is meant to be reusable across runs).
-        this.expectedRequestProcessingSecs = 0;
+        this.#expectedRequestProcessingSecs = 0;
     }
     /**
      * @inheritdoc
@@ -630,8 +767,17 @@ export class RequestQueue {
      * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
     async getInfo() {
-        checkStorageAccess();
-        return this.backend.getMetadata();
+        const transaction = activeStorageTransaction();
+        const metadata = await this.backend.getMetadata();
+        const bufferedCount = transaction ? this.bufferedRequests(transaction).size : 0;
+        if (bufferedCount > 0) {
+            return {
+                ...metadata,
+                totalRequestCount: metadata.totalRequestCount + bufferedCount,
+                pendingRequestCount: metadata.pendingRequestCount + bufferedCount,
+            };
+        }
+        return metadata;
     }
     /**
      * Fetches URLs from requestsFromUrl and returns them in format of list of requests
@@ -641,10 +787,10 @@ export class RequestQueue {
         // Download remote resource and parse URLs.
         let urlsArr;
         try {
-            urlsArr = await this._downloadListOfUrls({
+            urlsArr = await this.downloadListOfUrls({
                 url: requestsFromUrl,
                 urlRegExp: regex,
-                proxyUrl: (await this.proxyConfiguration?.newProxyInfo())?.url,
+                proxyUrl: (await this.#proxyConfiguration?.newProxyInfo())?.url,
             });
         }
         catch (err) {
@@ -676,10 +822,10 @@ export class RequestQueue {
     /**
      * @internal wraps public utility for mocking purposes
      */
-    async _downloadListOfUrls(options) {
+    async downloadListOfUrls(options) {
         return downloadListOfUrls({
             ...options,
-            httpClient: this.httpClient,
+            httpClient: this.#httpClient,
         });
     }
     /**
@@ -700,15 +846,10 @@ export class RequestQueue {
      * @param [options] Open Request Queue options.
      */
     static async open(identifier, options = {}) {
-        checkStorageAccess();
-        ow(options, ow.object.exactShape({
-            configuration: ow.optional.object.instanceOf(Configuration),
-            storageBackend: ow.optional.object,
-            proxyConfiguration: ow.optional.object,
-            httpClient: ow.optional.object,
-        }));
-        const storageBackend = options.storageBackend ?? serviceLocator.getStorageBackend();
-        const configuration = options.configuration ?? serviceLocator.getConfiguration();
+        tryCancel();
+        const parsedOptions = parseArgument(options, openOptionsSchema);
+        const storageBackend = parsedOptions.storageBackend ?? serviceLocator.getStorageBackend();
+        const configuration = parsedOptions.configuration ?? serviceLocator.getConfiguration();
         await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration });
         const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'RequestQueue');
         const queue = await serviceLocator
@@ -718,8 +859,8 @@ export class RequestQueue {
             backendOpener: () => storageBackend.createRequestQueueBackend(resolved),
             backendCacheKey: storageBackend.getStorageBackendCacheKey?.() ?? storageBackend.constructor.name,
         });
-        queue.proxyConfiguration = options.proxyConfiguration;
-        queue.httpClient = options.httpClient;
+        queue.#proxyConfiguration = parsedOptions.proxyConfiguration;
+        queue.#httpClient = parsedOptions.httpClient;
         return queue;
     }
 }