@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.131

package/session_pool/session_pool.js

@@ -1,31 +1,37 @@
-import { EventEmitter } from 'node:events';
 import { AsyncQueue } from '@sapphire/async-queue';
-import ow from 'ow';
-import { Configuration } from '../configuration.js';
-import { log as defaultLog } from '../log.js';
+import { z } from 'zod';
+import { EventType } from '../events/event_manager.js';
+import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from '../storages/key_value_store.js';
-import { BLOCKED_STATUS_CODES, MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
+import { parseArgument, schemas, validators } from '../validators.js';
+import { MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
+import { createDefaultSessionFingerprint } from './fingerprint.js';
 import { Session } from './session.js';
+const SESSION_REUSE_STRATEGIES = ['random', 'round-robin', 'use-until-failure'];
+// `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
+// copy), so class instances like loggers keep their prototype.
+const sessionPoolOptionsSchema = z.strictObject({
+    id: z.union([schemas.anyNumber, z.string()]).optional(),
+    maxPoolSize: schemas.anyNumber.default(MAX_POOL_SIZE),
+    persistStateKeyValueStoreId: z.string().optional(),
+    persistStateKey: z.string().optional(),
+    createSessionFunction: schemas.anyFunction.optional(),
+    sessionOptions: schemas.anyObject.default(() => ({})),
+    log: validators.logger.default(() => serviceLocator.getLogger()),
+    persistenceOptions: schemas.anyObject.default(() => ({ enable: true })),
+    sessionReuseStrategy: z.enum(SESSION_REUSE_STRATEGIES).default('random'),
+});
+const createSessionOptionsSchema = z.strictObject({
+    sessionOptions: schemas.anyObject.default(() => ({})),
+});
 /**
  * Handles the rotation, creation and persistence of user-like sessions.
  * Creates a pool of {@link Session} instances, that are randomly rotated.
  * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
  * Learn more in the {@doclink guides/session-management | Session management guide}.
  *
- * You can create one by calling the {@link SessionPool.open} function.
- *
- * Session pool is already integrated into crawlers, and it can significantly improve your scraper
- * performance with just 2 lines of code.
- *
- * **Example usage:**
- *
- * ```javascript
- * const crawler = new CheerioCrawler({
- *     useSessionPool: true,
- *     persistCookiesPerSession: true,
- *     // ...
- * })
- * ```
+ * Session pool is already integrated into crawlers and is always active.
+ * All public methods are lazy-initialized — the pool initializes itself on first use.
  *
  * You can configure the pool with many options. See the {@link SessionPoolOptions}.
  * Session pool is by default persisted in default {@link KeyValueStore}.
@@ -35,7 +41,7 @@ import { Session } from './session.js';
  * **Advanced usage:**
  *
  * ```javascript
- * const sessionPool = await SessionPool.open({
+ * const sessionPool = new SessionPool({
  *     maxPoolSize: 25,
  *     sessionOptions:{
  *          maxAgeSecs: 10,
@@ -71,94 +77,85 @@ import { Session } from './session.js';
  *
  * @category Scaling
  */
-export class SessionPool extends EventEmitter {
-    config;
-    log;
+export class SessionPool {
+    static #nextId = 0;
+    id;
+    #log;
+    #sessions = [];
+    // kept as TS-private: session_pool tests read/override the members below directly
     maxPoolSize;
     createSessionFunction;
     keyValueStore;
-    sessions = [];
     sessionMap = new Map();
     sessionOptions;
     persistStateKeyValueStoreId;
     persistStateKey;
-    _listener;
-    events;
-    blockedStatusCodes;
-    persistenceOptions;
-    isInitialized = false;
-    queue = new AsyncQueue();
-    /**
-     * @internal
-     */
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
-        super();
-        this.config = config;
-        ow(options, ow.object.exactShape({
-            maxPoolSize: ow.optional.number,
-            persistStateKeyValueStoreId: ow.optional.string,
-            persistStateKey: ow.optional.string,
-            createSessionFunction: ow.optional.function,
-            sessionOptions: ow.optional.object,
-            blockedStatusCodes: ow.optional.array.ofType(ow.number),
-            log: ow.optional.object,
-            persistenceOptions: ow.optional.object,
-        }));
-        const { maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey = PERSIST_STATE_KEY, createSessionFunction, sessionOptions = {}, blockedStatusCodes = BLOCKED_STATUS_CODES, log = defaultLog, persistenceOptions = {
-            enable: true,
-        }, } = options;
-        this.config = config;
-        this.blockedStatusCodes = blockedStatusCodes;
-        this.events = config.getEventManager();
-        this.log = log.child({ prefix: 'SessionPool' });
-        this.persistenceOptions = persistenceOptions;
+    #listener;
+    #events;
+    #persistenceOptions;
+    #sessionReuseStrategy;
+    #initPromise;
+    #queue = new AsyncQueue();
+    #roundRobinIndex = 0;
+    constructor(options = {}) {
+        const { id, maxPoolSize, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions, log, persistenceOptions, sessionReuseStrategy, } = parseArgument(options, sessionPoolOptionsSchema);
+        this.id = id != null ? String(id) : String(SessionPool.#nextId++);
+        this.#sessionReuseStrategy = sessionReuseStrategy;
+        this.#events = serviceLocator.getEventManager();
+        this.#log = log.child({ prefix: 'SessionPool' });
+        this.#persistenceOptions = persistenceOptions;
         // Pool Configuration
         this.maxPoolSize = maxPoolSize;
-        this.createSessionFunction = createSessionFunction || this._defaultCreateSessionFunction;
-        // Session configuration
+        this.createSessionFunction = createSessionFunction || this.defaultCreateSessionFunction;
+        // Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside
+        // `invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
+        // having to know about it.
         this.sessionOptions = {
             ...sessionOptions,
-            // the log needs to propagate to createSessionFunction as in "new Session({ ...sessionPool.sessionOptions })"
-            // and can't go inside _defaultCreateSessionFunction
-            log: this.log,
+            log: this.#log,
         };
         // Session keyValueStore
         this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
-        this.persistStateKey = persistStateKey;
+        this.persistStateKey = persistStateKey ?? `${PERSIST_STATE_KEY}_${this.id}`;
     }
     /**
      * Gets count of usable sessions in the pool.
      */
-    get usableSessionsCount() {
-        return this.sessions.filter((session) => session.isUsable()).length;
+    async usableSessionsCount() {
+        await this.ensureInitialized();
+        return this.#sessions.filter((session) => session.isUsable()).length;
     }
     /**
      * Gets count of retired sessions in the pool.
      */
-    get retiredSessionsCount() {
-        return this.sessions.filter((session) => !session.isUsable()).length;
+    async retiredSessionsCount() {
+        await this.ensureInitialized();
+        return this.#sessions.filter((session) => !session.isUsable()).length;
     }
     /**
      * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
-     * It is called automatically by the {@link SessionPool.open} function.
+     * Called automatically on first use of any public method.
      */
-    async initialize() {
-        if (this.isInitialized) {
-            return;
+    async ensureInitialized() {
+        if (!this.#initPromise) {
+            this.#initPromise = this.setupPool();
         }
-        this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId, { config: this.config });
-        if (!this.persistenceOptions.enable) {
-            this.isInitialized = true;
+        return this.#initPromise;
+    }
+    async setupPool() {
+        if (!this.#persistenceOptions.enable) {
             return;
         }
+        this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId ? { id: this.persistStateKeyValueStoreId } : null, {
+            configuration: serviceLocator.getConfiguration(),
+        });
         if (!this.persistStateKeyValueStoreId) {
-            this.log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
+            this.#log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
         }
         // in case of migration happened and SessionPool state should be restored from the keyValueStore.
-        await this._maybeLoadSessionPool();
-        this._listener = this.persistState.bind(this);
-        this.events.on("persistState" /* EventType.PERSIST_STATE */, this._listener);
-        this.isInitialized = true;
+        await this.maybeLoadSessionPool();
+        this.#listener = this.persistState.bind(this);
+        this.#events.on(EventType.PERSIST_STATE, this.#listener);
     }
     /**
      * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
@@ -167,7 +164,7 @@ export class SessionPool extends EventEmitter {
      * @param [options] The configuration options for the session being added to the session pool.
      */
     async addSession(options = {}) {
-        this._throwIfNotInitialized();
+        await this.ensureInitialized();
         const { id } = options;
         if (id) {
             const sessionExists = this.sessionMap.has(id);
@@ -175,12 +172,12 @@ export class SessionPool extends EventEmitter {
                 throw new Error(`Cannot add session with id '${id}' as it already exists in the pool`);
             }
         }
-        if (!this._hasSpaceForSession()) {
-            this._removeRetiredSessions();
+        if (!this.hasSpaceForSession()) {
+            this.removeRetiredSessions();
         }
-        const newSession = options instanceof Session ? options : await this.createSessionFunction(this, { sessionOptions: options });
-        this.log.debug(`Adding new Session - ${newSession.id}`);
-        this._addSession(newSession);
+        const newSession = options instanceof Session ? options : await this.invokeCreateSessionFunction(options);
+        this.#log.debug(`Adding new Session - ${newSession.id}`);
+        this.registerSession(newSession);
     }
     /**
      * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
@@ -189,9 +186,9 @@ export class SessionPool extends EventEmitter {
      * @param [options] The configuration options for the session being added to the session pool.
      */
     async newSession(sessionOptions) {
-        this._throwIfNotInitialized();
-        const newSession = await this.createSessionFunction(this, { sessionOptions });
-        this._addSession(newSession);
+        await this.ensureInitialized();
+        const newSession = await this.invokeCreateSessionFunction(sessionOptions);
+        this.registerSession(newSession);
         return newSession;
     }
     /**
@@ -202,47 +199,48 @@ export class SessionPool extends EventEmitter {
      * @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
      */
     async getSession(sessionId) {
-        await this.queue.wait();
+        await this.ensureInitialized();
+        await this.#queue.wait();
         try {
-            this._throwIfNotInitialized();
             if (sessionId) {
                 const session = this.sessionMap.get(sessionId);
-                if (session && session.isUsable())
+                if (session?.isUsable())
                     return session;
                 return undefined;
             }
-            if (this._hasSpaceForSession()) {
-                return await this._createSession();
-            }
-            const pickedSession = this._pickSession();
-            if (pickedSession.isUsable()) {
+            const pickedSession = this.pickSession();
+            if (pickedSession)
                 return pickedSession;
+            if (this.hasSpaceForSession()) {
+                return await this.createSession();
             }
-            this._removeRetiredSessions();
-            return await this._createSession();
+            this.removeRetiredSessions();
+            return await this.createSession();
         }
         finally {
-            this.queue.shift();
+            this.#queue.shift();
         }
     }
     /**
      * @param options - Override the persistence options provided in the constructor
      */
     async resetStore(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
+        if (!this.#persistenceOptions.enable && !options?.enable) {
             return;
         }
+        await this.ensureInitialized();
         await this.keyValueStore?.setValue(this.persistStateKey, null);
     }
     /**
      * Returns an object representing the internal state of the `SessionPool` instance.
      * Note that the object's fields can change in future releases.
      */
-    getState() {
+    async getState() {
+        await this.ensureInitialized();
         return {
-            usableSessionsCount: this.usableSessionsCount,
-            retiredSessionsCount: this.retiredSessionsCount,
-            sessions: this.sessions.map((session) => session.getState()),
+            usableSessionsCount: await this.usableSessionsCount(),
+            retiredSessionsCount: await this.retiredSessionsCount(),
+            sessions: this.#sessions.map((session) => session.getState()),
         };
     }
     /**
@@ -251,47 +249,43 @@ export class SessionPool extends EventEmitter {
      * @param options - Override the persistence options provided in the constructor
      */
     async persistState(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
+        if (!this.#persistenceOptions.enable && !options?.enable) {
             return;
         }
-        this.log.debug('Persisting state', {
+        await this.ensureInitialized();
+        this.#log.debug('Persisting state', {
             persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
             persistStateKey: this.persistStateKey,
         });
-        // use half the interval of `persistState` to avoid race conditions
-        const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
-        const timeoutSecs = persistStateIntervalMillis / 2_000;
         await this.keyValueStore
-            .setValue(this.persistStateKey, this.getState(), {
-            timeoutSecs,
-            doNotRetryTimeouts: true,
-        })
-            .catch((error) => this.log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
+            ?.setValue(this.persistStateKey, await this.getState())
+            .catch((error) => this.#log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
     }
     /**
      * Removes listener from `persistState` event.
      * This function should be called after you are done with using the `SessionPool` instance.
+     * @param options - Set `persistState` to false when the final state was already persisted by the event manager.
      */
-    async teardown() {
-        this.events.off("persistState" /* EventType.PERSIST_STATE */, this._listener);
-        await this.persistState();
-    }
-    /**
-     * SessionPool should not work before initialization.
-     */
-    _throwIfNotInitialized() {
-        if (!this.isInitialized)
-            throw new Error('SessionPool is not initialized.');
+    async teardown({ persistState = true } = {}) {
+        if (!this.#initPromise)
+            return;
+        await this.ensureInitialized();
+        if (this.#listener) {
+            this.#events.off(EventType.PERSIST_STATE, this.#listener);
+        }
+        if (persistState) {
+            await this.persistState();
+        }
     }
     /**
      * Removes retired `Session` instances from `SessionPool`.
      */
-    _removeRetiredSessions() {
-        this.sessions = this.sessions.filter((storedSession) => {
+    removeRetiredSessions() {
+        this.#sessions = this.#sessions.filter((storedSession) => {
             if (storedSession.isUsable())
                 return true;
             this.sessionMap.delete(storedSession.id);
-            this.log.debug(`Removed Session - ${storedSession.id}`);
+            this.#log.debug(`Removed Session - ${storedSession.id}`);
             return false;
         });
     }
@@ -299,89 +293,102 @@ export class SessionPool extends EventEmitter {
      * Adds `Session` instance to `SessionPool`.
      * @param newSession `Session` instance to be added.
      */
-    _addSession(newSession) {
-        this.sessions.push(newSession);
+    registerSession(newSession) {
+        this.#sessions.push(newSession);
         this.sessionMap.set(newSession.id, newSession);
     }
     /**
      * Gets random index.
      */
-    _getRandomIndex() {
-        return Math.floor(Math.random() * this.sessions.length);
+    getRandomIndex() {
+        return Math.floor(Math.random() * this.#sessions.length);
     }
     /**
      * Creates new session without any extra behavior.
-     * @param sessionPool
      * @param [options]
      * @param [options.sessionOptions] The configuration options for the session being created.
      * @returns New session.
      */
-    async _defaultCreateSessionFunction(sessionPool, options = {}) {
-        ow(options, ow.object.exactShape({ sessionOptions: ow.optional.object }));
-        const { sessionOptions = {} } = options;
-        return new Session({
+    async defaultCreateSessionFunction(options = {}) {
+        const { sessionOptions } = parseArgument(options, createSessionOptionsSchema);
+        return new Session(sessionOptions);
+    }
+    /**
+     * Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
+     * the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
+     *
+     * A default {@link SessionFingerprint} is generated up front (host OS as
+     * `platform`, a random valid `browser`/`device` for that platform). Pool-wide
+     * and per-call options override it, and a persisted fingerprint coming
+     * through `maybeLoadSessionPool` naturally wins because it arrives in
+     * `perCallOptions`.
+     */
+    async invokeCreateSessionFunction(perCallOptions) {
+        const sessionOptions = {
+            fingerprint: createDefaultSessionFingerprint(),
             ...this.sessionOptions,
-            ...sessionOptions,
-            sessionPool,
-        });
+            ...perCallOptions,
+        };
+        return this.createSessionFunction({ sessionOptions });
     }
     /**
      * Creates new session and adds it to the pool.
      * @returns Newly created `Session` instance.
      */
-    async _createSession() {
-        const newSession = await this.createSessionFunction(this);
-        this._addSession(newSession);
-        this.log.debug(`Created new Session - ${newSession.id}`);
+    async createSession() {
+        const newSession = await this.invokeCreateSessionFunction();
+        this.registerSession(newSession);
+        this.#log.debug(`Created new Session - ${newSession.id}`);
         return newSession;
     }
     /**
      * Decides whether there is enough space for creating new session.
      */
-    _hasSpaceForSession() {
-        return this.sessions.length < this.maxPoolSize;
+    hasSpaceForSession() {
+        return this.#sessions.length < this.maxPoolSize;
     }
     /**
-     * Picks random session from the `SessionPool`.
-     * @returns Picked `Session`.
+     * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
+     * Returns `undefined` when no session should be reused and a new one should be created instead.
      */
-    _pickSession() {
-        return this.sessions[this._getRandomIndex()]; // Or maybe we should let the developer to customize the picking algorithm
+    pickSession() {
+        if (this.#sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession())
+            return undefined;
+        if (this.#sessionReuseStrategy === 'use-until-failure') {
+            return this.#sessions.find((session) => session.isUsable());
+        }
+        let picked;
+        if (this.#sessionReuseStrategy === 'round-robin') {
+            const index = this.#roundRobinIndex % this.#sessions.length;
+            this.#roundRobinIndex = index + 1;
+            picked = this.#sessions[index];
+        }
+        else {
+            picked = this.#sessions[this.getRandomIndex()];
+        }
+        return picked.isUsable() ? picked : undefined;
     }
     /**
      * Potentially loads `SessionPool`.
      * If the state was persisted it loads the `SessionPool` from the persisted state.
      */
-    async _maybeLoadSessionPool() {
-        const loadedSessionPool = await this.keyValueStore.getValue(this.persistStateKey);
+    async maybeLoadSessionPool() {
+        const loadedSessionPool = await this.keyValueStore?.getValue(this.persistStateKey);
         if (!loadedSessionPool)
             return;
         // Invalidate old sessions and load active sessions only
-        this.log.debug('Recreating state from KeyValueStore', {
+        this.#log.debug('Recreating state from KeyValueStore', {
             persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
             persistStateKey: this.persistStateKey,
         });
         for (const sessionObject of loadedSessionPool.sessions) {
-            sessionObject.sessionPool = this;
             sessionObject.createdAt = new Date(sessionObject.createdAt);
             sessionObject.expiresAt = new Date(sessionObject.expiresAt);
-            const recreatedSession = await this.createSessionFunction(this, { sessionOptions: sessionObject });
+            const recreatedSession = await this.invokeCreateSessionFunction(sessionObject);
             if (recreatedSession.isUsable()) {
-                this._addSession(recreatedSession);
+                this.registerSession(recreatedSession);
             }
         }
-        this.log.debug(`${this.usableSessionsCount} active sessions loaded from KeyValueStore`);
-    }
-    /**
-     * Opens a SessionPool and returns a promise resolving to an instance
-     * of the {@link SessionPool} class that is already initialized.
-     *
-     * For more details and code examples, see the {@link SessionPool} class.
-     */
-    static async open(options, config) {
-        const sessionPool = new SessionPool(options, config);
-        await sessionPool.initialize();
-        return sessionPool;
+        this.#log.debug(`${this.#sessions.length} active sessions loaded from KeyValueStore`);
     }
 }
-//# sourceMappingURL=session_pool.js.map

package/storages/batched_adds.d.ts

@@ -0,0 +1,37 @@
+import type { ProcessedRequest } from '@crawlee/types';
+import type { Source } from '../request.js';
+import type { AddRequestsBatchedResult } from './request_queue.js';
+export interface DrainRequestBatchesOptions<TItem extends Source> {
+    /**
+     * The requests to add, already normalized by the caller. Consumed lazily: an unbounded or expensive
+     * iterable is only pulled from as far as the batching (and any `maxNewRequests` budget) requires.
+     */
+    items: AsyncGenerator<TItem>;
+    batchSize: number;
+    waitBetweenBatchesMillis: number;
+    waitForAllRequestsToBeAdded: boolean;
+    maxNewRequests?: number;
+    /**
+     * Adds a single chunk and reports what it processed.
+     *
+     * @param isInitial Whether this is the first chunk, which is added before this function returns. Later
+     *  chunks land in the background, which is why some callers cache only the first.
+     */
+    processChunk: (chunk: TItem[], isInitial: boolean) => Promise<ProcessedRequest[]>;
+    /**
+     * Called with the promise covering every chunk after the first, so the caller can keep its own
+     * `isFinished` honest while batches are still landing.
+     */
+    trackBackgroundBatches?: (batches: Promise<unknown>) => void;
+}
+/**
+ * Drives the chunk-by-chunk half of `addRequestsBatched`: the first chunk is added before returning and the
+ * rest continue in the background, paced by `waitBetweenBatchesMillis`.
+ *
+ * Callers differ only in how a chunk is added and how the input is normalized, so that is all
+ * {@link DrainRequestBatchesOptions} asks for - the budget arithmetic, the lazy chunking, the
+ * over-limit reporting and the transaction handling are identical for everyone and live here. In
+ * particular, every caller has to keep its background chunks out of a transaction they will outlive,
+ * so that is read from the ambient transaction rather than asked of the caller.
+ */
+export declare function drainRequestBatches<TItem extends Source>(options: DrainRequestBatchesOptions<TItem>): Promise<AddRequestsBatchedResult>;

package/storages/batched_adds.js

@@ -0,0 +1,73 @@
+import { setTimeout as sleep } from 'node:timers/promises';
+import { chunkedAsyncIterable, peekableAsyncIterable } from '../iterables.js';
+import { activeStorageTransaction, withDirectStorageAccess } from './transaction.js';
+/**
+ * Drives the chunk-by-chunk half of `addRequestsBatched`: the first chunk is added before returning and the
+ * rest continue in the background, paced by `waitBetweenBatchesMillis`.
+ *
+ * Callers differ only in how a chunk is added and how the input is normalized, so that is all
+ * {@link DrainRequestBatchesOptions} asks for - the budget arithmetic, the lazy chunking, the
+ * over-limit reporting and the transaction handling are identical for everyone and live here. In
+ * particular, every caller has to keep its background chunks out of a transaction they will outlive,
+ * so that is read from the ambient transaction rather than asked of the caller.
+ */
+export async function drainRequestBatches(options) {
+    const { items, batchSize, waitBetweenBatchesMillis, waitForAllRequestsToBeAdded, maxNewRequests, processChunk, trackBackgroundBatches, } = options;
+    const deferred = activeStorageTransaction()?.policy.requestQueue === 'deferred';
+    let remainingBudget = maxNewRequests ?? Infinity;
+    const requestsOverLimit = [];
+    // Never hand a chunk more than the budget allows, so an over-large final batch cannot overshoot.
+    const effectiveChunkSize = maxNewRequests !== undefined ? () => Math.min(batchSize, remainingBudget) : batchSize;
+    const chunks = peekableAsyncIterable(chunkedAsyncIterable(items, effectiveChunkSize));
+    const chunksIterator = chunks[Symbol.asyncIterator]();
+    const addChunk = async (chunk, isInitial) => {
+        const processedRequests = await processChunk(chunk, isInitial);
+        if (maxNewRequests !== undefined) {
+            remainingBudget -= processedRequests.filter((request) => !request.wasAlreadyPresent).length;
+        }
+        return processedRequests;
+    };
+    const buildResult = async (addedRequests, waitForAll) => {
+        if (maxNewRequests !== undefined) {
+            // `chunkedAsyncIterable` stops pulling once the budget-derived chunk size hits zero, so whatever
+            // is left is still sitting in `items` rather than in a chunk we have seen.
+            for await (const item of items) {
+                requestsOverLimit.push(item);
+            }
+        }
+        return { addedRequests, waitForAllRequestsToBeAdded: waitForAll, requestsOverLimit };
+    };
+    const initialChunk = await chunksIterator.peek();
+    if (initialChunk === undefined) {
+        return buildResult([], Promise.resolve([]));
+    }
+    const addedRequests = await addChunk(initialChunk, true);
+    await chunksIterator.next();
+    if ((await chunksIterator.peek()) === undefined) {
+        return buildResult(addedRequests, Promise.resolve([]));
+    }
+    const processRemainingChunks = async () => {
+        const added = [];
+        for await (const chunk of chunks) {
+            added.push(...(await addChunk(chunk, false)));
+            // Under `deferred` no chunk performs backend I/O, so pacing them would only stall the handler.
+            await sleep(deferred ? 0 : waitBetweenBatchesMillis);
+        }
+        return added;
+    };
+    // With a budget we must drain everything before we can report what went over it; under `deferred` a
+    // writer that finishes after commit would have nowhere to put its journal entries.
+    const awaitsRemainder = waitForAllRequestsToBeAdded || maxNewRequests !== undefined || deferred;
+    // An un-awaited writer outlives the transaction scope it inherits, so it must not record into a
+    // transaction that may already be closed. It writes directly - its write-through additions were never
+    // going to be rolled back anyway - which means the requests it adds are not journaled.
+    // See `StorageTransactionView.enqueuedUrls`.
+    const remainder = awaitsRemainder ? processRemainingChunks() : withDirectStorageAccess(processRemainingChunks);
+    // The caller is not obliged to await `remainder`, so give it a handler of its own - an unhandled
+    // rejection here would otherwise take the process down.
+    trackBackgroundBatches?.(remainder.catch(() => { }));
+    if (awaitsRemainder) {
+        addedRequests.push(...(await remainder));
+    }
+    return buildResult(addedRequests, remainder);
+}