@crawlee/core 4.0.0-beta.111 → 4.0.0-beta.113

package/errors.d.ts

@@ -42,6 +42,26 @@ export declare class RequestValidationError extends NonRetryableError {
 export declare class RetryRequestError extends Error {
     constructor(message?: string);
 }
+/**
+ * Thrown when a domain has rate-limited us and the request should simply be attempted again later.
+ *
+ * The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
+ * nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
+ * the domain's backoff expires, so retries are paced rather than immediate.
+ */
+export declare class RequestThrottledError extends RetryRequestError {
+    constructor(message?: string);
+}
+/**
+ * Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
+ * abandoned rather than kept waiting.
+ *
+ * Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
+ * The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
+ * resumes them once the domain recovers.
+ */
+export declare class PersistentRateLimitError extends CriticalError {
+}
 /**
  * Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
  *

package/errors.js

@@ -47,6 +47,28 @@ export class RetryRequestError extends Error {
         super(message ?? "Request is being retried at the user's request");
     }
 }
+/**
+ * Thrown when a domain has rate-limited us and the request should simply be attempted again later.
+ *
+ * The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
+ * nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
+ * the domain's backoff expires, so retries are paced rather than immediate.
+ */
+export class RequestThrottledError extends RetryRequestError {
+    constructor(message) {
+        super(message ?? 'Request is being retried later because its domain is rate-limiting us');
+    }
+}
+/**
+ * Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
+ * abandoned rather than kept waiting.
+ *
+ * Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
+ * The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
+ * resumes them once the domain recovers.
+ */
+export class PersistentRateLimitError extends CriticalError {
+}
 /**
  * Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
  *

package/http.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Parses a `Retry-After` response header into a delay in milliseconds.
+ *
+ * The header holds either a non-negative number of seconds or an HTTP-date.
+ * See [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After).
+ *
+ * @returns The delay in milliseconds, or `null` if the header is absent, unparseable, or already elapsed.
+ */
+export declare function parseRetryAfterHeader(value?: string | null): number | null;

package/http.js

@@ -0,0 +1,28 @@
+/**
+ * Parses a `Retry-After` response header into a delay in milliseconds.
+ *
+ * The header holds either a non-negative number of seconds or an HTTP-date.
+ * See [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After).
+ *
+ * @returns The delay in milliseconds, or `null` if the header is absent, unparseable, or already elapsed.
+ */
+export function parseRetryAfterHeader(value) {
+    if (!value) {
+        return null;
+    }
+    const trimmed = value.trim();
+    // Per the spec this is a `delay-seconds`: digits only, so a negative or fractional value is not one.
+    if (/^\d+$/.test(trimmed)) {
+        // `Retry-After: 0` names no future deadline, same as an HTTP-date that has already passed. Reporting it
+        // as a zero delay would leave the domain unthrottled while still counting as a rate-limit event, so the
+        // caller would defer the request for free and re-send it immediately.
+        const delayMs = Number(trimmed) * 1000;
+        return delayMs > 0 ? delayMs : null;
+    }
+    const date = Date.parse(trimmed);
+    if (!Number.isNaN(date)) {
+        const delayMs = date - Date.now();
+        return delayMs > 0 ? delayMs : null;
+    }
+    return null;
+}

package/index.d.ts

@@ -17,5 +17,6 @@ export * from './storages/index.js';
 export * from './memory-storage/index.js';
 export * from './validators.js';
 export * from './cookie_utils.js';
+export * from './http.js';
 export * from './recoverable_state.js';
 export type { StorageBackend } from '@crawlee/types';

package/index.js

@@ -17,4 +17,5 @@ export * from './storages/index.js';
 export * from './memory-storage/index.js';
 export * from './validators.js';
 export * from './cookie_utils.js';
+export * from './http.js';
 export * from './recoverable_state.js';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.111",
+    "version": "4.0.0-beta.113",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,9 +52,9 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.111",
-        "@crawlee/types": "4.0.0-beta.111",
-        "@crawlee/utils": "4.0.0-beta.111",
+        "@crawlee/fs-storage": "4.0.0-beta.113",
+        "@crawlee/types": "4.0.0-beta.113",
+        "@crawlee/utils": "4.0.0-beta.113",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "a6b7dec651fa7d81c61689e0a3cd0a8d06fdb586"
+    "gitHead": "a80d4b05a84ecc651dcd68fafd7a1ff35e6e7532"
 }

package/storages/batched_adds.d.ts

@@ -0,0 +1,37 @@
+import type { ProcessedRequest } from '@crawlee/types';
+import type { Source } from '../request.js';
+import type { AddRequestsBatchedResult } from './request_queue.js';
+export interface DrainRequestBatchesOptions<TItem extends Source> {
+    /**
+     * The requests to add, already normalized by the caller. Consumed lazily: an unbounded or expensive
+     * iterable is only pulled from as far as the batching (and any `maxNewRequests` budget) requires.
+     */
+    items: AsyncGenerator<TItem>;
+    batchSize: number;
+    waitBetweenBatchesMillis: number;
+    waitForAllRequestsToBeAdded: boolean;
+    maxNewRequests?: number;
+    /**
+     * Adds a single chunk and reports what it processed.
+     *
+     * @param isInitial Whether this is the first chunk, which is added before this function returns. Later
+     *  chunks land in the background, which is why some callers cache only the first.
+     */
+    processChunk: (chunk: TItem[], isInitial: boolean) => Promise<ProcessedRequest[]>;
+    /**
+     * Called with the promise covering every chunk after the first, so the caller can keep its own
+     * `isFinished` honest while batches are still landing.
+     */
+    trackBackgroundBatches?: (batches: Promise<unknown>) => void;
+}
+/**
+ * Drives the chunk-by-chunk half of `addRequestsBatched`: the first chunk is added before returning and the
+ * rest continue in the background, paced by `waitBetweenBatchesMillis`.
+ *
+ * Callers differ only in how a chunk is added and how the input is normalized, so that is all
+ * {@link DrainRequestBatchesOptions} asks for - the budget arithmetic, the lazy chunking, the
+ * over-limit reporting and the transaction handling are identical for everyone and live here. In
+ * particular, every caller has to keep its background chunks out of a transaction they will outlive,
+ * so that is read from the ambient transaction rather than asked of the caller.
+ */
+export declare function drainRequestBatches<TItem extends Source>(options: DrainRequestBatchesOptions<TItem>): Promise<AddRequestsBatchedResult>;

package/storages/batched_adds.js

@@ -0,0 +1,73 @@
+import { setTimeout as sleep } from 'node:timers/promises';
+import { chunkedAsyncIterable, peekableAsyncIterable } from '../iterables.js';
+import { activeStorageTransaction, withDirectStorageAccess } from './transaction.js';
+/**
+ * Drives the chunk-by-chunk half of `addRequestsBatched`: the first chunk is added before returning and the
+ * rest continue in the background, paced by `waitBetweenBatchesMillis`.
+ *
+ * Callers differ only in how a chunk is added and how the input is normalized, so that is all
+ * {@link DrainRequestBatchesOptions} asks for - the budget arithmetic, the lazy chunking, the
+ * over-limit reporting and the transaction handling are identical for everyone and live here. In
+ * particular, every caller has to keep its background chunks out of a transaction they will outlive,
+ * so that is read from the ambient transaction rather than asked of the caller.
+ */
+export async function drainRequestBatches(options) {
+    const { items, batchSize, waitBetweenBatchesMillis, waitForAllRequestsToBeAdded, maxNewRequests, processChunk, trackBackgroundBatches, } = options;
+    const deferred = activeStorageTransaction()?.policy.requestQueue === 'deferred';
+    let remainingBudget = maxNewRequests ?? Infinity;
+    const requestsOverLimit = [];
+    // Never hand a chunk more than the budget allows, so an over-large final batch cannot overshoot.
+    const effectiveChunkSize = maxNewRequests !== undefined ? () => Math.min(batchSize, remainingBudget) : batchSize;
+    const chunks = peekableAsyncIterable(chunkedAsyncIterable(items, effectiveChunkSize));
+    const chunksIterator = chunks[Symbol.asyncIterator]();
+    const addChunk = async (chunk, isInitial) => {
+        const processedRequests = await processChunk(chunk, isInitial);
+        if (maxNewRequests !== undefined) {
+            remainingBudget -= processedRequests.filter((request) => !request.wasAlreadyPresent).length;
+        }
+        return processedRequests;
+    };
+    const buildResult = async (addedRequests, waitForAll) => {
+        if (maxNewRequests !== undefined) {
+            // `chunkedAsyncIterable` stops pulling once the budget-derived chunk size hits zero, so whatever
+            // is left is still sitting in `items` rather than in a chunk we have seen.
+            for await (const item of items) {
+                requestsOverLimit.push(item);
+            }
+        }
+        return { addedRequests, waitForAllRequestsToBeAdded: waitForAll, requestsOverLimit };
+    };
+    const initialChunk = await chunksIterator.peek();
+    if (initialChunk === undefined) {
+        return buildResult([], Promise.resolve([]));
+    }
+    const addedRequests = await addChunk(initialChunk, true);
+    await chunksIterator.next();
+    if ((await chunksIterator.peek()) === undefined) {
+        return buildResult(addedRequests, Promise.resolve([]));
+    }
+    const processRemainingChunks = async () => {
+        const added = [];
+        for await (const chunk of chunks) {
+            added.push(...(await addChunk(chunk, false)));
+            // Under `deferred` no chunk performs backend I/O, so pacing them would only stall the handler.
+            await sleep(deferred ? 0 : waitBetweenBatchesMillis);
+        }
+        return added;
+    };
+    // With a budget we must drain everything before we can report what went over it; under `deferred` a
+    // writer that finishes after commit would have nowhere to put its journal entries.
+    const awaitsRemainder = waitForAllRequestsToBeAdded || maxNewRequests !== undefined || deferred;
+    // An un-awaited writer outlives the transaction scope it inherits, so it must not record into a
+    // transaction that may already be closed. It writes directly - its write-through additions were never
+    // going to be rolled back anyway - which means the requests it adds are not journaled.
+    // See `StorageTransactionView.enqueuedUrls`.
+    const remainder = awaitsRemainder ? processRemainingChunks() : withDirectStorageAccess(processRemainingChunks);
+    // The caller is not obliged to await `remainder`, so give it a handler of its own - an unhandled
+    // rejection here would otherwise take the process down.
+    trackBackgroundBatches?.(remainder.catch(() => { }));
+    if (awaitsRemainder) {
+        addedRequests.push(...(await remainder));
+    }
+    return buildResult(addedRequests, remainder);
+}

package/storages/index.d.ts

@@ -11,3 +11,4 @@ export * from './utils.js';
 export * from './transaction.js';
 export * from './sitemap_request_loader.js';
 export * from './request_manager_tandem.js';
+export * from './throttling_request_manager.js';

package/storages/index.js

@@ -9,3 +9,4 @@ export * from './utils.js';
 export * from './transaction.js';
 export * from './sitemap_request_loader.js';
 export * from './request_manager_tandem.js';
+export * from './throttling_request_manager.js';

package/storages/request_loader.d.ts

@@ -54,6 +54,11 @@ export interface IRequestLoader {
      * Resolves to `true` if the next call to {@link IRequestLoader.fetchNextRequest} function
      * would return `null`, otherwise it resolves to `false`.
      * Note that even if the loader is empty, there might be some pending requests currently being processed.
+     *
+     * This is a statement about what the *next fetch* would return, not about how much work is left, so it
+     * may report `true` while {@link IRequestLoader.getPendingCount} is non-zero - a loader that withholds
+     * requests for a while (as {@link ThrottlingRequestManager} does for a rate-limited domain) is empty
+     * for as long as it will not hand anything over. Use `isFinished()` to ask whether the work is done.
      */
     isEmpty(): Promise<boolean>;
     /**

package/storages/request_queue.js

@@ -1,15 +1,15 @@
 import { inspect } from 'node:util';
 import { isAsyncIterable, isIterable } from '@crawlee/utils/internal';
-import { downloadListOfUrls, sleep } from '@crawlee/utils';
+import { downloadListOfUrls } from '@crawlee/utils';
 import ow from 'ow';
 import { LruCache } from '@apify/datastructures';
 import { tryCancel } from '@apify/timeout';
 import { Configuration } from '../configuration.js';
 import { getObjectType } from '../debug.js';
-import { chunkedAsyncIterable, peekableAsyncIterable } from '../iterables.js';
 import { Request } from '../request.js';
 import { serviceLocator } from '../service_locator.js';
-import { activeStorageTransaction, rejectOperationInTransaction, withDirectStorageAccess } from './transaction.js';
+import { activeStorageTransaction, rejectOperationInTransaction } from './transaction.js';
+import { drainRequestBatches } from './batched_adds.js';
 import { StorageStatsTracker } from './storage_stats.js';
 import { resolveStorageIdentifier } from './storage_instance_manager.js';
 import { getRequestId, purgeDefaultStorages } from './utils.js';
@@ -438,8 +438,6 @@ export class RequestQueue {
      * @param options Options for the request queue
      */
     async addRequestsBatched(requests, options = {}) {
-        const transaction = activeStorageTransaction();
-        const deferred = transaction?.policy.requestQueue === 'deferred';
         ow(requests, ow.object
             .is((value) => isIterable(value) || isAsyncIterable(value))
             .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
@@ -476,103 +474,36 @@ export class RequestQueue {
                 }
             }
         }
-        const { batchSize = 1000, maxNewRequests = undefined } = options;
-        // Under `deferred` no chunk performs backend I/O, so pacing them would only stall the handler.
-        const waitBetweenBatchesMillis = deferred ? 0 : (options.waitBetweenBatchesMillis ?? 1000);
-        let remainingBudget = maxNewRequests ?? Infinity;
-        const requestsOverLimit = [];
-        // If there's a limit on the number of added requests, do not send batches bigger than the limit
-        const effectiveChunkSize = maxNewRequests !== undefined ? () => Math.min(batchSize, remainingBudget) : batchSize;
-        // Hold onto the underlying iterator so we can drain leftovers from it in buildResult
-        const requestIterator = generateRequests();
-        const chunks = peekableAsyncIterable(chunkedAsyncIterable(requestIterator, effectiveChunkSize));
-        const chunksIterator = chunks[Symbol.asyncIterator]();
-        /**
-         * Process a chunk: send it to the queue, then update the remaining budget if maxNewRequests is active.
-         *
-         * Requests the backend reports as unprocessed are warned about and skipped rather than retried:
-         * `unprocessedRequests` is what remains after the backend's own transient-error handling - a
-         * semantic rejection (e.g. a malformed `userData` shape) that re-sending would only re-poke.
-         * Retrying transient failures is the storage backend's job, not the frontend's.
-         */
-        const processChunk = async (chunk, cache = true) => {
-            const { processedRequests, unprocessedRequests } = await this.addRequests(chunk, {
-                forefront: options.forefront,
-                cache,
-            });
-            if (unprocessedRequests.length > 0) {
-                this.log.warning('Some requests were rejected by the request queue and will be skipped. ' +
-                    "This usually means the request data is malformed (e.g. an invalid 'userData' shape).", { unprocessedRequests });
-            }
-            if (maxNewRequests !== undefined) {
-                remainingBudget -= processedRequests.filter((r) => !r.wasAlreadyPresent).length;
-            }
-            return processedRequests;
-        };
-        /**
-         * Build the final result. When maxNewRequests is set, drains any remaining items
-         * from the underlying request iterator into requestsOverLimit.
-         *
-         * We accept the iterator explicitly (rather than closing over it) to make it obvious
-         * that this is the *same* iterator that `chunkedAsyncIterable` has been consuming —
-         * so only unconsumed items are drained. We drain `requestIterator` (not `chunks`)
-         * because `chunkedAsyncIterable` stops yielding when the budget-based chunk size
-         * drops to 0, leaving unconsumed items in the underlying iterator.
-         */
-        const buildResult = async (addedRequests, waitForAllRequestsToBeAdded, unconsumedIterator) => {
-            if (maxNewRequests !== undefined) {
-                for await (const request of unconsumedIterator) {
-                    requestsOverLimit.push(request);
+        return drainRequestBatches({
+            items: generateRequests(),
+            batchSize: options.batchSize ?? 1000,
+            waitBetweenBatchesMillis: options.waitBetweenBatchesMillis ?? 1000,
+            waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded ?? false,
+            maxNewRequests: options.maxNewRequests,
+            /**
+             * Requests the backend reports as unprocessed are warned about and skipped rather than retried:
+             * `unprocessedRequests` is what remains after the backend's own transient-error handling - a
+             * semantic rejection (e.g. a malformed `userData` shape) that re-sending would only re-poke.
+             * Retrying transient failures is the storage backend's job, not the frontend's.
+             */
+            processChunk: async (chunk, isInitial) => {
+                const { processedRequests, unprocessedRequests } = await this.addRequests(chunk, {
+                    forefront: options.forefront,
+                    cache: isInitial,
+                });
+                if (unprocessedRequests.length > 0) {
+                    this.log.warning('Some requests were rejected by the request queue and will be skipped. ' +
+                        "This usually means the request data is malformed (e.g. an invalid 'userData' shape).", { unprocessedRequests });
                 }
-            }
-            return { addedRequests, waitForAllRequestsToBeAdded, requestsOverLimit };
-        };
-        // Add initial batch to process right away
-        const initialChunk = await chunksIterator.peek();
-        if (initialChunk === undefined) {
-            return buildResult([], Promise.resolve([]), requestIterator);
-        }
-        const addedRequests = await processChunk(initialChunk);
-        await chunksIterator.next();
-        // If we have no more requests to add (either exhausted or budget hit), return immediately
-        if ((await chunksIterator.peek()) === undefined) {
-            return buildResult(addedRequests, Promise.resolve([]), requestIterator);
-        }
-        const processRemainingChunks = async () => {
-            const finalAddedRequests = [];
-            for await (const requestChunk of chunks) {
-                finalAddedRequests.push(...(await processChunk(requestChunk, false)));
-                await sleep(waitBetweenBatchesMillis);
-            }
-            return finalAddedRequests;
-        };
-        // maxNewRequests needs all batches to report skipped requests accurately; `deferred` needs them
-        // too - a writer that finishes after commit would have nowhere to put its journal entries.
-        const awaitsRemainingChunks = options.waitForAllRequestsToBeAdded || maxNewRequests !== undefined || deferred;
-        // eslint-disable-next-line no-async-promise-executor
-        const promise = new Promise(async (resolve) => {
-            if (awaitsRemainingChunks) {
-                // Awaited below, i.e. still within the caller's transaction scope, so the additions are
-                // journaled like the initial chunk - introspection must not depend on where the chunk
-                // boundary happened to fall.
-                resolve(await processRemainingChunks());
-            }
-            else {
-                // Nobody awaits this writer, so it outlives the transaction scope it inherits and must
-                // not record into a transaction that may already be closed. It writes directly - its
-                // write-through additions were never going to be rolled back anyway - which means the
-                // requests it adds are not journaled. See `StorageTransactionView.enqueuedUrls`.
-                resolve(await withDirectStorageAccess(processRemainingChunks));
-            }
-        });
-        this.inProgressRequestBatchCount += 1;
-        void promise.finally(() => {
-            this.inProgressRequestBatchCount -= 1;
+                return processedRequests;
+            },
+            trackBackgroundBatches: (batches) => {
+                this.inProgressRequestBatchCount += 1;
+                void batches.finally(() => {
+                    this.inProgressRequestBatchCount -= 1;
+                });
+            },
         });
-        if (awaitsRemainingChunks) {
-            addedRequests.push(...(await promise));
-        }
-        return buildResult(addedRequests, promise, requestIterator);
     }
     /**
      * Gets the request from the queue specified by its `uniqueKey`.

package/storages/throttling_request_manager.d.ts

@@ -0,0 +1,216 @@
+import type { Dictionary } from '@crawlee/types';
+import type { Configuration } from '../configuration.js';
+import type { Request, Source } from '../request.js';
+import type { IRequestManager, RequestsLike } from './request_manager.js';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';
+import type { StorageIdentifier } from './storage_instance_manager.js';
+import type { StorageOpenOptions } from './utils.js';
+/**
+ * Opens a request manager, matching the shape of storage `open` methods such as
+ * {@link RequestQueue.open|`RequestQueue.open`}.
+ *
+ * {@link ThrottlingRequestManager} calls this once per configured domain, so every per-domain queue shares the
+ * concrete type and storage backend of the manager being wrapped.
+ */
+export type RequestManagerOpener<T extends IRequestManager = IRequestManager> = (identifier: string | StorageIdentifier, options?: StorageOpenOptions) => Promise<T>;
+/**
+ * A request manager that can pace requests per domain, as {@link ThrottlingRequestManager} does.
+ *
+ * The crawlers detect this structurally rather than by type, so a wrapper can opt in by forwarding these three
+ * methods without {@link IRequestManager} having to know that throttling exists.
+ */
+export interface SupportsDomainThrottling {
+    /** @see {@link ThrottlingRequestManager.recordDomainDelay} */
+    recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
+    /** @see {@link ThrottlingRequestManager.setCrawlDelay} */
+    setCrawlDelay(url: string, delaySeconds: number): boolean;
+    /** @see {@link ThrottlingRequestManager.assertNoStalledDomains} */
+    assertNoStalledDomains(): Promise<void>;
+}
+/** Whether `manager` can pace requests per domain. */
+export declare function supportsDomainThrottling(manager: unknown): manager is SupportsDomainThrottling;
+/** Options for {@link ThrottlingRequestManager}. */
+export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRequestManager> {
+    /**
+     * The request manager to wrap, usually a {@link RequestQueue}. Requests for domains that are not throttled
+     * are stored here.
+     */
+    inner: T;
+    /**
+     * Hostnames to throttle. Matching is case-insensitive and exact - wildcards such as `*.example.com` are not
+     * supported, so list each subdomain you care about. Requests for any other domain bypass throttling entirely.
+     *
+     * An internationalized domain may be given in either its unicode or its punycode form, and an IPv6 address
+     * has to be bracketed (`[::1]`).
+     */
+    domains: string[];
+    /**
+     * Opens the per-domain queues, one per entry in `domains`, each under the alias `throttled-<domain>`.
+     * @default RequestQueue.open
+     */
+    requestManagerOpener?: RequestManagerOpener<T>;
+    /**
+     * The delay applied after a domain's first HTTP 429, doubled on each subsequent one.
+     * @default 2
+     */
+    baseDelaySecs?: number;
+    /**
+     * Upper bound on the delay between requests to a rate-limited domain, applied to both the exponential
+     * backoff and a `Retry-After` value.
+     * @default 60
+     */
+    maxDelaySecs?: number;
+    /**
+     * How long a domain may rate-limit us without a single request getting through before the crawl is
+     * abandoned with a {@link PersistentRateLimitError}.
+     *
+     * A domain that keeps answering 429 for this long is not going to be crawled by waiting longer - the
+     * concurrency is too high for it, or it has blocked us outright. Its requests are deliberately left in
+     * their queue, so re-running the crawl without purging storages picks them up once the domain recovers.
+     *
+     * A crawler running with `keepAlive` is exempt - outliving a domain that will not let us through is the
+     * whole point there.
+     * @default 900
+     */
+    maxDomainStallSecs?: number;
+}
+/**
+ * A request manager that wraps another one and paces requests per domain.
+ *
+ * Requests for the configured {@link ThrottlingRequestManagerOptions.domains|`domains`} are routed into their own
+ * queue when they are added, so each request lives in exactly one place and deduplication keeps working. Everything
+ * else goes to the wrapped manager untouched.
+ *
+ * {@link ThrottlingRequestManager.fetchNextRequest|`fetchNextRequest()`} serves the domain that has been waiting
+ * longest and skips any that are backing off, falling back to the wrapped manager. It never blocks: while every
+ * remaining request belongs to a throttled domain it returns `null` and {@link ThrottlingRequestManager.isEmpty}
+ * reports `true`, so the crawler idles instead of holding a concurrency slot open.
+ *
+ * Delays come from two places:
+ * - HTTP 429 responses, honouring `Retry-After` and otherwise backing off exponentially. The crawlers report these
+ *   automatically; a request that is throttled is retried later without counting against `maxRequestRetries` and
+ *   without penalising its session.
+ * - robots.txt `Crawl-delay` directives, when `respectRobotsTxtFile` is enabled.
+ *
+ * This is opt-in: throttling only happens for a domain you list explicitly.
+ *
+ * **Example usage:**
+ *
+ * ```ts
+ * const crawler = new CheerioCrawler({
+ *     requestManager: new ThrottlingRequestManager({
+ *         inner: await RequestQueue.open(),
+ *         domains: ['api.example.com', 'slow-site.org'],
+ *     }),
+ *     requestHandler: async ({ request }) => { ... },
+ * });
+ * ```
+ *
+ * @category Sources
+ */
+export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager, SupportsDomainThrottling {
+    private readonly config;
+    private readonly inner;
+    private readonly requestManagerOpener;
+    private readonly baseDelayMs;
+    private readonly maxDelayMs;
+    private readonly maxDomainStallMs;
+    private readonly domainStates;
+    private readonly subManagers;
+    private readonly log;
+    /**
+     * Sub-managers are keyed by a stable alias, so they outlive the process. They must therefore be reopened
+     * for every configured domain rather than created on first insert - otherwise a restart sees an empty map,
+     * reports the crawl finished, and strands whatever the previous run left in them.
+     */
+    private subManagersReady?;
+    /** Batches still being added in the background; keeps {@link ThrottlingRequestManager.isFinished} honest. */
+    private inProgressBatchCount;
+    private readonly warnedAbout;
+    private get hasThrottledDomains();
+    constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);
+    /** The wrapped manager, holding every request whose domain is not throttled. */
+    get innerManager(): T;
+    /** Warns once about sources that cannot be routed by domain, because their URLs are not known yet. */
+    private warnIfNotRoutable;
+    private warnOnce;
+    private extractDomain;
+    private getDomainState;
+    private selectManager;
+    /** Only valid once {@link ThrottlingRequestManager.ensureSubManagers} has resolved. */
+    private managerForUrl;
+    private ensureSubManagers;
+    private getSubManagers;
+    /** Configured domains that are not currently backing off, longest-overdue first. */
+    private fetchableDomains;
+    /**
+     * Records a 429 response and puts the URL's domain into backoff.
+     *
+     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     */
+    recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
+    /**
+     * Applies a robots.txt `Crawl-delay` to the URL's domain, as a minimum interval between dispatches.
+     *
+     * The first value wins, so a robots.txt re-fetch cannot change the cadence mid-crawl.
+     *
+     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     */
+    setCrawlDelay(url: string, delaySeconds: number): boolean;
+    /**
+     * Throws {@link PersistentRateLimitError} if any domain has been rate-limiting us past
+     * {@link ThrottlingRequestManagerOptions.maxDomainStallSecs|`maxDomainStallSecs`} without letting a single
+     * request through.
+     *
+     * A domain qualifies only while it still has queued requests and is actively rate-limiting - a domain that
+     * has simply run out of work is finished, not stalled, and one being waited out under a long robots.txt
+     * `Crawl-delay` is being obeyed, not stonewalled.
+     */
+    assertNoStalledDomains(): Promise<void>;
+    /** Records that a domain let a request through, which ends any rate-limit run stall detection was timing. */
+    private recordProgress;
+    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    /**
+     * Adds requests in batches, routing each one to the manager that owns its domain.
+     *
+     * Batching, validation, deduplication and `Retry-After`-free bookkeeping are all delegated to the target
+     * managers - this only decides where each request goes, one batch at a time, so a lazy or unbounded input
+     * iterable is never fully materialized.
+     */
+    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+    reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+    markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
+    getTotalCount(): Promise<number>;
+    getPendingCount(): Promise<number>;
+    getHandledCount(): Promise<number>;
+    /**
+     * Whether the next {@link ThrottlingRequestManager.fetchNextRequest} would return `null`.
+     *
+     * Requests waiting on a throttled domain count as unavailable, so a crawler whose task loop is gated on
+     * this idles for the backoff instead of spinning on a fetch that cannot succeed yet.
+     */
+    isEmpty(): Promise<boolean>;
+    /** Unlike {@link ThrottlingRequestManager.isEmpty}, throttled requests still count as outstanding work. */
+    isFinished(): Promise<boolean>;
+    /**
+     * Empties every manager and clears the accumulated backoff. A robots.txt `Crawl-delay` is a property of the
+     * site rather than of the run, so it survives.
+     */
+    purge(): Promise<void>;
+    setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+    private forEachManager;
+    private sumOverManagers;
+    private everyManager;
+    /**
+     * Returns the next request from a domain that is not backing off, or from the inner manager.
+     *
+     * Returns `null` while every remaining request belongs to a throttled domain - it never waits the backoff
+     * out, because a consumer parked in here holds a concurrency slot, which the autoscaler reads as spare
+     * capacity and answers by scaling up. Callers poll instead, and {@link ThrottlingRequestManager.isEmpty}
+     * reports `true` meanwhile so the crawler's task loop idles rather than spins.
+     */
+    fetchNextRequest<R extends Dictionary = Dictionary>(): Promise<Request<R> | null>;
+    [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
+    persistState(): Promise<void>;
+    drop(): Promise<void>;
+}

package/storages/throttling_request_manager.js

@@ -0,0 +1,453 @@
+import { URL } from 'node:url';
+import ow from 'ow';
+import { PersistentRateLimitError } from '../errors.js';
+import { asyncifyIterable } from '../iterables.js';
+import { serviceLocator } from '../service_locator.js';
+import { normalizeHostname } from '../url.js';
+import { drainRequestBatches } from './batched_adds.js';
+import { RequestQueue } from './request_queue.js';
+/** Whether `manager` can pace requests per domain. */
+export function supportsDomainThrottling(manager) {
+    const candidate = manager;
+    return (typeof candidate?.recordDomainDelay === 'function' &&
+        typeof candidate.setCrawlDelay === 'function' &&
+        typeof candidate.assertNoStalledDomains === 'function');
+}
+/** The moment a domain may be dispatched to again - whichever of its two independent clocks runs longer. */
+function throttledUntil(state) {
+    return Math.max(state.backoffUntil, state.crawlDelayUntil);
+}
+/**
+ * A request manager that wraps another one and paces requests per domain.
+ *
+ * Requests for the configured {@link ThrottlingRequestManagerOptions.domains|`domains`} are routed into their own
+ * queue when they are added, so each request lives in exactly one place and deduplication keeps working. Everything
+ * else goes to the wrapped manager untouched.
+ *
+ * {@link ThrottlingRequestManager.fetchNextRequest|`fetchNextRequest()`} serves the domain that has been waiting
+ * longest and skips any that are backing off, falling back to the wrapped manager. It never blocks: while every
+ * remaining request belongs to a throttled domain it returns `null` and {@link ThrottlingRequestManager.isEmpty}
+ * reports `true`, so the crawler idles instead of holding a concurrency slot open.
+ *
+ * Delays come from two places:
+ * - HTTP 429 responses, honouring `Retry-After` and otherwise backing off exponentially. The crawlers report these
+ *   automatically; a request that is throttled is retried later without counting against `maxRequestRetries` and
+ *   without penalising its session.
+ * - robots.txt `Crawl-delay` directives, when `respectRobotsTxtFile` is enabled.
+ *
+ * This is opt-in: throttling only happens for a domain you list explicitly.
+ *
+ * **Example usage:**
+ *
+ * ```ts
+ * const crawler = new CheerioCrawler({
+ *     requestManager: new ThrottlingRequestManager({
+ *         inner: await RequestQueue.open(),
+ *         domains: ['api.example.com', 'slow-site.org'],
+ *     }),
+ *     requestHandler: async ({ request }) => { ... },
+ * });
+ * ```
+ *
+ * @category Sources
+ */
+export class ThrottlingRequestManager {
+    config;
+    inner;
+    requestManagerOpener;
+    baseDelayMs;
+    maxDelayMs;
+    maxDomainStallMs;
+    domainStates = new Map();
+    subManagers = new Map();
+    log;
+    /**
+     * Sub-managers are keyed by a stable alias, so they outlive the process. They must therefore be reopened
+     * for every configured domain rather than created on first insert - otherwise a restart sees an empty map,
+     * reports the crawl finished, and strands whatever the previous run left in them.
+     */
+    subManagersReady;
+    /** Batches still being added in the background; keeps {@link ThrottlingRequestManager.isFinished} honest. */
+    inProgressBatchCount = 0;
+    warnedAbout = new Set();
+    get hasThrottledDomains() {
+        return this.domainStates.size > 0;
+    }
+    constructor(options, config = serviceLocator.getConfiguration()) {
+        this.config = config;
+        ow(options, ow.object.exactShape({
+            inner: ow.object,
+            domains: ow.array.ofType(ow.string.nonEmpty),
+            requestManagerOpener: ow.optional.function,
+            baseDelaySecs: ow.optional.number.positive,
+            maxDelaySecs: ow.optional.number.positive,
+            maxDomainStallSecs: ow.optional.number.positive,
+        }));
+        this.inner = options.inner;
+        this.requestManagerOpener =
+            options.requestManagerOpener ??
+                ((idOrAlias, opts) => RequestQueue.open(idOrAlias, opts));
+        this.baseDelayMs = (options.baseDelaySecs ?? 2) * 1000;
+        this.maxDelayMs = (options.maxDelaySecs ?? 60) * 1000;
+        this.maxDomainStallMs = (options.maxDomainStallSecs ?? 900) * 1000;
+        this.log = serviceLocator.getLogger().child({ prefix: 'ThrottlingRequestManager' });
+        for (const domain of options.domains) {
+            let hostname;
+            try {
+                // These are bare hostnames, so they only reach `URL` - and with it IDNA - via a synthetic URL.
+                hostname = normalizeHostname(new URL(`http://${domain}`).hostname);
+            }
+            catch {
+                throw new Error(`"${domain}" is not a valid hostname. The \`domains\` option takes bare hostnames such as ` +
+                    `"example.com"; an IPv6 address has to be bracketed, as in "[::1]".`);
+            }
+            this.domainStates.set(hostname, {
+                domain: hostname,
+                backoffUntil: 0,
+                crawlDelayUntil: 0,
+                backoffDecaysAt: 0,
+                consecutive429Count: 0,
+                crawlDelayMs: null,
+                rateLimitedSince: 0,
+                lastRateLimitedAt: 0,
+            });
+        }
+    }
+    /** The wrapped manager, holding every request whose domain is not throttled. */
+    get innerManager() {
+        return this.inner;
+    }
+    /** Warns once about sources that cannot be routed by domain, because their URLs are not known yet. */
+    warnIfNotRoutable(requestLike) {
+        if ('requestsFromUrl' in requestLike && requestLike.requestsFromUrl !== undefined && this.hasThrottledDomains) {
+            // The URL list is only fetched once the owning manager expands it, so we cannot know which domains
+            // it covers and cannot route it. Warn instead of silently exempting those URLs from throttling.
+            this.warnOnce('urlListNotRouted', `Requests loaded via \`requestsFromUrl\` cannot be routed to a per-domain queue, because their URLs ` +
+                `are not known at insertion time. They will be added to the inner request manager and will not ` +
+                `be throttled, even if they belong to a configured domain.`);
+        }
+    }
+    warnOnce(key, message) {
+        if (this.warnedAbout.has(key)) {
+            return;
+        }
+        this.warnedAbout.add(key);
+        this.log.warning(message);
+    }
+    extractDomain(url) {
+        try {
+            return normalizeHostname(new URL(url).hostname);
+        }
+        catch {
+            return '';
+        }
+    }
+    getDomainState(url) {
+        const domain = this.extractDomain(url);
+        return this.domainStates.get(domain) ?? null;
+    }
+    async selectManager(url) {
+        await this.ensureSubManagers();
+        return this.managerForUrl(url);
+    }
+    /** Only valid once {@link ThrottlingRequestManager.ensureSubManagers} has resolved. */
+    managerForUrl(url) {
+        return this.subManagers.get(this.extractDomain(url)) ?? this.inner;
+    }
+    async ensureSubManagers() {
+        this.subManagersReady ??= (async () => {
+            await Promise.all(Array.from(this.domainStates.keys(), async (domain) => {
+                const subManager = await this.requestManagerOpener(
+                // Backends use the alias as a directory name, and an IPv6 literal is full of characters
+                // Windows will not accept. Ordinary hostnames survive this untouched.
+                { alias: `throttled-${encodeURIComponent(domain)}` }, { configuration: this.config });
+                this.subManagers.set(domain, subManager);
+            }));
+        })();
+        await this.subManagersReady;
+    }
+    async getSubManagers() {
+        await this.ensureSubManagers();
+        return Array.from(this.subManagers.values());
+    }
+    /** Configured domains that are not currently backing off, longest-overdue first. */
+    fetchableDomains() {
+        const now = Date.now();
+        return Array.from(this.domainStates.values())
+            .filter((state) => now >= throttledUntil(state))
+            .sort((a, b) => throttledUntil(a) - throttledUntil(b))
+            .map((state) => state.domain);
+    }
+    /**
+     * Records a 429 response and puts the URL's domain into backoff.
+     *
+     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     */
+    recordDomainDelay(url, retryAfterMs) {
+        const state = this.getDomainState(url);
+        if (!state) {
+            return false;
+        }
+        const now = Date.now();
+        // Recorded before the burst suppression below, because a suppressed 429 is still the domain turning us
+        // away - which is exactly what stall detection needs to know about.
+        state.lastRateLimitedAt = now;
+        if (state.rateLimitedSince === 0) {
+            state.rateLimitedSince = now;
+        }
+        // Requests already in flight when the limit was hit all come back 429. They describe one rate-limit
+        // event, so only the first advances the backoff - otherwise concurrency alone drives the exponent.
+        // Only the backoff clock may suppress here: `crawlDelayUntil` is in the future after every dispatch,
+        // so consulting it would discard every 429 the domain ever sends, `Retry-After` included.
+        if (now < state.backoffUntil) {
+            return true;
+        }
+        // A domain that has served us for a full extra backoff window is no longer rate-limiting; start over
+        // rather than carrying the old exponent into an unrelated burst.
+        if (now >= state.backoffDecaysAt) {
+            state.consecutive429Count = 0;
+        }
+        state.consecutive429Count += 1;
+        const retryAfterGiven = retryAfterMs !== undefined && retryAfterMs !== null;
+        let delayMs = retryAfterGiven ? retryAfterMs : this.baseDelayMs * Math.pow(2, state.consecutive429Count - 1);
+        if (delayMs > this.maxDelayMs) {
+            const source = retryAfterGiven ? 'Retry-After header' : 'exponential backoff';
+            this.log.warning(`Capping ${source} delay of ${(delayMs / 1000).toFixed(1)}s for domain "${state.domain}" ` +
+                `to maxDelaySecs (${(this.maxDelayMs / 1000).toFixed(1)}s); the domain may continue to rate-limit. ` +
+                `Consider increasing maxDelaySecs if this recurs.`);
+            delayMs = this.maxDelayMs;
+        }
+        state.backoffUntil = now + delayMs;
+        state.backoffDecaysAt = state.backoffUntil + delayMs;
+        this.log.info(`Rate limit (429) detected for domain "${state.domain}" ` +
+            `(consecutive: ${state.consecutive429Count}, delay: ${(delayMs / 1000).toFixed(1)}s)`);
+        return true;
+    }
+    /**
+     * Applies a robots.txt `Crawl-delay` to the URL's domain, as a minimum interval between dispatches.
+     *
+     * The first value wins, so a robots.txt re-fetch cannot change the cadence mid-crawl.
+     *
+     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     */
+    setCrawlDelay(url, delaySeconds) {
+        const state = this.getDomainState(url);
+        if (!state) {
+            return false;
+        }
+        if (state.crawlDelayMs === null) {
+            state.crawlDelayMs = delaySeconds * 1000;
+            this.log.debug(`Set crawl-delay for domain "${state.domain}" to ${delaySeconds}s`);
+        }
+        return true;
+    }
+    /**
+     * Throws {@link PersistentRateLimitError} if any domain has been rate-limiting us past
+     * {@link ThrottlingRequestManagerOptions.maxDomainStallSecs|`maxDomainStallSecs`} without letting a single
+     * request through.
+     *
+     * A domain qualifies only while it still has queued requests and is actively rate-limiting - a domain that
+     * has simply run out of work is finished, not stalled, and one being waited out under a long robots.txt
+     * `Crawl-delay` is being obeyed, not stonewalled.
+     */
+    async assertNoStalledDomains() {
+        await this.ensureSubManagers();
+        const now = Date.now();
+        const candidates = Array.from(this.domainStates.values()).filter(
+        // Together: it is still turning us away, and has been doing so without a break for longer than the
+        // window. A domain that has simply been idle starts this clock at its first 429 rather than
+        // arriving with the idle time already on it.
+        (state) => state.rateLimitedSince !== 0 &&
+            now - state.lastRateLimitedAt <= this.maxDomainStallMs &&
+            now - state.rateLimitedSince > this.maxDomainStallMs);
+        const stalled = (await Promise.all(candidates.map(async (state) => ((await this.subManagers.get(state.domain).isEmpty()) ? null : state)))).filter((state) => state !== null);
+        if (stalled.length === 0) {
+            return;
+        }
+        const summary = stalled
+            .map((state) => `"${state.domain}" (${((now - state.rateLimitedSince) / 1000).toFixed(0)}s)`)
+            .join(', ');
+        throw new PersistentRateLimitError(`Giving up: ${summary} rate-limited every request for longer than maxDomainStallSecs ` +
+            `(${(this.maxDomainStallMs / 1000).toFixed(0)}s). Waiting longer will not help - lower the ` +
+            `crawler's concurrency, or drop these domains. Their requests are still queued, so re-running ` +
+            `without purging storages will resume them if the rate limit lifts.`);
+    }
+    /** Records that a domain let a request through, which ends any rate-limit run stall detection was timing. */
+    recordProgress(url) {
+        const state = this.getDomainState(url);
+        if (state) {
+            state.rateLimitedSince = 0;
+        }
+    }
+    // --- IRequestManager Implementation ---
+    async addRequest(requestLike, options) {
+        this.warnIfNotRoutable(requestLike);
+        const manager = await this.selectManager(requestLike.url ?? '');
+        return manager.addRequest(requestLike, options);
+    }
+    /**
+     * Adds requests in batches, routing each one to the manager that owns its domain.
+     *
+     * Batching, validation, deduplication and `Retry-After`-free bookkeeping are all delegated to the target
+     * managers - this only decides where each request goes, one batch at a time, so a lazy or unbounded input
+     * iterable is never fully materialized.
+     */
+    async addRequestsBatched(requests, options = {}) {
+        await this.ensureSubManagers();
+        // Normalized up front so the shared batching helper - and `requestsOverLimit` - only ever see `Source`.
+        async function* iterateRequests() {
+            for await (const request of asyncifyIterable(requests)) {
+                yield typeof request === 'string' ? { url: request } : request;
+            }
+        }
+        return drainRequestBatches({
+            items: iterateRequests(),
+            batchSize: options.batchSize ?? 1000,
+            waitBetweenBatchesMillis: options.waitBetweenBatchesMillis ?? 1000,
+            waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded ?? false,
+            maxNewRequests: options.maxNewRequests,
+            // Routing is the only thing this manager adds; the targets do the batching, validation and
+            // deduplication themselves.
+            processChunk: async (chunk) => {
+                const byManager = new Map();
+                for (const request of chunk) {
+                    this.warnIfNotRoutable(request);
+                    const manager = this.managerForUrl(request.url ?? '');
+                    const bucket = byManager.get(manager);
+                    if (bucket) {
+                        bucket.push(request);
+                    }
+                    else {
+                        byManager.set(manager, [request]);
+                    }
+                }
+                const results = await Promise.all(Array.from(byManager, ([manager, slice]) => manager.addRequestsBatched(slice, {
+                    forefront: options.forefront,
+                    // The slice is already one batch, and we need its results before releasing the next one.
+                    batchSize: slice.length,
+                    waitForAllRequestsToBeAdded: true,
+                })));
+                return results.flatMap((result) => result.addedRequests);
+            },
+            // Keeps the crawler from concluding it is finished while batches are still landing.
+            trackBackgroundBatches: (batches) => {
+                this.inProgressBatchCount += 1;
+                void batches.finally(() => {
+                    this.inProgressBatchCount -= 1;
+                });
+            },
+        });
+    }
+    async reclaimRequest(request, options) {
+        const manager = await this.selectManager(request.url);
+        return manager.reclaimRequest(request, options);
+    }
+    async markRequestAsHandled(request) {
+        const manager = await this.selectManager(request.url);
+        // Reached whether the request succeeded or ran out of retries; either way the domain answered us.
+        this.recordProgress(request.url);
+        return manager.markRequestAsHandled(request);
+    }
+    async getTotalCount() {
+        return this.sumOverManagers((manager) => manager.getTotalCount());
+    }
+    async getPendingCount() {
+        return this.sumOverManagers((manager) => manager.getPendingCount());
+    }
+    async getHandledCount() {
+        return this.sumOverManagers((manager) => manager.getHandledCount());
+    }
+    /**
+     * Whether the next {@link ThrottlingRequestManager.fetchNextRequest} would return `null`.
+     *
+     * Requests waiting on a throttled domain count as unavailable, so a crawler whose task loop is gated on
+     * this idles for the backoff instead of spinning on a fetch that cannot succeed yet.
+     */
+    async isEmpty() {
+        await this.ensureSubManagers();
+        const fetchable = [this.inner, ...this.fetchableDomains().map((domain) => this.subManagers.get(domain))];
+        const results = await Promise.all(fetchable.map((manager) => manager.isEmpty()));
+        return results.every(Boolean);
+    }
+    /** Unlike {@link ThrottlingRequestManager.isEmpty}, throttled requests still count as outstanding work. */
+    async isFinished() {
+        if (this.inProgressBatchCount > 0) {
+            return false;
+        }
+        return this.everyManager((manager) => manager.isFinished());
+    }
+    /**
+     * Empties every manager and clears the accumulated backoff. A robots.txt `Crawl-delay` is a property of the
+     * site rather than of the run, so it survives.
+     */
+    async purge() {
+        await this.forEachManager((manager) => manager.purge?.());
+        for (const state of this.domainStates.values()) {
+            state.consecutive429Count = 0;
+            state.backoffUntil = 0;
+            state.crawlDelayUntil = 0;
+            state.backoffDecaysAt = 0;
+            state.rateLimitedSince = 0;
+            state.lastRateLimitedAt = 0;
+        }
+    }
+    async setExpectedRequestProcessingTimeSecs(secs) {
+        await this.forEachManager((manager) => manager.setExpectedRequestProcessingTimeSecs?.(secs));
+    }
+    async forEachManager(fn) {
+        // `fn` targets optional members, so it may return nothing - the wrapper normalizes that for `Promise.all`.
+        await Promise.all([this.inner, ...(await this.getSubManagers())].map(async (manager) => fn(manager)));
+    }
+    async sumOverManagers(fn) {
+        const counts = await Promise.all([this.inner, ...(await this.getSubManagers())].map(fn));
+        return counts.reduce((a, b) => a + b, 0);
+    }
+    async everyManager(fn) {
+        const results = await Promise.all([this.inner, ...(await this.getSubManagers())].map(fn));
+        return results.every(Boolean);
+    }
+    /**
+     * Returns the next request from a domain that is not backing off, or from the inner manager.
+     *
+     * Returns `null` while every remaining request belongs to a throttled domain - it never waits the backoff
+     * out, because a consumer parked in here holds a concurrency slot, which the autoscaler reads as spare
+     * capacity and answers by scaling up. Callers poll instead, and {@link ThrottlingRequestManager.isEmpty}
+     * reports `true` meanwhile so the crawler's task loop idles rather than spins.
+     */
+    async fetchNextRequest() {
+        await this.ensureSubManagers();
+        for (const domain of this.fetchableDomains()) {
+            const state = this.domainStates.get(domain);
+            // Armed while the fetch below is still suspended, so that a concurrent `fetchNextRequest` cannot
+            // find the domain fetchable and dispatch into the same window - which would pace each task
+            // rather than the domain.
+            const crawlDelayBefore = state.crawlDelayUntil;
+            if (state.crawlDelayMs !== null) {
+                state.crawlDelayUntil = Date.now() + state.crawlDelayMs;
+            }
+            const request = await this.subManagers.get(domain).fetchNextRequest();
+            if (request) {
+                return request;
+            }
+            // No dispatch to pace, so the domain keeps its slot.
+            state.crawlDelayUntil = crawlDelayBefore;
+        }
+        return this.inner.fetchNextRequest();
+    }
+    async *[Symbol.asyncIterator]() {
+        while (true) {
+            const req = await this.fetchNextRequest();
+            if (!req)
+                break;
+            yield req;
+        }
+    }
+    async persistState() {
+        await this.forEachManager((manager) => manager.persistState?.());
+    }
+    async drop() {
+        await this.forEachManager((manager) => manager.drop?.());
+        this.subManagers.clear();
+        this.subManagersReady = undefined;
+    }
+}

package/url.d.ts

@@ -0,0 +1,9 @@
+/**
+ * The canonical form of a hostname: lower-case, punycode, and without the optional root dot.
+ *
+ * Pass both sides of a hostname comparison through this, so that a domain written as `háčky.cz` still matches the
+ * `xn--hky-ela4t.cz` that `URL` reports.
+ *
+ * @internal
+ */
+export declare function normalizeHostname(hostname: string): string;

package/url.js

@@ -0,0 +1,11 @@
+/**
+ * The canonical form of a hostname: lower-case, punycode, and without the optional root dot.
+ *
+ * Pass both sides of a hostname comparison through this, so that a domain written as `háčky.cz` still matches the
+ * `xn--hky-ela4t.cz` that `URL` reports.
+ *
+ * @internal
+ */
+export function normalizeHostname(hostname) {
+    return hostname.toLowerCase().replace(/\.$/, '');
+}