@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.130

package/storages/throttling_request_manager.js

@@ -0,0 +1,646 @@
+import { URL } from 'node:url';
+import { getDomain } from 'tldts';
+import { z } from 'zod';
+import { PersistentRateLimitError } from '../errors.js';
+import { asyncifyIterable } from '../iterables.js';
+import { serviceLocator } from '../service_locator.js';
+import { normalizeHostname } from '../url.js';
+import { parseArgument, schemas } from '../validators.js';
+import { drainRequestBatches } from './batched_adds.js';
+import { KeyValueStore } from './key_value_store.js';
+import { RequestQueue } from './request_queue.js';
+const throttlingRequestManagerOptionsSchema = z.strictObject({
+    inner: schemas.anyObject,
+    domains: z.union([schemas.arrayOf(z.string().nonempty(), 'non-empty strings'), z.literal('all')]),
+    requestManagerOpener: schemas.anyFunction.optional(),
+    baseDelaySecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
+    maxDelaySecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
+    maxDomainStallSecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
+    minCrawlDelaySecs: schemas.anyNumber
+        .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
+        .optional(),
+    throttleBy: z.enum(['hostname', 'registrableDomain']).optional(),
+    maxThrottledDomains: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
+    persistStateKey: z.string().nonempty().optional(),
+});
+/** Whether `manager` can pace requests per domain. */
+export function supportsDomainThrottling(manager) {
+    const candidate = manager;
+    return (typeof candidate?.recordDomainDelay === 'function' &&
+        typeof candidate.setCrawlDelay === 'function' &&
+        typeof candidate.assertNoStalledDomains === 'function');
+}
+/** The moment a domain may be dispatched to again - whichever of its two independent clocks runs longer. */
+function throttledUntil(state) {
+    return Math.max(state.backoffUntil, state.crawlDelayUntil);
+}
+/** How long a domain must be left alone after a dispatch: what it asked for, or our floor, whichever is longer. */
+function crawlDelayMs(state, minCrawlDelayMs) {
+    return Math.max(state.declaredCrawlDelayMs ?? 0, minCrawlDelayMs);
+}
+function newDomainState(domain) {
+    return {
+        domain,
+        backoffUntil: 0,
+        crawlDelayUntil: 0,
+        backoffDecaysAt: 0,
+        consecutive429Count: 0,
+        declaredCrawlDelayMs: null,
+        rateLimitedSince: 0,
+        lastRateLimitedAt: 0,
+    };
+}
+const DEFAULT_PERSIST_STATE_KEY = 'CRAWLEE_THROTTLED_DOMAINS';
+/**
+ * A request manager that wraps another one and paces requests per domain.
+ *
+ * Requests for a throttled domain are routed into their own queue when they are added, so each request lives in
+ * exactly one place and deduplication keeps working. Everything else goes to the wrapped manager untouched.
+ *
+ * {@link ThrottlingRequestManager.fetchNextRequest|`fetchNextRequest()`} serves the domain that has been waiting
+ * longest and skips any that are backing off, falling back to the wrapped manager. It never blocks: while every
+ * remaining request belongs to a throttled domain it returns `null` and {@link ThrottlingRequestManager.isEmpty}
+ * reports `true`, so the crawler idles instead of holding a concurrency slot open.
+ *
+ * Each throttled domain runs two independent clocks, and may be dispatched to once **both** have run out:
+ * - **Backoff**, set by HTTP 429 responses - honouring `Retry-After`, and otherwise doubling from `baseDelaySecs`.
+ *   Reactive and temporary: it decays once the domain stops turning us away. The crawlers report the 429s
+ *   themselves; a request held back this way is retried later without counting against `maxRequestRetries` and
+ *   without penalising its session.
+ * - **Crawl delay**, the minimum interval between two dispatches to the domain, armed after each one. Proactive
+ *   and constant: whatever the domain's robots.txt asks for, floored by
+ *   {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`}. Either may be absent, in
+ *   which case the other one is the delay.
+ *
+ * Which domains get those clocks is {@link ThrottlingRequestManagerOptions.domains|`domains`} - a list, or
+ * `'all'` for every domain the crawl encounters.
+ *
+ * **Example usage:**
+ *
+ * ```ts
+ * const crawler = new CheerioCrawler({
+ *     requestManager: new ThrottlingRequestManager({
+ *         inner: await RequestQueue.open(),
+ *         domains: ['api.example.com', 'slow-site.org'],
+ *     }),
+ *     requestHandler: async ({ request }) => { ... },
+ * });
+ * ```
+ *
+ * @category Sources
+ */
+export class ThrottlingRequestManager {
+    config;
+    #inner;
+    #requestManagerOpener;
+    #baseDelayMs;
+    #maxDelayMs;
+    #maxDomainStallMs;
+    #minCrawlDelayMs;
+    #throttlesEveryDomain;
+    #throttleBy;
+    #maxThrottledDomains;
+    #persistStateKey;
+    #subManagers = new Map();
+    // Not `#private`, unlike the rest: the tests reach for these two.
+    domainStates = new Map();
+    log;
+    /** Domains from the `domains` option, which are throttled whether or not the crawl ever visits them. */
+    #listedDomains = new Set();
+    /**
+     * Domains picked up at runtime under `domains: 'all'`. Persisted, because unlike the listed ones there
+     * is nothing to rediscover them from at startup - and a sub-queue nobody reopens is a sub-queue whose
+     * requests are never crawled.
+     */
+    #discoveredDomains = new Set();
+    /**
+     * Requests currently held by the consumer that came out of the wrapped manager rather than a sub-queue.
+     * They have to go back where they came from: routing them by domain would mark them handled in a queue
+     * that has never heard of them, leaving the wrapped manager to hand them out over and over.
+     */
+    #inFlightFromInner = new Set();
+    #domainListStore;
+    #lastDomainListWrite = Promise.resolve();
+    #queuedDomainListWrite;
+    /**
+     * Sub-managers are keyed by a stable alias, so with `purgeOnStart` disabled they outlive the process. They
+     * must therefore be reopened for every known domain rather than created on first insert - otherwise a
+     * restart sees an empty map, reports the crawl finished, and strands whatever the previous run left in them.
+     */
+    #subManagersReady;
+    /** Batches still being added in the background; keeps {@link ThrottlingRequestManager.isFinished} honest. */
+    #inProgressBatchCount = 0;
+    #warnedAbout = new Set();
+    /** Whether any domain at all may end up throttled - listed up front, or discovered as the crawl runs. */
+    get #throttlingEnabled() {
+        return this.#listedDomains.size > 0 || this.#throttlesEveryDomain;
+    }
+    constructor(options, config = serviceLocator.getConfiguration()) {
+        this.config = config;
+        parseArgument(options, throttlingRequestManagerOptionsSchema, 'ThrottlingRequestManagerOptions');
+        this.#inner = options.inner;
+        this.#requestManagerOpener =
+            options.requestManagerOpener ??
+                ((idOrAlias, opts) => RequestQueue.open(idOrAlias, opts));
+        this.#baseDelayMs = (options.baseDelaySecs ?? 2) * 1000;
+        this.#maxDelayMs = (options.maxDelaySecs ?? 60) * 1000;
+        this.#maxDomainStallMs = (options.maxDomainStallSecs ?? 900) * 1000;
+        this.#minCrawlDelayMs = (options.minCrawlDelaySecs ?? 0) * 1000;
+        this.#throttlesEveryDomain = options.domains === 'all';
+        this.#throttleBy = options.throttleBy ?? 'hostname';
+        this.#maxThrottledDomains = options.maxThrottledDomains ?? 100;
+        this.#persistStateKey = options.persistStateKey ?? DEFAULT_PERSIST_STATE_KEY;
+        this.log = serviceLocator.getLogger().child({ prefix: 'ThrottlingRequestManager' });
+        for (const domain of Array.isArray(options.domains) ? options.domains : []) {
+            let hostname;
+            try {
+                // These are bare hostnames, so they only reach `URL` - and with it IDNA - via a synthetic URL.
+                hostname = new URL(`http://${domain}`).hostname;
+            }
+            catch {
+                throw new Error(`"${domain}" is not a valid hostname. The \`domains\` option takes bare hostnames such as ` +
+                    `"example.com"; an IPv6 address has to be bracketed, as in "[::1]".`);
+            }
+            const key = this.#domainKey(hostname);
+            this.#listedDomains.add(key);
+            this.domainStates.set(key, newDomainState(key));
+        }
+    }
+    /**
+     * The key a URL's requests are grouped under - one delay clock and one sub-queue per key.
+     *
+     * @param hostname A hostname as `URL` reports it, so in punycode and possibly with a root dot.
+     */
+    #domainKey(hostname) {
+        const normalized = normalizeHostname(hostname);
+        if (this.#throttleBy === 'hostname') {
+            return normalized;
+        }
+        // No registrable domain to group by for an IP address or a single-label host such as `localhost`,
+        // so those stay paced per hostname.
+        return getDomain(normalized, { mixedInputs: false }) ?? normalized;
+    }
+    /** The wrapped manager, holding every request whose domain is not throttled. */
+    get innerManager() {
+        return this.#inner;
+    }
+    /** Warns once about sources that cannot be routed by domain, because their URLs are not known yet. */
+    #warnIfNotRoutable(requestLike) {
+        if ('requestsFromUrl' in requestLike && requestLike.requestsFromUrl !== undefined && this.#throttlingEnabled) {
+            // The URL list is only fetched once the owning manager expands it, so we cannot know which domains
+            // it covers and cannot route it. Warn instead of silently exempting those URLs from throttling.
+            this.#warnOnce('urlListNotRouted', `Requests loaded via \`requestsFromUrl\` cannot be routed to a per-domain queue, because their URLs ` +
+                `are not known at insertion time. They will be added to the inner request manager and will not ` +
+                `be throttled, even if they belong to a configured domain.`);
+        }
+    }
+    #warnOnce(key, message) {
+        if (this.#warnedAbout.has(key)) {
+            return;
+        }
+        this.#warnedAbout.add(key);
+        this.log.warning(message);
+    }
+    #extractDomain(url) {
+        try {
+            return this.#domainKey(new URL(url).hostname);
+        }
+        catch {
+            return '';
+        }
+    }
+    #getDomainState(url) {
+        const domain = this.#extractDomain(url);
+        return this.domainStates.get(domain) ?? null;
+    }
+    /**
+     * The manager that owns a URL's requests, opening a sub-queue for its domain if this is the first time we
+     * have seen it and every domain is throttled. `null` means the domain is new and `maxThrottledDomains`
+     * leaves no room for it.
+     */
+    async #selectManager(url) {
+        await this.#ensureSubManagers();
+        const domain = this.#extractDomain(url);
+        if (!domain || !(this.#listedDomains.has(domain) || this.#throttlesEveryDomain)) {
+            return this.#inner;
+        }
+        if (!this.#listedDomains.has(domain) && !this.#discoveredDomains.has(domain)) {
+            if (this.#discoveredDomains.size >= this.#maxThrottledDomains) {
+                return null;
+            }
+            this.#discoveredDomains.add(domain);
+            // Written before the request lands in the sub-queue: a crash in between would leave that queue
+            // with nothing to reopen it, and the crawl would silently drop everything in it.
+            await this.#persistDiscoveredDomains();
+        }
+        return this.#subManagerFor(domain);
+    }
+    async #selectManagerOrThrow(url) {
+        const manager = await this.#selectManager(url);
+        if (!manager) {
+            throw this.#throttledDomainLimitError([this.#extractDomain(url)]);
+        }
+        return manager;
+    }
+    #throttledDomainLimitError(domains) {
+        const [first, ...rest] = domains;
+        const named = rest.slice(0, 10);
+        const ellipsis = rest.length > named.length ? ', ...' : '';
+        const others = rest.length > 0 ? ` (and ${rest.length} other new domain(s): ${named.join(', ')}${ellipsis})` : '';
+        return new Error(`Refusing to throttle "${first}"${others}: ${this.#maxThrottledDomains} domains are already being ` +
+            `throttled (\`maxThrottledDomains\`). Each of them holds a request queue of its own, so a crawl ` +
+            `that keeps discovering new domains will bury the storage backend in them. Narrow the crawl down, ` +
+            `pace it with \`maxRequestsPerMinute\` instead, or raise \`maxThrottledDomains\` if you are ` +
+            `prepared to pay for it.`);
+    }
+    /** Opens the domain's sub-queue, or returns the one already opened (or being opened) for it. */
+    #subManagerFor(domain) {
+        let subManager = this.#subManagers.get(domain);
+        if (!subManager) {
+            subManager = this.#requestManagerOpener(
+            // Backends use the alias as a directory name, and an IPv6 literal is full of characters
+            // Windows will not accept. Ordinary hostnames survive this untouched.
+            { alias: `throttled-${encodeURIComponent(domain)}` }, { configuration: this.config });
+            this.#subManagers.set(domain, subManager);
+            this.#ensureDomainState(domain);
+        }
+        return subManager;
+    }
+    #ensureDomainState(domain) {
+        let state = this.domainStates.get(domain);
+        if (!state) {
+            state = newDomainState(domain);
+            this.domainStates.set(domain, state);
+        }
+        return state;
+    }
+    /**
+     * Coalesces the writes of the discovered domain list: callers that arrive while one is in flight share the
+     * single write queued behind it, which snapshots the set once it starts and so covers all of them.
+     */
+    async #persistDiscoveredDomains() {
+        this.#queuedDomainListWrite ??= this.#lastDomainListWrite
+            // A failed write must not poison the ones behind it - they will rewrite the whole set anyway.
+            .catch(() => { })
+            .then(async () => {
+            this.#queuedDomainListWrite = undefined;
+            await this.#domainListStore.setValue(this.#persistStateKey, Array.from(this.#discoveredDomains));
+        });
+        this.#lastDomainListWrite = this.#queuedDomainListWrite;
+        await this.#queuedDomainListWrite;
+    }
+    async #ensureSubManagers() {
+        this.#subManagersReady ??= (async () => {
+            if (this.#throttlesEveryDomain) {
+                this.#domainListStore = await KeyValueStore.open(null, { configuration: this.config });
+                for (const domain of (await this.#domainListStore.getValue(this.#persistStateKey)) ?? []) {
+                    this.#discoveredDomains.add(domain);
+                }
+            }
+            await Promise.all(Array.from([...this.#listedDomains, ...this.#discoveredDomains], async (domain) => this.#subManagerFor(domain)));
+        })();
+        await this.#subManagersReady;
+    }
+    async #getSubManagers() {
+        await this.#ensureSubManagers();
+        return Promise.all(this.#subManagers.values());
+    }
+    /**
+     * Throttled domains that are not currently backing off, longest-overdue first.
+     *
+     * A domain whose sub-queue has not been opened yet is skipped - a robots.txt `Crawl-delay` gives a domain a
+     * clock before its first request gives it a queue, and there is nothing to fetch from until then.
+     */
+    #fetchableDomains() {
+        const now = Date.now();
+        return Array.from(this.domainStates.values())
+            .filter((state) => now >= throttledUntil(state) && this.#subManagers.has(state.domain))
+            .sort((a, b) => throttledUntil(a) - throttledUntil(b))
+            .map((state) => state.domain);
+    }
+    /**
+     * Records a 429 response and puts the URL's domain into backoff.
+     *
+     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     */
+    recordDomainDelay(url, retryAfterMs) {
+        const state = this.#getDomainState(url);
+        if (!state) {
+            return false;
+        }
+        const now = Date.now();
+        // Recorded before the burst suppression below, because a suppressed 429 is still the domain turning us
+        // away - which is exactly what stall detection needs to know about.
+        state.lastRateLimitedAt = now;
+        if (state.rateLimitedSince === 0) {
+            state.rateLimitedSince = now;
+        }
+        // Requests already in flight when the limit was hit all come back 429. They describe one rate-limit
+        // event, so only the first advances the backoff - otherwise concurrency alone drives the exponent.
+        // Only the backoff clock may suppress here: `crawlDelayUntil` is in the future after every dispatch,
+        // so consulting it would discard every 429 the domain ever sends, `Retry-After` included.
+        if (now < state.backoffUntil) {
+            return true;
+        }
+        // A domain that has served us for a full extra backoff window is no longer rate-limiting; start over
+        // rather than carrying the old exponent into an unrelated burst.
+        if (now >= state.backoffDecaysAt) {
+            state.consecutive429Count = 0;
+        }
+        state.consecutive429Count += 1;
+        const retryAfterGiven = retryAfterMs !== undefined && retryAfterMs !== null;
+        let delayMs = retryAfterGiven ? retryAfterMs : this.#baseDelayMs * Math.pow(2, state.consecutive429Count - 1);
+        if (delayMs > this.#maxDelayMs) {
+            const source = retryAfterGiven ? 'Retry-After header' : 'exponential backoff';
+            this.log.warning(`Capping ${source} delay of ${(delayMs / 1000).toFixed(1)}s for domain "${state.domain}" ` +
+                `to maxDelaySecs (${(this.#maxDelayMs / 1000).toFixed(1)}s); the domain may continue to rate-limit. ` +
+                `Consider increasing maxDelaySecs if this recurs.`);
+            delayMs = this.#maxDelayMs;
+        }
+        state.backoffUntil = now + delayMs;
+        state.backoffDecaysAt = state.backoffUntil + delayMs;
+        this.log.info(`Rate limit (429) detected for domain "${state.domain}" ` +
+            `(consecutive: ${state.consecutive429Count}, delay: ${(delayMs / 1000).toFixed(1)}s)`);
+        return true;
+    }
+    /**
+     * Records the `Crawl-delay` a domain's robots.txt asked for, which becomes its crawl delay unless
+     * {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`} asks for longer.
+     *
+     * The first value wins, so a robots.txt re-fetch cannot change the cadence mid-crawl.
+     *
+     * @returns `false` if the domain is not throttled, in which case this is a no-op.
+     */
+    setCrawlDelay(url, delaySeconds) {
+        const domain = this.#extractDomain(url);
+        if (!domain || !(this.#listedDomains.has(domain) || this.#throttlesEveryDomain)) {
+            return false;
+        }
+        // The crawler reads robots.txt before it enqueues a domain's first request, so the clock can predate
+        // the sub-queue it will end up pacing.
+        const state = this.#ensureDomainState(domain);
+        if (state.declaredCrawlDelayMs === null) {
+            state.declaredCrawlDelayMs = delaySeconds * 1000;
+            this.log.debug(`Set crawl-delay for domain "${state.domain}" to ${delaySeconds}s`);
+        }
+        return true;
+    }
+    /**
+     * Throws {@link PersistentRateLimitError} if any domain has been rate-limiting us past
+     * {@link ThrottlingRequestManagerOptions.maxDomainStallSecs|`maxDomainStallSecs`} without letting a single
+     * request through.
+     *
+     * A domain qualifies only while it still has queued requests and is actively rate-limiting - a domain that
+     * has simply run out of work is finished, not stalled, and one being waited out under a long robots.txt
+     * `Crawl-delay` is being obeyed, not stonewalled.
+     */
+    async assertNoStalledDomains() {
+        await this.#ensureSubManagers();
+        const now = Date.now();
+        const candidates = Array.from(this.domainStates.values()).filter(
+        // Together: it is still turning us away, and has been doing so without a break for longer than the
+        // window. A domain that has simply been idle starts this clock at its first 429 rather than
+        // arriving with the idle time already on it.
+        (state) => state.rateLimitedSince !== 0 &&
+            now - state.lastRateLimitedAt <= this.#maxDomainStallMs &&
+            now - state.rateLimitedSince > this.#maxDomainStallMs);
+        const stalled = (await Promise.all(candidates.map(async (state) => {
+            const subManager = await this.#subManagers.get(state.domain);
+            return subManager && !(await subManager.isEmpty()) ? state : null;
+        }))).filter((state) => state !== null);
+        if (stalled.length === 0) {
+            return;
+        }
+        const summary = stalled
+            .map((state) => `"${state.domain}" (${((now - state.rateLimitedSince) / 1000).toFixed(0)}s)`)
+            .join(', ');
+        throw new PersistentRateLimitError(`Giving up: ${summary} rate-limited every request for longer than maxDomainStallSecs ` +
+            `(${(this.#maxDomainStallMs / 1000).toFixed(0)}s). Waiting longer will not help - lower the ` +
+            `crawler's concurrency, or drop these domains. Their requests are still queued, so re-running ` +
+            `with \`purgeOnStart\` disabled will resume them if the rate limit lifts.`);
+    }
+    /** Records that a domain let a request through, which ends any rate-limit run stall detection was timing. */
+    #recordProgress(url) {
+        const state = this.#getDomainState(url);
+        if (state) {
+            state.rateLimitedSince = 0;
+        }
+    }
+    // --- IRequestManager Implementation ---
+    async addRequest(requestLike, options) {
+        this.#warnIfNotRoutable(requestLike);
+        const manager = await this.#selectManagerOrThrow(requestLike.url ?? '');
+        return manager.addRequest(requestLike, options);
+    }
+    /**
+     * Adds requests in batches, routing each one to the manager that owns its domain.
+     *
+     * Batching, validation, deduplication and `Retry-After`-free bookkeeping are all delegated to the target
+     * managers - this only decides where each request goes, one batch at a time, so a lazy or unbounded input
+     * iterable is never fully materialized.
+     */
+    async addRequestsBatched(requests, options = {}) {
+        await this.#ensureSubManagers();
+        // Normalized up front so the shared batching helper - and `requestsOverLimit` - only ever see `Source`.
+        async function* iterateRequests() {
+            for await (const request of asyncifyIterable(requests)) {
+                yield typeof request === 'string' ? { url: request } : request;
+            }
+        }
+        return drainRequestBatches({
+            items: iterateRequests(),
+            batchSize: options.batchSize ?? 1000,
+            waitBetweenBatchesMillis: options.waitBetweenBatchesMillis ?? 1000,
+            waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded ?? false,
+            maxNewRequests: options.maxNewRequests,
+            // Routing is the only thing this manager adds; the targets do the batching, validation and
+            // deduplication themselves.
+            processChunk: async (chunk) => {
+                const byManager = new Map();
+                // Collected, not thrown on sight, so an overflow does not discard the requests that fit.
+                const overflowing = new Set();
+                for (const request of chunk) {
+                    this.#warnIfNotRoutable(request);
+                    const url = request.url ?? '';
+                    const manager = await this.#selectManager(url);
+                    if (!manager) {
+                        overflowing.add(this.#extractDomain(url));
+                        continue;
+                    }
+                    const bucket = byManager.get(manager);
+                    if (bucket) {
+                        bucket.push(request);
+                    }
+                    else {
+                        byManager.set(manager, [request]);
+                    }
+                }
+                const results = await Promise.all(Array.from(byManager, ([manager, slice]) => manager.addRequestsBatched(slice, {
+                    forefront: options.forefront,
+                    // The slice is already one batch, and we need its results before releasing the next one.
+                    batchSize: slice.length,
+                    waitForAllRequestsToBeAdded: true,
+                })));
+                if (overflowing.size > 0) {
+                    throw this.#throttledDomainLimitError(Array.from(overflowing));
+                }
+                return results.flatMap((result) => result.addedRequests);
+            },
+            // Keeps the crawler from concluding it is finished while batches are still landing.
+            trackBackgroundBatches: (batches) => {
+                this.#inProgressBatchCount += 1;
+                void batches.finally(() => {
+                    this.#inProgressBatchCount -= 1;
+                });
+            },
+        });
+    }
+    async reclaimRequest(request, options) {
+        const manager = await this.#managerHolding(request);
+        return manager.reclaimRequest(request, options);
+    }
+    async markRequestAsHandled(request) {
+        const manager = await this.#managerHolding(request);
+        // Reached whether the request succeeded or ran out of retries; either way the domain answered us.
+        this.#recordProgress(request.url);
+        return manager.markRequestAsHandled(request);
+    }
+    /**
+     * The manager a request in the consumer's hands has to be given back to - the one it was fetched from,
+     * which is only the same as the one its domain routes to if it was routed in the first place.
+     */
+    async #managerHolding(request) {
+        const key = request.id ?? request.uniqueKey;
+        if (this.#inFlightFromInner.delete(key)) {
+            return this.#inner;
+        }
+        return this.#selectManagerOrThrow(request.url);
+    }
+    async getTotalCount() {
+        return this.#sumOverManagers((manager) => manager.getTotalCount());
+    }
+    async getPendingCount() {
+        return this.#sumOverManagers((manager) => manager.getPendingCount());
+    }
+    async getHandledCount() {
+        return this.#sumOverManagers((manager) => manager.getHandledCount());
+    }
+    /**
+     * Whether the next {@link ThrottlingRequestManager.fetchNextRequest} would return `null`.
+     *
+     * Requests waiting on a throttled domain count as unavailable, so a crawler whose task loop is gated on
+     * this idles for the backoff instead of spinning on a fetch that cannot succeed yet.
+     */
+    async isEmpty() {
+        await this.#ensureSubManagers();
+        const fetchable = await Promise.all(this.#fetchableDomains().map(async (domain) => this.#subManagers.get(domain)));
+        const results = await Promise.all([this.#inner, ...fetchable].map(async (manager) => manager.isEmpty()));
+        return results.every(Boolean);
+    }
+    /** Unlike {@link ThrottlingRequestManager.isEmpty}, throttled requests still count as outstanding work. */
+    async isFinished() {
+        if (this.#inProgressBatchCount > 0) {
+            return false;
+        }
+        return this.#everyManager((manager) => manager.isFinished());
+    }
+    /**
+     * Empties every manager and clears the accumulated backoff. A robots.txt `Crawl-delay` is a property of the
+     * site rather than of the run, so it survives.
+     */
+    async purge() {
+        await this.#inner.purge?.();
+        await this.purgeDomainQueues();
+    }
+    /**
+     * Empties the per-domain queues, leaving the wrapped manager alone.
+     *
+     * Those queues are this manager's own no matter who owns the one it wraps, which is what makes this safe to
+     * call where a full {@link ThrottlingRequestManager.purge|`purge()`} would not be.
+     */
+    async purgeDomainQueues() {
+        const subManagers = await this.#getSubManagers();
+        await Promise.all(subManagers.map(async (manager) => manager.purge?.()));
+        for (const state of this.domainStates.values()) {
+            state.consecutive429Count = 0;
+            state.backoffUntil = 0;
+            state.crawlDelayUntil = 0;
+            state.backoffDecaysAt = 0;
+            state.rateLimitedSince = 0;
+            state.lastRateLimitedAt = 0;
+        }
+    }
+    async setExpectedRequestProcessingTimeSecs(secs) {
+        await this.#forEachManager((manager) => manager.setExpectedRequestProcessingTimeSecs?.(secs));
+    }
+    async #forEachManager(fn) {
+        // `fn` targets optional members, so it may return nothing - the wrapper normalizes that for `Promise.all`.
+        await Promise.all([this.#inner, ...(await this.#getSubManagers())].map(async (manager) => fn(manager)));
+    }
+    async #sumOverManagers(fn) {
+        const counts = await Promise.all([this.#inner, ...(await this.#getSubManagers())].map(fn));
+        return counts.reduce((a, b) => a + b, 0);
+    }
+    async #everyManager(fn) {
+        const results = await Promise.all([this.#inner, ...(await this.#getSubManagers())].map(fn));
+        return results.every(Boolean);
+    }
+    /**
+     * Returns the next request from a domain that is not backing off, or from the inner manager.
+     *
+     * Returns `null` while every remaining request belongs to a throttled domain - it never waits the backoff
+     * out, because a consumer parked in here holds a concurrency slot, which the autoscaler reads as spare
+     * capacity and answers by scaling up. Callers poll instead, and {@link ThrottlingRequestManager.isEmpty}
+     * reports `true` meanwhile so the crawler's task loop idles rather than spins.
+     */
+    async fetchNextRequest() {
+        await this.#ensureSubManagers();
+        for (const domain of this.#fetchableDomains()) {
+            const state = this.domainStates.get(domain);
+            // Armed while the fetch below is still suspended, so that a concurrent `fetchNextRequest` cannot
+            // find the domain fetchable and dispatch into the same window - which would pace each task
+            // rather than the domain.
+            const crawlDelayUntilBefore = state.crawlDelayUntil;
+            const delayMs = crawlDelayMs(state, this.#minCrawlDelayMs);
+            if (delayMs > 0) {
+                state.crawlDelayUntil = Date.now() + delayMs;
+            }
+            const request = await (await this.#subManagers.get(domain)).fetchNextRequest();
+            if (request) {
+                return request;
+            }
+            // No dispatch to pace, so the domain keeps its slot.
+            state.crawlDelayUntil = crawlDelayUntilBefore;
+        }
+        const request = await this.#inner.fetchNextRequest();
+        if (request !== null) {
+            this.#inFlightFromInner.add(request.id ?? request.uniqueKey);
+        }
+        if (request !== null && this.#throttlesEveryDomain) {
+            // Requests that were never routed by domain - a `RequestList`, a `requestsFromUrl` expansion - are
+            // handed out as fast as the crawler asks for them, because there is no per-domain queue to hold
+            // them back in.
+            this.#warnOnce('innerNotThrottled', `Requests read directly from the wrapped request manager (for instance from a \`RequestList\` or a ` +
+                `\`requestsFromUrl\` list) are not throttled, because they are not stored per domain. Enqueue ` +
+                `them through the crawler - \`crawler.run(requests)\` or \`crawler.addRequests()\` - to have ` +
+                `their domains paced.`);
+        }
+        return request;
+    }
+    async *[Symbol.asyncIterator]() {
+        while (true) {
+            const req = await this.fetchNextRequest();
+            if (!req)
+                break;
+            yield req;
+        }
+    }
+    async persistState() {
+        await this.#forEachManager((manager) => manager.persistState?.());
+    }
+    async drop() {
+        await this.#forEachManager((manager) => manager.drop?.());
+        this.#subManagers.clear();
+        this.#subManagersReady = undefined;
+    }
+}