@crawlee/core 4.0.0-beta.146 → 4.0.0-beta.147

package/storages/throttling_request_manager.js

@@ -1,16 +1,16 @@
 import { URL } from 'node:url';
 import { getDomain } from 'tldts';
 import { z } from 'zod';
-import { PersistentRateLimitError } from '../errors.js';
 import { asyncifyIterable } from '../iterables.js';
 import { serviceLocator } from '../service_locator.js';
 import { normalizeHostname } from '../url.js';
 import { parseArgument, schemas } from '../validators.js';
 import { drainRequestBatches } from './batched_adds.js';
 import { KeyValueStore } from './key_value_store.js';
+import { joinRequestSourceStatuses } from './request_loader.js';
 import { RequestQueue } from './request_queue.js';
 const throttlingRequestManagerOptionsSchema = z.strictObject({
-    inner: schemas.anyObject,
+    inner: z.union([schemas.anyObject, schemas.anyFunction]).optional(),
     domains: z.union([schemas.arrayOf(z.string().nonempty(), 'non-empty strings'), z.literal('all')]),
     requestManagerOpener: schemas.anyFunction.optional(),
     baseDelaySecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
@@ -23,13 +23,6 @@ const throttlingRequestManagerOptionsSchema = z.strictObject({
     maxThrottledDomains: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
     persistStateKey: z.string().nonempty().optional(),
 });
-/** Whether `manager` can pace requests per domain. */
-export function supportsDomainThrottling(manager) {
-    const candidate = manager;
-    return (typeof candidate?.recordDomainDelay === 'function' &&
-        typeof candidate.setCrawlDelay === 'function' &&
-        typeof candidate.assertNoStalledDomains === 'function');
-}
 /** The moment a domain may be dispatched to again - whichever of its two independent clocks runs longer. */
 function throttledUntil(state) {
     return Math.max(state.backoffUntil, state.crawlDelayUntil);
@@ -59,8 +52,9 @@ const DEFAULT_PERSIST_STATE_KEY = 'CRAWLEE_THROTTLED_DOMAINS';
  *
  * {@link ThrottlingRequestManager.fetchNextRequest|`fetchNextRequest()`} serves the domain that has been waiting
  * longest and skips any that are backing off, falling back to the wrapped manager. It never blocks: while every
- * remaining request belongs to a throttled domain it returns `null` and {@link ThrottlingRequestManager.isEmpty}
- * reports `true`, so the crawler idles instead of holding a concurrency slot open.
+ * remaining request belongs to a throttled domain it returns `null` and
+ * {@link ThrottlingRequestManager.checkReadiness|`checkReadiness()`} reports `waiting` with the moment the
+ * earliest of them comes due, so the crawler idles instead of holding a concurrency slot open.
  *
  * Each throttled domain runs two independent clocks, and may be dispatched to once **both** have run out:
  * - **Backoff**, set by HTTP 429 responses - honouring `Retry-After`, and otherwise doubling from `baseDelaySecs`.
@@ -75,12 +69,19 @@ const DEFAULT_PERSIST_STATE_KEY = 'CRAWLEE_THROTTLED_DOMAINS';
  * Which domains get those clocks is {@link ThrottlingRequestManagerOptions.domains|`domains`} - a list, or
  * `'all'` for every domain the crawl encounters.
  *
+ * Pass one as a crawler's `requestManager`; the `sameDomainDelaySecs` shorthand builds one with `domains: 'all'`
+ * and `throttleBy: 'registrableDomain'`. Construct it yourself to name the domains or tune the delays - one
+ * covering every domain also makes `sameDomainDelaySecs` land on it as a floor rather than adding a second pacer.
+ *
+ * Signals - 429s, robots.txt `Crawl-delay`, that floor - arrive through
+ * {@link IRequestManager.recordPacingSignal|`recordPacingSignal`}, which wrapping managers forward, so this
+ * works wherever it sits in a composition, including inside a {@link RequestManagerTandem}.
+ *
  * **Example usage:**
  *
  * ```ts
  * const crawler = new CheerioCrawler({
  *     requestManager: new ThrottlingRequestManager({
- *         inner: await RequestQueue.open(),
  *         domains: ['api.example.com', 'slow-site.org'],
  *     }),
  *     requestHandler: async ({ request }) => { ... },
@@ -91,7 +92,9 @@ const DEFAULT_PERSIST_STATE_KEY = 'CRAWLEE_THROTTLED_DOMAINS';
  */
 export class ThrottlingRequestManager {
     config;
-    #inner;
+    #innerFactory;
+    #innerPromise;
+    #resolvedInner;
     #requestManagerOpener;
     #baseDelayMs;
     #maxDelayMs;
@@ -128,8 +131,10 @@ export class ThrottlingRequestManager {
      * restart sees an empty map, reports the crawl finished, and strands whatever the previous run left in them.
      */
     #subManagersReady;
-    /** Batches still being added in the background; keeps {@link ThrottlingRequestManager.isFinished} honest. */
+    /** Batches still being added in the background; keeps {@link ThrottlingRequestManager.checkReadiness} honest. */
     #inProgressBatchCount = 0;
+    /** The latest {@link setExpectedRequestProcessingTimeSecs} hint, kept for a wrapped manager resolved later. */
+    #expectedRequestProcessingSecs;
     #warnedAbout = new Set();
     /** Whether any domain at all may end up throttled - listed up front, or discovered as the crawl runs. */
     get #throttlingEnabled() {
@@ -138,7 +143,19 @@ export class ThrottlingRequestManager {
     constructor(options, config = serviceLocator.getConfiguration()) {
         this.config = config;
         parseArgument(options, throttlingRequestManagerOptionsSchema, 'ThrottlingRequestManagerOptions');
-        this.#inner = options.inner;
+        if (options.inner === undefined) {
+            // Deferred like any other factory, so a manager nobody fetches from opens no storage. The opener is
+            // assigned below and read only when this runs.
+            this.#innerFactory = () => this.#requestManagerOpener(null, { configuration: this.config });
+        }
+        else if (typeof options.inner === 'function') {
+            this.#innerFactory = options.inner;
+        }
+        else {
+            // Nothing to open: resolved from the start, so `innerManager` and bookkeeping see it immediately.
+            this.#resolvedInner = options.inner;
+            this.#innerFactory = () => this.#resolvedInner;
+        }
         this.#requestManagerOpener =
             options.requestManagerOpener ??
                 ((idOrAlias, opts) => RequestQueue.open(idOrAlias, opts));
@@ -180,9 +197,30 @@ export class ThrottlingRequestManager {
         // so those stay paced per hostname.
         return getDomain(normalized, { mixedInputs: false }) ?? normalized;
     }
-    /** The wrapped manager, holding every request whose domain is not throttled. */
+    /**
+     * The wrapped manager, holding every request whose domain is not throttled. `undefined` until an `inner`
+     * passed as a factory is resolved - reading this never forces it, because a getter should not open a queue
+     * behind a caller's back.
+     */
     get innerManager() {
-        return this.#inner;
+        return this.#resolvedInner;
+    }
+    /**
+     * Resolves the wrapped manager, opening a factory `inner` on first use and memoizing it.
+     *
+     * Memoized for identity as much as for cost: {@link addRequestsBatched} groups by manager identity and
+     * {@link reclaimRequest} compares against it, so a fresh instance per call would split batches and reclaim
+     * requests into a manager that never handed them out.
+     */
+    async #getInner() {
+        if (this.#resolvedInner === undefined) {
+            this.#innerPromise ??= Promise.resolve(this.#innerFactory());
+            this.#resolvedInner = await this.#innerPromise;
+            if (this.#expectedRequestProcessingSecs !== undefined) {
+                await this.#resolvedInner.setExpectedRequestProcessingTimeSecs?.(this.#expectedRequestProcessingSecs);
+            }
+        }
+        return this.#resolvedInner;
     }
     /** Warns once about sources that cannot be routed by domain, because their URLs are not known yet. */
     #warnIfNotRoutable(requestLike) {
@@ -222,7 +260,7 @@ export class ThrottlingRequestManager {
         await this.#ensureSubManagers();
         const domain = this.#extractDomain(url);
         if (!domain || !(this.#listedDomains.has(domain) || this.#throttlesEveryDomain)) {
-            return this.#inner;
+            return this.#getInner();
         }
         if (!this.#listedDomains.has(domain) && !this.#discoveredDomains.has(domain)) {
             if (this.#discoveredDomains.size >= this.#maxThrottledDomains) {
@@ -319,11 +357,77 @@ export class ThrottlingRequestManager {
             .map((state) => state.domain);
     }
     /**
-     * Records a 429 response and puts the URL's domain into backoff.
+     * Records a pacing signal: a refusal puts the URL's domain into backoff, a declared interval becomes its
+     * crawl delay, and a crawl-wide floor raises {@link recordEverywhereFloor|the floor under all of them}.
      *
-     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     * @returns `false` if the domain the signal covers is not throttled, in which case this is a no-op.
+     * @throws If the signal's scope is one this manager cannot honour - see {@link assertScopeHonourable}.
+     * @inheritdoc
      */
-    recordDomainDelay(url, retryAfterMs) {
+    recordPacingSignal(signal) {
+        if (signal.reason === 'minIntervalEverywhere') {
+            return this.#recordEverywhereFloor(signal.intervalMs, signal.scope);
+        }
+        this.#assertScopeHonourable(signal.scope);
+        return signal.reason === 'rateLimited'
+            ? this.#recordRateLimit(signal.url, signal.waitMs)
+            : this.#recordDeclaredInterval(signal.url, signal.intervalMs);
+    }
+    /**
+     * The runtime form of {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`}: raises
+     * the floor under every throttled domain's crawl delay.
+     *
+     * @returns `false` if this manager paces nothing at all, so there is no floor to raise.
+     * @throws If it paces some domains but not all - holding back only those would leave the rest of what the
+     *  floor covers unpaced.
+     */
+    #recordEverywhereFloor(intervalMs, scope) {
+        // Before the scope check: a manager that paces nothing has no grouping worth objecting about, and
+        // `false` lets the caller pace it from outside.
+        if (!this.#throttlingEnabled) {
+            return false;
+        }
+        this.#assertScopeHonourable(scope);
+        if (!this.#throttlesEveryDomain) {
+            throw new Error(`Cannot honour a crawl-delay floor covering every domain: this manager only paces the domains ` +
+                `it was given (${Array.from(this.#listedDomains).join(', ')}), so everything else would run ` +
+                `unpaced. Set \`domains: 'all'\` to pace whatever the crawl encounters, or declare the floor ` +
+                `on this manager yourself via \`minCrawlDelaySecs\`.`);
+        }
+        this.#minCrawlDelayMs = Math.max(this.#minCrawlDelayMs, intervalMs);
+        this.log.debug(`Crawl-delay floor for every domain set to ${(this.#minCrawlDelayMs / 1000).toFixed(1)}s`);
+        return true;
+    }
+    /**
+     * Throws unless a signal scoped to `scope` can be honoured as declared or wider.
+     *
+     * Holding a domain back means holding its queue back, so
+     * {@link ThrottlingRequestManagerOptions.throttleBy|`throttleBy`} is the finest granularity this manager
+     * can express: a narrower scope is honoured by pacing the whole group, while a wider or unknown one must
+     * throw rather than under-apply, because pacing one group would leave the rest of what it covers unpaced.
+     */
+    #assertScopeHonourable(scope) {
+        // No scope means the reporter cannot tell how far the signal reaches, so our own grouping is as good
+        // an answer as there is.
+        if (scope === undefined || scope === this.#throttleBy) {
+            return;
+        }
+        if (scope === 'hostname' && this.#throttleBy === 'registrableDomain') {
+            // Only debug: grouping by registrable domain deliberately paces whole sites, subdomains included.
+            this.log.debug(`Applying a pacing signal scoped to "hostname" across the whole registrable domain, because that ` +
+                `is how this manager groups requests (\`throttleBy\`). Sibling subdomains are paced with it.`);
+            return;
+        }
+        throw new Error(`Cannot honour a pacing signal scoped to "${scope}": this manager groups requests by ` +
+            `"${this.#throttleBy}" and holds one request queue per group, so that is the widest scope it can ` +
+            `hold back at once. Applying the signal anyway would leave part of what it covers unpaced. ` +
+            (scope === 'registrableDomain'
+                ? 'Set `throttleBy: "registrableDomain"` to group requests that way.'
+                : 'This manager only understands the scopes "hostname" and "registrableDomain"; pace by ' +
+                    `"${scope}" with a request manager that groups requests by it.`));
+    }
+    /** Puts the URL's domain into backoff after a refusal, doubling from `baseDelaySecs` if it keeps refusing. */
+    #recordRateLimit(url, waitMs) {
         const state = this.#getDomainState(url);
         if (!state) {
             return false;
@@ -348,10 +452,10 @@ export class ThrottlingRequestManager {
             state.consecutive429Count = 0;
         }
         state.consecutive429Count += 1;
-        const retryAfterGiven = retryAfterMs !== undefined && retryAfterMs !== null;
-        let delayMs = retryAfterGiven ? retryAfterMs : this.#baseDelayMs * Math.pow(2, state.consecutive429Count - 1);
+        const waitGiven = waitMs !== undefined;
+        let delayMs = waitGiven ? waitMs : this.#baseDelayMs * Math.pow(2, state.consecutive429Count - 1);
         if (delayMs > this.#maxDelayMs) {
-            const source = retryAfterGiven ? 'Retry-After header' : 'exponential backoff';
+            const source = waitGiven ? 'requested wait' : 'exponential backoff';
             this.log.warning(`Capping ${source} delay of ${(delayMs / 1000).toFixed(1)}s for domain "${state.domain}" ` +
                 `to maxDelaySecs (${(this.#maxDelayMs / 1000).toFixed(1)}s); the domain may continue to rate-limit. ` +
                 `Consider increasing maxDelaySecs if this recurs.`);
@@ -364,14 +468,14 @@ export class ThrottlingRequestManager {
         return true;
     }
     /**
-     * Records the `Crawl-delay` a domain's robots.txt asked for, which becomes its crawl delay unless
+     * Records a declared minimum interval for a domain, which becomes its crawl delay unless
      * {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`} asks for longer.
      *
      * The first value wins, so a robots.txt re-fetch cannot change the cadence mid-crawl.
      *
      * @returns `false` if the domain is not throttled, in which case this is a no-op.
      */
-    setCrawlDelay(url, delaySeconds) {
+    #recordDeclaredInterval(url, intervalMs) {
         const domain = this.#extractDomain(url);
         if (!domain || !(this.#listedDomains.has(domain) || this.#throttlesEveryDomain)) {
             return false;
@@ -380,45 +484,11 @@ export class ThrottlingRequestManager {
         // the sub-queue it will end up pacing.
         const state = this.#ensureDomainState(domain);
         if (state.declaredCrawlDelayMs === null) {
-            state.declaredCrawlDelayMs = delaySeconds * 1000;
-            this.log.debug(`Set crawl-delay for domain "${state.domain}" to ${delaySeconds}s`);
+            state.declaredCrawlDelayMs = intervalMs;
+            this.log.debug(`Set crawl-delay for domain "${state.domain}" to ${(intervalMs / 1000).toFixed(1)}s`);
         }
         return true;
     }
-    /**
-     * Throws {@link PersistentRateLimitError} if any domain has been rate-limiting us past
-     * {@link ThrottlingRequestManagerOptions.maxDomainStallSecs|`maxDomainStallSecs`} without letting a single
-     * request through.
-     *
-     * A domain qualifies only while it still has queued requests and is actively rate-limiting - a domain that
-     * has simply run out of work is finished, not stalled, and one being waited out under a long robots.txt
-     * `Crawl-delay` is being obeyed, not stonewalled.
-     */
-    async assertNoStalledDomains() {
-        await this.#ensureSubManagers();
-        const now = Date.now();
-        const candidates = Array.from(this.domainStates.values()).filter(
-        // Together: it is still turning us away, and has been doing so without a break for longer than the
-        // window. A domain that has simply been idle starts this clock at its first 429 rather than
-        // arriving with the idle time already on it.
-        (state) => state.rateLimitedSince !== 0 &&
-            now - state.lastRateLimitedAt <= this.#maxDomainStallMs &&
-            now - state.rateLimitedSince > this.#maxDomainStallMs);
-        const stalled = (await Promise.all(candidates.map(async (state) => {
-            const subManager = await this.#subManagers.get(state.domain);
-            return subManager && !(await subManager.isEmpty()) ? state : null;
-        }))).filter((state) => state !== null);
-        if (stalled.length === 0) {
-            return;
-        }
-        const summary = stalled
-            .map((state) => `"${state.domain}" (${((now - state.rateLimitedSince) / 1000).toFixed(0)}s)`)
-            .join(', ');
-        throw new PersistentRateLimitError(`Giving up: ${summary} rate-limited every request for longer than maxDomainStallSecs ` +
-            `(${(this.#maxDomainStallMs / 1000).toFixed(0)}s). Waiting longer will not help - lower the ` +
-            `crawler's concurrency, or drop these domains. Their requests are still queued, so re-running ` +
-            `with \`purgeOnStart\` disabled will resume them if the rate limit lifts.`);
-    }
     /** Records that a domain let a request through, which ends any rate-limit run stall detection was timing. */
     #recordProgress(url) {
         const state = this.#getDomainState(url);
@@ -511,8 +581,9 @@ export class ThrottlingRequestManager {
      */
     async #managerHolding(request) {
         const key = request.id ?? request.uniqueKey;
+        // Only a fetch from the wrapped manager puts a key here, so it is necessarily resolved already.
         if (this.#inFlightFromInner.delete(key)) {
-            return this.#inner;
+            return this.#resolvedInner;
         }
         return this.#selectManagerOrThrow(request.url);
     }
@@ -526,39 +597,94 @@ export class ThrottlingRequestManager {
         return this.#sumOverManagers((manager) => manager.getHandledCount());
     }
     /**
-     * Whether the next {@link ThrottlingRequestManager.fetchNextRequest} would return `null`.
+     * Reports whether anything can be dispatched right now, and if not, when — or why never.
      *
-     * Requests waiting on a throttled domain count as unavailable, so a crawler whose task loop is gated on
-     * this idles for the backoff instead of spinning on a fetch that cannot succeed yet.
+     * One traversal of the domain clocks answers all of it: only domains whose delays have run out are probed,
+     * the rest merely contribute the moment they come due. Throttled requests count as outstanding work, so a
+     * crawler gated on this idles for the backoff instead of concluding it is done.
+     *
+     * `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,
+     * so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -
+     * see the traversal.
      */
-    async isEmpty() {
+    async checkReadiness() {
         await this.#ensureSubManagers();
-        const fetchable = await Promise.all(this.#fetchableDomains().map(async (domain) => this.#subManagers.get(domain)));
-        const results = await Promise.all([this.#inner, ...fetchable].map(async (manager) => manager.isEmpty()));
-        return results.every(Boolean);
-    }
-    /** Unlike {@link ThrottlingRequestManager.isEmpty}, throttled requests still count as outstanding work. */
-    async isFinished() {
-        if (this.#inProgressBatchCount > 0) {
-            return false;
+        const now = Date.now();
+        const dispatchable = [];
+        let readyAt;
+        let stallCandidates;
+        for (const state of this.domainStates.values()) {
+            // A `Crawl-delay` can give a domain a clock before its first request gives it a queue - nothing
+            // to fetch from and nothing to wait for until then.
+            if (!this.#subManagers.has(state.domain)) {
+                continue;
+            }
+            if (
+            // Together: still turning us away, and has been without a break for longer than the window - a
+            // domain that has merely been idle starts this clock at its first 429, not with idle time on it.
+            state.rateLimitedSince !== 0 &&
+                now - state.lastRateLimitedAt <= this.#maxDomainStallMs &&
+                now - state.rateLimitedSince > this.#maxDomainStallMs) {
+                // Deliberately not dispatchable: a domain that has refused every request for the whole window
+                // is not progress just because its backoff momentarily lapsed - that is what it does between 429s.
+                (stallCandidates ??= []).push(state);
+                continue;
+            }
+            const until = throttledUntil(state);
+            if (now >= until) {
+                dispatchable.push(this.#subManagers.get(state.domain));
+            }
+            else if (readyAt === undefined || until < readyAt) {
+                readyAt = until;
+            }
+        }
+        const probed = (await Promise.all([
+            (await this.#getInner()).checkReadiness(),
+            ...dispatchable.map(async (subManager) => (await subManager).checkReadiness()),
+        ])).reduce(joinRequestSourceStatuses);
+        // Anything dispatchable outranks a stalled or throttled domain, so we stop here without touching them.
+        if (probed.status === 'ready') {
+            return probed;
+        }
+        if (stallCandidates !== undefined) {
+            const stalled = (await Promise.all(stallCandidates.map(async (state) => (await (await this.#subManagers.get(state.domain)).checkReadiness()).status === 'ready'
+                ? state
+                : null))).filter((state) => state !== null);
+            if (stalled.length > 0) {
+                const summary = stalled
+                    .map((state) => `"${state.domain}" (${((now - state.rateLimitedSince) / 1000).toFixed(0)}s)`)
+                    .join(', ');
+                return {
+                    status: 'stalled',
+                    reason: `${summary} rate-limited every request for longer than maxDomainStallSecs ` +
+                        `(${(this.#maxDomainStallMs / 1000).toFixed(0)}s). Waiting longer will not help - lower the ` +
+                        `crawler's concurrency, or drop these domains. Their requests are still queued, so ` +
+                        `re-running with \`purgeOnStart\` disabled will resume them if the rate limit lifts.`,
+                };
+            }
         }
-        return this.#everyManager((manager) => manager.isFinished());
+        if (readyAt !== undefined) {
+            return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });
+        }
+        // Batches still landing in the background are work nobody can see in a queue yet.
+        if (this.#inProgressBatchCount > 0 && probed.status === 'finished') {
+            return { status: 'waiting' };
+        }
+        return probed;
     }
     /**
      * Empties every manager and clears the accumulated backoff. A robots.txt `Crawl-delay` is a property of the
      * site rather than of the run, so it survives.
      */
     async purge() {
-        await this.#inner.purge?.();
-        await this.purgeDomainQueues();
+        await this.#purgeDomainQueues();
+        await this.#resolvedInner?.purge?.();
     }
     /**
-     * Empties the per-domain queues, leaving the wrapped manager alone.
-     *
-     * Those queues are this manager's own no matter who owns the one it wraps, which is what makes this safe to
-     * call where a full {@link ThrottlingRequestManager.purge|`purge()`} would not be.
+     * Empties the per-domain queues, leaving the wrapped manager alone - those queues are this manager's own no
+     * matter who owns the one it wraps.
      */
-    async purgeDomainQueues() {
+    async #purgeDomainQueues() {
         const subManagers = await this.#getSubManagers();
         await Promise.all(subManagers.map(async (manager) => manager.purge?.()));
         for (const state of this.domainStates.values()) {
@@ -571,27 +697,35 @@ export class ThrottlingRequestManager {
         }
     }
     async setExpectedRequestProcessingTimeSecs(secs) {
+        // Remembered so a manager opened later still gets the hint, without opening one now just to pass it along.
+        this.#expectedRequestProcessingSecs = secs;
         await this.#forEachManager((manager) => manager.setExpectedRequestProcessingTimeSecs?.(secs));
     }
+    /**
+     * Runs `fn` over the sub-queues and, if it has been resolved, the wrapped manager - bookkeeping never forces
+     * a lazily-opened `inner`, since there is no point opening a queue purely to tell it something.
+     */
     async #forEachManager(fn) {
+        const managers = await this.#getSubManagers();
+        if (this.#resolvedInner !== undefined) {
+            managers.push(this.#resolvedInner);
+        }
         // `fn` targets optional members, so it may return nothing - the wrapper normalizes that for `Promise.all`.
-        await Promise.all([this.#inner, ...(await this.#getSubManagers())].map(async (manager) => fn(manager)));
+        await Promise.all(managers.map(async (manager) => fn(manager)));
     }
     async #sumOverManagers(fn) {
-        const counts = await Promise.all([this.#inner, ...(await this.#getSubManagers())].map(fn));
+        // Counts have to include the wrapped manager, so this one does resolve it.
+        const counts = await Promise.all([await this.#getInner(), ...(await this.#getSubManagers())].map(fn));
         return counts.reduce((a, b) => a + b, 0);
     }
-    async #everyManager(fn) {
-        const results = await Promise.all([this.#inner, ...(await this.#getSubManagers())].map(fn));
-        return results.every(Boolean);
-    }
     /**
      * Returns the next request from a domain that is not backing off, or from the inner manager.
      *
      * Returns `null` while every remaining request belongs to a throttled domain - it never waits the backoff
      * out, because a consumer parked in here holds a concurrency slot, which the autoscaler reads as spare
-     * capacity and answers by scaling up. Callers poll instead, and {@link ThrottlingRequestManager.isEmpty}
-     * reports `true` meanwhile so the crawler's task loop idles rather than spins.
+     * capacity and answers by scaling up. Callers poll instead, and
+     * {@link ThrottlingRequestManager.checkReadiness|`checkReadiness()`} reports `waiting` meanwhile so the
+     * crawler's task loop idles rather than spins.
      */
     async fetchNextRequest() {
         await this.#ensureSubManagers();
@@ -612,7 +746,7 @@ export class ThrottlingRequestManager {
             // No dispatch to pace, so the domain keeps its slot.
             state.crawlDelayUntil = crawlDelayUntilBefore;
         }
-        const request = await this.#inner.fetchNextRequest();
+        const request = await (await this.#getInner()).fetchNextRequest();
         if (request !== null) {
             this.#inFlightFromInner.add(request.id ?? request.uniqueKey);
         }