@crawlee/core 4.0.0-beta.123 → 4.0.0-beta.124

package/events/event_manager.js

@@ -18,7 +18,8 @@ export class EventManager {
     #persistStateIntervalMillis;
     constructor(options) {
         this.#persistStateIntervalMillis = options.persistStateIntervalMillis;
-        this.events.setMaxListeners(50);
+        // One MIGRATING listener per RequestQueue, and ThrottlingRequestManager opens one per domain.
+        this.events.setMaxListeners(150);
     }
     /**
      * Initializes the event manager by starting the `persistState` event interval.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.123",
+    "version": "4.0.0-beta.124",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,10 +52,10 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.123",
-        "@crawlee/http-client": "4.0.0-beta.123",
-        "@crawlee/types": "4.0.0-beta.123",
-        "@crawlee/utils": "4.0.0-beta.123",
+        "@crawlee/fs-storage": "4.0.0-beta.124",
+        "@crawlee/http-client": "4.0.0-beta.124",
+        "@crawlee/types": "4.0.0-beta.124",
+        "@crawlee/utils": "4.0.0-beta.124",
         "@sapphire/async-queue": "^1.5.5",
         "@vladfrangu/async_event_emitter": "^2.4.6",
         "content-type": "^1.0.5",
@@ -77,5 +77,5 @@
             }
         }
     },
-    "gitHead": "f77648095c6a3f5ed8815c7620ea765db430ae44"
+    "gitHead": "0694ee1b94c755b98141671baa93cc363f2bf8e3"
 }

package/storages/throttling_request_manager.d.ts

@@ -37,15 +37,58 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
      */
     inner: T;
     /**
-     * Hostnames to throttle. Matching is case-insensitive and exact - wildcards such as `*.example.com` are not
-     * supported, so list each subdomain you care about. Requests for any other domain bypass throttling entirely.
+     * Which domains to throttle: a list of hostnames, or `'all'` for every domain the crawl encounters.
      *
-     * An internationalized domain may be given in either its unicode or its punycode form, and an IPv6 address
-     * has to be bracketed (`[::1]`).
+     * Matching a listed hostname is case-insensitive and exact - wildcards such as `*.example.com` are not
+     * supported, so list each subdomain you care about (or set
+     * {@link ThrottlingRequestManagerOptions.throttleBy|`throttleBy: 'registrableDomain'`}). An
+     * internationalized domain may be given in either its unicode or its punycode form, and an IPv6 address has
+     * to be bracketed (`[::1]`). Requests for any other domain bypass throttling entirely.
+     *
+     * `'all'` gives each domain a queue of its own the first time it is seen, so that it can be held back
+     * without its requests being repeatedly popped and re-enqueued. One request queue per domain is not free,
+     * which is what {@link ThrottlingRequestManagerOptions.maxThrottledDomains|`maxThrottledDomains`} is
+     * there to bound.
+     */
+    domains: string[] | 'all';
+    /**
+     * A floor under the crawl delay of every throttled domain, in seconds - the proactive clock described on
+     * {@link ThrottlingRequestManager}. A domain whose robots.txt asks for a longer `Crawl-delay` gets the
+     * longer one; this is a minimum, not an override.
+     * @default 0
+     */
+    minCrawlDelaySecs?: number;
+    /**
+     * What counts as "the same domain": the exact hostname, or the registrable domain it belongs to
+     * (`example.com` for `www.example.com`, `a.example.co.uk` and so on). Hosts with no registrable domain -
+     * IP addresses, `localhost` - are always throttled per hostname.
+     *
+     * Grouping by registrable domain gives subdomains a single pair of clocks and a single queue, which is what
+     * you want when the pacing is there to be polite to one server rather than to satisfy a specific host's
+     * rate limit.
+     * @default 'hostname'
+     */
+    throttleBy?: 'hostname' | 'registrableDomain';
+    /**
+     * The most domains a run may throttle at once. Exceeding it throws, rather than silently letting the
+     * throttling lapse - one request queue per domain is not free, and a crawl that discovers domains without
+     * bound would drown the storage backend in them.
+     *
+     * Only domains discovered under `domains: 'all'` count against this; an explicit list is taken at face value.
+     * @default 100
      */
-    domains: string[];
+    maxThrottledDomains?: number;
     /**
-     * Opens the per-domain queues, one per entry in `domains`, each under the alias `throttled-<domain>`.
+     * The key under which the discovered domain list is kept in the default key-value store, so that a restart
+     * with `purgeOnStart` disabled reopens their queues instead of stranding whatever they still hold. Only
+     * written under `domains: 'all'`.
+     *
+     * Give each manager its own key when running several of them against the same storage.
+     * @default 'CRAWLEE_THROTTLED_DOMAINS'
+     */
+    persistStateKey?: string;
+    /**
+     * Opens the per-domain queues, one per throttled domain, each under the alias `throttled-<domain>`.
      * @default RequestQueue.open
      */
     requestManagerOpener?: RequestManagerOpener<T>;
@@ -77,22 +120,26 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
 /**
  * A request manager that wraps another one and paces requests per domain.
  *
- * Requests for the configured {@link ThrottlingRequestManagerOptions.domains|`domains`} are routed into their own
- * queue when they are added, so each request lives in exactly one place and deduplication keeps working. Everything
- * else goes to the wrapped manager untouched.
+ * Requests for a throttled domain are routed into their own queue when they are added, so each request lives in
+ * exactly one place and deduplication keeps working. Everything else goes to the wrapped manager untouched.
  *
  * {@link ThrottlingRequestManager.fetchNextRequest|`fetchNextRequest()`} serves the domain that has been waiting
  * longest and skips any that are backing off, falling back to the wrapped manager. It never blocks: while every
  * remaining request belongs to a throttled domain it returns `null` and {@link ThrottlingRequestManager.isEmpty}
  * reports `true`, so the crawler idles instead of holding a concurrency slot open.
  *
- * Delays come from two places:
- * - HTTP 429 responses, honouring `Retry-After` and otherwise backing off exponentially. The crawlers report these
- *   automatically; a request that is throttled is retried later without counting against `maxRequestRetries` and
+ * Each throttled domain runs two independent clocks, and may be dispatched to once **both** have run out:
+ * - **Backoff**, set by HTTP 429 responses - honouring `Retry-After`, and otherwise doubling from `baseDelaySecs`.
+ *   Reactive and temporary: it decays once the domain stops turning us away. The crawlers report the 429s
+ *   themselves; a request held back this way is retried later without counting against `maxRequestRetries` and
  *   without penalising its session.
- * - robots.txt `Crawl-delay` directives, when `respectRobotsTxtFile` is enabled.
+ * - **Crawl delay**, the minimum interval between two dispatches to the domain, armed after each one. Proactive
+ *   and constant: whatever the domain's robots.txt asks for, floored by
+ *   {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`}. Either may be absent, in
+ *   which case the other one is the delay.
  *
- * This is opt-in: throttling only happens for a domain you list explicitly.
+ * Which domains get those clocks is {@link ThrottlingRequestManagerOptions.domains|`domains`} - a list, or
+ * `'all'` for every domain the crawl encounters.
  *
  * **Example usage:**
  *
@@ -109,40 +156,13 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
  * @category Sources
  */
 export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager, SupportsDomainThrottling {
+    #private;
     private readonly config;
-    private readonly inner;
-    private readonly requestManagerOpener;
-    private readonly baseDelayMs;
-    private readonly maxDelayMs;
-    private readonly maxDomainStallMs;
     private readonly domainStates;
-    private readonly subManagers;
     private readonly log;
-    /**
-     * Sub-managers are keyed by a stable alias, so with `purgeOnStart` disabled they outlive the process. They
-     * must therefore be reopened for every configured domain rather than created on first insert - otherwise a
-     * restart sees an empty map, reports the crawl finished, and strands whatever the previous run left in them.
-     */
-    private subManagersReady?;
-    /** Batches still being added in the background; keeps {@link ThrottlingRequestManager.isFinished} honest. */
-    private inProgressBatchCount;
-    private readonly warnedAbout;
-    private get hasThrottledDomains();
     constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);
     /** The wrapped manager, holding every request whose domain is not throttled. */
     get innerManager(): T;
-    /** Warns once about sources that cannot be routed by domain, because their URLs are not known yet. */
-    private warnIfNotRoutable;
-    private warnOnce;
-    private extractDomain;
-    private getDomainState;
-    private selectManager;
-    /** Only valid once {@link ThrottlingRequestManager.ensureSubManagers} has resolved. */
-    private managerForUrl;
-    private ensureSubManagers;
-    private getSubManagers;
-    /** Configured domains that are not currently backing off, longest-overdue first. */
-    private fetchableDomains;
     /**
      * Records a 429 response and puts the URL's domain into backoff.
      *
@@ -150,11 +170,12 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
      */
     recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
     /**
-     * Applies a robots.txt `Crawl-delay` to the URL's domain, as a minimum interval between dispatches.
+     * Records the `Crawl-delay` a domain's robots.txt asked for, which becomes its crawl delay unless
+     * {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`} asks for longer.
      *
      * The first value wins, so a robots.txt re-fetch cannot change the cadence mid-crawl.
      *
-     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     * @returns `false` if the domain is not throttled, in which case this is a no-op.
      */
     setCrawlDelay(url: string, delaySeconds: number): boolean;
     /**
@@ -167,8 +188,6 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
      * `Crawl-delay` is being obeyed, not stonewalled.
      */
     assertNoStalledDomains(): Promise<void>;
-    /** Records that a domain let a request through, which ends any rate-limit run stall detection was timing. */
-    private recordProgress;
     addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
     /**
      * Adds requests in batches, routing each one to the manager that owns its domain.
@@ -197,10 +216,14 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
      * site rather than of the run, so it survives.
      */
     purge(): Promise<void>;
+    /**
+     * Empties the per-domain queues, leaving the wrapped manager alone.
+     *
+     * Those queues are this manager's own no matter who owns the one it wraps, which is what makes this safe to
+     * call where a full {@link ThrottlingRequestManager.purge|`purge()`} would not be.
+     */
+    purgeDomainQueues(): Promise<void>;
     setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
-    private forEachManager;
-    private sumOverManagers;
-    private everyManager;
     /**
      * Returns the next request from a domain that is not backing off, or from the inner manager.
      *

package/storages/throttling_request_manager.js

@@ -1,4 +1,5 @@
 import { URL } from 'node:url';
+import { getDomain } from 'tldts';
 import { z } from 'zod';
 import { PersistentRateLimitError } from '../errors.js';
 import { asyncifyIterable } from '../iterables.js';
@@ -6,14 +7,21 @@ import { serviceLocator } from '../service_locator.js';
 import { normalizeHostname } from '../url.js';
 import { parseArgument, schemas } from '../validators.js';
 import { drainRequestBatches } from './batched_adds.js';
+import { KeyValueStore } from './key_value_store.js';
 import { RequestQueue } from './request_queue.js';
 const throttlingRequestManagerOptionsSchema = z.strictObject({
     inner: schemas.anyObject,
-    domains: schemas.arrayOf(z.string().nonempty(), 'non-empty strings'),
+    domains: z.union([schemas.arrayOf(z.string().nonempty(), 'non-empty strings'), z.literal('all')]),
     requestManagerOpener: schemas.anyFunction.optional(),
     baseDelaySecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
     maxDelaySecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
     maxDomainStallSecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
+    minCrawlDelaySecs: schemas.anyNumber
+        .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
+        .optional(),
+    throttleBy: z.enum(['hostname', 'registrableDomain']).optional(),
+    maxThrottledDomains: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
+    persistStateKey: z.string().nonempty().optional(),
 });
 /** Whether `manager` can pace requests per domain. */
 export function supportsDomainThrottling(manager) {
@@ -26,25 +34,46 @@ export function supportsDomainThrottling(manager) {
 function throttledUntil(state) {
     return Math.max(state.backoffUntil, state.crawlDelayUntil);
 }
+/** How long a domain must be left alone after a dispatch: what it asked for, or our floor, whichever is longer. */
+function crawlDelayMs(state, minCrawlDelayMs) {
+    return Math.max(state.declaredCrawlDelayMs ?? 0, minCrawlDelayMs);
+}
+function newDomainState(domain) {
+    return {
+        domain,
+        backoffUntil: 0,
+        crawlDelayUntil: 0,
+        backoffDecaysAt: 0,
+        consecutive429Count: 0,
+        declaredCrawlDelayMs: null,
+        rateLimitedSince: 0,
+        lastRateLimitedAt: 0,
+    };
+}
+const DEFAULT_PERSIST_STATE_KEY = 'CRAWLEE_THROTTLED_DOMAINS';
 /**
  * A request manager that wraps another one and paces requests per domain.
  *
- * Requests for the configured {@link ThrottlingRequestManagerOptions.domains|`domains`} are routed into their own
- * queue when they are added, so each request lives in exactly one place and deduplication keeps working. Everything
- * else goes to the wrapped manager untouched.
+ * Requests for a throttled domain are routed into their own queue when they are added, so each request lives in
+ * exactly one place and deduplication keeps working. Everything else goes to the wrapped manager untouched.
  *
  * {@link ThrottlingRequestManager.fetchNextRequest|`fetchNextRequest()`} serves the domain that has been waiting
  * longest and skips any that are backing off, falling back to the wrapped manager. It never blocks: while every
  * remaining request belongs to a throttled domain it returns `null` and {@link ThrottlingRequestManager.isEmpty}
  * reports `true`, so the crawler idles instead of holding a concurrency slot open.
  *
- * Delays come from two places:
- * - HTTP 429 responses, honouring `Retry-After` and otherwise backing off exponentially. The crawlers report these
- *   automatically; a request that is throttled is retried later without counting against `maxRequestRetries` and
+ * Each throttled domain runs two independent clocks, and may be dispatched to once **both** have run out:
+ * - **Backoff**, set by HTTP 429 responses - honouring `Retry-After`, and otherwise doubling from `baseDelaySecs`.
+ *   Reactive and temporary: it decays once the domain stops turning us away. The crawlers report the 429s
+ *   themselves; a request held back this way is retried later without counting against `maxRequestRetries` and
  *   without penalising its session.
- * - robots.txt `Crawl-delay` directives, when `respectRobotsTxtFile` is enabled.
+ * - **Crawl delay**, the minimum interval between two dispatches to the domain, armed after each one. Proactive
+ *   and constant: whatever the domain's robots.txt asks for, floored by
+ *   {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`}. Either may be absent, in
+ *   which case the other one is the delay.
  *
- * This is opt-in: throttling only happens for a domain you list explicitly.
+ * Which domains get those clocks is {@link ThrottlingRequestManagerOptions.domains|`domains`} - a list, or
+ * `'all'` for every domain the crawl encounters.
  *
  * **Example usage:**
  *
@@ -62,121 +91,230 @@ function throttledUntil(state) {
  */
 export class ThrottlingRequestManager {
     config;
-    inner;
-    requestManagerOpener;
-    baseDelayMs;
-    maxDelayMs;
-    maxDomainStallMs;
+    #inner;
+    #requestManagerOpener;
+    #baseDelayMs;
+    #maxDelayMs;
+    #maxDomainStallMs;
+    #minCrawlDelayMs;
+    #throttlesEveryDomain;
+    #throttleBy;
+    #maxThrottledDomains;
+    #persistStateKey;
+    #subManagers = new Map();
+    // Not `#private`, unlike the rest: the tests reach for these two.
     domainStates = new Map();
-    subManagers = new Map();
     log;
+    /** Domains from the `domains` option, which are throttled whether or not the crawl ever visits them. */
+    #listedDomains = new Set();
+    /**
+     * Domains picked up at runtime under `domains: 'all'`. Persisted, because unlike the listed ones there
+     * is nothing to rediscover them from at startup - and a sub-queue nobody reopens is a sub-queue whose
+     * requests are never crawled.
+     */
+    #discoveredDomains = new Set();
+    /**
+     * Requests currently held by the consumer that came out of the wrapped manager rather than a sub-queue.
+     * They have to go back where they came from: routing them by domain would mark them handled in a queue
+     * that has never heard of them, leaving the wrapped manager to hand them out over and over.
+     */
+    #inFlightFromInner = new Set();
+    #domainListStore;
+    #lastDomainListWrite = Promise.resolve();
+    #queuedDomainListWrite;
     /**
      * Sub-managers are keyed by a stable alias, so with `purgeOnStart` disabled they outlive the process. They
-     * must therefore be reopened for every configured domain rather than created on first insert - otherwise a
+     * must therefore be reopened for every known domain rather than created on first insert - otherwise a
      * restart sees an empty map, reports the crawl finished, and strands whatever the previous run left in them.
      */
-    subManagersReady;
+    #subManagersReady;
     /** Batches still being added in the background; keeps {@link ThrottlingRequestManager.isFinished} honest. */
-    inProgressBatchCount = 0;
-    warnedAbout = new Set();
-    get hasThrottledDomains() {
-        return this.domainStates.size > 0;
+    #inProgressBatchCount = 0;
+    #warnedAbout = new Set();
+    /** Whether any domain at all may end up throttled - listed up front, or discovered as the crawl runs. */
+    get #throttlingEnabled() {
+        return this.#listedDomains.size > 0 || this.#throttlesEveryDomain;
     }
     constructor(options, config = serviceLocator.getConfiguration()) {
         this.config = config;
         parseArgument(options, throttlingRequestManagerOptionsSchema, 'ThrottlingRequestManagerOptions');
-        this.inner = options.inner;
-        this.requestManagerOpener =
+        this.#inner = options.inner;
+        this.#requestManagerOpener =
             options.requestManagerOpener ??
                 ((idOrAlias, opts) => RequestQueue.open(idOrAlias, opts));
-        this.baseDelayMs = (options.baseDelaySecs ?? 2) * 1000;
-        this.maxDelayMs = (options.maxDelaySecs ?? 60) * 1000;
-        this.maxDomainStallMs = (options.maxDomainStallSecs ?? 900) * 1000;
+        this.#baseDelayMs = (options.baseDelaySecs ?? 2) * 1000;
+        this.#maxDelayMs = (options.maxDelaySecs ?? 60) * 1000;
+        this.#maxDomainStallMs = (options.maxDomainStallSecs ?? 900) * 1000;
+        this.#minCrawlDelayMs = (options.minCrawlDelaySecs ?? 0) * 1000;
+        this.#throttlesEveryDomain = options.domains === 'all';
+        this.#throttleBy = options.throttleBy ?? 'hostname';
+        this.#maxThrottledDomains = options.maxThrottledDomains ?? 100;
+        this.#persistStateKey = options.persistStateKey ?? DEFAULT_PERSIST_STATE_KEY;
         this.log = serviceLocator.getLogger().child({ prefix: 'ThrottlingRequestManager' });
-        for (const domain of options.domains) {
+        for (const domain of Array.isArray(options.domains) ? options.domains : []) {
             let hostname;
             try {
                 // These are bare hostnames, so they only reach `URL` - and with it IDNA - via a synthetic URL.
-                hostname = normalizeHostname(new URL(`http://${domain}`).hostname);
+                hostname = new URL(`http://${domain}`).hostname;
             }
             catch {
                 throw new Error(`"${domain}" is not a valid hostname. The \`domains\` option takes bare hostnames such as ` +
                     `"example.com"; an IPv6 address has to be bracketed, as in "[::1]".`);
             }
-            this.domainStates.set(hostname, {
-                domain: hostname,
-                backoffUntil: 0,
-                crawlDelayUntil: 0,
-                backoffDecaysAt: 0,
-                consecutive429Count: 0,
-                crawlDelayMs: null,
-                rateLimitedSince: 0,
-                lastRateLimitedAt: 0,
-            });
+            const key = this.#domainKey(hostname);
+            this.#listedDomains.add(key);
+            this.domainStates.set(key, newDomainState(key));
         }
     }
+    /**
+     * The key a URL's requests are grouped under - one delay clock and one sub-queue per key.
+     *
+     * @param hostname A hostname as `URL` reports it, so in punycode and possibly with a root dot.
+     */
+    #domainKey(hostname) {
+        const normalized = normalizeHostname(hostname);
+        if (this.#throttleBy === 'hostname') {
+            return normalized;
+        }
+        // No registrable domain to group by for an IP address or a single-label host such as `localhost`,
+        // so those stay paced per hostname.
+        return getDomain(normalized, { mixedInputs: false }) ?? normalized;
+    }
     /** The wrapped manager, holding every request whose domain is not throttled. */
     get innerManager() {
-        return this.inner;
+        return this.#inner;
     }
     /** Warns once about sources that cannot be routed by domain, because their URLs are not known yet. */
-    warnIfNotRoutable(requestLike) {
-        if ('requestsFromUrl' in requestLike && requestLike.requestsFromUrl !== undefined && this.hasThrottledDomains) {
+    #warnIfNotRoutable(requestLike) {
+        if ('requestsFromUrl' in requestLike && requestLike.requestsFromUrl !== undefined && this.#throttlingEnabled) {
             // The URL list is only fetched once the owning manager expands it, so we cannot know which domains
             // it covers and cannot route it. Warn instead of silently exempting those URLs from throttling.
-            this.warnOnce('urlListNotRouted', `Requests loaded via \`requestsFromUrl\` cannot be routed to a per-domain queue, because their URLs ` +
+            this.#warnOnce('urlListNotRouted', `Requests loaded via \`requestsFromUrl\` cannot be routed to a per-domain queue, because their URLs ` +
                 `are not known at insertion time. They will be added to the inner request manager and will not ` +
                 `be throttled, even if they belong to a configured domain.`);
         }
     }
-    warnOnce(key, message) {
-        if (this.warnedAbout.has(key)) {
+    #warnOnce(key, message) {
+        if (this.#warnedAbout.has(key)) {
             return;
         }
-        this.warnedAbout.add(key);
+        this.#warnedAbout.add(key);
         this.log.warning(message);
     }
-    extractDomain(url) {
+    #extractDomain(url) {
         try {
-            return normalizeHostname(new URL(url).hostname);
+            return this.#domainKey(new URL(url).hostname);
         }
         catch {
             return '';
         }
     }
-    getDomainState(url) {
-        const domain = this.extractDomain(url);
+    #getDomainState(url) {
+        const domain = this.#extractDomain(url);
         return this.domainStates.get(domain) ?? null;
     }
-    async selectManager(url) {
-        await this.ensureSubManagers();
-        return this.managerForUrl(url);
-    }
-    /** Only valid once {@link ThrottlingRequestManager.ensureSubManagers} has resolved. */
-    managerForUrl(url) {
-        return this.subManagers.get(this.extractDomain(url)) ?? this.inner;
-    }
-    async ensureSubManagers() {
-        this.subManagersReady ??= (async () => {
-            await Promise.all(Array.from(this.domainStates.keys(), async (domain) => {
-                const subManager = await this.requestManagerOpener(
-                // Backends use the alias as a directory name, and an IPv6 literal is full of characters
-                // Windows will not accept. Ordinary hostnames survive this untouched.
-                { alias: `throttled-${encodeURIComponent(domain)}` }, { configuration: this.config });
-                this.subManagers.set(domain, subManager);
-            }));
+    /**
+     * The manager that owns a URL's requests, opening a sub-queue for its domain if this is the first time we
+     * have seen it and every domain is throttled. `null` means the domain is new and `maxThrottledDomains`
+     * leaves no room for it.
+     */
+    async #selectManager(url) {
+        await this.#ensureSubManagers();
+        const domain = this.#extractDomain(url);
+        if (!domain || !(this.#listedDomains.has(domain) || this.#throttlesEveryDomain)) {
+            return this.#inner;
+        }
+        if (!this.#listedDomains.has(domain) && !this.#discoveredDomains.has(domain)) {
+            if (this.#discoveredDomains.size >= this.#maxThrottledDomains) {
+                return null;
+            }
+            this.#discoveredDomains.add(domain);
+            // Written before the request lands in the sub-queue: a crash in between would leave that queue
+            // with nothing to reopen it, and the crawl would silently drop everything in it.
+            await this.#persistDiscoveredDomains();
+        }
+        return this.#subManagerFor(domain);
+    }
+    async #selectManagerOrThrow(url) {
+        const manager = await this.#selectManager(url);
+        if (!manager) {
+            throw this.#throttledDomainLimitError([this.#extractDomain(url)]);
+        }
+        return manager;
+    }
+    #throttledDomainLimitError(domains) {
+        const [first, ...rest] = domains;
+        const named = rest.slice(0, 10);
+        const ellipsis = rest.length > named.length ? ', ...' : '';
+        const others = rest.length > 0 ? ` (and ${rest.length} other new domain(s): ${named.join(', ')}${ellipsis})` : '';
+        return new Error(`Refusing to throttle "${first}"${others}: ${this.#maxThrottledDomains} domains are already being ` +
+            `throttled (\`maxThrottledDomains\`). Each of them holds a request queue of its own, so a crawl ` +
+            `that keeps discovering new domains will bury the storage backend in them. Narrow the crawl down, ` +
+            `pace it with \`maxRequestsPerMinute\` instead, or raise \`maxThrottledDomains\` if you are ` +
+            `prepared to pay for it.`);
+    }
+    /** Opens the domain's sub-queue, or returns the one already opened (or being opened) for it. */
+    #subManagerFor(domain) {
+        let subManager = this.#subManagers.get(domain);
+        if (!subManager) {
+            subManager = this.#requestManagerOpener(
+            // Backends use the alias as a directory name, and an IPv6 literal is full of characters
+            // Windows will not accept. Ordinary hostnames survive this untouched.
+            { alias: `throttled-${encodeURIComponent(domain)}` }, { configuration: this.config });
+            this.#subManagers.set(domain, subManager);
+            this.#ensureDomainState(domain);
+        }
+        return subManager;
+    }
+    #ensureDomainState(domain) {
+        let state = this.domainStates.get(domain);
+        if (!state) {
+            state = newDomainState(domain);
+            this.domainStates.set(domain, state);
+        }
+        return state;
+    }
+    /**
+     * Coalesces the writes of the discovered domain list: callers that arrive while one is in flight share the
+     * single write queued behind it, which snapshots the set once it starts and so covers all of them.
+     */
+    async #persistDiscoveredDomains() {
+        this.#queuedDomainListWrite ??= this.#lastDomainListWrite
+            // A failed write must not poison the ones behind it - they will rewrite the whole set anyway.
+            .catch(() => { })
+            .then(async () => {
+            this.#queuedDomainListWrite = undefined;
+            await this.#domainListStore.setValue(this.#persistStateKey, Array.from(this.#discoveredDomains));
+        });
+        this.#lastDomainListWrite = this.#queuedDomainListWrite;
+        await this.#queuedDomainListWrite;
+    }
+    async #ensureSubManagers() {
+        this.#subManagersReady ??= (async () => {
+            if (this.#throttlesEveryDomain) {
+                this.#domainListStore = await KeyValueStore.open(null, { configuration: this.config });
+                for (const domain of (await this.#domainListStore.getValue(this.#persistStateKey)) ?? []) {
+                    this.#discoveredDomains.add(domain);
+                }
+            }
+            await Promise.all(Array.from([...this.#listedDomains, ...this.#discoveredDomains], async (domain) => this.#subManagerFor(domain)));
         })();
-        await this.subManagersReady;
+        await this.#subManagersReady;
     }
-    async getSubManagers() {
-        await this.ensureSubManagers();
-        return Array.from(this.subManagers.values());
+    async #getSubManagers() {
+        await this.#ensureSubManagers();
+        return Promise.all(this.#subManagers.values());
     }
-    /** Configured domains that are not currently backing off, longest-overdue first. */
-    fetchableDomains() {
+    /**
+     * Throttled domains that are not currently backing off, longest-overdue first.
+     *
+     * A domain whose sub-queue has not been opened yet is skipped - a robots.txt `Crawl-delay` gives a domain a
+     * clock before its first request gives it a queue, and there is nothing to fetch from until then.
+     */
+    #fetchableDomains() {
         const now = Date.now();
         return Array.from(this.domainStates.values())
-            .filter((state) => now >= throttledUntil(state))
+            .filter((state) => now >= throttledUntil(state) && this.#subManagers.has(state.domain))
             .sort((a, b) => throttledUntil(a) - throttledUntil(b))
             .map((state) => state.domain);
     }
@@ -186,7 +324,7 @@ export class ThrottlingRequestManager {
      * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
      */
     recordDomainDelay(url, retryAfterMs) {
-        const state = this.getDomainState(url);
+        const state = this.#getDomainState(url);
         if (!state) {
             return false;
         }
@@ -211,13 +349,13 @@ export class ThrottlingRequestManager {
         }
         state.consecutive429Count += 1;
         const retryAfterGiven = retryAfterMs !== undefined && retryAfterMs !== null;
-        let delayMs = retryAfterGiven ? retryAfterMs : this.baseDelayMs * Math.pow(2, state.consecutive429Count - 1);
-        if (delayMs > this.maxDelayMs) {
+        let delayMs = retryAfterGiven ? retryAfterMs : this.#baseDelayMs * Math.pow(2, state.consecutive429Count - 1);
+        if (delayMs > this.#maxDelayMs) {
             const source = retryAfterGiven ? 'Retry-After header' : 'exponential backoff';
             this.log.warning(`Capping ${source} delay of ${(delayMs / 1000).toFixed(1)}s for domain "${state.domain}" ` +
-                `to maxDelaySecs (${(this.maxDelayMs / 1000).toFixed(1)}s); the domain may continue to rate-limit. ` +
+                `to maxDelaySecs (${(this.#maxDelayMs / 1000).toFixed(1)}s); the domain may continue to rate-limit. ` +
                 `Consider increasing maxDelaySecs if this recurs.`);
-            delayMs = this.maxDelayMs;
+            delayMs = this.#maxDelayMs;
         }
         state.backoffUntil = now + delayMs;
         state.backoffDecaysAt = state.backoffUntil + delayMs;
@@ -226,19 +364,23 @@ export class ThrottlingRequestManager {
         return true;
     }
     /**
-     * Applies a robots.txt `Crawl-delay` to the URL's domain, as a minimum interval between dispatches.
+     * Records the `Crawl-delay` a domain's robots.txt asked for, which becomes its crawl delay unless
+     * {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`} asks for longer.
      *
      * The first value wins, so a robots.txt re-fetch cannot change the cadence mid-crawl.
      *
-     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     * @returns `false` if the domain is not throttled, in which case this is a no-op.
      */
     setCrawlDelay(url, delaySeconds) {
-        const state = this.getDomainState(url);
-        if (!state) {
+        const domain = this.#extractDomain(url);
+        if (!domain || !(this.#listedDomains.has(domain) || this.#throttlesEveryDomain)) {
             return false;
         }
-        if (state.crawlDelayMs === null) {
-            state.crawlDelayMs = delaySeconds * 1000;
+        // The crawler reads robots.txt before it enqueues a domain's first request, so the clock can predate
+        // the sub-queue it will end up pacing.
+        const state = this.#ensureDomainState(domain);
+        if (state.declaredCrawlDelayMs === null) {
+            state.declaredCrawlDelayMs = delaySeconds * 1000;
             this.log.debug(`Set crawl-delay for domain "${state.domain}" to ${delaySeconds}s`);
         }
         return true;
@@ -253,16 +395,19 @@ export class ThrottlingRequestManager {
      * `Crawl-delay` is being obeyed, not stonewalled.
      */
     async assertNoStalledDomains() {
-        await this.ensureSubManagers();
+        await this.#ensureSubManagers();
         const now = Date.now();
         const candidates = Array.from(this.domainStates.values()).filter(
         // Together: it is still turning us away, and has been doing so without a break for longer than the
         // window. A domain that has simply been idle starts this clock at its first 429 rather than
         // arriving with the idle time already on it.
         (state) => state.rateLimitedSince !== 0 &&
-            now - state.lastRateLimitedAt <= this.maxDomainStallMs &&
-            now - state.rateLimitedSince > this.maxDomainStallMs);
-        const stalled = (await Promise.all(candidates.map(async (state) => ((await this.subManagers.get(state.domain).isEmpty()) ? null : state)))).filter((state) => state !== null);
+            now - state.lastRateLimitedAt <= this.#maxDomainStallMs &&
+            now - state.rateLimitedSince > this.#maxDomainStallMs);
+        const stalled = (await Promise.all(candidates.map(async (state) => {
+            const subManager = await this.#subManagers.get(state.domain);
+            return subManager && !(await subManager.isEmpty()) ? state : null;
+        }))).filter((state) => state !== null);
         if (stalled.length === 0) {
             return;
         }
@@ -270,21 +415,21 @@ export class ThrottlingRequestManager {
             .map((state) => `"${state.domain}" (${((now - state.rateLimitedSince) / 1000).toFixed(0)}s)`)
             .join(', ');
         throw new PersistentRateLimitError(`Giving up: ${summary} rate-limited every request for longer than maxDomainStallSecs ` +
-            `(${(this.maxDomainStallMs / 1000).toFixed(0)}s). Waiting longer will not help - lower the ` +
+            `(${(this.#maxDomainStallMs / 1000).toFixed(0)}s). Waiting longer will not help - lower the ` +
             `crawler's concurrency, or drop these domains. Their requests are still queued, so re-running ` +
-            `without purging storages will resume them if the rate limit lifts.`);
+            `with \`purgeOnStart\` disabled will resume them if the rate limit lifts.`);
     }
     /** Records that a domain let a request through, which ends any rate-limit run stall detection was timing. */
-    recordProgress(url) {
-        const state = this.getDomainState(url);
+    #recordProgress(url) {
+        const state = this.#getDomainState(url);
         if (state) {
             state.rateLimitedSince = 0;
         }
     }
     // --- IRequestManager Implementation ---
     async addRequest(requestLike, options) {
-        this.warnIfNotRoutable(requestLike);
-        const manager = await this.selectManager(requestLike.url ?? '');
+        this.#warnIfNotRoutable(requestLike);
+        const manager = await this.#selectManagerOrThrow(requestLike.url ?? '');
         return manager.addRequest(requestLike, options);
     }
     /**
@@ -295,7 +440,7 @@ export class ThrottlingRequestManager {
      * iterable is never fully materialized.
      */
     async addRequestsBatched(requests, options = {}) {
-        await this.ensureSubManagers();
+        await this.#ensureSubManagers();
         // Normalized up front so the shared batching helper - and `requestsOverLimit` - only ever see `Source`.
         async function* iterateRequests() {
             for await (const request of asyncifyIterable(requests)) {
@@ -312,9 +457,16 @@ export class ThrottlingRequestManager {
             // deduplication themselves.
             processChunk: async (chunk) => {
                 const byManager = new Map();
+                // Collected, not thrown on sight, so an overflow does not discard the requests that fit.
+                const overflowing = new Set();
                 for (const request of chunk) {
-                    this.warnIfNotRoutable(request);
-                    const manager = this.managerForUrl(request.url ?? '');
+                    this.#warnIfNotRoutable(request);
+                    const url = request.url ?? '';
+                    const manager = await this.#selectManager(url);
+                    if (!manager) {
+                        overflowing.add(this.#extractDomain(url));
+                        continue;
+                    }
                     const bucket = byManager.get(manager);
                     if (bucket) {
                         bucket.push(request);
@@ -329,35 +481,49 @@ export class ThrottlingRequestManager {
                     batchSize: slice.length,
                     waitForAllRequestsToBeAdded: true,
                 })));
+                if (overflowing.size > 0) {
+                    throw this.#throttledDomainLimitError(Array.from(overflowing));
+                }
                 return results.flatMap((result) => result.addedRequests);
             },
             // Keeps the crawler from concluding it is finished while batches are still landing.
             trackBackgroundBatches: (batches) => {
-                this.inProgressBatchCount += 1;
+                this.#inProgressBatchCount += 1;
                 void batches.finally(() => {
-                    this.inProgressBatchCount -= 1;
+                    this.#inProgressBatchCount -= 1;
                 });
             },
         });
     }
     async reclaimRequest(request, options) {
-        const manager = await this.selectManager(request.url);
+        const manager = await this.#managerHolding(request);
         return manager.reclaimRequest(request, options);
     }
     async markRequestAsHandled(request) {
-        const manager = await this.selectManager(request.url);
+        const manager = await this.#managerHolding(request);
         // Reached whether the request succeeded or ran out of retries; either way the domain answered us.
-        this.recordProgress(request.url);
+        this.#recordProgress(request.url);
         return manager.markRequestAsHandled(request);
     }
+    /**
+     * The manager a request in the consumer's hands has to be given back to - the one it was fetched from,
+     * which is only the same as the one its domain routes to if it was routed in the first place.
+     */
+    async #managerHolding(request) {
+        const key = request.id ?? request.uniqueKey;
+        if (this.#inFlightFromInner.delete(key)) {
+            return this.#inner;
+        }
+        return this.#selectManagerOrThrow(request.url);
+    }
     async getTotalCount() {
-        return this.sumOverManagers((manager) => manager.getTotalCount());
+        return this.#sumOverManagers((manager) => manager.getTotalCount());
     }
     async getPendingCount() {
-        return this.sumOverManagers((manager) => manager.getPendingCount());
+        return this.#sumOverManagers((manager) => manager.getPendingCount());
     }
     async getHandledCount() {
-        return this.sumOverManagers((manager) => manager.getHandledCount());
+        return this.#sumOverManagers((manager) => manager.getHandledCount());
     }
     /**
      * Whether the next {@link ThrottlingRequestManager.fetchNextRequest} would return `null`.
@@ -366,24 +532,35 @@ export class ThrottlingRequestManager {
      * this idles for the backoff instead of spinning on a fetch that cannot succeed yet.
      */
     async isEmpty() {
-        await this.ensureSubManagers();
-        const fetchable = [this.inner, ...this.fetchableDomains().map((domain) => this.subManagers.get(domain))];
-        const results = await Promise.all(fetchable.map((manager) => manager.isEmpty()));
+        await this.#ensureSubManagers();
+        const fetchable = await Promise.all(this.#fetchableDomains().map(async (domain) => this.#subManagers.get(domain)));
+        const results = await Promise.all([this.#inner, ...fetchable].map(async (manager) => manager.isEmpty()));
         return results.every(Boolean);
     }
     /** Unlike {@link ThrottlingRequestManager.isEmpty}, throttled requests still count as outstanding work. */
     async isFinished() {
-        if (this.inProgressBatchCount > 0) {
+        if (this.#inProgressBatchCount > 0) {
             return false;
         }
-        return this.everyManager((manager) => manager.isFinished());
+        return this.#everyManager((manager) => manager.isFinished());
     }
     /**
      * Empties every manager and clears the accumulated backoff. A robots.txt `Crawl-delay` is a property of the
      * site rather than of the run, so it survives.
      */
     async purge() {
-        await this.forEachManager((manager) => manager.purge?.());
+        await this.#inner.purge?.();
+        await this.purgeDomainQueues();
+    }
+    /**
+     * Empties the per-domain queues, leaving the wrapped manager alone.
+     *
+     * Those queues are this manager's own no matter who owns the one it wraps, which is what makes this safe to
+     * call where a full {@link ThrottlingRequestManager.purge|`purge()`} would not be.
+     */
+    async purgeDomainQueues() {
+        const subManagers = await this.#getSubManagers();
+        await Promise.all(subManagers.map(async (manager) => manager.purge?.()));
         for (const state of this.domainStates.values()) {
             state.consecutive429Count = 0;
             state.backoffUntil = 0;
@@ -394,18 +571,18 @@ export class ThrottlingRequestManager {
         }
     }
     async setExpectedRequestProcessingTimeSecs(secs) {
-        await this.forEachManager((manager) => manager.setExpectedRequestProcessingTimeSecs?.(secs));
+        await this.#forEachManager((manager) => manager.setExpectedRequestProcessingTimeSecs?.(secs));
     }
-    async forEachManager(fn) {
+    async #forEachManager(fn) {
         // `fn` targets optional members, so it may return nothing - the wrapper normalizes that for `Promise.all`.
-        await Promise.all([this.inner, ...(await this.getSubManagers())].map(async (manager) => fn(manager)));
+        await Promise.all([this.#inner, ...(await this.#getSubManagers())].map(async (manager) => fn(manager)));
     }
-    async sumOverManagers(fn) {
-        const counts = await Promise.all([this.inner, ...(await this.getSubManagers())].map(fn));
+    async #sumOverManagers(fn) {
+        const counts = await Promise.all([this.#inner, ...(await this.#getSubManagers())].map(fn));
         return counts.reduce((a, b) => a + b, 0);
     }
-    async everyManager(fn) {
-        const results = await Promise.all([this.inner, ...(await this.getSubManagers())].map(fn));
+    async #everyManager(fn) {
+        const results = await Promise.all([this.#inner, ...(await this.#getSubManagers())].map(fn));
         return results.every(Boolean);
     }
     /**
@@ -417,24 +594,38 @@ export class ThrottlingRequestManager {
      * reports `true` meanwhile so the crawler's task loop idles rather than spins.
      */
     async fetchNextRequest() {
-        await this.ensureSubManagers();
-        for (const domain of this.fetchableDomains()) {
+        await this.#ensureSubManagers();
+        for (const domain of this.#fetchableDomains()) {
             const state = this.domainStates.get(domain);
             // Armed while the fetch below is still suspended, so that a concurrent `fetchNextRequest` cannot
             // find the domain fetchable and dispatch into the same window - which would pace each task
             // rather than the domain.
-            const crawlDelayBefore = state.crawlDelayUntil;
-            if (state.crawlDelayMs !== null) {
-                state.crawlDelayUntil = Date.now() + state.crawlDelayMs;
+            const crawlDelayUntilBefore = state.crawlDelayUntil;
+            const delayMs = crawlDelayMs(state, this.#minCrawlDelayMs);
+            if (delayMs > 0) {
+                state.crawlDelayUntil = Date.now() + delayMs;
             }
-            const request = await this.subManagers.get(domain).fetchNextRequest();
+            const request = await (await this.#subManagers.get(domain)).fetchNextRequest();
             if (request) {
                 return request;
             }
             // No dispatch to pace, so the domain keeps its slot.
-            state.crawlDelayUntil = crawlDelayBefore;
+            state.crawlDelayUntil = crawlDelayUntilBefore;
+        }
+        const request = await this.#inner.fetchNextRequest();
+        if (request !== null) {
+            this.#inFlightFromInner.add(request.id ?? request.uniqueKey);
+        }
+        if (request !== null && this.#throttlesEveryDomain) {
+            // Requests that were never routed by domain - a `RequestList`, a `requestsFromUrl` expansion - are
+            // handed out as fast as the crawler asks for them, because there is no per-domain queue to hold
+            // them back in.
+            this.#warnOnce('innerNotThrottled', `Requests read directly from the wrapped request manager (for instance from a \`RequestList\` or a ` +
+                `\`requestsFromUrl\` list) are not throttled, because they are not stored per domain. Enqueue ` +
+                `them through the crawler - \`crawler.run(requests)\` or \`crawler.addRequests()\` - to have ` +
+                `their domains paced.`);
         }
-        return this.inner.fetchNextRequest();
+        return request;
     }
     async *[Symbol.asyncIterator]() {
         while (true) {
@@ -445,11 +636,11 @@ export class ThrottlingRequestManager {
         }
     }
     async persistState() {
-        await this.forEachManager((manager) => manager.persistState?.());
+        await this.#forEachManager((manager) => manager.persistState?.());
     }
     async drop() {
-        await this.forEachManager((manager) => manager.drop?.());
-        this.subManagers.clear();
-        this.subManagersReady = undefined;
+        await this.#forEachManager((manager) => manager.drop?.());
+        this.#subManagers.clear();
+        this.#subManagersReady = undefined;
     }
 }