@crawlee/core 4.0.0-beta.122 → 4.0.0-beta.124

package/enqueue_links/enqueue_links.js

@@ -1,8 +1,4 @@
 import { getDomain } from 'tldts';
-import { z } from 'zod';
-import { Request } from '../request.js';
-import { parseArgument, schemas } from '../validators.js';
-import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, filterRequestOptionsByPatterns, urlPatternSchema, } from './shared.js';
 /**
  * The different enqueueing strategies available.
  *
@@ -55,159 +51,6 @@ export var EnqueueStrategy;
      */
     EnqueueStrategy["SameOrigin"] = "same-origin";
 })(EnqueueStrategy || (EnqueueStrategy = {}));
-// `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
-// copy), so `userData` keeps its identity for the enqueued requests.
-const enqueueLinksOptionsSchema = z.strictObject({
-    urls: schemas.arrayOf(z.string(), 'strings'),
-    requestManager: schemas.objectWithKeys(['addRequestsBatched']),
-    robotsTxtFile: schemas.objectWithKeys(['isAllowed']).optional(),
-    respectRobotsTxtFile: z.union([z.boolean(), z.strictObject({ userAgent: z.string().optional() })]).optional(),
-    onSkippedRequest: schemas.anyFunction.optional(),
-    forefront: z.boolean().optional(),
-    skipNavigation: z.boolean().optional(),
-    sessionId: z.string().optional(),
-    limit: schemas.anyNumber.optional(),
-    selector: z.string().optional(),
-    baseUrl: z.string().optional(),
-    userData: schemas.anyObject.optional(),
-    label: z.string().optional(),
-    include: schemas.arrayOf(urlPatternSchema, 'URL patterns').min(1).optional(),
-    exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
-    transformRequestFunction: schemas.anyFunction.optional(),
-    strategy: z.enum(EnqueueStrategy).optional(),
-    waitForAllRequestsToBeAdded: z.boolean().optional(),
-});
-/**
- * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
- * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
- *
- * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
- *
- * **Example usage**
- *
- * ```javascript
- * await enqueueLinks({
- *   urls: aListOfFoundUrls,
- *   requestManager,
- *   selector: 'a.product-detail',
- *   include: [
- *       'https://www.example.com/handbags/*',
- *       'https://www.example.com/purses/*'
- *   ],
- * });
- * ```
- *
- * @param options All `enqueueLinks()` parameters are passed via an options object.
- * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
- */
-export async function enqueueLinks(options) {
-    if (!options || Object.keys(options).length === 0) {
-        throw new RangeError([
-            'enqueueLinks() was called without the required options. You can only do that when you use the `crawlingContext.enqueueLinks()` method in request handlers.',
-            'Check out our guide on how to use enqueueLinks() here: https://crawlee.dev/js/docs/examples/crawl-relative-links',
-        ].join('\n'));
-    }
-    const parsedOptions = parseArgument(options, enqueueLinksOptionsSchema, 'EnqueueLinksOptions');
-    const { requestManager, limit, urls, include, exclude, transformRequestFunction, forefront, waitForAllRequestsToBeAdded, robotsTxtFile, onSkippedRequest, } = parsedOptions;
-    const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
-    const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
-    // The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
-    // (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
-    parsedOptions.strategy ??= EnqueueStrategy.SameHostname;
-    const enqueueStrategyPatterns = [];
-    if (parsedOptions.baseUrl) {
-        const url = new URL(parsedOptions.baseUrl);
-        switch (parsedOptions.strategy) {
-            case EnqueueStrategy.SameHostname:
-                // We need to get the origin of the passed in domain in the event someone sets baseUrl
-                // to an url like https://example.com/deep/default/path and one of the found urls is an
-                // absolute relative path (/path/to/page)
-                enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
-                break;
-            case EnqueueStrategy.SameDomain: {
-                // Get the actual hostname from the base url
-                const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
-                if (baseUrlHostname) {
-                    // We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
-                    url.hostname = baseUrlHostname;
-                    enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) }, { glob: ignoreHttpSchema(`${url.origin}/**`) });
-                }
-                else {
-                    // We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
-                    // as SameDomainAndSubdomain
-                    enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
-                }
-                break;
-            }
-            case EnqueueStrategy.SameOrigin: {
-                // The same behavior as SameHostname, but respecting the protocol of the URL
-                enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
-                break;
-            }
-            case EnqueueStrategy.All:
-            default:
-                enqueueStrategyPatterns.push({ glob: `http{s,}://**` });
-                break;
-        }
-    }
-    async function reportSkippedRequests(skippedRequests, reason) {
-        if (onSkippedRequest && skippedRequests.length > 0) {
-            await Promise.all(skippedRequests.map((request) => {
-                return onSkippedRequest({
-                    url: request.url,
-                    reason: request.skippedReason ?? reason,
-                });
-            }));
-        }
-    }
-    let requestOptions = createRequestOptions(urls, parsedOptions);
-    if (robotsTxtFile && parsedOptions.respectRobotsTxtFile !== false) {
-        const robotsUserAgent = typeof parsedOptions.respectRobotsTxtFile === 'object'
-            ? (parsedOptions.respectRobotsTxtFile.userAgent ?? '*')
-            : '*';
-        const skippedRequests = [];
-        requestOptions = requestOptions.filter((request) => {
-            if (robotsTxtFile.isAllowed(request.url, robotsUserAgent)) {
-                return true;
-            }
-            skippedRequests.push(request);
-            return false;
-        });
-        await reportSkippedRequests(skippedRequests, 'robotsTxt');
-    }
-    async function createFilteredRequests() {
-        const skippedRequests = [];
-        // Step 1: Filter request options by exclude patterns, user include patterns, and strategy patterns.
-        let filteredOptions;
-        if (urlPatternObjects.length === 0) {
-            filteredOptions = filterRequestOptionsByPatterns(requestOptions, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, urlExcludePatternObjects, parsedOptions.strategy, (url) => skippedRequests.push(url));
-        }
-        else {
-            // Filter by user patterns first (with exclude)
-            const afterUserPatterns = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects, urlExcludePatternObjects, parsedOptions.strategy, (url) => skippedRequests.push(url));
-            // ...then filter by the enqueue links strategy (making this an AND check)
-            filteredOptions = filterRequestOptionsByPatterns(afterUserPatterns, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], parsedOptions.strategy, (url) => skippedRequests.push(url));
-        }
-        await reportSkippedRequests(skippedRequests.map((url) => ({ url })), 'filters');
-        // Step 2: Apply transformRequestFunction on request options - it has the highest priority
-        if (transformRequestFunction) {
-            const skippedByTransform = [];
-            filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
-            await reportSkippedRequests(skippedByTransform, 'transform');
-        }
-        // Step 3: Create Request instances from the final request options
-        return filteredOptions.map((opts) => new Request(opts));
-    }
-    const { addedRequests, requestsOverLimit } = await requestManager.addRequestsBatched(await createFilteredRequests(), {
-        forefront,
-        waitForAllRequestsToBeAdded,
-        maxNewRequests: limit,
-    });
-    if (requestsOverLimit?.length !== undefined && requestsOverLimit.length > 0) {
-        await reportSkippedRequests(requestsOverLimit.map((r) => ({ url: typeof r === 'string' ? r : r.url })), 'enqueueLimit');
-    }
-    return { processedRequests: addedRequests, unprocessedRequests: [] };
-}
 /**
  * @internal
  * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
@@ -242,6 +85,41 @@ export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalR
     // before actually finding the urls
     return originalUrlOrigin;
 }
+/**
+ * @internal
+ * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
+ */
+export function buildEnqueueStrategyPatterns(baseUrl, strategy) {
+    const url = new URL(baseUrl);
+    switch (strategy) {
+        case EnqueueStrategy.SameHostname:
+            // We need to get the origin of the passed in domain in the event someone sets baseUrl
+            // to an url like https://example.com/deep/default/path and one of the found urls is an
+            // absolute relative path (/path/to/page)
+            return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
+        case EnqueueStrategy.SameDomain: {
+            // Get the actual hostname from the base url
+            const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
+            if (baseUrlHostname) {
+                // We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
+                url.hostname = baseUrlHostname;
+                return [
+                    { glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) },
+                    { glob: ignoreHttpSchema(`${url.origin}/**`) },
+                ];
+            }
+            // We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
+            // as SameDomainAndSubdomain
+            return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
+        }
+        case EnqueueStrategy.SameOrigin:
+            // The same behavior as SameHostname, but respecting the protocol of the URL
+            return [{ glob: `${url.origin}/**` }];
+        case EnqueueStrategy.All:
+        default:
+            return [{ glob: `http{s,}://**` }];
+    }
+}
 /**
  * Internal function that changes the enqueue glob patterns to match both http and https
  */

package/enqueue_links/shared.d.ts

@@ -1,7 +1,7 @@
-import type { Awaitable } from '@crawlee/types';
+import type { Awaitable, Dictionary } from '@crawlee/types';
 import { z } from 'zod';
 import type { RequestOptions } from '../request.js';
-import type { EnqueueLinksOptions } from './enqueue_links.js';
+import type { EnqueueStrategyOption } from './enqueue_links.js';
 export { tryAbsoluteURL } from '@crawlee/utils/internal';
 export interface UrlPatternObject {
     glob?: string;
@@ -59,11 +59,18 @@ export declare function constructUrlPatternObjects(patterns: readonly UrlPattern
  * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
  * @ignore
  */
-export declare function filterRequestOptionsByPatterns(requestOptions: RequestOptions[], includePatterns: UrlPatternObject[] | undefined, excludePatterns?: UrlPatternObject[], strategy?: EnqueueLinksOptions['strategy'], onSkippedUrl?: (url: string) => void): RequestOptions[];
+export declare function filterRequestOptionsByPatterns(requestOptions: RequestOptions[], includePatterns: UrlPatternObject[] | undefined, excludePatterns?: UrlPatternObject[], strategy?: EnqueueStrategyOption, onSkippedUrl?: (url: string) => void): RequestOptions[];
 /**
  * @ignore
  */
-export declare function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl' | 'skipNavigation' | 'sessionId' | 'strategy'>): RequestOptions[];
+export declare function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: {
+    label?: string;
+    userData?: Dictionary;
+    baseUrl?: string;
+    skipNavigation?: boolean;
+    sessionId?: string;
+    strategy?: EnqueueStrategyOption;
+}): RequestOptions[];
 /**
  * Takes a {@link RequestOptions} object and changes its attributes in a desired way. This user-function is used
  * by {@link enqueueLinks} to modify request options before they are converted to {@link Request} instances.

package/enqueue_links/shared.js

@@ -144,6 +144,16 @@ export function filterRequestOptionsByPatterns(requestOptions, includePatterns,
     })
         .filter((opts) => opts !== null);
 }
+function isAbsoluteUrl(url) {
+    try {
+        // eslint-disable-next-line no-new
+        new URL(url);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
 /**
  * @ignore
  */
@@ -161,7 +171,12 @@ export function createRequestOptions(sources, options = {}) {
         }
     })
         .map((requestOptions) => {
-        requestOptions.url = new URL(requestOptions.url, options.baseUrl).href;
+        // Leave already-absolute URLs untouched - re-deriving them via `new URL()` would normalize them
+        // (e.g. adding a trailing slash to a bare domain), which is surprising for URLs that didn't need
+        // resolving against `baseUrl` in the first place.
+        if (!isAbsoluteUrl(requestOptions.url)) {
+            requestOptions.url = new URL(requestOptions.url, options.baseUrl).href;
+        }
         requestOptions.userData ??= options.userData ?? {};
         if (typeof options.label === 'string') {
             requestOptions.userData = {

package/events/event_manager.js

@@ -18,7 +18,8 @@ export class EventManager {
     #persistStateIntervalMillis;
     constructor(options) {
         this.#persistStateIntervalMillis = options.persistStateIntervalMillis;
-        this.events.setMaxListeners(50);
+        // One MIGRATING listener per RequestQueue, and ThrottlingRequestManager opens one per domain.
+        this.events.setMaxListeners(150);
     }
     /**
      * Initializes the event manager by starting the `persistState` event interval.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.122",
+    "version": "4.0.0-beta.124",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,10 +52,10 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.122",
-        "@crawlee/http-client": "4.0.0-beta.122",
-        "@crawlee/types": "4.0.0-beta.122",
-        "@crawlee/utils": "4.0.0-beta.122",
+        "@crawlee/fs-storage": "4.0.0-beta.124",
+        "@crawlee/http-client": "4.0.0-beta.124",
+        "@crawlee/types": "4.0.0-beta.124",
+        "@crawlee/utils": "4.0.0-beta.124",
         "@sapphire/async-queue": "^1.5.5",
         "@vladfrangu/async_event_emitter": "^2.4.6",
         "content-type": "^1.0.5",
@@ -77,5 +77,5 @@
             }
         }
     },
-    "gitHead": "2c3e87fefdb9e1fca4c144f03167d8524cfdc2e5"
+    "gitHead": "0694ee1b94c755b98141671baa93cc363f2bf8e3"
 }

package/request.d.ts

@@ -1,6 +1,6 @@
 import type { BinaryLike } from 'node:crypto';
 import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
-import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
+import type { EnqueueStrategyOption } from './enqueue_links/enqueue_links.js';
 import type { SkippedRequestReason } from './enqueue_links/shared.js';
 export declare enum RequestState {
     UNPROCESSED = 0,
@@ -281,7 +281,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
     /** @internal */
     lockExpiresAt?: Date;
     /** @internal */
-    enqueueStrategy?: EnqueueLinksOptions['strategy'];
+    enqueueStrategy?: EnqueueStrategyOption;
 }
 export interface PushErrorMessageOptions {
     /**

package/router.d.ts

@@ -65,7 +65,7 @@ export declare function validateUserData(label: string | symbol, schema: Standar
  * `Record<string, ...>`), any string or symbol label is accepted, preserving the original behaviour.
  */
 export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | symbol : (keyof Routes & string) | symbol;
-export interface RouterHandler<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends Router<Context, Routes> {
+export interface RouterHandler<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends Router<Context, Routes> {
     (ctx: Context): Awaitable<void>;
 }
 export type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;
@@ -214,7 +214,7 @@ export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary
  * });
  * ```
  */
-export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
+export declare class Router<Context extends RestrictedCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
     #private;
     /**
      * use Router.create() instead!
@@ -299,8 +299,8 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
      * await crawler.run();
      * ```
      */
-    static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
-    static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
-    static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
+    static create<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+    static create<Context extends RestrictedCrawlingContext = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+    static create<Context extends RestrictedCrawlingContext = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
 }
 export {};

package/storages/throttling_request_manager.d.ts

@@ -37,15 +37,58 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
      */
     inner: T;
     /**
-     * Hostnames to throttle. Matching is case-insensitive and exact - wildcards such as `*.example.com` are not
-     * supported, so list each subdomain you care about. Requests for any other domain bypass throttling entirely.
+     * Which domains to throttle: a list of hostnames, or `'all'` for every domain the crawl encounters.
      *
-     * An internationalized domain may be given in either its unicode or its punycode form, and an IPv6 address
-     * has to be bracketed (`[::1]`).
+     * Matching a listed hostname is case-insensitive and exact - wildcards such as `*.example.com` are not
+     * supported, so list each subdomain you care about (or set
+     * {@link ThrottlingRequestManagerOptions.throttleBy|`throttleBy: 'registrableDomain'`}). An
+     * internationalized domain may be given in either its unicode or its punycode form, and an IPv6 address has
+     * to be bracketed (`[::1]`). Requests for any other domain bypass throttling entirely.
+     *
+     * `'all'` gives each domain a queue of its own the first time it is seen, so that it can be held back
+     * without its requests being repeatedly popped and re-enqueued. One request queue per domain is not free,
+     * which is what {@link ThrottlingRequestManagerOptions.maxThrottledDomains|`maxThrottledDomains`} is
+     * there to bound.
+     */
+    domains: string[] | 'all';
+    /**
+     * A floor under the crawl delay of every throttled domain, in seconds - the proactive clock described on
+     * {@link ThrottlingRequestManager}. A domain whose robots.txt asks for a longer `Crawl-delay` gets the
+     * longer one; this is a minimum, not an override.
+     * @default 0
+     */
+    minCrawlDelaySecs?: number;
+    /**
+     * What counts as "the same domain": the exact hostname, or the registrable domain it belongs to
+     * (`example.com` for `www.example.com`, `a.example.co.uk` and so on). Hosts with no registrable domain -
+     * IP addresses, `localhost` - are always throttled per hostname.
+     *
+     * Grouping by registrable domain gives subdomains a single pair of clocks and a single queue, which is what
+     * you want when the pacing is there to be polite to one server rather than to satisfy a specific host's
+     * rate limit.
+     * @default 'hostname'
+     */
+    throttleBy?: 'hostname' | 'registrableDomain';
+    /**
+     * The most domains a run may throttle at once. Exceeding it throws, rather than silently letting the
+     * throttling lapse - one request queue per domain is not free, and a crawl that discovers domains without
+     * bound would drown the storage backend in them.
+     *
+     * Only domains discovered under `domains: 'all'` count against this; an explicit list is taken at face value.
+     * @default 100
      */
-    domains: string[];
+    maxThrottledDomains?: number;
     /**
-     * Opens the per-domain queues, one per entry in `domains`, each under the alias `throttled-<domain>`.
+     * The key under which the discovered domain list is kept in the default key-value store, so that a restart
+     * with `purgeOnStart` disabled reopens their queues instead of stranding whatever they still hold. Only
+     * written under `domains: 'all'`.
+     *
+     * Give each manager its own key when running several of them against the same storage.
+     * @default 'CRAWLEE_THROTTLED_DOMAINS'
+     */
+    persistStateKey?: string;
+    /**
+     * Opens the per-domain queues, one per throttled domain, each under the alias `throttled-<domain>`.
      * @default RequestQueue.open
      */
     requestManagerOpener?: RequestManagerOpener<T>;
@@ -77,22 +120,26 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
 /**
  * A request manager that wraps another one and paces requests per domain.
  *
- * Requests for the configured {@link ThrottlingRequestManagerOptions.domains|`domains`} are routed into their own
- * queue when they are added, so each request lives in exactly one place and deduplication keeps working. Everything
- * else goes to the wrapped manager untouched.
+ * Requests for a throttled domain are routed into their own queue when they are added, so each request lives in
+ * exactly one place and deduplication keeps working. Everything else goes to the wrapped manager untouched.
  *
  * {@link ThrottlingRequestManager.fetchNextRequest|`fetchNextRequest()`} serves the domain that has been waiting
  * longest and skips any that are backing off, falling back to the wrapped manager. It never blocks: while every
  * remaining request belongs to a throttled domain it returns `null` and {@link ThrottlingRequestManager.isEmpty}
  * reports `true`, so the crawler idles instead of holding a concurrency slot open.
  *
- * Delays come from two places:
- * - HTTP 429 responses, honouring `Retry-After` and otherwise backing off exponentially. The crawlers report these
- *   automatically; a request that is throttled is retried later without counting against `maxRequestRetries` and
+ * Each throttled domain runs two independent clocks, and may be dispatched to once **both** have run out:
+ * - **Backoff**, set by HTTP 429 responses - honouring `Retry-After`, and otherwise doubling from `baseDelaySecs`.
+ *   Reactive and temporary: it decays once the domain stops turning us away. The crawlers report the 429s
+ *   themselves; a request held back this way is retried later without counting against `maxRequestRetries` and
  *   without penalising its session.
- * - robots.txt `Crawl-delay` directives, when `respectRobotsTxtFile` is enabled.
+ * - **Crawl delay**, the minimum interval between two dispatches to the domain, armed after each one. Proactive
+ *   and constant: whatever the domain's robots.txt asks for, floored by
+ *   {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`}. Either may be absent, in
+ *   which case the other one is the delay.
  *
- * This is opt-in: throttling only happens for a domain you list explicitly.
+ * Which domains get those clocks is {@link ThrottlingRequestManagerOptions.domains|`domains`} - a list, or
+ * `'all'` for every domain the crawl encounters.
  *
  * **Example usage:**
  *
@@ -109,40 +156,13 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
  * @category Sources
  */
 export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager, SupportsDomainThrottling {
+    #private;
     private readonly config;
-    private readonly inner;
-    private readonly requestManagerOpener;
-    private readonly baseDelayMs;
-    private readonly maxDelayMs;
-    private readonly maxDomainStallMs;
     private readonly domainStates;
-    private readonly subManagers;
     private readonly log;
-    /**
-     * Sub-managers are keyed by a stable alias, so with `purgeOnStart` disabled they outlive the process. They
-     * must therefore be reopened for every configured domain rather than created on first insert - otherwise a
-     * restart sees an empty map, reports the crawl finished, and strands whatever the previous run left in them.
-     */
-    private subManagersReady?;
-    /** Batches still being added in the background; keeps {@link ThrottlingRequestManager.isFinished} honest. */
-    private inProgressBatchCount;
-    private readonly warnedAbout;
-    private get hasThrottledDomains();
     constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);
     /** The wrapped manager, holding every request whose domain is not throttled. */
     get innerManager(): T;
-    /** Warns once about sources that cannot be routed by domain, because their URLs are not known yet. */
-    private warnIfNotRoutable;
-    private warnOnce;
-    private extractDomain;
-    private getDomainState;
-    private selectManager;
-    /** Only valid once {@link ThrottlingRequestManager.ensureSubManagers} has resolved. */
-    private managerForUrl;
-    private ensureSubManagers;
-    private getSubManagers;
-    /** Configured domains that are not currently backing off, longest-overdue first. */
-    private fetchableDomains;
     /**
      * Records a 429 response and puts the URL's domain into backoff.
      *
@@ -150,11 +170,12 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
      */
     recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
     /**
-     * Applies a robots.txt `Crawl-delay` to the URL's domain, as a minimum interval between dispatches.
+     * Records the `Crawl-delay` a domain's robots.txt asked for, which becomes its crawl delay unless
+     * {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`} asks for longer.
      *
      * The first value wins, so a robots.txt re-fetch cannot change the cadence mid-crawl.
      *
-     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     * @returns `false` if the domain is not throttled, in which case this is a no-op.
      */
     setCrawlDelay(url: string, delaySeconds: number): boolean;
     /**
@@ -167,8 +188,6 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
      * `Crawl-delay` is being obeyed, not stonewalled.
      */
     assertNoStalledDomains(): Promise<void>;
-    /** Records that a domain let a request through, which ends any rate-limit run stall detection was timing. */
-    private recordProgress;
     addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
     /**
      * Adds requests in batches, routing each one to the manager that owns its domain.
@@ -197,10 +216,14 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
      * site rather than of the run, so it survives.
      */
     purge(): Promise<void>;
+    /**
+     * Empties the per-domain queues, leaving the wrapped manager alone.
+     *
+     * Those queues are this manager's own no matter who owns the one it wraps, which is what makes this safe to
+     * call where a full {@link ThrottlingRequestManager.purge|`purge()`} would not be.
+     */
+    purgeDomainQueues(): Promise<void>;
     setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
-    private forEachManager;
-    private sumOverManagers;
-    private everyManager;
     /**
      * Returns the next request from a domain that is not backing off, or from the inner manager.
      *