@crawlee/core 4.0.0-beta.123 → 4.0.0-beta.125

package/storages/throttling_request_manager.d.ts

@@ -37,15 +37,58 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
      */
     inner: T;
     /**
-     * Hostnames to throttle. Matching is case-insensitive and exact - wildcards such as `*.example.com` are not
-     * supported, so list each subdomain you care about. Requests for any other domain bypass throttling entirely.
+     * Which domains to throttle: a list of hostnames, or `'all'` for every domain the crawl encounters.
      *
-     * An internationalized domain may be given in either its unicode or its punycode form, and an IPv6 address
-     * has to be bracketed (`[::1]`).
+     * Matching a listed hostname is case-insensitive and exact - wildcards such as `*.example.com` are not
+     * supported, so list each subdomain you care about (or set
+     * {@link ThrottlingRequestManagerOptions.throttleBy|`throttleBy: 'registrableDomain'`}). An
+     * internationalized domain may be given in either its unicode or its punycode form, and an IPv6 address has
+     * to be bracketed (`[::1]`). Requests for any other domain bypass throttling entirely.
+     *
+     * `'all'` gives each domain a queue of its own the first time it is seen, so that it can be held back
+     * without its requests being repeatedly popped and re-enqueued. One request queue per domain is not free,
+     * which is what {@link ThrottlingRequestManagerOptions.maxThrottledDomains|`maxThrottledDomains`} is
+     * there to bound.
+     */
+    domains: string[] | 'all';
+    /**
+     * A floor under the crawl delay of every throttled domain, in seconds - the proactive clock described on
+     * {@link ThrottlingRequestManager}. A domain whose robots.txt asks for a longer `Crawl-delay` gets the
+     * longer one; this is a minimum, not an override.
+     * @default 0
+     */
+    minCrawlDelaySecs?: number;
+    /**
+     * What counts as "the same domain": the exact hostname, or the registrable domain it belongs to
+     * (`example.com` for `www.example.com`, `a.example.co.uk` and so on). Hosts with no registrable domain -
+     * IP addresses, `localhost` - are always throttled per hostname.
+     *
+     * Grouping by registrable domain gives subdomains a single pair of clocks and a single queue, which is what
+     * you want when the pacing is there to be polite to one server rather than to satisfy a specific host's
+     * rate limit.
+     * @default 'hostname'
+     */
+    throttleBy?: 'hostname' | 'registrableDomain';
+    /**
+     * The most domains a run may throttle at once. Exceeding it throws, rather than silently letting the
+     * throttling lapse - one request queue per domain is not free, and a crawl that discovers domains without
+     * bound would drown the storage backend in them.
+     *
+     * Only domains discovered under `domains: 'all'` count against this; an explicit list is taken at face value.
+     * @default 100
      */
-    domains: string[];
+    maxThrottledDomains?: number;
     /**
-     * Opens the per-domain queues, one per entry in `domains`, each under the alias `throttled-<domain>`.
+     * The key under which the discovered domain list is kept in the default key-value store, so that a restart
+     * with `purgeOnStart` disabled reopens their queues instead of stranding whatever they still hold. Only
+     * written under `domains: 'all'`.
+     *
+     * Give each manager its own key when running several of them against the same storage.
+     * @default 'CRAWLEE_THROTTLED_DOMAINS'
+     */
+    persistStateKey?: string;
+    /**
+     * Opens the per-domain queues, one per throttled domain, each under the alias `throttled-<domain>`.
      * @default RequestQueue.open
      */
     requestManagerOpener?: RequestManagerOpener<T>;
@@ -77,22 +120,26 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
 /**
  * A request manager that wraps another one and paces requests per domain.
  *
- * Requests for the configured {@link ThrottlingRequestManagerOptions.domains|`domains`} are routed into their own
- * queue when they are added, so each request lives in exactly one place and deduplication keeps working. Everything
- * else goes to the wrapped manager untouched.
+ * Requests for a throttled domain are routed into their own queue when they are added, so each request lives in
+ * exactly one place and deduplication keeps working. Everything else goes to the wrapped manager untouched.
  *
  * {@link ThrottlingRequestManager.fetchNextRequest|`fetchNextRequest()`} serves the domain that has been waiting
  * longest and skips any that are backing off, falling back to the wrapped manager. It never blocks: while every
  * remaining request belongs to a throttled domain it returns `null` and {@link ThrottlingRequestManager.isEmpty}
  * reports `true`, so the crawler idles instead of holding a concurrency slot open.
  *
- * Delays come from two places:
- * - HTTP 429 responses, honouring `Retry-After` and otherwise backing off exponentially. The crawlers report these
- *   automatically; a request that is throttled is retried later without counting against `maxRequestRetries` and
+ * Each throttled domain runs two independent clocks, and may be dispatched to once **both** have run out:
+ * - **Backoff**, set by HTTP 429 responses - honouring `Retry-After`, and otherwise doubling from `baseDelaySecs`.
+ *   Reactive and temporary: it decays once the domain stops turning us away. The crawlers report the 429s
+ *   themselves; a request held back this way is retried later without counting against `maxRequestRetries` and
  *   without penalising its session.
- * - robots.txt `Crawl-delay` directives, when `respectRobotsTxtFile` is enabled.
+ * - **Crawl delay**, the minimum interval between two dispatches to the domain, armed after each one. Proactive
+ *   and constant: whatever the domain's robots.txt asks for, floored by
+ *   {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`}. Either may be absent, in
+ *   which case the other one is the delay.
  *
- * This is opt-in: throttling only happens for a domain you list explicitly.
+ * Which domains get those clocks is {@link ThrottlingRequestManagerOptions.domains|`domains`} - a list, or
+ * `'all'` for every domain the crawl encounters.
  *
  * **Example usage:**
  *
@@ -109,40 +156,13 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
  * @category Sources
  */
 export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager, SupportsDomainThrottling {
+    #private;
     private readonly config;
-    private readonly inner;
-    private readonly requestManagerOpener;
-    private readonly baseDelayMs;
-    private readonly maxDelayMs;
-    private readonly maxDomainStallMs;
     private readonly domainStates;
-    private readonly subManagers;
     private readonly log;
-    /**
-     * Sub-managers are keyed by a stable alias, so with `purgeOnStart` disabled they outlive the process. They
-     * must therefore be reopened for every configured domain rather than created on first insert - otherwise a
-     * restart sees an empty map, reports the crawl finished, and strands whatever the previous run left in them.
-     */
-    private subManagersReady?;
-    /** Batches still being added in the background; keeps {@link ThrottlingRequestManager.isFinished} honest. */
-    private inProgressBatchCount;
-    private readonly warnedAbout;
-    private get hasThrottledDomains();
     constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);
     /** The wrapped manager, holding every request whose domain is not throttled. */
     get innerManager(): T;
-    /** Warns once about sources that cannot be routed by domain, because their URLs are not known yet. */
-    private warnIfNotRoutable;
-    private warnOnce;
-    private extractDomain;
-    private getDomainState;
-    private selectManager;
-    /** Only valid once {@link ThrottlingRequestManager.ensureSubManagers} has resolved. */
-    private managerForUrl;
-    private ensureSubManagers;
-    private getSubManagers;
-    /** Configured domains that are not currently backing off, longest-overdue first. */
-    private fetchableDomains;
     /**
      * Records a 429 response and puts the URL's domain into backoff.
      *
@@ -150,11 +170,12 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
      */
     recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
     /**
-     * Applies a robots.txt `Crawl-delay` to the URL's domain, as a minimum interval between dispatches.
+     * Records the `Crawl-delay` a domain's robots.txt asked for, which becomes its crawl delay unless
+     * {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`} asks for longer.
      *
      * The first value wins, so a robots.txt re-fetch cannot change the cadence mid-crawl.
      *
-     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     * @returns `false` if the domain is not throttled, in which case this is a no-op.
      */
     setCrawlDelay(url: string, delaySeconds: number): boolean;
     /**
@@ -167,8 +188,6 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
      * `Crawl-delay` is being obeyed, not stonewalled.
      */
     assertNoStalledDomains(): Promise<void>;
-    /** Records that a domain let a request through, which ends any rate-limit run stall detection was timing. */
-    private recordProgress;
     addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
     /**
      * Adds requests in batches, routing each one to the manager that owns its domain.
@@ -197,10 +216,14 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
      * site rather than of the run, so it survives.
      */
     purge(): Promise<void>;
+    /**
+     * Empties the per-domain queues, leaving the wrapped manager alone.
+     *
+     * Those queues are this manager's own no matter who owns the one it wraps, which is what makes this safe to
+     * call where a full {@link ThrottlingRequestManager.purge|`purge()`} would not be.
+     */
+    purgeDomainQueues(): Promise<void>;
     setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
-    private forEachManager;
-    private sumOverManagers;
-    private everyManager;
     /**
      * Returns the next request from a domain that is not backing off, or from the inner manager.
      *