@crawlee/core 4.0.0-beta.146 → 4.0.0-beta.147

package/storages/throttling_request_manager.d.ts

@@ -1,7 +1,8 @@
 import type { Dictionary } from '@crawlee/types';
 import type { Configuration } from '../configuration.js';
 import type { Request, Source } from '../request.js';
-import type { IRequestManager, RequestsLike } from './request_manager.js';
+import type { RequestSourceStatus } from './request_loader.js';
+import type { IRequestManager, PacingSignal, RequestsLike } from './request_manager.js';
 import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';
 import type { StorageIdentifier } from './storage_instance_manager.js';
 import type { StorageOpenOptions } from './utils.js';
@@ -12,30 +13,18 @@ import type { StorageOpenOptions } from './utils.js';
  * {@link ThrottlingRequestManager} calls this once per configured domain, so every per-domain queue shares the
  * concrete type and storage backend of the manager being wrapped.
  */
-export type RequestManagerOpener<T extends IRequestManager = IRequestManager> = (identifier: string | StorageIdentifier, options?: StorageOpenOptions) => Promise<T>;
-/**
- * A request manager that can pace requests per domain, as {@link ThrottlingRequestManager} does.
- *
- * The crawlers detect this structurally rather than by type, so a wrapper can opt in by forwarding these three
- * methods without {@link IRequestManager} having to know that throttling exists.
- */
-export interface SupportsDomainThrottling {
-    /** @see {@link ThrottlingRequestManager.recordDomainDelay} */
-    recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
-    /** @see {@link ThrottlingRequestManager.setCrawlDelay} */
-    setCrawlDelay(url: string, delaySeconds: number): boolean;
-    /** @see {@link ThrottlingRequestManager.assertNoStalledDomains} */
-    assertNoStalledDomains(): Promise<void>;
-}
-/** Whether `manager` can pace requests per domain. */
-export declare function supportsDomainThrottling(manager: unknown): manager is SupportsDomainThrottling;
+export type RequestManagerOpener<T extends IRequestManager = IRequestManager> = (identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions) => Promise<T>;
 /** Options for {@link ThrottlingRequestManager}. */
 export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRequestManager> {
     /**
      * The request manager to wrap, usually a {@link RequestQueue}. Requests for domains that are not throttled
-     * are stored here.
+     * are stored here. May be a factory, so that the throttler can be constructed synchronously and the manager
+     * under it opened only on first use.
+     *
+     * Omitted, the default request queue is opened on first use through
+     * {@link ThrottlingRequestManagerOptions.requestManagerOpener|`requestManagerOpener`}.
      */
-    inner: T;
+    inner?: T | (() => T | Promise<T>);
     /**
      * Which domains to throttle: a list of hostnames, or `'all'` for every domain the crawl encounters.
      *
@@ -54,7 +43,8 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
     /**
      * A floor under the crawl delay of every throttled domain, in seconds - the proactive clock described on
      * {@link ThrottlingRequestManager}. A domain whose robots.txt asks for a longer `Crawl-delay` gets the
-     * longer one; this is a minimum, not an override.
+     * longer one; this is a minimum, not an override. A `minIntervalEverywhere` {@link PacingSignal} raises
+     * this floor at runtime, and never lowers it.
      * @default 0
      */
     minCrawlDelaySecs?: number;
@@ -125,8 +115,9 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
  *
  * {@link ThrottlingRequestManager.fetchNextRequest|`fetchNextRequest()`} serves the domain that has been waiting
  * longest and skips any that are backing off, falling back to the wrapped manager. It never blocks: while every
- * remaining request belongs to a throttled domain it returns `null` and {@link ThrottlingRequestManager.isEmpty}
- * reports `true`, so the crawler idles instead of holding a concurrency slot open.
+ * remaining request belongs to a throttled domain it returns `null` and
+ * {@link ThrottlingRequestManager.checkReadiness|`checkReadiness()`} reports `waiting` with the moment the
+ * earliest of them comes due, so the crawler idles instead of holding a concurrency slot open.
  *
  * Each throttled domain runs two independent clocks, and may be dispatched to once **both** have run out:
  * - **Backoff**, set by HTTP 429 responses - honouring `Retry-After`, and otherwise doubling from `baseDelaySecs`.
@@ -141,12 +132,19 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
  * Which domains get those clocks is {@link ThrottlingRequestManagerOptions.domains|`domains`} - a list, or
  * `'all'` for every domain the crawl encounters.
  *
+ * Pass one as a crawler's `requestManager`; the `sameDomainDelaySecs` shorthand builds one with `domains: 'all'`
+ * and `throttleBy: 'registrableDomain'`. Construct it yourself to name the domains or tune the delays - one
+ * covering every domain also makes `sameDomainDelaySecs` land on it as a floor rather than adding a second pacer.
+ *
+ * Signals - 429s, robots.txt `Crawl-delay`, that floor - arrive through
+ * {@link IRequestManager.recordPacingSignal|`recordPacingSignal`}, which wrapping managers forward, so this
+ * works wherever it sits in a composition, including inside a {@link RequestManagerTandem}.
+ *
  * **Example usage:**
  *
  * ```ts
  * const crawler = new CheerioCrawler({
  *     requestManager: new ThrottlingRequestManager({
- *         inner: await RequestQueue.open(),
  *         domains: ['api.example.com', 'slow-site.org'],
  *     }),
  *     requestHandler: async ({ request }) => { ... },
@@ -155,39 +153,27 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
  *
  * @category Sources
  */
-export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager, SupportsDomainThrottling {
+export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager {
     #private;
     private readonly config;
     private readonly domainStates;
     private readonly log;
     constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);
-    /** The wrapped manager, holding every request whose domain is not throttled. */
-    get innerManager(): T;
-    /**
-     * Records a 429 response and puts the URL's domain into backoff.
-     *
-     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
-     */
-    recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
     /**
-     * Records the `Crawl-delay` a domain's robots.txt asked for, which becomes its crawl delay unless
-     * {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`} asks for longer.
-     *
-     * The first value wins, so a robots.txt re-fetch cannot change the cadence mid-crawl.
-     *
-     * @returns `false` if the domain is not throttled, in which case this is a no-op.
+     * The wrapped manager, holding every request whose domain is not throttled. `undefined` until an `inner`
+     * passed as a factory is resolved - reading this never forces it, because a getter should not open a queue
+     * behind a caller's back.
      */
-    setCrawlDelay(url: string, delaySeconds: number): boolean;
+    get innerManager(): T | undefined;
     /**
-     * Throws {@link PersistentRateLimitError} if any domain has been rate-limiting us past
-     * {@link ThrottlingRequestManagerOptions.maxDomainStallSecs|`maxDomainStallSecs`} without letting a single
-     * request through.
+     * Records a pacing signal: a refusal puts the URL's domain into backoff, a declared interval becomes its
+     * crawl delay, and a crawl-wide floor raises {@link recordEverywhereFloor|the floor under all of them}.
      *
-     * A domain qualifies only while it still has queued requests and is actively rate-limiting - a domain that
-     * has simply run out of work is finished, not stalled, and one being waited out under a long robots.txt
-     * `Crawl-delay` is being obeyed, not stonewalled.
+     * @returns `false` if the domain the signal covers is not throttled, in which case this is a no-op.
+     * @throws If the signal's scope is one this manager cannot honour - see {@link assertScopeHonourable}.
+     * @inheritdoc
      */
-    assertNoStalledDomains(): Promise<void>;
+    recordPacingSignal(signal: PacingSignal): boolean;
     addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
     /**
      * Adds requests in batches, routing each one to the manager that owns its domain.
@@ -203,34 +189,31 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
     getPendingCount(): Promise<number>;
     getHandledCount(): Promise<number>;
     /**
-     * Whether the next {@link ThrottlingRequestManager.fetchNextRequest} would return `null`.
+     * Reports whether anything can be dispatched right now, and if not, when — or why never.
      *
-     * Requests waiting on a throttled domain count as unavailable, so a crawler whose task loop is gated on
-     * this idles for the backoff instead of spinning on a fetch that cannot succeed yet.
+     * One traversal of the domain clocks answers all of it: only domains whose delays have run out are probed,
+     * the rest merely contribute the moment they come due. Throttled requests count as outstanding work, so a
+     * crawler gated on this idles for the backoff instead of concluding it is done.
+     *
+     * `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,
+     * so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -
+     * see the traversal.
      */
-    isEmpty(): Promise<boolean>;
-    /** Unlike {@link ThrottlingRequestManager.isEmpty}, throttled requests still count as outstanding work. */
-    isFinished(): Promise<boolean>;
+    checkReadiness(): Promise<RequestSourceStatus>;
     /**
      * Empties every manager and clears the accumulated backoff. A robots.txt `Crawl-delay` is a property of the
      * site rather than of the run, so it survives.
      */
     purge(): Promise<void>;
-    /**
-     * Empties the per-domain queues, leaving the wrapped manager alone.
-     *
-     * Those queues are this manager's own no matter who owns the one it wraps, which is what makes this safe to
-     * call where a full {@link ThrottlingRequestManager.purge|`purge()`} would not be.
-     */
-    purgeDomainQueues(): Promise<void>;
     setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
     /**
      * Returns the next request from a domain that is not backing off, or from the inner manager.
      *
      * Returns `null` while every remaining request belongs to a throttled domain - it never waits the backoff
      * out, because a consumer parked in here holds a concurrency slot, which the autoscaler reads as spare
-     * capacity and answers by scaling up. Callers poll instead, and {@link ThrottlingRequestManager.isEmpty}
-     * reports `true` meanwhile so the crawler's task loop idles rather than spins.
+     * capacity and answers by scaling up. Callers poll instead, and
+     * {@link ThrottlingRequestManager.checkReadiness|`checkReadiness()`} reports `waiting` meanwhile so the
+     * crawler's task loop idles rather than spins.
      */
     fetchNextRequest<R extends Dictionary = Dictionary>(): Promise<Request<R> | null>;
     [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;