@crawlee/basic 4.0.0-beta.18 → 4.0.0-beta.181

package/internals/throttling_request_manager.d.ts

@@ -0,0 +1,216 @@
+import type { Dictionary } from '@crawlee/types';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, Configuration, IRequestManager, PacingSignal, RequestQueueOperationInfo, RequestQueueOperationOptions, RequestsLike, Source, StorageIdentifier, RequestSourceStatus, StorageOpenOptions } from '@crawlee/core';
+import { Request } from '@crawlee/core';
+/**
+ * Opens a request manager, matching the shape of storage `open` methods such as
+ * {@link RequestQueue.open|`RequestQueue.open`}.
+ *
+ * {@link ThrottlingRequestManager} calls this once per configured domain, so every per-domain queue shares the
+ * concrete type and storage backend of the manager being wrapped.
+ */
+export type RequestManagerOpener<T extends IRequestManager = IRequestManager> = (identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions) => Promise<T>;
+/** Options for {@link ThrottlingRequestManager}. */
+export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRequestManager> {
+    /**
+     * The request manager to wrap, usually a {@link RequestQueue}. Requests for domains that are not throttled
+     * are stored here. May be a factory, so that the throttler can be constructed synchronously and the manager
+     * under it opened only on first use.
+     *
+     * Omitted, the default request queue is opened on first use through
+     * {@link ThrottlingRequestManagerOptions.requestManagerOpener|`requestManagerOpener`}.
+     */
+    inner?: T | (() => T | Promise<T>);
+    /**
+     * Which domains to throttle: a list of hostnames, or `'all'` for every domain the crawl encounters.
+     *
+     * Matching a listed hostname is case-insensitive and exact - wildcards such as `*.example.com` are not
+     * supported, so list each subdomain you care about (or set
+     * {@link ThrottlingRequestManagerOptions.throttleBy|`throttleBy: 'registrableDomain'`}). An
+     * internationalized domain may be given in either its unicode or its punycode form, and an IPv6 address has
+     * to be bracketed (`[::1]`). Requests for any other domain bypass throttling entirely.
+     *
+     * `'all'` gives each domain a queue of its own the first time it is seen, so that it can be held back
+     * without its requests being repeatedly popped and re-enqueued. One request queue per domain is not free,
+     * which is what {@link ThrottlingRequestManagerOptions.maxThrottledDomains|`maxThrottledDomains`} is
+     * there to bound.
+     */
+    domains: string[] | 'all';
+    /**
+     * A floor under the crawl delay of every throttled domain, in seconds - the proactive clock described on
+     * {@link ThrottlingRequestManager}. A domain whose robots.txt asks for a longer `Crawl-delay` gets the
+     * longer one; this is a minimum, not an override. A `minIntervalEverywhere` {@link PacingSignal} raises
+     * this floor at runtime, and never lowers it.
+     * @default 0
+     */
+    minCrawlDelaySecs?: number;
+    /**
+     * What counts as "the same domain": the exact hostname, or the registrable domain it belongs to
+     * (`example.com` for `www.example.com`, `a.example.co.uk` and so on). Hosts with no registrable domain -
+     * IP addresses, `localhost` - are always throttled per hostname.
+     *
+     * Grouping by registrable domain gives subdomains a single pair of clocks and a single queue, which is what
+     * you want when the pacing is there to be polite to one server rather than to satisfy a specific host's
+     * rate limit.
+     * @default 'hostname'
+     */
+    throttleBy?: 'hostname' | 'registrableDomain';
+    /**
+     * The most domains a run may throttle at once. Exceeding it throws, rather than silently letting the
+     * throttling lapse - one request queue per domain is not free, and a crawl that discovers domains without
+     * bound would drown the storage backend in them.
+     *
+     * Only domains discovered under `domains: 'all'` count against this; an explicit list is taken at face value.
+     * @default 100
+     */
+    maxThrottledDomains?: number;
+    /**
+     * The key under which the discovered domain list is kept in the default key-value store, so that a restart
+     * with `purgeOnStart` disabled reopens their queues instead of stranding whatever they still hold. Only
+     * written under `domains: 'all'`.
+     *
+     * Give each manager its own key when running several of them against the same storage.
+     * @default 'CRAWLEE_THROTTLED_DOMAINS'
+     */
+    persistStateKey?: string;
+    /**
+     * Opens the per-domain queues, one per throttled domain, each under the alias `throttled-<domain>`.
+     * @default RequestQueue.open
+     */
+    requestManagerOpener?: RequestManagerOpener<T>;
+    /**
+     * The delay applied after a domain's first HTTP 429, doubled on each subsequent one.
+     * @default 2
+     */
+    baseDelaySecs?: number;
+    /**
+     * Upper bound on the delay between requests to a rate-limited domain, applied to both the exponential
+     * backoff and a `Retry-After` value.
+     * @default 60
+     */
+    maxDelaySecs?: number;
+    /**
+     * How long a domain may rate-limit us without a single request getting through before the crawl is
+     * abandoned with a {@link PersistentRateLimitError}.
+     *
+     * A domain that keeps answering 429 for this long is not going to be crawled by waiting longer - the
+     * concurrency is too high for it, or it has blocked us outright. Its requests are deliberately left in
+     * their queue, so re-running the crawl with `purgeOnStart` disabled picks them up once the domain recovers.
+     *
+     * A crawler running with `keepAlive` is exempt - outliving a domain that will not let us through is the
+     * whole point there.
+     * @default 900
+     */
+    maxDomainStallSecs?: number;
+}
+/**
+ * A request manager that wraps another one and paces requests per domain.
+ *
+ * Requests for a throttled domain are routed into their own queue when they are added, so each request lives in
+ * exactly one place and deduplication keeps working. Everything else goes to the wrapped manager untouched.
+ *
+ * {@link ThrottlingRequestManager.fetchNextRequest|`fetchNextRequest()`} serves the domain that has been waiting
+ * longest and skips any that are backing off, falling back to the wrapped manager. It never blocks: while every
+ * remaining request belongs to a throttled domain it returns `null` and
+ * {@link ThrottlingRequestManager.checkReadiness|`checkReadiness()`} reports `waiting` with the moment the
+ * earliest of them comes due, so the crawler idles instead of holding a concurrency slot open.
+ *
+ * Each throttled domain runs two independent clocks, and may be dispatched to once **both** have run out:
+ * - **Backoff**, set by HTTP 429 responses - honouring `Retry-After`, and otherwise doubling from `baseDelaySecs`.
+ *   Reactive and temporary: it decays once the domain stops turning us away. The crawlers report the 429s
+ *   themselves; a request held back this way is retried later without counting against `maxRequestRetries` and
+ *   without penalising its session.
+ * - **Crawl delay**, the minimum interval between two dispatches to the domain, armed after each one. Proactive
+ *   and constant: whatever the domain's robots.txt asks for, floored by
+ *   {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`}. Either may be absent, in
+ *   which case the other one is the delay.
+ *
+ * Which domains get those clocks is {@link ThrottlingRequestManagerOptions.domains|`domains`} - a list, or
+ * `'all'` for every domain the crawl encounters.
+ *
+ * Pass one as a crawler's `requestManager`; the `sameDomainDelaySecs` shorthand builds one with `domains: 'all'`
+ * and `throttleBy: 'registrableDomain'`. Construct it yourself to name the domains or tune the delays - one
+ * covering every domain also makes `sameDomainDelaySecs` land on it as a floor rather than adding a second pacer.
+ *
+ * Signals - 429s, robots.txt `Crawl-delay`, that floor - arrive through
+ * {@link IRequestManager.recordPacingSignal|`recordPacingSignal`}, which wrapping managers forward, so this
+ * works wherever it sits in a composition, including inside a {@link RequestManagerTandem}.
+ *
+ * **Example usage:**
+ *
+ * ```ts
+ * const crawler = new CheerioCrawler({
+ *     requestManager: new ThrottlingRequestManager({
+ *         domains: ['api.example.com', 'slow-site.org'],
+ *     }),
+ *     requestHandler: async ({ request }) => { ... },
+ * });
+ * ```
+ *
+ * @category Sources
+ */
+export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager {
+    #private;
+    private readonly config;
+    private readonly domainStates;
+    private readonly log;
+    constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);
+    /**
+     * The wrapped manager, holding every request whose domain is not throttled. `undefined` until an `inner`
+     * passed as a factory is resolved - reading this never forces it, because a getter should not open a queue
+     * behind a caller's back.
+     */
+    get innerManager(): T | undefined;
+    /**
+     * Records a pacing signal: a refusal puts the URL's domain into backoff, a declared interval becomes its
+     * crawl delay, and a crawl-wide floor raises {@link recordEverywhereFloor|the floor under all of them}.
+     *
+     * @returns `false` if the domain the signal covers is not throttled, in which case this is a no-op.
+     * @throws If the signal's scope is one this manager cannot honour - see {@link assertScopeHonourable}.
+     * @inheritdoc
+     */
+    recordPacingSignal(signal: PacingSignal): boolean;
+    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    /**
+     * Adds requests in batches, routing each one to the manager that owns its domain.
+     *
+     * Batching, validation, deduplication and `Retry-After`-free bookkeeping are all delegated to the target
+     * managers - this only decides where each request goes, one batch at a time, so a lazy or unbounded input
+     * iterable is never fully materialized.
+     */
+    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+    reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+    markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
+    getTotalCount(): Promise<number>;
+    getPendingCount(): Promise<number>;
+    getHandledCount(): Promise<number>;
+    /**
+     * Reports whether anything can be dispatched right now, and if not, when — or why never.
+     *
+     * One traversal of the domain clocks answers all of it: only domains whose delays have run out are probed,
+     * the rest merely contribute the moment they come due. Throttled requests count as outstanding work, so a
+     * crawler gated on this idles for the backoff instead of concluding it is done.
+     *
+     * `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,
+     * so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -
+     * see the traversal.
+     */
+    checkReadiness(): Promise<RequestSourceStatus>;
+    /**
+     * Empties every manager and clears the accumulated backoff. A robots.txt `Crawl-delay` is a property of the
+     * site rather than of the run, so it survives.
+     */
+    purge(): Promise<void>;
+    setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+    /**
+     * Returns the next request from a domain that is not backing off, or from the inner manager.
+     *
+     * Returns `null` while every remaining request belongs to a throttled domain - it never waits the backoff
+     * out, because a consumer parked in here holds a concurrency slot, which the autoscaler reads as spare
+     * capacity and answers by scaling up. Callers poll instead, and
+     * {@link ThrottlingRequestManager.checkReadiness|`checkReadiness()`} reports `waiting` meanwhile so the
+     * crawler's task loop idles rather than spins.
+     */
+    fetchNextRequest<R extends Dictionary = Dictionary>(): Promise<Request<R> | null>;
+    [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
+    drop(): Promise<void>;
+}