@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.131

package/storages/storage_instance_manager.d.ts

@@ -0,0 +1,87 @@
+import type { Constructor, DatasetBackend, KeyValueStoreBackend, RequestQueueBackend, StorageBackend, StorageIdentifier } from '@crawlee/types';
+export type { StorageIdentifier } from '@crawlee/types';
+/**
+ * Matches an `IStorage` – a storage "frontend" (Dataset, KeyValueStore, RequestQueue).
+ */
+export interface IStorage {
+    id: string;
+    name?: string;
+}
+type Hashable = string;
+/**
+ * Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
+ *
+ * A single instance manages all storage types. Instances are cached by
+ * `(storageClass, id/name/alias, backendCacheKey)` so the same storage is never opened twice.
+ *
+ * The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
+ * and a pre-bound `backendOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
+ * assigns a reserved default alias.
+ */
+export declare class StorageInstanceManager {
+    #private;
+    /**
+     * Open (or retrieve from cache) a storage instance.
+     *
+     * @param cls               The storage class constructor (e.g. `Dataset`, `KeyValueStore`, `RequestQueue`).
+     * @param id                Storage ID (mutually exclusive with `name` and `alias`).
+     * @param name              Storage name (mutually exclusive with `id` and `alias`).
+     * @param alias             Run-scoped alias (mutually exclusive with `id` and `name`).
+     *                          Automatically assigned when no identifier is provided.
+     * @param backendOpener      A **lazy** factory that creates the sub-backend.
+     *                          Only called on a cache miss.
+     * @param backendCacheKey    Opaque key identifying the storage backend, so that the same logical
+     *                          storage opened through different clients is cached separately.
+     */
+    openStorage<TStorage extends IStorage>(cls: Constructor<TStorage>, { id, name, alias, backendOpener, backendCacheKey, }: (ExplicitStorageIdentifier | DefaultStorageIdentifier) & {
+        backendOpener: () => Promise<DatasetBackend | KeyValueStoreBackend | RequestQueueBackend>;
+        backendCacheKey: Hashable;
+    }): Promise<TStorage>;
+    /**
+     * Remove a storage instance from the cache (called from `storage.drop()`).
+     */
+    removeFromCache(instance: IStorage): void;
+    /**
+     * Clear the entire cache. Also calls `clearCache()` on any cached KeyValueStore
+     * instances (duck-typed to avoid importing KeyValueStore and circular dependencies).
+     * Called during service locator reset.
+     */
+    clearCache(): void;
+}
+/**
+ * A storage identifier where exactly one of `id`, `name`, or `alias` is specified.
+ * Produced by {@link resolveStorageIdentifier} from ambiguous user input.
+ */
+export type ExplicitStorageIdentifier = {
+    id: string;
+    name?: never;
+    alias?: never;
+} | {
+    id?: never;
+    name: string;
+    alias?: never;
+} | {
+    id?: never;
+    name?: never;
+    alias: string;
+};
+/**
+ * Represents the case where no identifier was provided — the caller wants the default storage.
+ */
+export interface DefaultStorageIdentifier {
+    id?: never;
+    name?: never;
+    alias?: never;
+}
+/**
+ * Decompose a user-provided `identifier` (the `Dataset.open()` / `KeyValueStore.open()` /
+ * `RequestQueue.open()` argument) into separate `id`, `name`, and `alias` fields that
+ * the `StorageInstanceManager` and `StorageBackend.create*Client` expect.
+ *
+ * - `null` / `undefined` / `{}` → default storage alias
+ * - `string` → resolved via `storageExists` (ID-first, then name)
+ * - `{ id }` → `{ id }`
+ * - `{ name }` → `{ name }`
+ * - `{ alias }` → `{ alias }`
+ */
+export declare function resolveStorageIdentifier(identifier: string | StorageIdentifier | null | undefined, storageBackend: StorageBackend, storageType: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<ExplicitStorageIdentifier>;

package/storages/storage_instance_manager.js

@@ -0,0 +1,256 @@
+import { AsyncQueue } from '@sapphire/async-queue';
+/** Reserved alias for the default (unnamed) storage. */
+const DEFAULT_STORAGE_ALIAS = '__default__';
+/**
+ * Three-tier cache for storage instances, modelled after crawlee-python's `_StorageCache`.
+ *
+ * Each tier maps `[storageClass][key][backendCacheKey] → instance`:
+ *   - `byId`    — keyed by the backend-assigned storage id
+ *   - `byName`  — keyed by the persistent storage name
+ *   - `byAlias` — keyed by a run-scoped alias (e.g. `'__default__'` for unnamed storages)
+ */
+class StorageCache {
+    byId = new Map();
+    byName = new Map();
+    byAlias = new Map();
+    get(cls, { id, name, alias, backendCacheKey, }) {
+        for (const [tier, key] of [
+            [this.byId, id],
+            [this.byName, name],
+            [this.byAlias, alias],
+        ]) {
+            if (key === undefined)
+                continue;
+            const cached = tier.get(cls)?.get(key)?.get(backendCacheKey);
+            if (cached) {
+                if (cached instanceof cls) {
+                    return cached;
+                }
+                throw new Error('Cached storage instance type mismatch.');
+            }
+        }
+        return undefined;
+    }
+    /** Write a single entry into a given tier. */
+    setInMap(tier, cls, key, instance, backendCacheKey) {
+        if (!tier.has(cls))
+            tier.set(cls, new Map());
+        const keyMap = tier.get(cls);
+        if (!keyMap.has(key))
+            keyMap.set(key, new Map());
+        keyMap.get(key).set(backendCacheKey, instance);
+    }
+    /**
+     * Cache an instance under its actual id, name, and an optional alias.
+     */
+    set(cls, instance, backendCacheKey, alias) {
+        // Always cache by id.
+        this.setInMap(this.byId, cls, instance.id, instance, backendCacheKey);
+        // Cache by name — only for named storages.
+        if (instance.name) {
+            this.setInMap(this.byName, cls, instance.name, instance, backendCacheKey);
+        }
+        // Cache by alias — only for unnamed storages opened via alias.
+        if (alias !== undefined) {
+            this.setInMap(this.byAlias, cls, alias, instance, backendCacheKey);
+        }
+    }
+    removeFromCache(instance) {
+        const storageType = instance.constructor;
+        for (const tier of [this.byId, this.byName, this.byAlias]) {
+            const classMap = tier.get(storageType);
+            if (!classMap)
+                continue;
+            for (const keyMap of classMap.values()) {
+                for (const [cacheKey, cached] of keyMap) {
+                    if (cached === instance) {
+                        keyMap.delete(cacheKey);
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Ensure that the same string is not used as both a name and an alias for the same
+     * storage class + backend combination. Mirrors crawlee-python's `_check_name_alias_conflict`.
+     */
+    checkNameAliasConflict(cls, { name, alias, backendCacheKey }) {
+        if (alias) {
+            const existingByName = this.byName.get(cls)?.get(alias)?.get(backendCacheKey);
+            if (existingByName) {
+                throw new Error(`Cannot open storage with alias "${alias}" because a named storage with the same identifier already exists.`);
+            }
+        }
+        if (name) {
+            const existingByAlias = this.byAlias.get(cls)?.get(name)?.get(backendCacheKey);
+            if (existingByAlias) {
+                throw new Error(`Cannot open storage with name "${name}" because an alias storage with the same identifier already exists.` +
+                    ` If you meant to open the alias storage, use { alias: "${name}" } instead.`);
+            }
+        }
+    }
+    /** Iterate all cached instances across all storage types. */
+    *allValues() {
+        const seen = new Set();
+        for (const classMap of this.byId.values()) {
+            for (const keyMap of classMap.values()) {
+                for (const instance of keyMap.values()) {
+                    if (!seen.has(instance)) {
+                        seen.add(instance);
+                        yield instance;
+                    }
+                }
+            }
+        }
+    }
+    clear() {
+        this.byId.clear();
+        this.byName.clear();
+        this.byAlias.clear();
+    }
+}
+/**
+ * Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
+ *
+ * A single instance manages all storage types. Instances are cached by
+ * `(storageClass, id/name/alias, backendCacheKey)` so the same storage is never opened twice.
+ *
+ * The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
+ * and a pre-bound `backendOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
+ * assigns a reserved default alias.
+ */
+export class StorageInstanceManager {
+    #cache = new StorageCache();
+    #openerLocks = new Map();
+    /**
+     * Open (or retrieve from cache) a storage instance.
+     *
+     * @param cls               The storage class constructor (e.g. `Dataset`, `KeyValueStore`, `RequestQueue`).
+     * @param id                Storage ID (mutually exclusive with `name` and `alias`).
+     * @param name              Storage name (mutually exclusive with `id` and `alias`).
+     * @param alias             Run-scoped alias (mutually exclusive with `id` and `name`).
+     *                          Automatically assigned when no identifier is provided.
+     * @param backendOpener      A **lazy** factory that creates the sub-backend.
+     *                          Only called on a cache miss.
+     * @param backendCacheKey    Opaque key identifying the storage backend, so that the same logical
+     *                          storage opened through different clients is cached separately.
+     */
+    async openStorage(cls, { id, name, alias, backendOpener, backendCacheKey, }) {
+        // Auto-set alias='__default__' when no parameters are specified (mirrors crawlee-python).
+        if (!id && !name && !alias) {
+            alias = DEFAULT_STORAGE_ALIAS;
+        }
+        // Fast-path cache check (no lock).
+        if (alias !== undefined) {
+            const cached = this.#cache.get(cls, { alias, backendCacheKey });
+            if (cached)
+                return cached;
+        }
+        else if (id) {
+            const cached = this.#cache.get(cls, { id, backendCacheKey });
+            if (cached)
+                return cached;
+        }
+        else if (name) {
+            const cached = this.#cache.get(cls, { name, backendCacheKey });
+            if (cached)
+                return cached;
+        }
+        const identifierKey = id ?? name ?? alias ?? DEFAULT_STORAGE_ALIAS;
+        const lockKey = `${cls.name}:${identifierKey}:${backendCacheKey}`;
+        if (!this.#openerLocks.has(lockKey)) {
+            this.#openerLocks.set(lockKey, new AsyncQueue());
+        }
+        const queue = this.#openerLocks.get(lockKey);
+        await queue.wait();
+        try {
+            // Double-check cache under lock (another caller may have filled it while we waited).
+            if (alias !== undefined) {
+                const cached = this.#cache.get(cls, { alias, backendCacheKey });
+                if (cached)
+                    return cached;
+            }
+            else if (id) {
+                const cached = this.#cache.get(cls, { id, backendCacheKey });
+                if (cached)
+                    return cached;
+            }
+            else if (name) {
+                const cached = this.#cache.get(cls, { name, backendCacheKey });
+                if (cached)
+                    return cached;
+            }
+            // Prevent the same string from being used as both a name and an alias.
+            this.#cache.checkNameAliasConflict(cls, { name, alias, backendCacheKey });
+            // Cache miss — create the sub-backend and storage instance.
+            const subBackend = await backendOpener();
+            const storageInfo = await subBackend.getMetadata();
+            // Storage frontends are thin wrappers over the backend. We hand them the resolved metadata
+            // we just fetched (so `id`/`name` etc. are available synchronously) along with the backend.
+            const instance = new cls({ metadata: storageInfo, backend: subBackend });
+            // Atomic cache writes (no awaits between these).
+            this.#cache.set(cls, instance, backendCacheKey, alias);
+            return instance;
+        }
+        finally {
+            queue.shift();
+            // Clean up idle locks so the map doesn't grow unboundedly
+            // (mirrors crawlee-python's WeakValueDictionary behaviour).
+            if (queue.remaining === 0) {
+                this.#openerLocks.delete(lockKey);
+            }
+        }
+    }
+    /**
+     * Remove a storage instance from the cache (called from `storage.drop()`).
+     */
+    removeFromCache(instance) {
+        this.#cache.removeFromCache(instance);
+    }
+    /**
+     * Clear the entire cache. Also calls `clearCache()` on any cached KeyValueStore
+     * instances (duck-typed to avoid importing KeyValueStore and circular dependencies).
+     * Called during service locator reset.
+     */
+    clearCache() {
+        for (const instance of this.#cache.allValues()) {
+            if ('clearCache' in instance && typeof instance.clearCache === 'function') {
+                instance.clearCache();
+            }
+        }
+        this.#cache.clear();
+    }
+}
+/**
+ * Decompose a user-provided `identifier` (the `Dataset.open()` / `KeyValueStore.open()` /
+ * `RequestQueue.open()` argument) into separate `id`, `name`, and `alias` fields that
+ * the `StorageInstanceManager` and `StorageBackend.create*Client` expect.
+ *
+ * - `null` / `undefined` / `{}` → default storage alias
+ * - `string` → resolved via `storageExists` (ID-first, then name)
+ * - `{ id }` → `{ id }`
+ * - `{ name }` → `{ name }`
+ * - `{ alias }` → `{ alias }`
+ */
+export async function resolveStorageIdentifier(identifier, storageBackend, storageType) {
+    if (identifier === null || identifier === undefined) {
+        return { alias: DEFAULT_STORAGE_ALIAS };
+    }
+    if (typeof identifier === 'string') {
+        if (storageBackend.storageExists && (await storageBackend.storageExists(identifier, storageType))) {
+            return { id: identifier };
+        }
+        return { name: identifier };
+    }
+    if (identifier.id) {
+        return { id: identifier.id };
+    }
+    if (identifier.name) {
+        return { name: identifier.name };
+    }
+    if ('alias' in identifier && identifier.alias) {
+        return { alias: identifier.alias };
+    }
+    // Empty object — treated as default storage.
+    return { alias: DEFAULT_STORAGE_ALIAS };
+}

package/storages/storage_stats.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Backend-independent usage counters tracked by the storage frontend classes
+ * ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}).
+ *
+ * These count the operations the frontend issues against its underlying storage backend, so they are
+ * meaningful for any storage backend (memory, file system, cloud). They are tallied per client call
+ * — e.g. iterating a key-value store increments `readCount` once per record fetched and `listCount`
+ * once per listed page. Backend-specific figures that the frontend cannot compute (such as the number
+ * of bytes stored) are intentionally not included here; read those from the backend's own API instead.
+ */
+/** Usage counters for a {@link Dataset}. */
+export interface DatasetStats {
+    /** Number of read operations issued to the dataset client (e.g. `getData`). */
+    readCount: number;
+    /** Number of write operations issued to the dataset client (e.g. `pushData`). */
+    writeCount: number;
+}
+/** Usage counters for a {@link KeyValueStore}. */
+export interface KeyValueStoreStats {
+    /** Number of read operations issued to the key-value store client (e.g. `getValue`). */
+    readCount: number;
+    /** Number of write operations issued to the key-value store client (e.g. `setValue`). */
+    writeCount: number;
+    /** Number of delete operations issued to the key-value store client (e.g. `deleteValue`). */
+    deleteCount: number;
+    /** Number of listing operations issued to the key-value store client (e.g. `listKeys`). */
+    listCount: number;
+}
+/** Usage counters for a {@link RequestQueue}. */
+export interface RequestQueueStats {
+    /** Number of write operations issued to the request queue client (add / handle / reclaim). */
+    writeCount: number;
+    /** Number of queue-head reads issued to the request queue client (`fetchNextRequest`). */
+    headItemReadCount: number;
+}
+/**
+ * A tiny mutable counter that the storage frontends increment on each client call and expose through
+ * a read-only `stats` snapshot. Generic over the concrete counter shape so each storage type gets only
+ * the buckets that make sense for it.
+ */
+export declare class StorageStatsTracker<T extends Record<keyof T, number>> {
+    #private;
+    constructor(initial: T);
+    /** Increment a counter bucket by `by` (default `1`). */
+    add(key: keyof T, by?: number): void;
+    /** Return a snapshot of the current counters. The returned object is a copy and safe to keep. */
+    get current(): T;
+}

package/storages/storage_stats.js

@@ -0,0 +1,29 @@
+/**
+ * Backend-independent usage counters tracked by the storage frontend classes
+ * ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}).
+ *
+ * These count the operations the frontend issues against its underlying storage backend, so they are
+ * meaningful for any storage backend (memory, file system, cloud). They are tallied per client call
+ * — e.g. iterating a key-value store increments `readCount` once per record fetched and `listCount`
+ * once per listed page. Backend-specific figures that the frontend cannot compute (such as the number
+ * of bytes stored) are intentionally not included here; read those from the backend's own API instead.
+ */
+/**
+ * A tiny mutable counter that the storage frontends increment on each client call and expose through
+ * a read-only `stats` snapshot. Generic over the concrete counter shape so each storage type gets only
+ * the buckets that make sense for it.
+ */
+export class StorageStatsTracker {
+    #counters;
+    constructor(initial) {
+        this.#counters = { ...initial };
+    }
+    /** Increment a counter bucket by `by` (default `1`). */
+    add(key, by = 1) {
+        this.#counters[key] += by;
+    }
+    /** Return a snapshot of the current counters. The returned object is a copy and safe to keep. */
+    get current() {
+        return { ...this.#counters };
+    }
+}

package/storages/throttling_request_manager.d.ts

@@ -0,0 +1,239 @@
+import type { Dictionary } from '@crawlee/types';
+import type { Configuration } from '../configuration.js';
+import type { Request, Source } from '../request.js';
+import type { IRequestManager, RequestsLike } from './request_manager.js';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';
+import type { StorageIdentifier } from './storage_instance_manager.js';
+import type { StorageOpenOptions } from './utils.js';
+/**
+ * Opens a request manager, matching the shape of storage `open` methods such as
+ * {@link RequestQueue.open|`RequestQueue.open`}.
+ *
+ * {@link ThrottlingRequestManager} calls this once per configured domain, so every per-domain queue shares the
+ * concrete type and storage backend of the manager being wrapped.
+ */
+export type RequestManagerOpener<T extends IRequestManager = IRequestManager> = (identifier: string | StorageIdentifier, options?: StorageOpenOptions) => Promise<T>;
+/**
+ * A request manager that can pace requests per domain, as {@link ThrottlingRequestManager} does.
+ *
+ * The crawlers detect this structurally rather than by type, so a wrapper can opt in by forwarding these three
+ * methods without {@link IRequestManager} having to know that throttling exists.
+ */
+export interface SupportsDomainThrottling {
+    /** @see {@link ThrottlingRequestManager.recordDomainDelay} */
+    recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
+    /** @see {@link ThrottlingRequestManager.setCrawlDelay} */
+    setCrawlDelay(url: string, delaySeconds: number): boolean;
+    /** @see {@link ThrottlingRequestManager.assertNoStalledDomains} */
+    assertNoStalledDomains(): Promise<void>;
+}
+/** Whether `manager` can pace requests per domain. */
+export declare function supportsDomainThrottling(manager: unknown): manager is SupportsDomainThrottling;
+/** Options for {@link ThrottlingRequestManager}. */
+export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRequestManager> {
+    /**
+     * The request manager to wrap, usually a {@link RequestQueue}. Requests for domains that are not throttled
+     * are stored here.
+     */
+    inner: T;
+    /**
+     * Which domains to throttle: a list of hostnames, or `'all'` for every domain the crawl encounters.
+     *
+     * Matching a listed hostname is case-insensitive and exact - wildcards such as `*.example.com` are not
+     * supported, so list each subdomain you care about (or set
+     * {@link ThrottlingRequestManagerOptions.throttleBy|`throttleBy: 'registrableDomain'`}). An
+     * internationalized domain may be given in either its unicode or its punycode form, and an IPv6 address has
+     * to be bracketed (`[::1]`). Requests for any other domain bypass throttling entirely.
+     *
+     * `'all'` gives each domain a queue of its own the first time it is seen, so that it can be held back
+     * without its requests being repeatedly popped and re-enqueued. One request queue per domain is not free,
+     * which is what {@link ThrottlingRequestManagerOptions.maxThrottledDomains|`maxThrottledDomains`} is
+     * there to bound.
+     */
+    domains: string[] | 'all';
+    /**
+     * A floor under the crawl delay of every throttled domain, in seconds - the proactive clock described on
+     * {@link ThrottlingRequestManager}. A domain whose robots.txt asks for a longer `Crawl-delay` gets the
+     * longer one; this is a minimum, not an override.
+     * @default 0
+     */
+    minCrawlDelaySecs?: number;
+    /**
+     * What counts as "the same domain": the exact hostname, or the registrable domain it belongs to
+     * (`example.com` for `www.example.com`, `a.example.co.uk` and so on). Hosts with no registrable domain -
+     * IP addresses, `localhost` - are always throttled per hostname.
+     *
+     * Grouping by registrable domain gives subdomains a single pair of clocks and a single queue, which is what
+     * you want when the pacing is there to be polite to one server rather than to satisfy a specific host's
+     * rate limit.
+     * @default 'hostname'
+     */
+    throttleBy?: 'hostname' | 'registrableDomain';
+    /**
+     * The most domains a run may throttle at once. Exceeding it throws, rather than silently letting the
+     * throttling lapse - one request queue per domain is not free, and a crawl that discovers domains without
+     * bound would drown the storage backend in them.
+     *
+     * Only domains discovered under `domains: 'all'` count against this; an explicit list is taken at face value.
+     * @default 100
+     */
+    maxThrottledDomains?: number;
+    /**
+     * The key under which the discovered domain list is kept in the default key-value store, so that a restart
+     * with `purgeOnStart` disabled reopens their queues instead of stranding whatever they still hold. Only
+     * written under `domains: 'all'`.
+     *
+     * Give each manager its own key when running several of them against the same storage.
+     * @default 'CRAWLEE_THROTTLED_DOMAINS'
+     */
+    persistStateKey?: string;
+    /**
+     * Opens the per-domain queues, one per throttled domain, each under the alias `throttled-<domain>`.
+     * @default RequestQueue.open
+     */
+    requestManagerOpener?: RequestManagerOpener<T>;
+    /**
+     * The delay applied after a domain's first HTTP 429, doubled on each subsequent one.
+     * @default 2
+     */
+    baseDelaySecs?: number;
+    /**
+     * Upper bound on the delay between requests to a rate-limited domain, applied to both the exponential
+     * backoff and a `Retry-After` value.
+     * @default 60
+     */
+    maxDelaySecs?: number;
+    /**
+     * How long a domain may rate-limit us without a single request getting through before the crawl is
+     * abandoned with a {@link PersistentRateLimitError}.
+     *
+     * A domain that keeps answering 429 for this long is not going to be crawled by waiting longer - the
+     * concurrency is too high for it, or it has blocked us outright. Its requests are deliberately left in
+     * their queue, so re-running the crawl with `purgeOnStart` disabled picks them up once the domain recovers.
+     *
+     * A crawler running with `keepAlive` is exempt - outliving a domain that will not let us through is the
+     * whole point there.
+     * @default 900
+     */
+    maxDomainStallSecs?: number;
+}
+/**
+ * A request manager that wraps another one and paces requests per domain.
+ *
+ * Requests for a throttled domain are routed into their own queue when they are added, so each request lives in
+ * exactly one place and deduplication keeps working. Everything else goes to the wrapped manager untouched.
+ *
+ * {@link ThrottlingRequestManager.fetchNextRequest|`fetchNextRequest()`} serves the domain that has been waiting
+ * longest and skips any that are backing off, falling back to the wrapped manager. It never blocks: while every
+ * remaining request belongs to a throttled domain it returns `null` and {@link ThrottlingRequestManager.isEmpty}
+ * reports `true`, so the crawler idles instead of holding a concurrency slot open.
+ *
+ * Each throttled domain runs two independent clocks, and may be dispatched to once **both** have run out:
+ * - **Backoff**, set by HTTP 429 responses - honouring `Retry-After`, and otherwise doubling from `baseDelaySecs`.
+ *   Reactive and temporary: it decays once the domain stops turning us away. The crawlers report the 429s
+ *   themselves; a request held back this way is retried later without counting against `maxRequestRetries` and
+ *   without penalising its session.
+ * - **Crawl delay**, the minimum interval between two dispatches to the domain, armed after each one. Proactive
+ *   and constant: whatever the domain's robots.txt asks for, floored by
+ *   {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`}. Either may be absent, in
+ *   which case the other one is the delay.
+ *
+ * Which domains get those clocks is {@link ThrottlingRequestManagerOptions.domains|`domains`} - a list, or
+ * `'all'` for every domain the crawl encounters.
+ *
+ * **Example usage:**
+ *
+ * ```ts
+ * const crawler = new CheerioCrawler({
+ *     requestManager: new ThrottlingRequestManager({
+ *         inner: await RequestQueue.open(),
+ *         domains: ['api.example.com', 'slow-site.org'],
+ *     }),
+ *     requestHandler: async ({ request }) => { ... },
+ * });
+ * ```
+ *
+ * @category Sources
+ */
+export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager, SupportsDomainThrottling {
+    #private;
+    private readonly config;
+    private readonly domainStates;
+    private readonly log;
+    constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);
+    /** The wrapped manager, holding every request whose domain is not throttled. */
+    get innerManager(): T;
+    /**
+     * Records a 429 response and puts the URL's domain into backoff.
+     *
+     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     */
+    recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
+    /**
+     * Records the `Crawl-delay` a domain's robots.txt asked for, which becomes its crawl delay unless
+     * {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`} asks for longer.
+     *
+     * The first value wins, so a robots.txt re-fetch cannot change the cadence mid-crawl.
+     *
+     * @returns `false` if the domain is not throttled, in which case this is a no-op.
+     */
+    setCrawlDelay(url: string, delaySeconds: number): boolean;
+    /**
+     * Throws {@link PersistentRateLimitError} if any domain has been rate-limiting us past
+     * {@link ThrottlingRequestManagerOptions.maxDomainStallSecs|`maxDomainStallSecs`} without letting a single
+     * request through.
+     *
+     * A domain qualifies only while it still has queued requests and is actively rate-limiting - a domain that
+     * has simply run out of work is finished, not stalled, and one being waited out under a long robots.txt
+     * `Crawl-delay` is being obeyed, not stonewalled.
+     */
+    assertNoStalledDomains(): Promise<void>;
+    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    /**
+     * Adds requests in batches, routing each one to the manager that owns its domain.
+     *
+     * Batching, validation, deduplication and `Retry-After`-free bookkeeping are all delegated to the target
+     * managers - this only decides where each request goes, one batch at a time, so a lazy or unbounded input
+     * iterable is never fully materialized.
+     */
+    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+    reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+    markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
+    getTotalCount(): Promise<number>;
+    getPendingCount(): Promise<number>;
+    getHandledCount(): Promise<number>;
+    /**
+     * Whether the next {@link ThrottlingRequestManager.fetchNextRequest} would return `null`.
+     *
+     * Requests waiting on a throttled domain count as unavailable, so a crawler whose task loop is gated on
+     * this idles for the backoff instead of spinning on a fetch that cannot succeed yet.
+     */
+    isEmpty(): Promise<boolean>;
+    /** Unlike {@link ThrottlingRequestManager.isEmpty}, throttled requests still count as outstanding work. */
+    isFinished(): Promise<boolean>;
+    /**
+     * Empties every manager and clears the accumulated backoff. A robots.txt `Crawl-delay` is a property of the
+     * site rather than of the run, so it survives.
+     */
+    purge(): Promise<void>;
+    /**
+     * Empties the per-domain queues, leaving the wrapped manager alone.
+     *
+     * Those queues are this manager's own no matter who owns the one it wraps, which is what makes this safe to
+     * call where a full {@link ThrottlingRequestManager.purge|`purge()`} would not be.
+     */
+    purgeDomainQueues(): Promise<void>;
+    setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+    /**
+     * Returns the next request from a domain that is not backing off, or from the inner manager.
+     *
+     * Returns `null` while every remaining request belongs to a throttled domain - it never waits the backoff
+     * out, because a consumer parked in here holds a concurrency slot, which the autoscaler reads as spare
+     * capacity and answers by scaling up. Callers poll instead, and {@link ThrottlingRequestManager.isEmpty}
+     * reports `true` meanwhile so the crawler's task loop idles rather than spins.
+     */
+    fetchNextRequest<R extends Dictionary = Dictionary>(): Promise<Request<R> | null>;
+    [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
+    persistState(): Promise<void>;
+    drop(): Promise<void>;
+}