@crawlee/core 4.0.0-rc.0 → 4.0.0-rc.1

package/storages/sitemap_request_loader.js

@@ -1,438 +0,0 @@
-import { Transform } from 'node:stream';
-import { EnqueueStrategy, parseSitemap } from '@crawlee/utils';
-import { minimatch } from 'minimatch';
-import { z } from 'zod';
-import { constructUrlPatternObjects, urlPatternSchema } from '../enqueue_links/shared.js';
-import { EventType } from '../events/event_manager.js';
-import { Request } from '../request.js';
-import { serviceLocator } from '../service_locator.js';
-import { parseArgument, schemas } from '../validators.js';
-import { KeyValueStore } from './key_value_store.js';
-import { purgeDefaultStorages } from './utils.js';
-const sitemapRequestLoaderOptionsSchema = z.strictObject({
-    sitemapUrls: schemas.arrayOf(z.string(), 'strings'),
-    proxyUrl: z.string().optional(),
-    persistStateKey: z.string().optional(),
-    signal: z.unknown().optional(),
-    timeoutMillis: schemas.anyNumber.optional(),
-    maxBufferSize: schemas.anyNumber.default(200),
-    enqueueStrategy: z.enum(EnqueueStrategy).default(EnqueueStrategy.SameHostname),
-    parseSitemapOptions: z.looseObject({}).optional(),
-    include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
-    exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
-    persistenceOptions: z.looseObject({}).optional(),
-    httpClient: schemas.httpClient.optional(),
-});
-/** @internal */
-const STATE_PERSISTENCE_KEY = 'SITEMAP_REQUEST_LOADER_STATE';
-/**
- * A list of URLs to crawl parsed from a sitemap.
- *
- * The loading of the sitemap is performed in the background so that crawling can start before the sitemap is fully loaded.
- */
-export class SitemapRequestLoader {
-    /**
-     * Set of URLs that were returned by `fetchNextRequest()` and not marked as handled yet.
-     * @internal
-     */
-    inProgress = new Set();
-    /**
-     * Map of returned Request objects that have not been marked as handled yet.
-     *
-     * We use this to persist custom user fields on the in-progress requests.
-     */
-    #requestData = new Map();
-    /**
-     * Object for keeping track of the sitemap parsing progress.
-     */
-    #sitemapParsingProgress = {
-        /**
-         * URL of the sitemap that is currently being parsed. `null` if no sitemap is being parsed.
-         */
-        inProgressSitemapUrl: null,
-        /**
-         * Buffer for URLs from the currently parsed sitemap. Used for tracking partially loaded sitemaps across migrations.
-         */
-        inProgressEntries: new Set(),
-        /**
-         * Set of sitemap URLs that have not been parsed yet. If the set is empty and `inProgressSitemapUrl` is `null`, the sitemap loading is finished.
-         */
-        pendingSitemapUrls: new Set(),
-    };
-    /**
-     * Object stream of URLs parsed from the sitemaps.
-     * Using `highWaterMark`, this can manage the speed of the sitemap loading.
-     *
-     * Fetch the next URL to be processed using `fetchNextRequest()`.
-     */
-    #urlQueueStream;
-    /**
-     * Indicates whether the request list sitemap loading was aborted.
-     *
-     * If the loading was aborted before the sitemaps were fully loaded, the request list might be missing some URLs.
-     * The `isSitemapFullyLoaded` method can be used to check if the sitemaps were fully loaded.
-     *
-     * If the loading is aborted and all the requests are handled, `isFinished()` will return `true`.
-     */
-    #abortLoading = false;
-    /** Number of URLs that were marked as handled */
-    #handledUrlCount = 0;
-    #persistStateKey;
-    #store;
-    #closed = false;
-    /**
-     * Proxy URL to be used for sitemap loading.
-     */
-    #proxyUrl;
-    /**
-     * Enqueue strategy applied to sitemap-derived URLs and stamped onto the emitted `Request` objects.
-     */
-    #enqueueStrategy;
-    /**
-     * Logger instance.
-     */
-    #log;
-    #urlExcludePatternObjects = [];
-    #urlPatternObjects = [];
-    /** EventManager used to handle persistence */
-    #events;
-    #persistenceOptions;
-    /** @internal */
-    constructor(options) {
-        const { include, exclude, persistStateKey, persistenceOptions, proxyUrl, maxBufferSize, sitemapUrls, enqueueStrategy, } = parseArgument(options, sitemapRequestLoaderOptionsSchema, 'SitemapRequestLoaderOptions');
-        this.#log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestLoader' });
-        if (exclude?.length) {
-            this.#urlExcludePatternObjects.push(...constructUrlPatternObjects(exclude));
-        }
-        if (include?.length) {
-            this.#urlPatternObjects.push(...constructUrlPatternObjects(include));
-        }
-        this.#persistStateKey = persistStateKey;
-        this.#persistenceOptions = { enable: true, ...persistenceOptions };
-        this.#proxyUrl = proxyUrl;
-        this.#enqueueStrategy = enqueueStrategy;
-        this.#urlQueueStream = this.createNewStream(maxBufferSize);
-        this.#sitemapParsingProgress.pendingSitemapUrls = new Set(sitemapUrls);
-        this.#events = serviceLocator.getEventManager();
-        this.persistState = this.persistState.bind(this);
-    }
-    /**
-     * Creates a new object stream with the specified highWaterMark.
-     * @param highWaterMark High water mark for the stream (the maximum number of objects the stream will buffer).
-     * @returns A new object stream.
-     */
-    createNewStream(highWaterMark) {
-        return new Transform({
-            objectMode: true,
-            highWaterMark,
-        }).pause();
-    }
-    /**
-     * Returns a function that checks whether the provided pattern matches the closure URL.
-     * @param url URL to be checked.
-     * @returns A matcher function that checks whether the pattern matches the closure URL.
-     */
-    matchesUrl(url) {
-        return (patternObject) => {
-            const { regexp, glob } = patternObject;
-            const matchesRegex = (regexp && url.match(regexp)) || false;
-            const matchesGlob = (glob && minimatch(url, glob, { nocase: true })) || false;
-            return Boolean(matchesRegex || matchesGlob);
-        };
-    }
-    /**
-     * Checks whether the URL matches the `include` / `exclude` patterns provided in the `options`.
-     * @param url URL to be checked.
-     * @returns `true` if the URL matches the patterns, `false` otherwise.
-     */
-    isUrlMatchingPatterns(url) {
-        return (!this.#urlExcludePatternObjects.some(this.matchesUrl(url)) &&
-            (this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(this.matchesUrl(url))));
-    }
-    /**
-     * Adds a URL to the queue of parsed URLs.
-     *
-     * Blocks if the stream is full until it is drained.
-     */
-    async pushNextUrl(url) {
-        return new Promise((resolve) => {
-            if (this.#closed || (url && !this.isUrlMatchingPatterns(url))) {
-                resolve();
-                return;
-            }
-            if (!this.#urlQueueStream.push(url)) {
-                // This doesn't work with the 'drain' event (it's not emitted for some reason).
-                this.#urlQueueStream.once('readdata', () => {
-                    resolve();
-                });
-            }
-            else {
-                resolve();
-            }
-        });
-    }
-    /**
-     * Reads the next URL from the queue of parsed URLs.
-     *
-     * If the stream is empty, blocks until a new URL is pushed.
-     * @returns The next URL from the queue or `null` if we have read all URLs.
-     */
-    async readNextUrl() {
-        return new Promise((resolve) => {
-            if (this.#closed) {
-                resolve(null);
-                return;
-            }
-            const result = this.#urlQueueStream.read();
-            if (!result && !this.isSitemapFullyLoaded()) {
-                this.#urlQueueStream.once('readable', () => {
-                    const nextUrl = this.#urlQueueStream.read();
-                    resolve(nextUrl);
-                });
-            }
-            else {
-                resolve(result);
-            }
-            this.#urlQueueStream.emit('readdata');
-        });
-    }
-    /**
-     * Indicates whether the background processing of sitemap contents has successfully finished.
-     *
-     * If this is `false`, the background processing is either still in progress or was aborted.
-     */
-    isSitemapFullyLoaded() {
-        return (this.#sitemapParsingProgress.inProgressSitemapUrl === null &&
-            this.#sitemapParsingProgress.pendingSitemapUrls.size === 0);
-    }
-    /**
-     * Start processing the sitemaps and loading the URLs.
-     *
-     * Resolves once all the sitemaps URLs have been fully loaded (sets `isSitemapFullyLoaded` to `true`).
-     */
-    async load({ parseSitemapOptions, }) {
-        while (!this.isSitemapFullyLoaded() && !this.#abortLoading) {
-            const sitemapUrl = this.#sitemapParsingProgress.inProgressSitemapUrl ??
-                this.#sitemapParsingProgress.pendingSitemapUrls.values().next().value;
-            try {
-                for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.#proxyUrl, {
-                    ...parseSitemapOptions,
-                    maxDepth: 0,
-                    emitNestedSitemaps: true,
-                    enqueueStrategy: this.#enqueueStrategy,
-                })) {
-                    if (!item.originSitemapUrl) {
-                        // This is a nested sitemap
-                        this.#sitemapParsingProgress.pendingSitemapUrls.add(item.loc);
-                        continue;
-                    }
-                    if (!this.#sitemapParsingProgress.inProgressEntries.has(item.loc)) {
-                        await this.pushNextUrl(item.loc);
-                        this.#sitemapParsingProgress.inProgressEntries.add(item.loc);
-                    }
-                }
-            }
-            catch (e) {
-                this.#log.error('Error loading sitemap contents:', e);
-            }
-            this.#sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
-            this.#sitemapParsingProgress.inProgressEntries.clear();
-            this.#sitemapParsingProgress.inProgressSitemapUrl = null;
-        }
-        this.#urlQueueStream.end();
-    }
-    /**
-     * Open a sitemap and start processing it.
-     *
-     * Resolves to a new instance of `SitemapRequestLoader`, which **might not be fully loaded yet** - i.e. the sitemap might still be loading in the background.
-     *
-     * Track the loading progress using the `isSitemapFullyLoaded` property.
-     */
-    static async open(options) {
-        const { httpClient, ...restOptions } = options;
-        const requestList = new SitemapRequestLoader({
-            ...restOptions,
-            persistStateKey: options.persistStateKey ?? STATE_PERSISTENCE_KEY,
-        });
-        await requestList.restoreState();
-        void requestList.load({
-            parseSitemapOptions: { logger: serviceLocator.getLogger(), ...options.parseSitemapOptions, httpClient },
-        });
-        if (requestList.#persistenceOptions.enable) {
-            requestList.#events.on(EventType.PERSIST_STATE, requestList.persistState);
-        }
-        options?.signal?.addEventListener('abort', () => {
-            requestList.#abortLoading = true;
-        });
-        if (options.timeoutMillis) {
-            setTimeout(() => {
-                requestList.#abortLoading = true;
-            }, options.timeoutMillis);
-        }
-        return requestList;
-    }
-    /**
-     * @inheritDoc
-     */
-    async getTotalCount() {
-        // Total known so far = not-yet-fetched (still buffered in the stream) + in-progress (fetched but not
-        // yet handled) + already handled.
-        return this.#urlQueueStream.readableLength + this.inProgress.size + this.#handledUrlCount;
-    }
-    /**
-     * @inheritDoc
-     */
-    async getPendingCount() {
-        // Pending = everything not yet handled = not-yet-fetched + in-progress.
-        return this.#urlQueueStream.readableLength + this.inProgress.size;
-    }
-    /**
-     * Combines this list with a request manager (a {@link RequestQueue} by default) into a
-     * {@link RequestManagerTandem}, allowing requests to be added and reclaimed while still
-     * being read from this list first.
-     */
-    async toTandem(requestManager) {
-        // Import here to avoid circular imports.
-        const { RequestManagerTandem } = await import('./request_manager_tandem.js');
-        const { RequestQueue } = await import('./request_queue.js');
-        return new RequestManagerTandem(this, requestManager ?? (await RequestQueue.open()));
-    }
-    /**
-     * @inheritDoc
-     */
-    async isFinished() {
-        return ((await this.isEmpty()) && this.inProgress.size === 0 && (this.isSitemapFullyLoaded() || this.#abortLoading));
-    }
-    /**
-     * @inheritDoc
-     */
-    async isEmpty() {
-        return this.#urlQueueStream.readableLength === 0;
-    }
-    /**
-     * @inheritDoc
-     */
-    async getHandledCount() {
-        return this.#handledUrlCount;
-    }
-    /**
-     * @inheritDoc
-     */
-    async persistState() {
-        if (this.#persistStateKey === undefined) {
-            return;
-        }
-        this.#store ??= await KeyValueStore.open();
-        const urlQueue = [];
-        while (this.#urlQueueStream.readableLength > 0) {
-            const url = this.#urlQueueStream.read();
-            if (url === null) {
-                break;
-            }
-            urlQueue.push(url);
-        }
-        // Create a new stream, as we have read all the URLs from the current one.
-        // Pushing the urls back to the original stream might not be possible if it has been ended.
-        const previousStream = this.#urlQueueStream;
-        const newStream = this.createNewStream(previousStream.readableHighWaterMark);
-        for (const url of urlQueue) {
-            newStream.push(url);
-        }
-        if (previousStream.writableEnded) {
-            newStream.end();
-        }
-        this.#urlQueueStream = newStream;
-        // A `pushNextUrl()` call may be blocked on backpressure, waiting for a `readdata` event on the
-        // previous stream. That event is only ever emitted by `readNextUrl()` on the current stream, so
-        // after the swap the waiter would never be notified and the background sitemap loading would hang.
-        // Re-emit `readdata` on the previous stream to release any such pending waiter (its URL has already
-        // been transferred to the new stream above).
-        previousStream.emit('readdata');
-        await this.#store.setValue(this.#persistStateKey, {
-            sitemapParsingProgress: {
-                pendingSitemapUrls: Array.from(this.#sitemapParsingProgress.pendingSitemapUrls),
-                inProgressSitemapUrl: this.#sitemapParsingProgress.inProgressSitemapUrl,
-                inProgressEntries: Array.from(this.#sitemapParsingProgress.inProgressEntries),
-            },
-            // Re-queue in-progress requests to the front so they are retried if the state is restored.
-            urlQueue: [...this.inProgress, ...urlQueue],
-            requestData: Array.from(this.#requestData.entries()),
-            abortLoading: this.#abortLoading,
-            closed: this.#closed,
-        });
-    }
-    async restoreState() {
-        await purgeDefaultStorages({ onlyPurgeOnce: true });
-        if (this.#persistStateKey === undefined) {
-            return;
-        }
-        this.#store ??= await KeyValueStore.open();
-        const state = await this.#store.getValue(this.#persistStateKey);
-        if (state === null) {
-            return;
-        }
-        this.#sitemapParsingProgress = {
-            pendingSitemapUrls: new Set(state.sitemapParsingProgress.pendingSitemapUrls),
-            inProgressSitemapUrl: state.sitemapParsingProgress.inProgressSitemapUrl,
-            inProgressEntries: new Set(state.sitemapParsingProgress.inProgressEntries),
-        };
-        this.#requestData = new Map(state.requestData ?? []);
-        for (const url of state.urlQueue) {
-            this.#urlQueueStream.push(url);
-        }
-        this.#abortLoading = state.abortLoading;
-        this.#closed = state.closed;
-    }
-    /**
-     * @inheritDoc
-     */
-    async fetchNextRequest() {
-        const nextUrl = await this.readNextUrl();
-        if (!nextUrl) {
-            return null;
-        }
-        // A restored in-progress request already has its Request data; don't overwrite it.
-        if (!this.#requestData.has(nextUrl)) {
-            this.#requestData.set(nextUrl, new Request({ url: nextUrl, enqueueStrategy: this.#enqueueStrategy }));
-        }
-        this.inProgress.add(nextUrl);
-        return this.#requestData.get(nextUrl);
-    }
-    /**
-     * @inheritDoc
-     */
-    async *[Symbol.asyncIterator]() {
-        while (!(await this.isFinished())) {
-            const request = await this.fetchNextRequest();
-            if (!request)
-                break;
-            yield request;
-        }
-    }
-    /**
-     * Aborts the internal sitemap loading, stops the processing of the sitemap contents and drops all the pending URLs.
-     *
-     * Calling `fetchNextRequest()` after this method will always return `null`.
-     */
-    async teardown() {
-        this.#closed = true;
-        this.#abortLoading = true;
-        this.#events.off(EventType.PERSIST_STATE, this.persistState);
-        await this.persistState();
-        this.#urlQueueStream.emit('readdata'); // unblocks the potentially waiting `pushNextUrl` call
-    }
-    /**
-     * @inheritDoc
-     */
-    async markRequestAsHandled(request) {
-        this.#handledUrlCount += 1;
-        this.ensureInProgress(request.url);
-        this.inProgress.delete(request.url);
-        this.#requestData.delete(request.url);
-    }
-    ensureInProgress(url) {
-        if (!this.inProgress.has(url)) {
-            throw new Error(`The request is not being processed (url: ${url})`);
-        }
-    }
-}

package/storages/throttling_request_manager.d.ts

@@ -1,239 +0,0 @@
-import type { Dictionary } from '@crawlee/types';
-import type { Configuration } from '../configuration.js';
-import type { Request, Source } from '../request.js';
-import type { IRequestManager, RequestsLike } from './request_manager.js';
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';
-import type { StorageIdentifier } from './storage_instance_manager.js';
-import type { StorageOpenOptions } from './utils.js';
-/**
- * Opens a request manager, matching the shape of storage `open` methods such as
- * {@link RequestQueue.open|`RequestQueue.open`}.
- *
- * {@link ThrottlingRequestManager} calls this once per configured domain, so every per-domain queue shares the
- * concrete type and storage backend of the manager being wrapped.
- */
-export type RequestManagerOpener<T extends IRequestManager = IRequestManager> = (identifier: string | StorageIdentifier, options?: StorageOpenOptions) => Promise<T>;
-/**
- * A request manager that can pace requests per domain, as {@link ThrottlingRequestManager} does.
- *
- * The crawlers detect this structurally rather than by type, so a wrapper can opt in by forwarding these three
- * methods without {@link IRequestManager} having to know that throttling exists.
- */
-export interface SupportsDomainThrottling {
-    /** @see {@link ThrottlingRequestManager.recordDomainDelay} */
-    recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
-    /** @see {@link ThrottlingRequestManager.setCrawlDelay} */
-    setCrawlDelay(url: string, delaySeconds: number): boolean;
-    /** @see {@link ThrottlingRequestManager.assertNoStalledDomains} */
-    assertNoStalledDomains(): Promise<void>;
-}
-/** Whether `manager` can pace requests per domain. */
-export declare function supportsDomainThrottling(manager: unknown): manager is SupportsDomainThrottling;
-/** Options for {@link ThrottlingRequestManager}. */
-export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRequestManager> {
-    /**
-     * The request manager to wrap, usually a {@link RequestQueue}. Requests for domains that are not throttled
-     * are stored here.
-     */
-    inner: T;
-    /**
-     * Which domains to throttle: a list of hostnames, or `'all'` for every domain the crawl encounters.
-     *
-     * Matching a listed hostname is case-insensitive and exact - wildcards such as `*.example.com` are not
-     * supported, so list each subdomain you care about (or set
-     * {@link ThrottlingRequestManagerOptions.throttleBy|`throttleBy: 'registrableDomain'`}). An
-     * internationalized domain may be given in either its unicode or its punycode form, and an IPv6 address has
-     * to be bracketed (`[::1]`). Requests for any other domain bypass throttling entirely.
-     *
-     * `'all'` gives each domain a queue of its own the first time it is seen, so that it can be held back
-     * without its requests being repeatedly popped and re-enqueued. One request queue per domain is not free,
-     * which is what {@link ThrottlingRequestManagerOptions.maxThrottledDomains|`maxThrottledDomains`} is
-     * there to bound.
-     */
-    domains: string[] | 'all';
-    /**
-     * A floor under the crawl delay of every throttled domain, in seconds - the proactive clock described on
-     * {@link ThrottlingRequestManager}. A domain whose robots.txt asks for a longer `Crawl-delay` gets the
-     * longer one; this is a minimum, not an override.
-     * @default 0
-     */
-    minCrawlDelaySecs?: number;
-    /**
-     * What counts as "the same domain": the exact hostname, or the registrable domain it belongs to
-     * (`example.com` for `www.example.com`, `a.example.co.uk` and so on). Hosts with no registrable domain -
-     * IP addresses, `localhost` - are always throttled per hostname.
-     *
-     * Grouping by registrable domain gives subdomains a single pair of clocks and a single queue, which is what
-     * you want when the pacing is there to be polite to one server rather than to satisfy a specific host's
-     * rate limit.
-     * @default 'hostname'
-     */
-    throttleBy?: 'hostname' | 'registrableDomain';
-    /**
-     * The most domains a run may throttle at once. Exceeding it throws, rather than silently letting the
-     * throttling lapse - one request queue per domain is not free, and a crawl that discovers domains without
-     * bound would drown the storage backend in them.
-     *
-     * Only domains discovered under `domains: 'all'` count against this; an explicit list is taken at face value.
-     * @default 100
-     */
-    maxThrottledDomains?: number;
-    /**
-     * The key under which the discovered domain list is kept in the default key-value store, so that a restart
-     * with `purgeOnStart` disabled reopens their queues instead of stranding whatever they still hold. Only
-     * written under `domains: 'all'`.
-     *
-     * Give each manager its own key when running several of them against the same storage.
-     * @default 'CRAWLEE_THROTTLED_DOMAINS'
-     */
-    persistStateKey?: string;
-    /**
-     * Opens the per-domain queues, one per throttled domain, each under the alias `throttled-<domain>`.
-     * @default RequestQueue.open
-     */
-    requestManagerOpener?: RequestManagerOpener<T>;
-    /**
-     * The delay applied after a domain's first HTTP 429, doubled on each subsequent one.
-     * @default 2
-     */
-    baseDelaySecs?: number;
-    /**
-     * Upper bound on the delay between requests to a rate-limited domain, applied to both the exponential
-     * backoff and a `Retry-After` value.
-     * @default 60
-     */
-    maxDelaySecs?: number;
-    /**
-     * How long a domain may rate-limit us without a single request getting through before the crawl is
-     * abandoned with a {@link PersistentRateLimitError}.
-     *
-     * A domain that keeps answering 429 for this long is not going to be crawled by waiting longer - the
-     * concurrency is too high for it, or it has blocked us outright. Its requests are deliberately left in
-     * their queue, so re-running the crawl with `purgeOnStart` disabled picks them up once the domain recovers.
-     *
-     * A crawler running with `keepAlive` is exempt - outliving a domain that will not let us through is the
-     * whole point there.
-     * @default 900
-     */
-    maxDomainStallSecs?: number;
-}
-/**
- * A request manager that wraps another one and paces requests per domain.
- *
- * Requests for a throttled domain are routed into their own queue when they are added, so each request lives in
- * exactly one place and deduplication keeps working. Everything else goes to the wrapped manager untouched.
- *
- * {@link ThrottlingRequestManager.fetchNextRequest|`fetchNextRequest()`} serves the domain that has been waiting
- * longest and skips any that are backing off, falling back to the wrapped manager. It never blocks: while every
- * remaining request belongs to a throttled domain it returns `null` and {@link ThrottlingRequestManager.isEmpty}
- * reports `true`, so the crawler idles instead of holding a concurrency slot open.
- *
- * Each throttled domain runs two independent clocks, and may be dispatched to once **both** have run out:
- * - **Backoff**, set by HTTP 429 responses - honouring `Retry-After`, and otherwise doubling from `baseDelaySecs`.
- *   Reactive and temporary: it decays once the domain stops turning us away. The crawlers report the 429s
- *   themselves; a request held back this way is retried later without counting against `maxRequestRetries` and
- *   without penalising its session.
- * - **Crawl delay**, the minimum interval between two dispatches to the domain, armed after each one. Proactive
- *   and constant: whatever the domain's robots.txt asks for, floored by
- *   {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`}. Either may be absent, in
- *   which case the other one is the delay.
- *
- * Which domains get those clocks is {@link ThrottlingRequestManagerOptions.domains|`domains`} - a list, or
- * `'all'` for every domain the crawl encounters.
- *
- * **Example usage:**
- *
- * ```ts
- * const crawler = new CheerioCrawler({
- *     requestManager: new ThrottlingRequestManager({
- *         inner: await RequestQueue.open(),
- *         domains: ['api.example.com', 'slow-site.org'],
- *     }),
- *     requestHandler: async ({ request }) => { ... },
- * });
- * ```
- *
- * @category Sources
- */
-export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager, SupportsDomainThrottling {
-    #private;
-    private readonly config;
-    private readonly domainStates;
-    private readonly log;
-    constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);
-    /** The wrapped manager, holding every request whose domain is not throttled. */
-    get innerManager(): T;
-    /**
-     * Records a 429 response and puts the URL's domain into backoff.
-     *
-     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
-     */
-    recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
-    /**
-     * Records the `Crawl-delay` a domain's robots.txt asked for, which becomes its crawl delay unless
-     * {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`} asks for longer.
-     *
-     * The first value wins, so a robots.txt re-fetch cannot change the cadence mid-crawl.
-     *
-     * @returns `false` if the domain is not throttled, in which case this is a no-op.
-     */
-    setCrawlDelay(url: string, delaySeconds: number): boolean;
-    /**
-     * Throws {@link PersistentRateLimitError} if any domain has been rate-limiting us past
-     * {@link ThrottlingRequestManagerOptions.maxDomainStallSecs|`maxDomainStallSecs`} without letting a single
-     * request through.
-     *
-     * A domain qualifies only while it still has queued requests and is actively rate-limiting - a domain that
-     * has simply run out of work is finished, not stalled, and one being waited out under a long robots.txt
-     * `Crawl-delay` is being obeyed, not stonewalled.
-     */
-    assertNoStalledDomains(): Promise<void>;
-    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
-    /**
-     * Adds requests in batches, routing each one to the manager that owns its domain.
-     *
-     * Batching, validation, deduplication and `Retry-After`-free bookkeeping are all delegated to the target
-     * managers - this only decides where each request goes, one batch at a time, so a lazy or unbounded input
-     * iterable is never fully materialized.
-     */
-    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
-    reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
-    markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
-    getTotalCount(): Promise<number>;
-    getPendingCount(): Promise<number>;
-    getHandledCount(): Promise<number>;
-    /**
-     * Whether the next {@link ThrottlingRequestManager.fetchNextRequest} would return `null`.
-     *
-     * Requests waiting on a throttled domain count as unavailable, so a crawler whose task loop is gated on
-     * this idles for the backoff instead of spinning on a fetch that cannot succeed yet.
-     */
-    isEmpty(): Promise<boolean>;
-    /** Unlike {@link ThrottlingRequestManager.isEmpty}, throttled requests still count as outstanding work. */
-    isFinished(): Promise<boolean>;
-    /**
-     * Empties every manager and clears the accumulated backoff. A robots.txt `Crawl-delay` is a property of the
-     * site rather than of the run, so it survives.
-     */
-    purge(): Promise<void>;
-    /**
-     * Empties the per-domain queues, leaving the wrapped manager alone.
-     *
-     * Those queues are this manager's own no matter who owns the one it wraps, which is what makes this safe to
-     * call where a full {@link ThrottlingRequestManager.purge|`purge()`} would not be.
-     */
-    purgeDomainQueues(): Promise<void>;
-    setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
-    /**
-     * Returns the next request from a domain that is not backing off, or from the inner manager.
-     *
-     * Returns `null` while every remaining request belongs to a throttled domain - it never waits the backoff
-     * out, because a consumer parked in here holds a concurrency slot, which the autoscaler reads as spare
-     * capacity and answers by scaling up. Callers poll instead, and {@link ThrottlingRequestManager.isEmpty}
-     * reports `true` meanwhile so the crawler's task loop idles rather than spins.
-     */
-    fetchNextRequest<R extends Dictionary = Dictionary>(): Promise<Request<R> | null>;
-    [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
-    persistState(): Promise<void>;
-    drop(): Promise<void>;
-}