@crawlee/core 4.0.0-beta.12 → 4.0.0-beta.121

package/storages/throttling_request_manager.js

@@ -0,0 +1,453 @@
+import { URL } from 'node:url';
+import ow from 'ow';
+import { PersistentRateLimitError } from '../errors.js';
+import { asyncifyIterable } from '../iterables.js';
+import { serviceLocator } from '../service_locator.js';
+import { normalizeHostname } from '../url.js';
+import { drainRequestBatches } from './batched_adds.js';
+import { RequestQueue } from './request_queue.js';
+/** Whether `manager` can pace requests per domain. */
+export function supportsDomainThrottling(manager) {
+    const candidate = manager;
+    return (typeof candidate?.recordDomainDelay === 'function' &&
+        typeof candidate.setCrawlDelay === 'function' &&
+        typeof candidate.assertNoStalledDomains === 'function');
+}
+/** The moment a domain may be dispatched to again - whichever of its two independent clocks runs longer. */
+function throttledUntil(state) {
+    return Math.max(state.backoffUntil, state.crawlDelayUntil);
+}
+/**
+ * A request manager that wraps another one and paces requests per domain.
+ *
+ * Requests for the configured {@link ThrottlingRequestManagerOptions.domains|`domains`} are routed into their own
+ * queue when they are added, so each request lives in exactly one place and deduplication keeps working. Everything
+ * else goes to the wrapped manager untouched.
+ *
+ * {@link ThrottlingRequestManager.fetchNextRequest|`fetchNextRequest()`} serves the domain that has been waiting
+ * longest and skips any that are backing off, falling back to the wrapped manager. It never blocks: while every
+ * remaining request belongs to a throttled domain it returns `null` and {@link ThrottlingRequestManager.isEmpty}
+ * reports `true`, so the crawler idles instead of holding a concurrency slot open.
+ *
+ * Delays come from two places:
+ * - HTTP 429 responses, honouring `Retry-After` and otherwise backing off exponentially. The crawlers report these
+ *   automatically; a request that is throttled is retried later without counting against `maxRequestRetries` and
+ *   without penalising its session.
+ * - robots.txt `Crawl-delay` directives, when `respectRobotsTxtFile` is enabled.
+ *
+ * This is opt-in: throttling only happens for a domain you list explicitly.
+ *
+ * **Example usage:**
+ *
+ * ```ts
+ * const crawler = new CheerioCrawler({
+ *     requestManager: new ThrottlingRequestManager({
+ *         inner: await RequestQueue.open(),
+ *         domains: ['api.example.com', 'slow-site.org'],
+ *     }),
+ *     requestHandler: async ({ request }) => { ... },
+ * });
+ * ```
+ *
+ * @category Sources
+ */
+export class ThrottlingRequestManager {
+    config;
+    inner;
+    requestManagerOpener;
+    baseDelayMs;
+    maxDelayMs;
+    maxDomainStallMs;
+    domainStates = new Map();
+    subManagers = new Map();
+    log;
+    /**
+     * Sub-managers are keyed by a stable alias, so with `purgeOnStart` disabled they outlive the process. They
+     * must therefore be reopened for every configured domain rather than created on first insert - otherwise a
+     * restart sees an empty map, reports the crawl finished, and strands whatever the previous run left in them.
+     */
+    subManagersReady;
+    /** Batches still being added in the background; keeps {@link ThrottlingRequestManager.isFinished} honest. */
+    inProgressBatchCount = 0;
+    warnedAbout = new Set();
+    get hasThrottledDomains() {
+        return this.domainStates.size > 0;
+    }
+    constructor(options, config = serviceLocator.getConfiguration()) {
+        this.config = config;
+        ow(options, ow.object.exactShape({
+            inner: ow.object,
+            domains: ow.array.ofType(ow.string.nonEmpty),
+            requestManagerOpener: ow.optional.function,
+            baseDelaySecs: ow.optional.number.positive,
+            maxDelaySecs: ow.optional.number.positive,
+            maxDomainStallSecs: ow.optional.number.positive,
+        }));
+        this.inner = options.inner;
+        this.requestManagerOpener =
+            options.requestManagerOpener ??
+                ((idOrAlias, opts) => RequestQueue.open(idOrAlias, opts));
+        this.baseDelayMs = (options.baseDelaySecs ?? 2) * 1000;
+        this.maxDelayMs = (options.maxDelaySecs ?? 60) * 1000;
+        this.maxDomainStallMs = (options.maxDomainStallSecs ?? 900) * 1000;
+        this.log = serviceLocator.getLogger().child({ prefix: 'ThrottlingRequestManager' });
+        for (const domain of options.domains) {
+            let hostname;
+            try {
+                // These are bare hostnames, so they only reach `URL` - and with it IDNA - via a synthetic URL.
+                hostname = normalizeHostname(new URL(`http://${domain}`).hostname);
+            }
+            catch {
+                throw new Error(`"${domain}" is not a valid hostname. The \`domains\` option takes bare hostnames such as ` +
+                    `"example.com"; an IPv6 address has to be bracketed, as in "[::1]".`);
+            }
+            this.domainStates.set(hostname, {
+                domain: hostname,
+                backoffUntil: 0,
+                crawlDelayUntil: 0,
+                backoffDecaysAt: 0,
+                consecutive429Count: 0,
+                crawlDelayMs: null,
+                rateLimitedSince: 0,
+                lastRateLimitedAt: 0,
+            });
+        }
+    }
+    /** The wrapped manager, holding every request whose domain is not throttled. */
+    get innerManager() {
+        return this.inner;
+    }
+    /** Warns once about sources that cannot be routed by domain, because their URLs are not known yet. */
+    warnIfNotRoutable(requestLike) {
+        if ('requestsFromUrl' in requestLike && requestLike.requestsFromUrl !== undefined && this.hasThrottledDomains) {
+            // The URL list is only fetched once the owning manager expands it, so we cannot know which domains
+            // it covers and cannot route it. Warn instead of silently exempting those URLs from throttling.
+            this.warnOnce('urlListNotRouted', `Requests loaded via \`requestsFromUrl\` cannot be routed to a per-domain queue, because their URLs ` +
+                `are not known at insertion time. They will be added to the inner request manager and will not ` +
+                `be throttled, even if they belong to a configured domain.`);
+        }
+    }
+    warnOnce(key, message) {
+        if (this.warnedAbout.has(key)) {
+            return;
+        }
+        this.warnedAbout.add(key);
+        this.log.warning(message);
+    }
+    extractDomain(url) {
+        try {
+            return normalizeHostname(new URL(url).hostname);
+        }
+        catch {
+            return '';
+        }
+    }
+    getDomainState(url) {
+        const domain = this.extractDomain(url);
+        return this.domainStates.get(domain) ?? null;
+    }
+    async selectManager(url) {
+        await this.ensureSubManagers();
+        return this.managerForUrl(url);
+    }
+    /** Only valid once {@link ThrottlingRequestManager.ensureSubManagers} has resolved. */
+    managerForUrl(url) {
+        return this.subManagers.get(this.extractDomain(url)) ?? this.inner;
+    }
+    async ensureSubManagers() {
+        this.subManagersReady ??= (async () => {
+            await Promise.all(Array.from(this.domainStates.keys(), async (domain) => {
+                const subManager = await this.requestManagerOpener(
+                // Backends use the alias as a directory name, and an IPv6 literal is full of characters
+                // Windows will not accept. Ordinary hostnames survive this untouched.
+                { alias: `throttled-${encodeURIComponent(domain)}` }, { configuration: this.config });
+                this.subManagers.set(domain, subManager);
+            }));
+        })();
+        await this.subManagersReady;
+    }
+    async getSubManagers() {
+        await this.ensureSubManagers();
+        return Array.from(this.subManagers.values());
+    }
+    /** Configured domains that are not currently backing off, longest-overdue first. */
+    fetchableDomains() {
+        const now = Date.now();
+        return Array.from(this.domainStates.values())
+            .filter((state) => now >= throttledUntil(state))
+            .sort((a, b) => throttledUntil(a) - throttledUntil(b))
+            .map((state) => state.domain);
+    }
+    /**
+     * Records a 429 response and puts the URL's domain into backoff.
+     *
+     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     */
+    recordDomainDelay(url, retryAfterMs) {
+        const state = this.getDomainState(url);
+        if (!state) {
+            return false;
+        }
+        const now = Date.now();
+        // Recorded before the burst suppression below, because a suppressed 429 is still the domain turning us
+        // away - which is exactly what stall detection needs to know about.
+        state.lastRateLimitedAt = now;
+        if (state.rateLimitedSince === 0) {
+            state.rateLimitedSince = now;
+        }
+        // Requests already in flight when the limit was hit all come back 429. They describe one rate-limit
+        // event, so only the first advances the backoff - otherwise concurrency alone drives the exponent.
+        // Only the backoff clock may suppress here: `crawlDelayUntil` is in the future after every dispatch,
+        // so consulting it would discard every 429 the domain ever sends, `Retry-After` included.
+        if (now < state.backoffUntil) {
+            return true;
+        }
+        // A domain that has served us for a full extra backoff window is no longer rate-limiting; start over
+        // rather than carrying the old exponent into an unrelated burst.
+        if (now >= state.backoffDecaysAt) {
+            state.consecutive429Count = 0;
+        }
+        state.consecutive429Count += 1;
+        const retryAfterGiven = retryAfterMs !== undefined && retryAfterMs !== null;
+        let delayMs = retryAfterGiven ? retryAfterMs : this.baseDelayMs * Math.pow(2, state.consecutive429Count - 1);
+        if (delayMs > this.maxDelayMs) {
+            const source = retryAfterGiven ? 'Retry-After header' : 'exponential backoff';
+            this.log.warning(`Capping ${source} delay of ${(delayMs / 1000).toFixed(1)}s for domain "${state.domain}" ` +
+                `to maxDelaySecs (${(this.maxDelayMs / 1000).toFixed(1)}s); the domain may continue to rate-limit. ` +
+                `Consider increasing maxDelaySecs if this recurs.`);
+            delayMs = this.maxDelayMs;
+        }
+        state.backoffUntil = now + delayMs;
+        state.backoffDecaysAt = state.backoffUntil + delayMs;
+        this.log.info(`Rate limit (429) detected for domain "${state.domain}" ` +
+            `(consecutive: ${state.consecutive429Count}, delay: ${(delayMs / 1000).toFixed(1)}s)`);
+        return true;
+    }
+    /**
+     * Applies a robots.txt `Crawl-delay` to the URL's domain, as a minimum interval between dispatches.
+     *
+     * The first value wins, so a robots.txt re-fetch cannot change the cadence mid-crawl.
+     *
+     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     */
+    setCrawlDelay(url, delaySeconds) {
+        const state = this.getDomainState(url);
+        if (!state) {
+            return false;
+        }
+        if (state.crawlDelayMs === null) {
+            state.crawlDelayMs = delaySeconds * 1000;
+            this.log.debug(`Set crawl-delay for domain "${state.domain}" to ${delaySeconds}s`);
+        }
+        return true;
+    }
+    /**
+     * Throws {@link PersistentRateLimitError} if any domain has been rate-limiting us past
+     * {@link ThrottlingRequestManagerOptions.maxDomainStallSecs|`maxDomainStallSecs`} without letting a single
+     * request through.
+     *
+     * A domain qualifies only while it still has queued requests and is actively rate-limiting - a domain that
+     * has simply run out of work is finished, not stalled, and one being waited out under a long robots.txt
+     * `Crawl-delay` is being obeyed, not stonewalled.
+     */
+    async assertNoStalledDomains() {
+        await this.ensureSubManagers();
+        const now = Date.now();
+        const candidates = Array.from(this.domainStates.values()).filter(
+        // Together: it is still turning us away, and has been doing so without a break for longer than the
+        // window. A domain that has simply been idle starts this clock at its first 429 rather than
+        // arriving with the idle time already on it.
+        (state) => state.rateLimitedSince !== 0 &&
+            now - state.lastRateLimitedAt <= this.maxDomainStallMs &&
+            now - state.rateLimitedSince > this.maxDomainStallMs);
+        const stalled = (await Promise.all(candidates.map(async (state) => ((await this.subManagers.get(state.domain).isEmpty()) ? null : state)))).filter((state) => state !== null);
+        if (stalled.length === 0) {
+            return;
+        }
+        const summary = stalled
+            .map((state) => `"${state.domain}" (${((now - state.rateLimitedSince) / 1000).toFixed(0)}s)`)
+            .join(', ');
+        throw new PersistentRateLimitError(`Giving up: ${summary} rate-limited every request for longer than maxDomainStallSecs ` +
+            `(${(this.maxDomainStallMs / 1000).toFixed(0)}s). Waiting longer will not help - lower the ` +
+            `crawler's concurrency, or drop these domains. Their requests are still queued, so re-running ` +
+            `without purging storages will resume them if the rate limit lifts.`);
+    }
+    /** Records that a domain let a request through, which ends any rate-limit run stall detection was timing. */
+    recordProgress(url) {
+        const state = this.getDomainState(url);
+        if (state) {
+            state.rateLimitedSince = 0;
+        }
+    }
+    // --- IRequestManager Implementation ---
+    async addRequest(requestLike, options) {
+        this.warnIfNotRoutable(requestLike);
+        const manager = await this.selectManager(requestLike.url ?? '');
+        return manager.addRequest(requestLike, options);
+    }
+    /**
+     * Adds requests in batches, routing each one to the manager that owns its domain.
+     *
+     * Batching, validation, deduplication and `Retry-After`-free bookkeeping are all delegated to the target
+     * managers - this only decides where each request goes, one batch at a time, so a lazy or unbounded input
+     * iterable is never fully materialized.
+     */
+    async addRequestsBatched(requests, options = {}) {
+        await this.ensureSubManagers();
+        // Normalized up front so the shared batching helper - and `requestsOverLimit` - only ever see `Source`.
+        async function* iterateRequests() {
+            for await (const request of asyncifyIterable(requests)) {
+                yield typeof request === 'string' ? { url: request } : request;
+            }
+        }
+        return drainRequestBatches({
+            items: iterateRequests(),
+            batchSize: options.batchSize ?? 1000,
+            waitBetweenBatchesMillis: options.waitBetweenBatchesMillis ?? 1000,
+            waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded ?? false,
+            maxNewRequests: options.maxNewRequests,
+            // Routing is the only thing this manager adds; the targets do the batching, validation and
+            // deduplication themselves.
+            processChunk: async (chunk) => {
+                const byManager = new Map();
+                for (const request of chunk) {
+                    this.warnIfNotRoutable(request);
+                    const manager = this.managerForUrl(request.url ?? '');
+                    const bucket = byManager.get(manager);
+                    if (bucket) {
+                        bucket.push(request);
+                    }
+                    else {
+                        byManager.set(manager, [request]);
+                    }
+                }
+                const results = await Promise.all(Array.from(byManager, ([manager, slice]) => manager.addRequestsBatched(slice, {
+                    forefront: options.forefront,
+                    // The slice is already one batch, and we need its results before releasing the next one.
+                    batchSize: slice.length,
+                    waitForAllRequestsToBeAdded: true,
+                })));
+                return results.flatMap((result) => result.addedRequests);
+            },
+            // Keeps the crawler from concluding it is finished while batches are still landing.
+            trackBackgroundBatches: (batches) => {
+                this.inProgressBatchCount += 1;
+                void batches.finally(() => {
+                    this.inProgressBatchCount -= 1;
+                });
+            },
+        });
+    }
+    async reclaimRequest(request, options) {
+        const manager = await this.selectManager(request.url);
+        return manager.reclaimRequest(request, options);
+    }
+    async markRequestAsHandled(request) {
+        const manager = await this.selectManager(request.url);
+        // Reached whether the request succeeded or ran out of retries; either way the domain answered us.
+        this.recordProgress(request.url);
+        return manager.markRequestAsHandled(request);
+    }
+    async getTotalCount() {
+        return this.sumOverManagers((manager) => manager.getTotalCount());
+    }
+    async getPendingCount() {
+        return this.sumOverManagers((manager) => manager.getPendingCount());
+    }
+    async getHandledCount() {
+        return this.sumOverManagers((manager) => manager.getHandledCount());
+    }
+    /**
+     * Whether the next {@link ThrottlingRequestManager.fetchNextRequest} would return `null`.
+     *
+     * Requests waiting on a throttled domain count as unavailable, so a crawler whose task loop is gated on
+     * this idles for the backoff instead of spinning on a fetch that cannot succeed yet.
+     */
+    async isEmpty() {
+        await this.ensureSubManagers();
+        const fetchable = [this.inner, ...this.fetchableDomains().map((domain) => this.subManagers.get(domain))];
+        const results = await Promise.all(fetchable.map((manager) => manager.isEmpty()));
+        return results.every(Boolean);
+    }
+    /** Unlike {@link ThrottlingRequestManager.isEmpty}, throttled requests still count as outstanding work. */
+    async isFinished() {
+        if (this.inProgressBatchCount > 0) {
+            return false;
+        }
+        return this.everyManager((manager) => manager.isFinished());
+    }
+    /**
+     * Empties every manager and clears the accumulated backoff. A robots.txt `Crawl-delay` is a property of the
+     * site rather than of the run, so it survives.
+     */
+    async purge() {
+        await this.forEachManager((manager) => manager.purge?.());
+        for (const state of this.domainStates.values()) {
+            state.consecutive429Count = 0;
+            state.backoffUntil = 0;
+            state.crawlDelayUntil = 0;
+            state.backoffDecaysAt = 0;
+            state.rateLimitedSince = 0;
+            state.lastRateLimitedAt = 0;
+        }
+    }
+    async setExpectedRequestProcessingTimeSecs(secs) {
+        await this.forEachManager((manager) => manager.setExpectedRequestProcessingTimeSecs?.(secs));
+    }
+    async forEachManager(fn) {
+        // `fn` targets optional members, so it may return nothing - the wrapper normalizes that for `Promise.all`.
+        await Promise.all([this.inner, ...(await this.getSubManagers())].map(async (manager) => fn(manager)));
+    }
+    async sumOverManagers(fn) {
+        const counts = await Promise.all([this.inner, ...(await this.getSubManagers())].map(fn));
+        return counts.reduce((a, b) => a + b, 0);
+    }
+    async everyManager(fn) {
+        const results = await Promise.all([this.inner, ...(await this.getSubManagers())].map(fn));
+        return results.every(Boolean);
+    }
+    /**
+     * Returns the next request from a domain that is not backing off, or from the inner manager.
+     *
+     * Returns `null` while every remaining request belongs to a throttled domain - it never waits the backoff
+     * out, because a consumer parked in here holds a concurrency slot, which the autoscaler reads as spare
+     * capacity and answers by scaling up. Callers poll instead, and {@link ThrottlingRequestManager.isEmpty}
+     * reports `true` meanwhile so the crawler's task loop idles rather than spins.
+     */
+    async fetchNextRequest() {
+        await this.ensureSubManagers();
+        for (const domain of this.fetchableDomains()) {
+            const state = this.domainStates.get(domain);
+            // Armed while the fetch below is still suspended, so that a concurrent `fetchNextRequest` cannot
+            // find the domain fetchable and dispatch into the same window - which would pace each task
+            // rather than the domain.
+            const crawlDelayBefore = state.crawlDelayUntil;
+            if (state.crawlDelayMs !== null) {
+                state.crawlDelayUntil = Date.now() + state.crawlDelayMs;
+            }
+            const request = await this.subManagers.get(domain).fetchNextRequest();
+            if (request) {
+                return request;
+            }
+            // No dispatch to pace, so the domain keeps its slot.
+            state.crawlDelayUntil = crawlDelayBefore;
+        }
+        return this.inner.fetchNextRequest();
+    }
+    async *[Symbol.asyncIterator]() {
+        while (true) {
+            const req = await this.fetchNextRequest();
+            if (!req)
+                break;
+            yield req;
+        }
+    }
+    async persistState() {
+        await this.forEachManager((manager) => manager.persistState?.());
+    }
+    async drop() {
+        await this.forEachManager((manager) => manager.drop?.());
+        this.subManagers.clear();
+        this.subManagersReady = undefined;
+    }
+}