@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.0

package/storages/storage_instance_manager.js

@@ -120,8 +120,8 @@ class StorageCache {
  * assigns a reserved default alias.
  */
 export class StorageInstanceManager {
-    cache = new StorageCache();
-    openerLocks = new Map();
+    #cache = new StorageCache();
+    #openerLocks = new Map();
     /**
      * Open (or retrieve from cache) a storage instance.
      *
@@ -142,46 +142,46 @@ export class StorageInstanceManager {
         }
         // Fast-path cache check (no lock).
         if (alias !== undefined) {
-            const cached = this.cache.get(cls, { alias, backendCacheKey });
+            const cached = this.#cache.get(cls, { alias, backendCacheKey });
             if (cached)
                 return cached;
         }
         else if (id) {
-            const cached = this.cache.get(cls, { id, backendCacheKey });
+            const cached = this.#cache.get(cls, { id, backendCacheKey });
             if (cached)
                 return cached;
         }
         else if (name) {
-            const cached = this.cache.get(cls, { name, backendCacheKey });
+            const cached = this.#cache.get(cls, { name, backendCacheKey });
             if (cached)
                 return cached;
         }
         const identifierKey = id ?? name ?? alias ?? DEFAULT_STORAGE_ALIAS;
         const lockKey = `${cls.name}:${identifierKey}:${backendCacheKey}`;
-        if (!this.openerLocks.has(lockKey)) {
-            this.openerLocks.set(lockKey, new AsyncQueue());
+        if (!this.#openerLocks.has(lockKey)) {
+            this.#openerLocks.set(lockKey, new AsyncQueue());
         }
-        const queue = this.openerLocks.get(lockKey);
+        const queue = this.#openerLocks.get(lockKey);
         await queue.wait();
         try {
             // Double-check cache under lock (another caller may have filled it while we waited).
             if (alias !== undefined) {
-                const cached = this.cache.get(cls, { alias, backendCacheKey });
+                const cached = this.#cache.get(cls, { alias, backendCacheKey });
                 if (cached)
                     return cached;
             }
             else if (id) {
-                const cached = this.cache.get(cls, { id, backendCacheKey });
+                const cached = this.#cache.get(cls, { id, backendCacheKey });
                 if (cached)
                     return cached;
             }
             else if (name) {
-                const cached = this.cache.get(cls, { name, backendCacheKey });
+                const cached = this.#cache.get(cls, { name, backendCacheKey });
                 if (cached)
                     return cached;
             }
             // Prevent the same string from being used as both a name and an alias.
-            this.cache.checkNameAliasConflict(cls, { name, alias, backendCacheKey });
+            this.#cache.checkNameAliasConflict(cls, { name, alias, backendCacheKey });
             // Cache miss — create the sub-backend and storage instance.
             const subBackend = await backendOpener();
             const storageInfo = await subBackend.getMetadata();
@@ -189,7 +189,7 @@ export class StorageInstanceManager {
             // we just fetched (so `id`/`name` etc. are available synchronously) along with the backend.
             const instance = new cls({ metadata: storageInfo, backend: subBackend });
             // Atomic cache writes (no awaits between these).
-            this.cache.set(cls, instance, backendCacheKey, alias);
+            this.#cache.set(cls, instance, backendCacheKey, alias);
             return instance;
         }
         finally {
@@ -197,7 +197,7 @@ export class StorageInstanceManager {
             // Clean up idle locks so the map doesn't grow unboundedly
             // (mirrors crawlee-python's WeakValueDictionary behaviour).
             if (queue.remaining === 0) {
-                this.openerLocks.delete(lockKey);
+                this.#openerLocks.delete(lockKey);
             }
         }
     }
@@ -205,7 +205,7 @@ export class StorageInstanceManager {
      * Remove a storage instance from the cache (called from `storage.drop()`).
      */
     removeFromCache(instance) {
-        this.cache.removeFromCache(instance);
+        this.#cache.removeFromCache(instance);
     }
     /**
      * Clear the entire cache. Also calls `clearCache()` on any cached KeyValueStore
@@ -213,12 +213,12 @@ export class StorageInstanceManager {
      * Called during service locator reset.
      */
     clearCache() {
-        for (const instance of this.cache.allValues()) {
+        for (const instance of this.#cache.allValues()) {
             if ('clearCache' in instance && typeof instance.clearCache === 'function') {
                 instance.clearCache();
             }
         }
-        this.cache.clear();
+        this.#cache.clear();
     }
 }
 /**

package/storages/storage_stats.d.ts

@@ -39,7 +39,7 @@ export interface RequestQueueStats {
  * the buckets that make sense for it.
  */
 export declare class StorageStatsTracker<T extends Record<keyof T, number>> {
-    private readonly counters;
+    #private;
     constructor(initial: T);
     /** Increment a counter bucket by `by` (default `1`). */
     add(key: keyof T, by?: number): void;

package/storages/storage_stats.js

@@ -14,16 +14,16 @@
  * the buckets that make sense for it.
  */
 export class StorageStatsTracker {
-    counters;
+    #counters;
     constructor(initial) {
-        this.counters = { ...initial };
+        this.#counters = { ...initial };
     }
     /** Increment a counter bucket by `by` (default `1`). */
     add(key, by = 1) {
-        this.counters[key] += by;
+        this.#counters[key] += by;
     }
     /** Return a snapshot of the current counters. The returned object is a copy and safe to keep. */
     get current() {
-        return { ...this.counters };
+        return { ...this.#counters };
     }
 }

package/storages/throttling_request_manager.d.ts

@@ -0,0 +1,239 @@
+import type { Dictionary } from '@crawlee/types';
+import type { Configuration } from '../configuration.js';
+import type { Request, Source } from '../request.js';
+import type { IRequestManager, RequestsLike } from './request_manager.js';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';
+import type { StorageIdentifier } from './storage_instance_manager.js';
+import type { StorageOpenOptions } from './utils.js';
+/**
+ * Opens a request manager, matching the shape of storage `open` methods such as
+ * {@link RequestQueue.open|`RequestQueue.open`}.
+ *
+ * {@link ThrottlingRequestManager} calls this once per configured domain, so every per-domain queue shares the
+ * concrete type and storage backend of the manager being wrapped.
+ */
+export type RequestManagerOpener<T extends IRequestManager = IRequestManager> = (identifier: string | StorageIdentifier, options?: StorageOpenOptions) => Promise<T>;
+/**
+ * A request manager that can pace requests per domain, as {@link ThrottlingRequestManager} does.
+ *
+ * The crawlers detect this structurally rather than by type, so a wrapper can opt in by forwarding these three
+ * methods without {@link IRequestManager} having to know that throttling exists.
+ */
+export interface SupportsDomainThrottling {
+    /** @see {@link ThrottlingRequestManager.recordDomainDelay} */
+    recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
+    /** @see {@link ThrottlingRequestManager.setCrawlDelay} */
+    setCrawlDelay(url: string, delaySeconds: number): boolean;
+    /** @see {@link ThrottlingRequestManager.assertNoStalledDomains} */
+    assertNoStalledDomains(): Promise<void>;
+}
+/** Whether `manager` can pace requests per domain. */
+export declare function supportsDomainThrottling(manager: unknown): manager is SupportsDomainThrottling;
+/** Options for {@link ThrottlingRequestManager}. */
+export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRequestManager> {
+    /**
+     * The request manager to wrap, usually a {@link RequestQueue}. Requests for domains that are not throttled
+     * are stored here.
+     */
+    inner: T;
+    /**
+     * Which domains to throttle: a list of hostnames, or `'all'` for every domain the crawl encounters.
+     *
+     * Matching a listed hostname is case-insensitive and exact - wildcards such as `*.example.com` are not
+     * supported, so list each subdomain you care about (or set
+     * {@link ThrottlingRequestManagerOptions.throttleBy|`throttleBy: 'registrableDomain'`}). An
+     * internationalized domain may be given in either its unicode or its punycode form, and an IPv6 address has
+     * to be bracketed (`[::1]`). Requests for any other domain bypass throttling entirely.
+     *
+     * `'all'` gives each domain a queue of its own the first time it is seen, so that it can be held back
+     * without its requests being repeatedly popped and re-enqueued. One request queue per domain is not free,
+     * which is what {@link ThrottlingRequestManagerOptions.maxThrottledDomains|`maxThrottledDomains`} is
+     * there to bound.
+     */
+    domains: string[] | 'all';
+    /**
+     * A floor under the crawl delay of every throttled domain, in seconds - the proactive clock described on
+     * {@link ThrottlingRequestManager}. A domain whose robots.txt asks for a longer `Crawl-delay` gets the
+     * longer one; this is a minimum, not an override.
+     * @default 0
+     */
+    minCrawlDelaySecs?: number;
+    /**
+     * What counts as "the same domain": the exact hostname, or the registrable domain it belongs to
+     * (`example.com` for `www.example.com`, `a.example.co.uk` and so on). Hosts with no registrable domain -
+     * IP addresses, `localhost` - are always throttled per hostname.
+     *
+     * Grouping by registrable domain gives subdomains a single pair of clocks and a single queue, which is what
+     * you want when the pacing is there to be polite to one server rather than to satisfy a specific host's
+     * rate limit.
+     * @default 'hostname'
+     */
+    throttleBy?: 'hostname' | 'registrableDomain';
+    /**
+     * The most domains a run may throttle at once. Exceeding it throws, rather than silently letting the
+     * throttling lapse - one request queue per domain is not free, and a crawl that discovers domains without
+     * bound would drown the storage backend in them.
+     *
+     * Only domains discovered under `domains: 'all'` count against this; an explicit list is taken at face value.
+     * @default 100
+     */
+    maxThrottledDomains?: number;
+    /**
+     * The key under which the discovered domain list is kept in the default key-value store, so that a restart
+     * with `purgeOnStart` disabled reopens their queues instead of stranding whatever they still hold. Only
+     * written under `domains: 'all'`.
+     *
+     * Give each manager its own key when running several of them against the same storage.
+     * @default 'CRAWLEE_THROTTLED_DOMAINS'
+     */
+    persistStateKey?: string;
+    /**
+     * Opens the per-domain queues, one per throttled domain, each under the alias `throttled-<domain>`.
+     * @default RequestQueue.open
+     */
+    requestManagerOpener?: RequestManagerOpener<T>;
+    /**
+     * The delay applied after a domain's first HTTP 429, doubled on each subsequent one.
+     * @default 2
+     */
+    baseDelaySecs?: number;
+    /**
+     * Upper bound on the delay between requests to a rate-limited domain, applied to both the exponential
+     * backoff and a `Retry-After` value.
+     * @default 60
+     */
+    maxDelaySecs?: number;
+    /**
+     * How long a domain may rate-limit us without a single request getting through before the crawl is
+     * abandoned with a {@link PersistentRateLimitError}.
+     *
+     * A domain that keeps answering 429 for this long is not going to be crawled by waiting longer - the
+     * concurrency is too high for it, or it has blocked us outright. Its requests are deliberately left in
+     * their queue, so re-running the crawl with `purgeOnStart` disabled picks them up once the domain recovers.
+     *
+     * A crawler running with `keepAlive` is exempt - outliving a domain that will not let us through is the
+     * whole point there.
+     * @default 900
+     */
+    maxDomainStallSecs?: number;
+}
+/**
+ * A request manager that wraps another one and paces requests per domain.
+ *
+ * Requests for a throttled domain are routed into their own queue when they are added, so each request lives in
+ * exactly one place and deduplication keeps working. Everything else goes to the wrapped manager untouched.
+ *
+ * {@link ThrottlingRequestManager.fetchNextRequest|`fetchNextRequest()`} serves the domain that has been waiting
+ * longest and skips any that are backing off, falling back to the wrapped manager. It never blocks: while every
+ * remaining request belongs to a throttled domain it returns `null` and {@link ThrottlingRequestManager.isEmpty}
+ * reports `true`, so the crawler idles instead of holding a concurrency slot open.
+ *
+ * Each throttled domain runs two independent clocks, and may be dispatched to once **both** have run out:
+ * - **Backoff**, set by HTTP 429 responses - honouring `Retry-After`, and otherwise doubling from `baseDelaySecs`.
+ *   Reactive and temporary: it decays once the domain stops turning us away. The crawlers report the 429s
+ *   themselves; a request held back this way is retried later without counting against `maxRequestRetries` and
+ *   without penalising its session.
+ * - **Crawl delay**, the minimum interval between two dispatches to the domain, armed after each one. Proactive
+ *   and constant: whatever the domain's robots.txt asks for, floored by
+ *   {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`}. Either may be absent, in
+ *   which case the other one is the delay.
+ *
+ * Which domains get those clocks is {@link ThrottlingRequestManagerOptions.domains|`domains`} - a list, or
+ * `'all'` for every domain the crawl encounters.
+ *
+ * **Example usage:**
+ *
+ * ```ts
+ * const crawler = new CheerioCrawler({
+ *     requestManager: new ThrottlingRequestManager({
+ *         inner: await RequestQueue.open(),
+ *         domains: ['api.example.com', 'slow-site.org'],
+ *     }),
+ *     requestHandler: async ({ request }) => { ... },
+ * });
+ * ```
+ *
+ * @category Sources
+ */
+export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager, SupportsDomainThrottling {
+    #private;
+    private readonly config;
+    private readonly domainStates;
+    private readonly log;
+    constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);
+    /** The wrapped manager, holding every request whose domain is not throttled. */
+    get innerManager(): T;
+    /**
+     * Records a 429 response and puts the URL's domain into backoff.
+     *
+     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     */
+    recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
+    /**
+     * Records the `Crawl-delay` a domain's robots.txt asked for, which becomes its crawl delay unless
+     * {@link ThrottlingRequestManagerOptions.minCrawlDelaySecs|`minCrawlDelaySecs`} asks for longer.
+     *
+     * The first value wins, so a robots.txt re-fetch cannot change the cadence mid-crawl.
+     *
+     * @returns `false` if the domain is not throttled, in which case this is a no-op.
+     */
+    setCrawlDelay(url: string, delaySeconds: number): boolean;
+    /**
+     * Throws {@link PersistentRateLimitError} if any domain has been rate-limiting us past
+     * {@link ThrottlingRequestManagerOptions.maxDomainStallSecs|`maxDomainStallSecs`} without letting a single
+     * request through.
+     *
+     * A domain qualifies only while it still has queued requests and is actively rate-limiting - a domain that
+     * has simply run out of work is finished, not stalled, and one being waited out under a long robots.txt
+     * `Crawl-delay` is being obeyed, not stonewalled.
+     */
+    assertNoStalledDomains(): Promise<void>;
+    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    /**
+     * Adds requests in batches, routing each one to the manager that owns its domain.
+     *
+     * Batching, validation, deduplication and `Retry-After`-free bookkeeping are all delegated to the target
+     * managers - this only decides where each request goes, one batch at a time, so a lazy or unbounded input
+     * iterable is never fully materialized.
+     */
+    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+    reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+    markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
+    getTotalCount(): Promise<number>;
+    getPendingCount(): Promise<number>;
+    getHandledCount(): Promise<number>;
+    /**
+     * Whether the next {@link ThrottlingRequestManager.fetchNextRequest} would return `null`.
+     *
+     * Requests waiting on a throttled domain count as unavailable, so a crawler whose task loop is gated on
+     * this idles for the backoff instead of spinning on a fetch that cannot succeed yet.
+     */
+    isEmpty(): Promise<boolean>;
+    /** Unlike {@link ThrottlingRequestManager.isEmpty}, throttled requests still count as outstanding work. */
+    isFinished(): Promise<boolean>;
+    /**
+     * Empties every manager and clears the accumulated backoff. A robots.txt `Crawl-delay` is a property of the
+     * site rather than of the run, so it survives.
+     */
+    purge(): Promise<void>;
+    /**
+     * Empties the per-domain queues, leaving the wrapped manager alone.
+     *
+     * Those queues are this manager's own no matter who owns the one it wraps, which is what makes this safe to
+     * call where a full {@link ThrottlingRequestManager.purge|`purge()`} would not be.
+     */
+    purgeDomainQueues(): Promise<void>;
+    setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+    /**
+     * Returns the next request from a domain that is not backing off, or from the inner manager.
+     *
+     * Returns `null` while every remaining request belongs to a throttled domain - it never waits the backoff
+     * out, because a consumer parked in here holds a concurrency slot, which the autoscaler reads as spare
+     * capacity and answers by scaling up. Callers poll instead, and {@link ThrottlingRequestManager.isEmpty}
+     * reports `true` meanwhile so the crawler's task loop idles rather than spins.
+     */
+    fetchNextRequest<R extends Dictionary = Dictionary>(): Promise<Request<R> | null>;
+    [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
+    persistState(): Promise<void>;
+    drop(): Promise<void>;
+}