@crawlee/core 4.0.0-beta.12 → 4.0.0-beta.121

package/storages/storage_instance_manager.js

@@ -0,0 +1,256 @@
+import { AsyncQueue } from '@sapphire/async-queue';
+/** Reserved alias for the default (unnamed) storage. */
+const DEFAULT_STORAGE_ALIAS = '__default__';
+/**
+ * Three-tier cache for storage instances, modelled after crawlee-python's `_StorageCache`.
+ *
+ * Each tier maps `[storageClass][key][backendCacheKey] → instance`:
+ *   - `byId`    — keyed by the backend-assigned storage id
+ *   - `byName`  — keyed by the persistent storage name
+ *   - `byAlias` — keyed by a run-scoped alias (e.g. `'__default__'` for unnamed storages)
+ */
+class StorageCache {
+    byId = new Map();
+    byName = new Map();
+    byAlias = new Map();
+    get(cls, { id, name, alias, backendCacheKey, }) {
+        for (const [tier, key] of [
+            [this.byId, id],
+            [this.byName, name],
+            [this.byAlias, alias],
+        ]) {
+            if (key === undefined)
+                continue;
+            const cached = tier.get(cls)?.get(key)?.get(backendCacheKey);
+            if (cached) {
+                if (cached instanceof cls) {
+                    return cached;
+                }
+                throw new Error('Cached storage instance type mismatch.');
+            }
+        }
+        return undefined;
+    }
+    /** Write a single entry into a given tier. */
+    setInMap(tier, cls, key, instance, backendCacheKey) {
+        if (!tier.has(cls))
+            tier.set(cls, new Map());
+        const keyMap = tier.get(cls);
+        if (!keyMap.has(key))
+            keyMap.set(key, new Map());
+        keyMap.get(key).set(backendCacheKey, instance);
+    }
+    /**
+     * Cache an instance under its actual id, name, and an optional alias.
+     */
+    set(cls, instance, backendCacheKey, alias) {
+        // Always cache by id.
+        this.setInMap(this.byId, cls, instance.id, instance, backendCacheKey);
+        // Cache by name — only for named storages.
+        if (instance.name) {
+            this.setInMap(this.byName, cls, instance.name, instance, backendCacheKey);
+        }
+        // Cache by alias — only for unnamed storages opened via alias.
+        if (alias !== undefined) {
+            this.setInMap(this.byAlias, cls, alias, instance, backendCacheKey);
+        }
+    }
+    removeFromCache(instance) {
+        const storageType = instance.constructor;
+        for (const tier of [this.byId, this.byName, this.byAlias]) {
+            const classMap = tier.get(storageType);
+            if (!classMap)
+                continue;
+            for (const keyMap of classMap.values()) {
+                for (const [cacheKey, cached] of keyMap) {
+                    if (cached === instance) {
+                        keyMap.delete(cacheKey);
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Ensure that the same string is not used as both a name and an alias for the same
+     * storage class + backend combination. Mirrors crawlee-python's `_check_name_alias_conflict`.
+     */
+    checkNameAliasConflict(cls, { name, alias, backendCacheKey }) {
+        if (alias) {
+            const existingByName = this.byName.get(cls)?.get(alias)?.get(backendCacheKey);
+            if (existingByName) {
+                throw new Error(`Cannot open storage with alias "${alias}" because a named storage with the same identifier already exists.`);
+            }
+        }
+        if (name) {
+            const existingByAlias = this.byAlias.get(cls)?.get(name)?.get(backendCacheKey);
+            if (existingByAlias) {
+                throw new Error(`Cannot open storage with name "${name}" because an alias storage with the same identifier already exists.` +
+                    ` If you meant to open the alias storage, use { alias: "${name}" } instead.`);
+            }
+        }
+    }
+    /** Iterate all cached instances across all storage types. */
+    *allValues() {
+        const seen = new Set();
+        for (const classMap of this.byId.values()) {
+            for (const keyMap of classMap.values()) {
+                for (const instance of keyMap.values()) {
+                    if (!seen.has(instance)) {
+                        seen.add(instance);
+                        yield instance;
+                    }
+                }
+            }
+        }
+    }
+    clear() {
+        this.byId.clear();
+        this.byName.clear();
+        this.byAlias.clear();
+    }
+}
+/**
+ * Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
+ *
+ * A single instance manages all storage types. Instances are cached by
+ * `(storageClass, id/name/alias, backendCacheKey)` so the same storage is never opened twice.
+ *
+ * The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
+ * and a pre-bound `backendOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
+ * assigns a reserved default alias.
+ */
+export class StorageInstanceManager {
+    #cache = new StorageCache();
+    #openerLocks = new Map();
+    /**
+     * Open (or retrieve from cache) a storage instance.
+     *
+     * @param cls               The storage class constructor (e.g. `Dataset`, `KeyValueStore`, `RequestQueue`).
+     * @param id                Storage ID (mutually exclusive with `name` and `alias`).
+     * @param name              Storage name (mutually exclusive with `id` and `alias`).
+     * @param alias             Run-scoped alias (mutually exclusive with `id` and `name`).
+     *                          Automatically assigned when no identifier is provided.
+     * @param backendOpener      A **lazy** factory that creates the sub-backend.
+     *                          Only called on a cache miss.
+     * @param backendCacheKey    Opaque key identifying the storage backend, so that the same logical
+     *                          storage opened through different clients is cached separately.
+     */
+    async openStorage(cls, { id, name, alias, backendOpener, backendCacheKey, }) {
+        // Auto-set alias='__default__' when no parameters are specified (mirrors crawlee-python).
+        if (!id && !name && !alias) {
+            alias = DEFAULT_STORAGE_ALIAS;
+        }
+        // Fast-path cache check (no lock).
+        if (alias !== undefined) {
+            const cached = this.#cache.get(cls, { alias, backendCacheKey });
+            if (cached)
+                return cached;
+        }
+        else if (id) {
+            const cached = this.#cache.get(cls, { id, backendCacheKey });
+            if (cached)
+                return cached;
+        }
+        else if (name) {
+            const cached = this.#cache.get(cls, { name, backendCacheKey });
+            if (cached)
+                return cached;
+        }
+        const identifierKey = id ?? name ?? alias ?? DEFAULT_STORAGE_ALIAS;
+        const lockKey = `${cls.name}:${identifierKey}:${backendCacheKey}`;
+        if (!this.#openerLocks.has(lockKey)) {
+            this.#openerLocks.set(lockKey, new AsyncQueue());
+        }
+        const queue = this.#openerLocks.get(lockKey);
+        await queue.wait();
+        try {
+            // Double-check cache under lock (another caller may have filled it while we waited).
+            if (alias !== undefined) {
+                const cached = this.#cache.get(cls, { alias, backendCacheKey });
+                if (cached)
+                    return cached;
+            }
+            else if (id) {
+                const cached = this.#cache.get(cls, { id, backendCacheKey });
+                if (cached)
+                    return cached;
+            }
+            else if (name) {
+                const cached = this.#cache.get(cls, { name, backendCacheKey });
+                if (cached)
+                    return cached;
+            }
+            // Prevent the same string from being used as both a name and an alias.
+            this.#cache.checkNameAliasConflict(cls, { name, alias, backendCacheKey });
+            // Cache miss — create the sub-backend and storage instance.
+            const subBackend = await backendOpener();
+            const storageInfo = await subBackend.getMetadata();
+            // Storage frontends are thin wrappers over the backend. We hand them the resolved metadata
+            // we just fetched (so `id`/`name` etc. are available synchronously) along with the backend.
+            const instance = new cls({ metadata: storageInfo, backend: subBackend });
+            // Atomic cache writes (no awaits between these).
+            this.#cache.set(cls, instance, backendCacheKey, alias);
+            return instance;
+        }
+        finally {
+            queue.shift();
+            // Clean up idle locks so the map doesn't grow unboundedly
+            // (mirrors crawlee-python's WeakValueDictionary behaviour).
+            if (queue.remaining === 0) {
+                this.#openerLocks.delete(lockKey);
+            }
+        }
+    }
+    /**
+     * Remove a storage instance from the cache (called from `storage.drop()`).
+     */
+    removeFromCache(instance) {
+        this.#cache.removeFromCache(instance);
+    }
+    /**
+     * Clear the entire cache. Also calls `clearCache()` on any cached KeyValueStore
+     * instances (duck-typed to avoid importing KeyValueStore and circular dependencies).
+     * Called during service locator reset.
+     */
+    clearCache() {
+        for (const instance of this.#cache.allValues()) {
+            if ('clearCache' in instance && typeof instance.clearCache === 'function') {
+                instance.clearCache();
+            }
+        }
+        this.#cache.clear();
+    }
+}
+/**
+ * Decompose a user-provided `identifier` (the `Dataset.open()` / `KeyValueStore.open()` /
+ * `RequestQueue.open()` argument) into separate `id`, `name`, and `alias` fields that
+ * the `StorageInstanceManager` and `StorageBackend.create*Client` expect.
+ *
+ * - `null` / `undefined` / `{}` → default storage alias
+ * - `string` → resolved via `storageExists` (ID-first, then name)
+ * - `{ id }` → `{ id }`
+ * - `{ name }` → `{ name }`
+ * - `{ alias }` → `{ alias }`
+ */
+export async function resolveStorageIdentifier(identifier, storageBackend, storageType) {
+    if (identifier === null || identifier === undefined) {
+        return { alias: DEFAULT_STORAGE_ALIAS };
+    }
+    if (typeof identifier === 'string') {
+        if (storageBackend.storageExists && (await storageBackend.storageExists(identifier, storageType))) {
+            return { id: identifier };
+        }
+        return { name: identifier };
+    }
+    if (identifier.id) {
+        return { id: identifier.id };
+    }
+    if (identifier.name) {
+        return { name: identifier.name };
+    }
+    if ('alias' in identifier && identifier.alias) {
+        return { alias: identifier.alias };
+    }
+    // Empty object — treated as default storage.
+    return { alias: DEFAULT_STORAGE_ALIAS };
+}

package/storages/storage_stats.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Backend-independent usage counters tracked by the storage frontend classes
+ * ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}).
+ *
+ * These count the operations the frontend issues against its underlying storage backend, so they are
+ * meaningful for any storage backend (memory, file system, cloud). They are tallied per client call
+ * — e.g. iterating a key-value store increments `readCount` once per record fetched and `listCount`
+ * once per listed page. Backend-specific figures that the frontend cannot compute (such as the number
+ * of bytes stored) are intentionally not included here; read those from the backend's own API instead.
+ */
+/** Usage counters for a {@link Dataset}. */
+export interface DatasetStats {
+    /** Number of read operations issued to the dataset client (e.g. `getData`). */
+    readCount: number;
+    /** Number of write operations issued to the dataset client (e.g. `pushData`). */
+    writeCount: number;
+}
+/** Usage counters for a {@link KeyValueStore}. */
+export interface KeyValueStoreStats {
+    /** Number of read operations issued to the key-value store client (e.g. `getValue`). */
+    readCount: number;
+    /** Number of write operations issued to the key-value store client (e.g. `setValue`). */
+    writeCount: number;
+    /** Number of delete operations issued to the key-value store client (e.g. `deleteValue`). */
+    deleteCount: number;
+    /** Number of listing operations issued to the key-value store client (e.g. `listKeys`). */
+    listCount: number;
+}
+/** Usage counters for a {@link RequestQueue}. */
+export interface RequestQueueStats {
+    /** Number of write operations issued to the request queue client (add / handle / reclaim). */
+    writeCount: number;
+    /** Number of queue-head reads issued to the request queue client (`fetchNextRequest`). */
+    headItemReadCount: number;
+}
+/**
+ * A tiny mutable counter that the storage frontends increment on each client call and expose through
+ * a read-only `stats` snapshot. Generic over the concrete counter shape so each storage type gets only
+ * the buckets that make sense for it.
+ */
+export declare class StorageStatsTracker<T extends Record<keyof T, number>> {
+    #private;
+    constructor(initial: T);
+    /** Increment a counter bucket by `by` (default `1`). */
+    add(key: keyof T, by?: number): void;
+    /** Return a snapshot of the current counters. The returned object is a copy and safe to keep. */
+    get current(): T;
+}

package/storages/storage_stats.js

@@ -0,0 +1,29 @@
+/**
+ * Backend-independent usage counters tracked by the storage frontend classes
+ * ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}).
+ *
+ * These count the operations the frontend issues against its underlying storage backend, so they are
+ * meaningful for any storage backend (memory, file system, cloud). They are tallied per client call
+ * — e.g. iterating a key-value store increments `readCount` once per record fetched and `listCount`
+ * once per listed page. Backend-specific figures that the frontend cannot compute (such as the number
+ * of bytes stored) are intentionally not included here; read those from the backend's own API instead.
+ */
+/**
+ * A tiny mutable counter that the storage frontends increment on each client call and expose through
+ * a read-only `stats` snapshot. Generic over the concrete counter shape so each storage type gets only
+ * the buckets that make sense for it.
+ */
+export class StorageStatsTracker {
+    #counters;
+    constructor(initial) {
+        this.#counters = { ...initial };
+    }
+    /** Increment a counter bucket by `by` (default `1`). */
+    add(key, by = 1) {
+        this.#counters[key] += by;
+    }
+    /** Return a snapshot of the current counters. The returned object is a copy and safe to keep. */
+    get current() {
+        return { ...this.#counters };
+    }
+}

package/storages/throttling_request_manager.d.ts

@@ -0,0 +1,216 @@
+import type { Dictionary } from '@crawlee/types';
+import type { Configuration } from '../configuration.js';
+import type { Request, Source } from '../request.js';
+import type { IRequestManager, RequestsLike } from './request_manager.js';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';
+import type { StorageIdentifier } from './storage_instance_manager.js';
+import type { StorageOpenOptions } from './utils.js';
+/**
+ * Opens a request manager, matching the shape of storage `open` methods such as
+ * {@link RequestQueue.open|`RequestQueue.open`}.
+ *
+ * {@link ThrottlingRequestManager} calls this once per configured domain, so every per-domain queue shares the
+ * concrete type and storage backend of the manager being wrapped.
+ */
+export type RequestManagerOpener<T extends IRequestManager = IRequestManager> = (identifier: string | StorageIdentifier, options?: StorageOpenOptions) => Promise<T>;
+/**
+ * A request manager that can pace requests per domain, as {@link ThrottlingRequestManager} does.
+ *
+ * The crawlers detect this structurally rather than by type, so a wrapper can opt in by forwarding these three
+ * methods without {@link IRequestManager} having to know that throttling exists.
+ */
+export interface SupportsDomainThrottling {
+    /** @see {@link ThrottlingRequestManager.recordDomainDelay} */
+    recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
+    /** @see {@link ThrottlingRequestManager.setCrawlDelay} */
+    setCrawlDelay(url: string, delaySeconds: number): boolean;
+    /** @see {@link ThrottlingRequestManager.assertNoStalledDomains} */
+    assertNoStalledDomains(): Promise<void>;
+}
+/** Whether `manager` can pace requests per domain. */
+export declare function supportsDomainThrottling(manager: unknown): manager is SupportsDomainThrottling;
+/** Options for {@link ThrottlingRequestManager}. */
+export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRequestManager> {
+    /**
+     * The request manager to wrap, usually a {@link RequestQueue}. Requests for domains that are not throttled
+     * are stored here.
+     */
+    inner: T;
+    /**
+     * Hostnames to throttle. Matching is case-insensitive and exact - wildcards such as `*.example.com` are not
+     * supported, so list each subdomain you care about. Requests for any other domain bypass throttling entirely.
+     *
+     * An internationalized domain may be given in either its unicode or its punycode form, and an IPv6 address
+     * has to be bracketed (`[::1]`).
+     */
+    domains: string[];
+    /**
+     * Opens the per-domain queues, one per entry in `domains`, each under the alias `throttled-<domain>`.
+     * @default RequestQueue.open
+     */
+    requestManagerOpener?: RequestManagerOpener<T>;
+    /**
+     * The delay applied after a domain's first HTTP 429, doubled on each subsequent one.
+     * @default 2
+     */
+    baseDelaySecs?: number;
+    /**
+     * Upper bound on the delay between requests to a rate-limited domain, applied to both the exponential
+     * backoff and a `Retry-After` value.
+     * @default 60
+     */
+    maxDelaySecs?: number;
+    /**
+     * How long a domain may rate-limit us without a single request getting through before the crawl is
+     * abandoned with a {@link PersistentRateLimitError}.
+     *
+     * A domain that keeps answering 429 for this long is not going to be crawled by waiting longer - the
+     * concurrency is too high for it, or it has blocked us outright. Its requests are deliberately left in
+     * their queue, so re-running the crawl with `purgeOnStart` disabled picks them up once the domain recovers.
+     *
+     * A crawler running with `keepAlive` is exempt - outliving a domain that will not let us through is the
+     * whole point there.
+     * @default 900
+     */
+    maxDomainStallSecs?: number;
+}
+/**
+ * A request manager that wraps another one and paces requests per domain.
+ *
+ * Requests for the configured {@link ThrottlingRequestManagerOptions.domains|`domains`} are routed into their own
+ * queue when they are added, so each request lives in exactly one place and deduplication keeps working. Everything
+ * else goes to the wrapped manager untouched.
+ *
+ * {@link ThrottlingRequestManager.fetchNextRequest|`fetchNextRequest()`} serves the domain that has been waiting
+ * longest and skips any that are backing off, falling back to the wrapped manager. It never blocks: while every
+ * remaining request belongs to a throttled domain it returns `null` and {@link ThrottlingRequestManager.isEmpty}
+ * reports `true`, so the crawler idles instead of holding a concurrency slot open.
+ *
+ * Delays come from two places:
+ * - HTTP 429 responses, honouring `Retry-After` and otherwise backing off exponentially. The crawlers report these
+ *   automatically; a request that is throttled is retried later without counting against `maxRequestRetries` and
+ *   without penalising its session.
+ * - robots.txt `Crawl-delay` directives, when `respectRobotsTxtFile` is enabled.
+ *
+ * This is opt-in: throttling only happens for a domain you list explicitly.
+ *
+ * **Example usage:**
+ *
+ * ```ts
+ * const crawler = new CheerioCrawler({
+ *     requestManager: new ThrottlingRequestManager({
+ *         inner: await RequestQueue.open(),
+ *         domains: ['api.example.com', 'slow-site.org'],
+ *     }),
+ *     requestHandler: async ({ request }) => { ... },
+ * });
+ * ```
+ *
+ * @category Sources
+ */
+export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager, SupportsDomainThrottling {
+    private readonly config;
+    private readonly inner;
+    private readonly requestManagerOpener;
+    private readonly baseDelayMs;
+    private readonly maxDelayMs;
+    private readonly maxDomainStallMs;
+    private readonly domainStates;
+    private readonly subManagers;
+    private readonly log;
+    /**
+     * Sub-managers are keyed by a stable alias, so with `purgeOnStart` disabled they outlive the process. They
+     * must therefore be reopened for every configured domain rather than created on first insert - otherwise a
+     * restart sees an empty map, reports the crawl finished, and strands whatever the previous run left in them.
+     */
+    private subManagersReady?;
+    /** Batches still being added in the background; keeps {@link ThrottlingRequestManager.isFinished} honest. */
+    private inProgressBatchCount;
+    private readonly warnedAbout;
+    private get hasThrottledDomains();
+    constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);
+    /** The wrapped manager, holding every request whose domain is not throttled. */
+    get innerManager(): T;
+    /** Warns once about sources that cannot be routed by domain, because their URLs are not known yet. */
+    private warnIfNotRoutable;
+    private warnOnce;
+    private extractDomain;
+    private getDomainState;
+    private selectManager;
+    /** Only valid once {@link ThrottlingRequestManager.ensureSubManagers} has resolved. */
+    private managerForUrl;
+    private ensureSubManagers;
+    private getSubManagers;
+    /** Configured domains that are not currently backing off, longest-overdue first. */
+    private fetchableDomains;
+    /**
+     * Records a 429 response and puts the URL's domain into backoff.
+     *
+     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     */
+    recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
+    /**
+     * Applies a robots.txt `Crawl-delay` to the URL's domain, as a minimum interval between dispatches.
+     *
+     * The first value wins, so a robots.txt re-fetch cannot change the cadence mid-crawl.
+     *
+     * @returns `false` if the domain is not configured for throttling, in which case this is a no-op.
+     */
+    setCrawlDelay(url: string, delaySeconds: number): boolean;
+    /**
+     * Throws {@link PersistentRateLimitError} if any domain has been rate-limiting us past
+     * {@link ThrottlingRequestManagerOptions.maxDomainStallSecs|`maxDomainStallSecs`} without letting a single
+     * request through.
+     *
+     * A domain qualifies only while it still has queued requests and is actively rate-limiting - a domain that
+     * has simply run out of work is finished, not stalled, and one being waited out under a long robots.txt
+     * `Crawl-delay` is being obeyed, not stonewalled.
+     */
+    assertNoStalledDomains(): Promise<void>;
+    /** Records that a domain let a request through, which ends any rate-limit run stall detection was timing. */
+    private recordProgress;
+    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    /**
+     * Adds requests in batches, routing each one to the manager that owns its domain.
+     *
+     * Batching, validation, deduplication and `Retry-After`-free bookkeeping are all delegated to the target
+     * managers - this only decides where each request goes, one batch at a time, so a lazy or unbounded input
+     * iterable is never fully materialized.
+     */
+    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+    reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+    markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
+    getTotalCount(): Promise<number>;
+    getPendingCount(): Promise<number>;
+    getHandledCount(): Promise<number>;
+    /**
+     * Whether the next {@link ThrottlingRequestManager.fetchNextRequest} would return `null`.
+     *
+     * Requests waiting on a throttled domain count as unavailable, so a crawler whose task loop is gated on
+     * this idles for the backoff instead of spinning on a fetch that cannot succeed yet.
+     */
+    isEmpty(): Promise<boolean>;
+    /** Unlike {@link ThrottlingRequestManager.isEmpty}, throttled requests still count as outstanding work. */
+    isFinished(): Promise<boolean>;
+    /**
+     * Empties every manager and clears the accumulated backoff. A robots.txt `Crawl-delay` is a property of the
+     * site rather than of the run, so it survives.
+     */
+    purge(): Promise<void>;
+    setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+    private forEachManager;
+    private sumOverManagers;
+    private everyManager;
+    /**
+     * Returns the next request from a domain that is not backing off, or from the inner manager.
+     *
+     * Returns `null` while every remaining request belongs to a throttled domain - it never waits the backoff
+     * out, because a consumer parked in here holds a concurrency slot, which the autoscaler reads as spare
+     * capacity and answers by scaling up. Callers poll instead, and {@link ThrottlingRequestManager.isEmpty}
+     * reports `true` meanwhile so the crawler's task loop idles rather than spins.
+     */
+    fetchNextRequest<R extends Dictionary = Dictionary>(): Promise<Request<R> | null>;
+    [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
+    persistState(): Promise<void>;
+    drop(): Promise<void>;
+}