@crawlee/core 4.0.0-beta.11 → 4.0.0-beta.111

package/storages/storage_instance_manager.js

@@ -0,0 +1,256 @@
+import { AsyncQueue } from '@sapphire/async-queue';
+/** Reserved alias for the default (unnamed) storage. */
+const DEFAULT_STORAGE_ALIAS = '__default__';
+/**
+ * Three-tier cache for storage instances, modelled after crawlee-python's `_StorageCache`.
+ *
+ * Each tier maps `[storageClass][key][backendCacheKey] → instance`:
+ *   - `byId`    — keyed by the backend-assigned storage id
+ *   - `byName`  — keyed by the persistent storage name
+ *   - `byAlias` — keyed by a run-scoped alias (e.g. `'__default__'` for unnamed storages)
+ */
+class StorageCache {
+    byId = new Map();
+    byName = new Map();
+    byAlias = new Map();
+    get(cls, { id, name, alias, backendCacheKey, }) {
+        for (const [tier, key] of [
+            [this.byId, id],
+            [this.byName, name],
+            [this.byAlias, alias],
+        ]) {
+            if (key === undefined)
+                continue;
+            const cached = tier.get(cls)?.get(key)?.get(backendCacheKey);
+            if (cached) {
+                if (cached instanceof cls) {
+                    return cached;
+                }
+                throw new Error('Cached storage instance type mismatch.');
+            }
+        }
+        return undefined;
+    }
+    /** Write a single entry into a given tier. */
+    setInMap(tier, cls, key, instance, backendCacheKey) {
+        if (!tier.has(cls))
+            tier.set(cls, new Map());
+        const keyMap = tier.get(cls);
+        if (!keyMap.has(key))
+            keyMap.set(key, new Map());
+        keyMap.get(key).set(backendCacheKey, instance);
+    }
+    /**
+     * Cache an instance under its actual id, name, and an optional alias.
+     */
+    set(cls, instance, backendCacheKey, alias) {
+        // Always cache by id.
+        this.setInMap(this.byId, cls, instance.id, instance, backendCacheKey);
+        // Cache by name — only for named storages.
+        if (instance.name) {
+            this.setInMap(this.byName, cls, instance.name, instance, backendCacheKey);
+        }
+        // Cache by alias — only for unnamed storages opened via alias.
+        if (alias !== undefined) {
+            this.setInMap(this.byAlias, cls, alias, instance, backendCacheKey);
+        }
+    }
+    removeFromCache(instance) {
+        const storageType = instance.constructor;
+        for (const tier of [this.byId, this.byName, this.byAlias]) {
+            const classMap = tier.get(storageType);
+            if (!classMap)
+                continue;
+            for (const keyMap of classMap.values()) {
+                for (const [cacheKey, cached] of keyMap) {
+                    if (cached === instance) {
+                        keyMap.delete(cacheKey);
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Ensure that the same string is not used as both a name and an alias for the same
+     * storage class + backend combination. Mirrors crawlee-python's `_check_name_alias_conflict`.
+     */
+    checkNameAliasConflict(cls, { name, alias, backendCacheKey }) {
+        if (alias) {
+            const existingByName = this.byName.get(cls)?.get(alias)?.get(backendCacheKey);
+            if (existingByName) {
+                throw new Error(`Cannot open storage with alias "${alias}" because a named storage with the same identifier already exists.`);
+            }
+        }
+        if (name) {
+            const existingByAlias = this.byAlias.get(cls)?.get(name)?.get(backendCacheKey);
+            if (existingByAlias) {
+                throw new Error(`Cannot open storage with name "${name}" because an alias storage with the same identifier already exists.` +
+                    ` If you meant to open the alias storage, use { alias: "${name}" } instead.`);
+            }
+        }
+    }
+    /** Iterate all cached instances across all storage types. */
+    *allValues() {
+        const seen = new Set();
+        for (const classMap of this.byId.values()) {
+            for (const keyMap of classMap.values()) {
+                for (const instance of keyMap.values()) {
+                    if (!seen.has(instance)) {
+                        seen.add(instance);
+                        yield instance;
+                    }
+                }
+            }
+        }
+    }
+    clear() {
+        this.byId.clear();
+        this.byName.clear();
+        this.byAlias.clear();
+    }
+}
+/**
+ * Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
+ *
+ * A single instance manages all storage types. Instances are cached by
+ * `(storageClass, id/name/alias, backendCacheKey)` so the same storage is never opened twice.
+ *
+ * The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
+ * and a pre-bound `backendOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
+ * assigns a reserved default alias.
+ */
+export class StorageInstanceManager {
+    #cache = new StorageCache();
+    #openerLocks = new Map();
+    /**
+     * Open (or retrieve from cache) a storage instance.
+     *
+     * @param cls               The storage class constructor (e.g. `Dataset`, `KeyValueStore`, `RequestQueue`).
+     * @param id                Storage ID (mutually exclusive with `name` and `alias`).
+     * @param name              Storage name (mutually exclusive with `id` and `alias`).
+     * @param alias             Run-scoped alias (mutually exclusive with `id` and `name`).
+     *                          Automatically assigned when no identifier is provided.
+     * @param backendOpener      A **lazy** factory that creates the sub-backend.
+     *                          Only called on a cache miss.
+     * @param backendCacheKey    Opaque key identifying the storage backend, so that the same logical
+     *                          storage opened through different clients is cached separately.
+     */
+    async openStorage(cls, { id, name, alias, backendOpener, backendCacheKey, }) {
+        // Auto-set alias='__default__' when no parameters are specified (mirrors crawlee-python).
+        if (!id && !name && !alias) {
+            alias = DEFAULT_STORAGE_ALIAS;
+        }
+        // Fast-path cache check (no lock).
+        if (alias !== undefined) {
+            const cached = this.#cache.get(cls, { alias, backendCacheKey });
+            if (cached)
+                return cached;
+        }
+        else if (id) {
+            const cached = this.#cache.get(cls, { id, backendCacheKey });
+            if (cached)
+                return cached;
+        }
+        else if (name) {
+            const cached = this.#cache.get(cls, { name, backendCacheKey });
+            if (cached)
+                return cached;
+        }
+        const identifierKey = id ?? name ?? alias ?? DEFAULT_STORAGE_ALIAS;
+        const lockKey = `${cls.name}:${identifierKey}:${backendCacheKey}`;
+        if (!this.#openerLocks.has(lockKey)) {
+            this.#openerLocks.set(lockKey, new AsyncQueue());
+        }
+        const queue = this.#openerLocks.get(lockKey);
+        await queue.wait();
+        try {
+            // Double-check cache under lock (another caller may have filled it while we waited).
+            if (alias !== undefined) {
+                const cached = this.#cache.get(cls, { alias, backendCacheKey });
+                if (cached)
+                    return cached;
+            }
+            else if (id) {
+                const cached = this.#cache.get(cls, { id, backendCacheKey });
+                if (cached)
+                    return cached;
+            }
+            else if (name) {
+                const cached = this.#cache.get(cls, { name, backendCacheKey });
+                if (cached)
+                    return cached;
+            }
+            // Prevent the same string from being used as both a name and an alias.
+            this.#cache.checkNameAliasConflict(cls, { name, alias, backendCacheKey });
+            // Cache miss — create the sub-backend and storage instance.
+            const subBackend = await backendOpener();
+            const storageInfo = await subBackend.getMetadata();
+            // Storage frontends are thin wrappers over the backend. We hand them the resolved metadata
+            // we just fetched (so `id`/`name` etc. are available synchronously) along with the backend.
+            const instance = new cls({ metadata: storageInfo, backend: subBackend });
+            // Atomic cache writes (no awaits between these).
+            this.#cache.set(cls, instance, backendCacheKey, alias);
+            return instance;
+        }
+        finally {
+            queue.shift();
+            // Clean up idle locks so the map doesn't grow unboundedly
+            // (mirrors crawlee-python's WeakValueDictionary behaviour).
+            if (queue.remaining === 0) {
+                this.#openerLocks.delete(lockKey);
+            }
+        }
+    }
+    /**
+     * Remove a storage instance from the cache (called from `storage.drop()`).
+     */
+    removeFromCache(instance) {
+        this.#cache.removeFromCache(instance);
+    }
+    /**
+     * Clear the entire cache. Also calls `clearCache()` on any cached KeyValueStore
+     * instances (duck-typed to avoid importing KeyValueStore and circular dependencies).
+     * Called during service locator reset.
+     */
+    clearCache() {
+        for (const instance of this.#cache.allValues()) {
+            if ('clearCache' in instance && typeof instance.clearCache === 'function') {
+                instance.clearCache();
+            }
+        }
+        this.#cache.clear();
+    }
+}
+/**
+ * Decompose a user-provided `identifier` (the `Dataset.open()` / `KeyValueStore.open()` /
+ * `RequestQueue.open()` argument) into separate `id`, `name`, and `alias` fields that
+ * the `StorageInstanceManager` and `StorageBackend.create*Client` expect.
+ *
+ * - `null` / `undefined` / `{}` → default storage alias
+ * - `string` → resolved via `storageExists` (ID-first, then name)
+ * - `{ id }` → `{ id }`
+ * - `{ name }` → `{ name }`
+ * - `{ alias }` → `{ alias }`
+ */
+export async function resolveStorageIdentifier(identifier, storageBackend, storageType) {
+    if (identifier === null || identifier === undefined) {
+        return { alias: DEFAULT_STORAGE_ALIAS };
+    }
+    if (typeof identifier === 'string') {
+        if (storageBackend.storageExists && (await storageBackend.storageExists(identifier, storageType))) {
+            return { id: identifier };
+        }
+        return { name: identifier };
+    }
+    if (identifier.id) {
+        return { id: identifier.id };
+    }
+    if (identifier.name) {
+        return { name: identifier.name };
+    }
+    if ('alias' in identifier && identifier.alias) {
+        return { alias: identifier.alias };
+    }
+    // Empty object — treated as default storage.
+    return { alias: DEFAULT_STORAGE_ALIAS };
+}

package/storages/storage_stats.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Backend-independent usage counters tracked by the storage frontend classes
+ * ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}).
+ *
+ * These count the operations the frontend issues against its underlying storage backend, so they are
+ * meaningful for any storage backend (memory, file system, cloud). They are tallied per client call
+ * — e.g. iterating a key-value store increments `readCount` once per record fetched and `listCount`
+ * once per listed page. Backend-specific figures that the frontend cannot compute (such as the number
+ * of bytes stored) are intentionally not included here; read those from the backend's own API instead.
+ */
+/** Usage counters for a {@link Dataset}. */
+export interface DatasetStats {
+    /** Number of read operations issued to the dataset client (e.g. `getData`). */
+    readCount: number;
+    /** Number of write operations issued to the dataset client (e.g. `pushData`). */
+    writeCount: number;
+}
+/** Usage counters for a {@link KeyValueStore}. */
+export interface KeyValueStoreStats {
+    /** Number of read operations issued to the key-value store client (e.g. `getValue`). */
+    readCount: number;
+    /** Number of write operations issued to the key-value store client (e.g. `setValue`). */
+    writeCount: number;
+    /** Number of delete operations issued to the key-value store client (e.g. `deleteValue`). */
+    deleteCount: number;
+    /** Number of listing operations issued to the key-value store client (e.g. `listKeys`). */
+    listCount: number;
+}
+/** Usage counters for a {@link RequestQueue}. */
+export interface RequestQueueStats {
+    /** Number of write operations issued to the request queue client (add / handle / reclaim). */
+    writeCount: number;
+    /** Number of queue-head reads issued to the request queue client (`fetchNextRequest`). */
+    headItemReadCount: number;
+}
+/**
+ * A tiny mutable counter that the storage frontends increment on each client call and expose through
+ * a read-only `stats` snapshot. Generic over the concrete counter shape so each storage type gets only
+ * the buckets that make sense for it.
+ */
+export declare class StorageStatsTracker<T extends Record<keyof T, number>> {
+    #private;
+    constructor(initial: T);
+    /** Increment a counter bucket by `by` (default `1`). */
+    add(key: keyof T, by?: number): void;
+    /** Return a snapshot of the current counters. The returned object is a copy and safe to keep. */
+    get current(): T;
+}

package/storages/storage_stats.js

@@ -0,0 +1,29 @@
+/**
+ * Backend-independent usage counters tracked by the storage frontend classes
+ * ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}).
+ *
+ * These count the operations the frontend issues against its underlying storage backend, so they are
+ * meaningful for any storage backend (memory, file system, cloud). They are tallied per client call
+ * — e.g. iterating a key-value store increments `readCount` once per record fetched and `listCount`
+ * once per listed page. Backend-specific figures that the frontend cannot compute (such as the number
+ * of bytes stored) are intentionally not included here; read those from the backend's own API instead.
+ */
+/**
+ * A tiny mutable counter that the storage frontends increment on each client call and expose through
+ * a read-only `stats` snapshot. Generic over the concrete counter shape so each storage type gets only
+ * the buckets that make sense for it.
+ */
+export class StorageStatsTracker {
+    #counters;
+    constructor(initial) {
+        this.#counters = { ...initial };
+    }
+    /** Increment a counter bucket by `by` (default `1`). */
+    add(key, by = 1) {
+        this.#counters[key] += by;
+    }
+    /** Return a snapshot of the current counters. The returned object is a copy and safe to keep. */
+    get current() {
+        return { ...this.#counters };
+    }
+}

package/storages/transaction.d.ts

@@ -0,0 +1,252 @@
+import type { Awaitable, Dictionary } from '@crawlee/types';
+import type { RecordOptions } from './key_value_store.js';
+/**
+ * Governs whether writes of a given storage type performed inside a {@link StorageTransaction} are
+ * applied immediately (`writeThrough`) or recorded and replayed on commit (`deferred`).
+ */
+export type StorageWriteMode = 'deferred' | 'writeThrough';
+/**
+ * Per-storage-type write policy of a {@link StorageTransaction}. Datasets and key-value stores are
+ * always `deferred` and not configurable — deferring is the only safe mode for non-idempotent writes,
+ * and {@link withDirectStorageAccess} covers one-off immediate writes.
+ */
+export interface StorageWritePolicy {
+    /**
+     * Write mode for request queue additions. Note that this is a *write policy* for the queue, not the
+     * queue instance itself (which is the top-level `requestQueue` crawler option).
+     *
+     * - `writeThrough` (default): requests are added immediately and are **not** rolled back with the
+     *   transaction. This is safe (additions are deduplicated by `uniqueKey`, so a retry is idempotent)
+     *   and keeps new requests visible to the crawler while the handler still runs.
+     * - `deferred`: requests are only added when the transaction commits — strict all-or-nothing
+     *   semantics, at the cost of the crawler not seeing them until the handler finishes.
+     */
+    requestQueue: StorageWriteMode;
+}
+export type StorageTransactionState = 'open' | 'committing' | 'committed' | 'failed' | 'rolledBack';
+/**
+ * A storage frontend that can record operations in a transaction journal.
+ * @internal
+ */
+export interface TransactionParticipant {
+    /**
+     * Replay the given buffered journal entries (all recorded by this participant) into the real storage
+     * backend. Called during commit, with the transaction already in the `committing` state, so the
+     * replayed operations pass through.
+     */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
+}
+/**
+ * A single dataset write (`pushData`) recorded in a transaction journal.
+ */
+export interface DatasetJournalEntry {
+    type: 'dataset';
+    /** @internal **/
+    participant: TransactionParticipant;
+    storageId: string;
+    /** The pushed items, captured by `structuredClone` at write time. */
+    items: Dictionary[];
+    recordedAt: Date;
+}
+/**
+ * A single key-value store write (`setValue`) recorded in a transaction journal.
+ */
+export interface KeyValueStoreJournalEntry {
+    type: 'keyValueStore';
+    /** @internal **/
+    participant: TransactionParticipant;
+    storageId: string;
+    key: string;
+    /** The original, pre-serialization value captured by `structuredClone`; `null` denotes a deletion. */
+    value: unknown;
+    options?: RecordOptions;
+}
+/**
+ * A request recorded in a transaction journal.
+ */
+export interface JournaledRequest {
+    url: string;
+    uniqueKey: string;
+    label?: string;
+    /**
+     * A full JSON snapshot of the request for the commit replay. Only present for buffered additions —
+     * deduplicated and write-through ones are journaled for introspection only.
+     */
+    snapshot?: Dictionary;
+}
+/**
+ * A batch of request queue additions recorded in a transaction journal.
+ */
+export interface RequestQueueJournalEntry {
+    type: 'requestQueue';
+    /** @internal **/
+    participant: TransactionParticipant;
+    requests: JournaledRequest[];
+    forefront: boolean;
+    /** Write-through entries were applied immediately; they are never replayed. */
+    writeThrough: boolean;
+}
+export type JournalEntry = DatasetJournalEntry | KeyValueStoreJournalEntry | RequestQueueJournalEntry;
+/**
+ * A read-only view of a {@link StorageTransaction}: only the journal-backed introspection accessors,
+ * without the lifecycle methods. The accessors are synchronous and expose the original pre-serialization
+ * values. They cover every write recorded while the transaction was open, under either write policy —
+ * with the one exception noted on {@link StorageTransactionView.enqueuedUrls|`enqueuedUrls`}. A view
+ * is valid until the transaction is disposed.
+ */
+export interface StorageTransactionView {
+    readonly state: StorageTransactionState;
+    /** Items pushed to datasets during the transaction, in push order. */
+    readonly datasetItems: {
+        item: Dictionary;
+        datasetId: string;
+    }[];
+    /**
+     * URLs enqueued to request queues during the transaction, under either write policy. Recorded as
+     * requested, so duplicate, already-present and backend-rejected URLs are included.
+     *
+     * One gap: unless a caller of `addRequestsBatched()` waits for every chunk
+     * (`waitForAllRequestsToBeAdded` or `maxNewRequests`, both of which {@link enqueueLinks} sets when
+     * a crawl limit applies), the chunks after the first are added by a background writer that outlives
+     * the transaction and is not recorded here.
+     */
+    readonly enqueuedUrls: {
+        url: string;
+        label?: string;
+    }[];
+    /** Key-value store changes made during the transaction, keyed by store id, last write per key. */
+    readonly keyValueStoreChanges: Record<string, Record<string, {
+        changedValue: unknown;
+        options?: RecordOptions;
+    }>>;
+}
+export interface StorageTransactionOptions {
+    /** Overrides of the per-storage-type write policy. See {@link StorageWritePolicy}. */
+    policy?: Partial<StorageWritePolicy>;
+    /**
+     * How long a commit may take before it fails, in milliseconds. There is no automatic retry — the
+     * replay of dataset items is not idempotent.
+     * @default 300000
+     */
+    commitTimeoutMillis?: number;
+}
+/**
+ * A storage transaction scoped to a request's lifecycle. Writes made through the storage frontends
+ * ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}) while the transaction is active
+ * are recorded rather than applied; on {@link StorageTransaction.commit|`commit()`} they are replayed
+ * into real storage, on {@link StorageTransaction.rollback|`rollback()`} they are dropped. Reads consult
+ * the recorded writes first, so a handler sees its own writes.
+ *
+ * Create one with {@link createStorageTransaction} (explicit commit/rollback) or
+ * {@link withStorageTransaction} (scoped sugar). Crawlers open one automatically around every request
+ * handler unless `transactionalStorage: false` is set.
+ */
+export declare class StorageTransaction implements StorageTransactionView {
+    #private;
+    /** The ordered, append-only journal — the source of truth for commit, introspection and reads. */
+    readonly journal: JournalEntry[];
+    /** Per-storage-type write policy. */
+    readonly policy: StorageWritePolicy;
+    /** @internal */
+    constructor(options?: StorageTransactionOptions);
+    get state(): StorageTransactionState;
+    /**
+     * `true` only while `state === 'open'`. This is the single predicate every storage operation
+     * consults — operations performed after the transaction is closed pass through to the real backend.
+     */
+    get isActive(): boolean;
+    /** Runs `callback` with this transaction installed in the async context. */
+    run<T>(callback: () => Awaitable<T>): Promise<T>;
+    /**
+     * Records a write operation in the journal.
+     * @internal
+     */
+    recordJournalEntry(entry: JournalEntry): void;
+    /**
+     * Replays the journaled writes into real storage. A no-op unless the transaction is `open`.
+     *
+     * The transaction transitions to `committing` *before* anything is flushed, so a commit that throws
+     * partway lands in `failed` (never back in `open`) and subsequent storage operations pass through
+     * rather than recording into a dead transaction. Delivery is at-least-once — a commit that fails
+     * partway may have applied some of the writes already.
+     */
+    commit(): Promise<void>;
+    private flush;
+    /**
+     * Discards the journaled writes. A no-op unless the transaction is `open` — in particular, calling it
+     * after a successful `commit()` (which the crawler's error handling can legitimately do) does nothing
+     * and never throws.
+     */
+    rollback(): void;
+    /**
+     * Releases the journal and the write-time snapshots it holds. Must be called for *every* terminal
+     * state, `failed` included. Idempotent, never throws, and does not change `state`. Any
+     * {@link StorageTransactionView} of this transaction is only valid until this is called.
+     */
+    dispose(): void;
+    get datasetItems(): {
+        item: Dictionary;
+        datasetId: string;
+    }[];
+    get enqueuedUrls(): {
+        url: string;
+        label?: string;
+    }[];
+    get keyValueStoreChanges(): Record<string, Record<string, {
+        changedValue: unknown;
+        options?: RecordOptions;
+    }>>;
+}
+/**
+ * Opens a {@link StorageTransaction} without running anything yet. The caller owns the outcome:
+ * `run()`, then `commit()` or `rollback()`, and always `dispose()` when done. For the common
+ * open-run-commit flow, prefer {@link withStorageTransaction}.
+ */
+export declare function createStorageTransaction(options?: StorageTransactionOptions): StorageTransaction;
+/**
+ * Runs `callback` inside a new {@link StorageTransaction}: storage writes made in the callback are
+ * committed when it returns and rolled back when it throws. If a transaction is already active in the
+ * current async context, it is reused and its outcome is left to its owner (and `options` are ignored)
+ * — there are no nested transaction semantics.
+ */
+export declare function withStorageTransaction<T>(callback: (transaction: StorageTransaction) => Awaitable<T>, options?: StorageTransactionOptions): Promise<T>;
+/**
+ * Runs `callback` outside of any storage transaction — the per-call-site escape hatch. Storage operations
+ * made inside it hit the real backend directly, are not rolled back, and operations that a transaction
+ * rejects (`drop`, stream-valued `setValue`, request queue internals, ...) are permitted.
+ */
+export declare function withDirectStorageAccess<T>(callback: () => Awaitable<T>): Promise<T>;
+/**
+ * The per-operation hook consulted by every storage frontend method: performs the cancellation check
+ * that aborts storage operations when the request handler times out, and returns the active storage
+ * transaction. Returns `undefined` when there is no transaction in the async context *or* when it is no
+ * longer open — operations on a closed transaction deliberately pass through to the real backend.
+ * @internal
+ */
+export declare function activeStorageTransaction(): StorageTransaction | undefined;
+/**
+ * Returns the transaction installed in the current async context, regardless of its state. Used by the
+ * crawler to drive the outcome of the transaction it opened.
+ * @internal
+ */
+export declare function currentStorageTransaction(): StorageTransaction | undefined;
+/**
+ * Captures a value at write time, so that later mutations of the caller's object affect neither the
+ * read-your-own-writes reads nor the commit replay. `structuredClone` for fidelity (`Date`, `Map`, `Set`,
+ * typed arrays, `undefined`); values it cannot handle fall back to the JSON round-trip the storage
+ * backends perform anyway.
+ * @internal
+ */
+export declare function snapshotValue<T>(value: T): T;
+/**
+ * The guard for operations that cannot be performed inside a storage transaction: throws when one is
+ * active, and performs the per-operation cancellation check either way.
+ * @internal
+ */
+export declare function rejectOperationInTransaction(operation: string, reason?: string): void;
+/**
+ * Builds the "operation not allowed in a transaction" error, for a call site that has already
+ * established a transaction is active and so wants to `throw` unconditionally.
+ * @internal
+ */
+export declare function operationRejectedInTransaction(operation: string, reason?: string): Error;