@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

package/storages/storage_instance_manager.d.ts

@@ -7,7 +7,6 @@ export interface IStorage {
     id: string;
     name?: string;
 }
-type Hashable = string;
 /**
  * Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
  *
@@ -19,8 +18,7 @@ type Hashable = string;
  * assigns a reserved default alias.
  */
 export declare class StorageInstanceManager {
-    private readonly cache;
-    private readonly openerLocks;
+    #private;
     /**
      * Open (or retrieve from cache) a storage instance.
      *
@@ -36,7 +34,7 @@ export declare class StorageInstanceManager {
      */
     openStorage<TStorage extends IStorage>(cls: Constructor<TStorage>, { id, name, alias, backendOpener, backendCacheKey, }: (ExplicitStorageIdentifier | DefaultStorageIdentifier) & {
         backendOpener: () => Promise<DatasetBackend | KeyValueStoreBackend | RequestQueueBackend>;
-        backendCacheKey: Hashable;
+        backendCacheKey: string;
     }): Promise<TStorage>;
     /**
      * Remove a storage instance from the cache (called from `storage.drop()`).

package/storages/storage_instance_manager.js

@@ -32,7 +32,7 @@ class StorageCache {
         return undefined;
     }
     /** Write a single entry into a given tier. */
-    setInMap(tier, cls, key, instance, backendCacheKey) {
+    #setInMap(tier, cls, key, instance, backendCacheKey) {
         if (!tier.has(cls))
             tier.set(cls, new Map());
         const keyMap = tier.get(cls);
@@ -45,14 +45,14 @@ class StorageCache {
      */
     set(cls, instance, backendCacheKey, alias) {
         // Always cache by id.
-        this.setInMap(this.byId, cls, instance.id, instance, backendCacheKey);
+        this.#setInMap(this.byId, cls, instance.id, instance, backendCacheKey);
         // Cache by name — only for named storages.
         if (instance.name) {
-            this.setInMap(this.byName, cls, instance.name, instance, backendCacheKey);
+            this.#setInMap(this.byName, cls, instance.name, instance, backendCacheKey);
         }
         // Cache by alias — only for unnamed storages opened via alias.
         if (alias !== undefined) {
-            this.setInMap(this.byAlias, cls, alias, instance, backendCacheKey);
+            this.#setInMap(this.byAlias, cls, alias, instance, backendCacheKey);
         }
     }
     removeFromCache(instance) {
@@ -120,8 +120,8 @@ class StorageCache {
  * assigns a reserved default alias.
  */
 export class StorageInstanceManager {
-    cache = new StorageCache();
-    openerLocks = new Map();
+    #cache = new StorageCache();
+    #openerLocks = new Map();
     /**
      * Open (or retrieve from cache) a storage instance.
      *
@@ -142,46 +142,46 @@ export class StorageInstanceManager {
         }
         // Fast-path cache check (no lock).
         if (alias !== undefined) {
-            const cached = this.cache.get(cls, { alias, backendCacheKey });
+            const cached = this.#cache.get(cls, { alias, backendCacheKey });
             if (cached)
                 return cached;
         }
         else if (id) {
-            const cached = this.cache.get(cls, { id, backendCacheKey });
+            const cached = this.#cache.get(cls, { id, backendCacheKey });
             if (cached)
                 return cached;
         }
         else if (name) {
-            const cached = this.cache.get(cls, { name, backendCacheKey });
+            const cached = this.#cache.get(cls, { name, backendCacheKey });
             if (cached)
                 return cached;
         }
         const identifierKey = id ?? name ?? alias ?? DEFAULT_STORAGE_ALIAS;
         const lockKey = `${cls.name}:${identifierKey}:${backendCacheKey}`;
-        if (!this.openerLocks.has(lockKey)) {
-            this.openerLocks.set(lockKey, new AsyncQueue());
+        if (!this.#openerLocks.has(lockKey)) {
+            this.#openerLocks.set(lockKey, new AsyncQueue());
         }
-        const queue = this.openerLocks.get(lockKey);
+        const queue = this.#openerLocks.get(lockKey);
         await queue.wait();
         try {
             // Double-check cache under lock (another caller may have filled it while we waited).
             if (alias !== undefined) {
-                const cached = this.cache.get(cls, { alias, backendCacheKey });
+                const cached = this.#cache.get(cls, { alias, backendCacheKey });
                 if (cached)
                     return cached;
             }
             else if (id) {
-                const cached = this.cache.get(cls, { id, backendCacheKey });
+                const cached = this.#cache.get(cls, { id, backendCacheKey });
                 if (cached)
                     return cached;
             }
             else if (name) {
-                const cached = this.cache.get(cls, { name, backendCacheKey });
+                const cached = this.#cache.get(cls, { name, backendCacheKey });
                 if (cached)
                     return cached;
             }
             // Prevent the same string from being used as both a name and an alias.
-            this.cache.checkNameAliasConflict(cls, { name, alias, backendCacheKey });
+            this.#cache.checkNameAliasConflict(cls, { name, alias, backendCacheKey });
             // Cache miss — create the sub-backend and storage instance.
             const subBackend = await backendOpener();
             const storageInfo = await subBackend.getMetadata();
@@ -189,7 +189,7 @@ export class StorageInstanceManager {
             // we just fetched (so `id`/`name` etc. are available synchronously) along with the backend.
             const instance = new cls({ metadata: storageInfo, backend: subBackend });
             // Atomic cache writes (no awaits between these).
-            this.cache.set(cls, instance, backendCacheKey, alias);
+            this.#cache.set(cls, instance, backendCacheKey, alias);
             return instance;
         }
         finally {
@@ -197,7 +197,7 @@ export class StorageInstanceManager {
             // Clean up idle locks so the map doesn't grow unboundedly
             // (mirrors crawlee-python's WeakValueDictionary behaviour).
             if (queue.remaining === 0) {
-                this.openerLocks.delete(lockKey);
+                this.#openerLocks.delete(lockKey);
             }
         }
     }
@@ -205,7 +205,7 @@ export class StorageInstanceManager {
      * Remove a storage instance from the cache (called from `storage.drop()`).
      */
     removeFromCache(instance) {
-        this.cache.removeFromCache(instance);
+        this.#cache.removeFromCache(instance);
     }
     /**
      * Clear the entire cache. Also calls `clearCache()` on any cached KeyValueStore
@@ -213,12 +213,12 @@ export class StorageInstanceManager {
      * Called during service locator reset.
      */
     clearCache() {
-        for (const instance of this.cache.allValues()) {
+        for (const instance of this.#cache.allValues()) {
             if ('clearCache' in instance && typeof instance.clearCache === 'function') {
                 instance.clearCache();
             }
         }
-        this.cache.clear();
+        this.#cache.clear();
     }
 }
 /**

package/storages/storage_stats.d.ts

@@ -39,7 +39,7 @@ export interface RequestQueueStats {
  * the buckets that make sense for it.
  */
 export declare class StorageStatsTracker<T extends Record<keyof T, number>> {
-    private readonly counters;
+    #private;
     constructor(initial: T);
     /** Increment a counter bucket by `by` (default `1`). */
     add(key: keyof T, by?: number): void;

package/storages/storage_stats.js

@@ -14,16 +14,16 @@
  * the buckets that make sense for it.
  */
 export class StorageStatsTracker {
-    counters;
+    #counters;
     constructor(initial) {
-        this.counters = { ...initial };
+        this.#counters = { ...initial };
     }
     /** Increment a counter bucket by `by` (default `1`). */
     add(key, by = 1) {
-        this.counters[key] += by;
+        this.#counters[key] += by;
     }
     /** Return a snapshot of the current counters. The returned object is a copy and safe to keep. */
     get current() {
-        return { ...this.counters };
+        return { ...this.#counters };
     }
 }

package/storages/transaction.d.ts

@@ -0,0 +1,270 @@
+import type { Awaitable, Dictionary, RequestSchema } from '@crawlee/types';
+import type { RecordOptions } from './key_value_store.js';
+/**
+ * Governs whether writes of a given storage type performed inside a {@link StorageTransaction} are
+ * applied immediately (`writeThrough`) or recorded and replayed on commit (`deferred`).
+ */
+export type StorageWriteMode = 'deferred' | 'writeThrough';
+/**
+ * Per-storage-type write policy of a {@link StorageTransaction}. Datasets and key-value stores are
+ * always `deferred` and not configurable — deferring is the only safe mode for non-idempotent writes,
+ * and {@link withDirectStorageAccess} covers one-off immediate writes.
+ */
+export interface StorageWritePolicy {
+    /**
+     * Write mode for request queue additions. Note that this is a *write policy* for the queue, not the
+     * queue instance itself (which is the top-level `requestQueue` crawler option).
+     *
+     * - `writeThrough` (default): requests are added immediately and are **not** rolled back with the
+     *   transaction. This is safe (additions are deduplicated by `uniqueKey`, so a retry is idempotent)
+     *   and keeps new requests visible to the crawler while the handler still runs.
+     * - `deferred`: requests are only added when the transaction commits — strict all-or-nothing
+     *   semantics, at the cost of the crawler not seeing them until the handler finishes.
+     */
+    requestQueue: StorageWriteMode;
+}
+export type StorageTransactionState = 'open' | 'committing' | 'committed' | 'failed' | 'rolledBack';
+/**
+ * A storage frontend that can record operations in a transaction journal.
+ * @internal
+ */
+export interface TransactionParticipant {
+    /**
+     * Replay the given buffered journal entries (all recorded by this participant) into the real storage
+     * backend. Called during commit, with the transaction already in the `committing` state, so the
+     * replayed operations pass through.
+     */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
+}
+/**
+ * A single dataset write (`pushData`) recorded in a transaction journal.
+ * @internal
+ */
+export interface DatasetJournalEntry {
+    type: 'dataset';
+    /** @internal **/
+    participant: TransactionParticipant;
+    storageId: string;
+    /** The pushed items, captured by `structuredClone` at write time. */
+    items: Dictionary[];
+    recordedAt: Date;
+}
+/**
+ * A single key-value store write (`setValue`) recorded in a transaction journal.
+ * @internal
+ */
+export interface KeyValueStoreJournalEntry {
+    type: 'keyValueStore';
+    /** @internal **/
+    participant: TransactionParticipant;
+    storageId: string;
+    key: string;
+    /** The original, pre-serialization value captured by `structuredClone`; `null` denotes a deletion. */
+    value: unknown;
+    options?: RecordOptions;
+}
+/**
+ * A request recorded in a transaction journal.
+ * @internal
+ */
+export interface JournaledRequest {
+    url: string;
+    uniqueKey: string;
+    label?: string;
+    /**
+     * A full JSON snapshot of the request for the commit replay. Only present for buffered additions —
+     * deduplicated and write-through ones are journaled for introspection only.
+     */
+    snapshot?: RequestSchema;
+}
+/**
+ * A batch of request queue additions recorded in a transaction journal.
+ * @internal
+ */
+export interface RequestQueueJournalEntry {
+    type: 'requestQueue';
+    /** @internal **/
+    participant: TransactionParticipant;
+    requests: JournaledRequest[];
+    forefront: boolean;
+    /** Write-through entries were applied immediately; they are never replayed. */
+    writeThrough: boolean;
+}
+/** @internal */
+export type JournalEntry = DatasetJournalEntry | KeyValueStoreJournalEntry | RequestQueueJournalEntry;
+/**
+ * A read-only view of a {@link StorageTransaction}: only the journal-backed introspection accessors,
+ * without the lifecycle methods. The accessors are synchronous and expose the original pre-serialization
+ * values. They cover every write recorded while the transaction was open, under either write policy —
+ * with the one exception noted on {@link StorageTransactionView.enqueuedUrls|`enqueuedUrls`}. A view
+ * is valid until the transaction is disposed.
+ */
+export interface StorageTransactionView {
+    readonly state: StorageTransactionState;
+    /** Items pushed to datasets during the transaction, in push order. */
+    readonly datasetItems: {
+        item: Dictionary;
+        datasetId: string;
+    }[];
+    /**
+     * URLs enqueued to request queues during the transaction, under either write policy. Recorded as
+     * requested, so duplicate, already-present and backend-rejected URLs are included.
+     *
+     * One gap: unless a caller of `addRequestsBatched()` waits for every chunk
+     * (`waitForAllRequestsToBeAdded` or `maxNewRequests`, both of which {@link enqueueLinks} sets when
+     * a crawl limit applies), the chunks after the first are added by a background writer that outlives
+     * the transaction and is not recorded here.
+     */
+    readonly enqueuedUrls: {
+        url: string;
+        label?: string;
+    }[];
+    /** Key-value store changes made during the transaction, keyed by store id, last write per key. */
+    readonly keyValueStoreChanges: Record<string, Record<string, {
+        changedValue: unknown;
+        options?: RecordOptions;
+    }>>;
+}
+export interface StorageTransactionOptions {
+    /** Overrides of the per-storage-type write policy. See {@link StorageWritePolicy}. */
+    policy?: Partial<StorageWritePolicy>;
+    /**
+     * How long a commit may take before it fails, in milliseconds. There is no automatic retry — the
+     * replay of dataset items is not idempotent.
+     * @default 300000
+     */
+    commitTimeoutMillis?: number;
+}
+/**
+ * A storage transaction scoped to a request's lifecycle. Writes made through the storage frontends
+ * ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}) while the transaction is active
+ * are recorded rather than applied; on {@link StorageTransaction.commit|`commit()`} they are replayed
+ * into real storage, on {@link StorageTransaction.rollback|`rollback()`} they are dropped. Reads consult
+ * the recorded writes first, so a handler sees its own writes.
+ *
+ * Create one with {@link createStorageTransaction} (explicit commit/rollback) or
+ * {@link withStorageTransaction} (scoped sugar). Crawlers open one automatically around every request
+ * handler unless `transactionalStorage: false` is set.
+ */
+export declare class StorageTransaction implements StorageTransactionView {
+    #private;
+    /**
+     * The ordered, append-only journal — the source of truth for commit, introspection and reads.
+     * @internal
+     */
+    readonly journal: JournalEntry[];
+    /** @internal */
+    constructor(options?: StorageTransactionOptions);
+    /**
+     * Per-storage-type write policy.
+     * @internal
+     */
+    get policy(): StorageWritePolicy;
+    get state(): StorageTransactionState;
+    /**
+     * `true` only while `state === 'open'`. This is the single predicate every storage operation
+     * consults — operations performed after the transaction is closed pass through to the real backend.
+     */
+    get isActive(): boolean;
+    /** Runs `callback` with this transaction installed in the async context. */
+    run<T>(callback: () => Awaitable<T>): Promise<T>;
+    /**
+     * Records a write operation in the journal.
+     * @internal
+     */
+    recordJournalEntry(entry: JournalEntry): void;
+    /**
+     * Registers a callback to run once a commit has been attempted, whether it succeeded (`error` is
+     * `undefined`) or failed - the only point at which a deferred write can be reacted to where it was
+     * made. Callbacks run in registration order; the first to throw propagates, the rest do not run, and
+     * on a failed commit its error replaces the commit error. Not run on rollback: nothing was written.
+     */
+    afterCommit(callback: (error?: Error) => Awaitable<void>): void;
+    /**
+     * Replays the journaled writes into real storage. A no-op unless the transaction is `open`.
+     *
+     * The transaction transitions to `committing` *before* anything is flushed, so a commit that throws
+     * partway lands in `failed` (never back in `open`) and subsequent storage operations pass through
+     * rather than recording into a dead transaction. Delivery is at-least-once — a commit that fails
+     * partway may have applied some of the writes already.
+     */
+    commit(): Promise<void>;
+    /**
+     * Discards the journaled writes. A no-op unless the transaction is `open` — in particular, calling it
+     * after a successful `commit()` (which the crawler's error handling can legitimately do) does nothing
+     * and never throws.
+     */
+    rollback(): void;
+    /**
+     * Releases the journal, the write-time snapshots it holds and any registered commit callbacks. Must
+     * be called for *every* terminal state, `failed` included. Idempotent, never throws, and does not
+     * change `state`. Any {@link StorageTransactionView} of this transaction is only valid until this
+     * is called.
+     */
+    dispose(): void;
+    get datasetItems(): {
+        item: Dictionary;
+        datasetId: string;
+    }[];
+    get enqueuedUrls(): {
+        url: string;
+        label?: string;
+    }[];
+    get keyValueStoreChanges(): Record<string, Record<string, {
+        changedValue: unknown;
+        options?: RecordOptions;
+    }>>;
+}
+/**
+ * Opens a {@link StorageTransaction} without running anything yet. The caller owns the outcome:
+ * `run()`, then `commit()` or `rollback()`, and always `dispose()` when done. For the common
+ * open-run-commit flow, prefer {@link withStorageTransaction}.
+ */
+export declare function createStorageTransaction(options?: StorageTransactionOptions): StorageTransaction;
+/**
+ * Runs `callback` inside a new {@link StorageTransaction}: storage writes made in the callback are
+ * committed when it returns and rolled back when it throws. If a transaction is already active in the
+ * current async context, it is reused and its outcome is left to its owner (and `options` are ignored)
+ * — there are no nested transaction semantics.
+ */
+export declare function withStorageTransaction<T>(callback: (transaction: StorageTransaction) => Awaitable<T>, options?: StorageTransactionOptions): Promise<T>;
+/**
+ * Runs `callback` outside of any storage transaction — the per-call-site escape hatch. Storage operations
+ * made inside it hit the real backend directly, are not rolled back, and operations that a transaction
+ * rejects (`drop`, stream-valued `setValue`, request queue internals, ...) are permitted.
+ */
+export declare function withDirectStorageAccess<T>(callback: () => Awaitable<T>): Promise<T>;
+/**
+ * The per-operation hook consulted by every storage frontend method: performs the cancellation check
+ * that aborts storage operations when the request handler times out, and returns the active storage
+ * transaction. Returns `undefined` when there is no transaction in the async context *or* when it is no
+ * longer open — operations on a closed transaction deliberately pass through to the real backend.
+ * @internal
+ */
+export declare function activeStorageTransaction(): StorageTransaction | undefined;
+/**
+ * Returns the transaction installed in the current async context, regardless of its state. Used by the
+ * crawler to drive the outcome of the transaction it opened.
+ * @internal
+ */
+export declare function currentStorageTransaction(): StorageTransaction | undefined;
+/**
+ * Captures a value at write time, so that later mutations of the caller's object affect neither the
+ * read-your-own-writes reads nor the commit replay. `structuredClone` for fidelity (`Date`, `Map`, `Set`,
+ * typed arrays, `undefined`); values it cannot handle fall back to the JSON round-trip the storage
+ * backends perform anyway.
+ * @internal
+ */
+export declare function snapshotValue<T>(value: T): T;
+/**
+ * The guard for operations that cannot be performed inside a storage transaction: throws when one is
+ * active, and performs the per-operation cancellation check either way.
+ * @internal
+ */
+export declare function rejectOperationInTransaction(operation: string, reason?: string): void;
+/**
+ * Builds the "operation not allowed in a transaction" error, for a call site that has already
+ * established a transaction is active and so wants to `throw` unconditionally.
+ * @internal
+ */
+export declare function operationRejectedInTransaction(operation: string, reason?: string): Error;