@crawlee/core 4.0.0-rc.0 → 4.0.0-rc.1

package/storages/transaction.js

@@ -1,6 +1,7 @@
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { addTimeoutToPromise, storage as timeoutStorage, tryCancel } from '@apify/timeout';
 import { serviceLocator } from '../service_locator.js';
+import { AfterCommitError, NonRetryableError } from '../errors.js';
 const DEFAULT_STORAGE_WRITE_POLICY = { requestQueue: 'writeThrough' };
 const DEFAULT_COMMIT_TIMEOUT_MILLIS = 300_000;
 const transactionStorage = new AsyncLocalStorage();
@@ -17,18 +18,29 @@ const COMMIT_ORDER = ['keyValueStore', 'requestQueue', 'dataset'];
  * handler unless `transactionalStorage: false` is set.
  */
 export class StorageTransaction {
-    /** The ordered, append-only journal — the source of truth for commit, introspection and reads. */
+    /**
+     * The ordered, append-only journal — the source of truth for commit, introspection and reads.
+     * @internal
+     */
     journal = [];
-    /** Per-storage-type write policy. */
-    policy;
+    #policy;
     #commitTimeoutMillis;
     #state = 'open';
     #disposed = false;
+    /** Callbacks registered via `afterCommit()`, run in registration order once a commit has been attempted. */
+    #afterCommitCallbacks = [];
     /** @internal */
     constructor(options = {}) {
-        this.policy = { ...DEFAULT_STORAGE_WRITE_POLICY, ...options.policy };
+        this.#policy = { ...DEFAULT_STORAGE_WRITE_POLICY, ...options.policy };
         this.#commitTimeoutMillis = options.commitTimeoutMillis ?? DEFAULT_COMMIT_TIMEOUT_MILLIS;
     }
+    /**
+     * Per-storage-type write policy.
+     * @internal
+     */
+    get policy() {
+        return this.#policy;
+    }
     get state() {
         return this.#state;
     }
@@ -53,6 +65,18 @@ export class StorageTransaction {
         }
         this.journal.push(entry);
     }
+    /**
+     * Registers a callback to run once a commit has been attempted, whether it succeeded (`error` is
+     * `undefined`) or failed - the only point at which a deferred write can be reacted to where it was
+     * made. Callbacks run in registration order; the first to throw propagates, the rest do not run, and
+     * on a failed commit its error replaces the commit error. Not run on rollback: nothing was written.
+     */
+    afterCommit(callback) {
+        if (!this.isActive) {
+            throw new Error(`Cannot register a commit callback on a transaction in the '${this.#state}' state`);
+        }
+        this.#afterCommitCallbacks.push(callback);
+    }
     /**
      * Replays the journaled writes into real storage. A no-op unless the transaction is `open`.
      *
@@ -71,15 +95,34 @@ export class StorageTransaction {
             // on every operation - and `@apify/timeout` shares one `AbortController` across nested
             // frames, so a request-handler timeout that already fired would abort the commit of a
             // handler that succeeded. Hence a fresh timeout context, which also provides the time bound.
-            await timeoutStorage.exit(async () => addTimeoutToPromise(async () => this.flush(), this.#commitTimeoutMillis, `Committing the storage transaction timed out after ${this.#commitTimeoutMillis / 1000} seconds.`));
-            this.#state = 'committed';
+            await timeoutStorage.exit(async () => addTimeoutToPromise(async () => this.#flush(), this.#commitTimeoutMillis, `Committing the storage transaction timed out after ${this.#commitTimeoutMillis / 1000} seconds.`));
         }
         catch (error) {
             this.#state = 'failed';
-            throw error;
+            // Callbacks are handed an `Error`, so a backend that rejects with anything else gets wrapped.
+            const commitError = error instanceof Error ? error : new Error(String(error), { cause: error });
+            // A callback that throws here propagates instead: it was handed the failure and decided what
+            // the request should fail with.
+            await this.#runAfterCommitCallbacks(commitError);
+            throw commitError;
+        }
+        this.#state = 'committed';
+        try {
+            await this.#runAfterCommitCallbacks();
         }
+        catch (error) {
+            throw error instanceof NonRetryableError ? error : new AfterCommitError(error);
+        }
+    }
+    /** Fresh timeout context for the same reason as the flush. */
+    async #runAfterCommitCallbacks(error) {
+        await timeoutStorage.exit(async () => {
+            for (const callback of this.#afterCommitCallbacks) {
+                await callback(error);
+            }
+        });
     }
-    async flush() {
+    async #flush() {
         // Each participating frontend replays all of its buffered entries in one call. Frontends are
         // ordered by storage type: key-value stores and request queues first (idempotent under retry),
         // datasets last (not idempotent), minimizing the blast radius of a partial commit failure.
@@ -112,9 +155,10 @@ export class StorageTransaction {
         this.#state = 'rolledBack';
     }
     /**
-     * Releases the journal and the write-time snapshots it holds. Must be called for *every* terminal
-     * state, `failed` included. Idempotent, never throws, and does not change `state`. Any
-     * {@link StorageTransactionView} of this transaction is only valid until this is called.
+     * Releases the journal, the write-time snapshots it holds and any registered commit callbacks. Must
+     * be called for *every* terminal state, `failed` included. Idempotent, never throws, and does not
+     * change `state`. Any {@link StorageTransactionView} of this transaction is only valid until this
+     * is called.
      */
     dispose() {
         if (this.#disposed) {
@@ -134,6 +178,7 @@ export class StorageTransaction {
         }
         this.#disposed = true;
         this.journal.length = 0;
+        this.#afterCommitCallbacks.length = 0;
     }
     get datasetItems() {
         return this.journal.flatMap((entry) => entry.type === 'dataset' ? entry.items.map((item) => ({ item, datasetId: entry.storageId })) : []);

package/storages/utils.d.ts

@@ -16,7 +16,7 @@ interface PurgeDefaultStorageOptions {
 /**
  * Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
  * Purging empties the storages that belong to a single run — the default one and every alias-keyed one —
- * keeping only INPUT.json in the default KV store. Named storages persist across runs and are not touched.
+ * Named storages persist across runs and are not touched.
  *
  * Purging of storages is happening automatically when we run our crawler (or when we open some storage
  * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@link Configuration}
@@ -31,7 +31,7 @@ export declare function purgeDefaultStorages(options?: PurgeDefaultStorageOption
 /**
  * Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
  * Purging empties the storages that belong to a single run — the default one and every alias-keyed one —
- * keeping only INPUT.json in the default KV store. Named storages persist across runs and are not touched.
+ * Named storages persist across runs and are not touched.
  *
  * Purging of storages is happening automatically when we run our crawler (or when we open some storage
  * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@link Configuration}

package/validators.d.ts

@@ -1,4 +1,5 @@
-export { ArgumentValidationError, parseArgument } from '@crawlee/utils';
+export { ArgumentValidationError } from '@crawlee/utils';
+export { parseArgument } from '@crawlee/utils/internal';
 export { schemas } from '@crawlee/utils/internal';
 /** @internal */
 export declare const validators: {
@@ -21,5 +22,5 @@ export declare const validators: {
 // @ts-ignore optional peer dependency or compatibility with es2022
     logger: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-    httpClient: import("zod").ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>;
+    httpClient: import("zod").ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>;
 };

package/validators.js

@@ -1,12 +1,13 @@
 import { schemas } from '@crawlee/utils/internal';
-export { ArgumentValidationError, parseArgument } from '@crawlee/utils';
+export { ArgumentValidationError } from '@crawlee/utils';
+export { parseArgument } from '@crawlee/utils/internal';
 export { schemas } from '@crawlee/utils/internal';
 /** @internal */
 export const validators = {
     // Naming it browser page for future proofing with Playwright
     browserPage: schemas.objectWithKeys(['goto', 'evaluate', '$', 'on'], 'Expected a Puppeteer Page, got something else.'),
     proxyConfiguration: schemas.objectWithKeys(['newProxyInfo'], "Expected an object implementing the IProxyConfiguration interface (missing 'newProxyInfo'), got something else."),
-    requestList: schemas.objectWithKeys(['fetchNextRequest', 'persistState'], 'Expected a RequestList, got something else.'),
+    requestList: schemas.objectWithKeys(['fetchNextRequest', 'checkReadiness'], 'Expected a RequestList, got something else.'),
     requestQueue: schemas.objectWithKeys(['fetchNextRequest', 'addRequest'], 'Expected a RequestQueue, got something else.'),
     browserPool: schemas.objectWithKeys(['newPage', 'closePage', 'extractPageState', 'injectPageState'], "Expected an object implementing the IBrowserPool interface (missing one of 'newPage', 'closePage', 'extractPageState', 'injectPageState'), got something else."),
     sessionPool: schemas.objectWithKeys(['getSession'], "Expected an object implementing the ISessionPool interface (missing 'getSession'), got something else."),

package/autoscaling/autoscaled_pool.d.ts

@@ -1,195 +0,0 @@
-import type { ConcurrencyConsumer, IConcurrencySystem } from './concurrency_system.js';
-import type { CrawleeLogger } from '../log.js';
-/**
- * The two predicates that steer a task loop: *is there work ready?* and *are we done?* These are the parts of the loop
- * a caller legitimately overrides, as opposed to the task itself (`runTaskFunction`), which the loop's driver owns.
- */
-export interface TaskLoopPredicates {
-    /**
-     * A function that indicates whether `runTaskFunction` should be called.
-     * This function is called every time there is free capacity for a new task and it should
-     * indicate whether it should start a new task or not by resolving to either `true` or `false`.
-     * Besides its obvious use, it is also useful for task throttling to save resources.
-     */
-    isTaskReadyFunction?: () => Promise<boolean>;
-    /**
-     * A function that is called only when there are no tasks to be processed.
-     * If it resolves to `true` then the run finishes. Being called only
-     * when there are no tasks being processed means that as long as `isTaskReadyFunction()`
-     * keeps resolving to `true`, `isFinishedFunction()` will never be called.
-     */
-    isFinishedFunction?: () => Promise<boolean>;
-}
-/** @internal */
-export interface AutoscaledPoolOptions extends TaskLoopPredicates {
-    /**
-     * The governor that decides whether there is free compute for one more task. Typically a
-     * {@link ConcurrencySystem}, but any {@link IConcurrencySystem} works. Share a single instance across
-     * multiple pools (and therefore multiple crawlers) to cap their *combined* concurrency against one budget.
-     *
-     * All concurrency/scaling/snapshotter configuration lives on the governor — the pool only owns the task loop and
-     * its cadence.
-     */
-    concurrencySystem: IConcurrencySystem;
-    /**
-     * Who this pool is, presented to the governor on every capacity query and booking so that a shared one can tell
-     * several pools apart. Worth naming meaningfully — a governor that allocates per consumer reports this `id`.
-     */
-    consumer: ConcurrencyConsumer;
-    /**
-     * A function that performs an asynchronous resource-intensive task.
-     * The function must either be labeled `async` or return a promise.
-     */
-    runTaskFunction?: () => Promise<unknown>;
-    /**
-     * Indicates how often the pool should call the `runTaskFunction()` to start a new task, in seconds.
-     * This has no effect on starting new tasks immediately after a task completes.
-     * @default 0.5
-     */
-    maybeRunIntervalSecs?: number;
-    /**
-     * Timeout in which the `runTaskFunction` needs to finish, given in seconds.
-     * @default 0
-     */
-    taskTimeoutSecs?: number;
-    log?: CrawleeLogger;
-}
-/**
- * Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
- * The pool only starts new tasks while its {@link IConcurrencySystem|concurrency system} reports free capacity —
- * that governor is what monitors CPU, memory and event loop load and autoscales the concurrency budget.
- *
- * Before running the pool, you need to implement the following three functions:
- * {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`},
- * {@link TaskLoopPredicates.isTaskReadyFunction|`isTaskReadyFunction`} and
- * {@link TaskLoopPredicates.isFinishedFunction|`isFinishedFunction`}.
- *
- * The auto-scaled pool is started by calling the {@link AutoscaledPool.run} function.
- * The pool periodically queries `isTaskReadyFunction` for more tasks, managing optimal concurrency, until the function
- * resolves to `false`. The pool then queries `isFinishedFunction`. If it resolves to `true`, the run finishes after all
- * running tasks complete. If it resolves to `false`, it assumes there will be more tasks available later and keeps
- * periodically querying for tasks.
- * If any of the tasks throws then the {@link AutoscaledPool.run} function rejects the promise with an error.
- *
- * The pool evaluates whether it should start a new task every time one of the tasks finishes
- * and also in the interval set by the `options.maybeRunIntervalSecs` parameter.
- *
- * **Example usage:**
- *
- * ```javascript
- * const concurrencySystem = new ConcurrencySystem({ maxConcurrency: 50 });
- * await concurrencySystem.start();
- *
- * const pool = new AutoscaledPool({
- *     concurrencySystem,
- *     consumer: { id: 'my-pool' },
- *     runTaskFunction: async () => {
- *         // Run some resource-intensive asynchronous operation here.
- *     },
- *     isTaskReadyFunction: async () => {
- *         // Tell the pool whether more tasks are ready to be processed.
- *         // Return true or false
- *     },
- *     isFinishedFunction: async () => {
- *         // Tell the pool whether it should finish
- *         // or wait for more tasks to become available.
- *         // Return true or false
- *     }
- * });
- *
- * try {
- *     await pool.run();
- * } finally {
- *     await concurrencySystem.stop();
- * }
- * ```
- *
- * @internal
- */
-export declare class AutoscaledPool {
-    #private;
-    constructor(options: AutoscaledPoolOptions);
-    /**
-     * The governor backing this pool, as supplied to the constructor — exposed as the read-only
-     * {@link IConcurrencySystem} contract.
-     *
-     * This and the two getters below are telemetry only: concurrency is configured and tuned on the concrete
-     * {@link ConcurrencySystem} its owner holds, never through the pool.
-     */
-    get system(): IConcurrencySystem;
-    /** The estimated number of parallel tasks the governor can currently support. */
-    get desiredConcurrency(): number;
-    /**
-     * The number of parallel tasks currently booked against the governor. When it is shared, this counts every
-     * borrowing pool's tasks, not just this one's.
-     */
-    get currentConcurrency(): number;
-    /**
-     * Runs the auto-scaled pool. Returns a promise that gets resolved or rejected once
-     * all the tasks are finished or one of them fails.
-     *
-     * Throws if the {@link IConcurrencySystem|concurrency system} it borrows was never started — the pool assumes
-     * a running governor and cannot start one it does not own.
-     */
-    run(): Promise<void>;
-    /**
-     * Aborts the run of the auto-scaled pool and destroys it. The promise returned from
-     * the {@link AutoscaledPool.run} function will immediately resolve, no more new tasks
-     * will be spawned and all running tasks will be left in their current state.
-     *
-     * Due to the nature of the tasks, auto-scaled pool cannot reliably guarantee abortion
-     * of all the running tasks, therefore, no abortion is attempted and some of the tasks
-     * may finish, while others may not. Essentially, auto-scaled pool doesn't care about
-     * their state after the invocation of `.abort()`, but that does not mean that some
-     * parts of their asynchronous chains of commands will not execute.
-     */
-    abort(): Promise<void>;
-    /**
-     * Prevents the auto-scaled pool from starting new tasks, but allows the running ones to finish
-     * (unlike abort, which terminates them). Used together with {@link AutoscaledPool.resume}
-     *
-     * The function's promise will resolve once all running tasks have completed and the pool
-     * is effectively idle. If the `timeoutSecs` argument is provided, the promise will reject
-     * with a timeout error after the `timeoutSecs` seconds.
-     *
-     * The promise returned from the {@link AutoscaledPool.run} function will not resolve
-     * when `.pause()` is invoked (unlike abort, which resolves it).
-     *
-     * > *NOTE:* Pausing the pool does not suspend the (possibly shared) {@link ConcurrencySystem} — its
-     * autoscaling and resource monitoring keep running, since other pools borrowing it may still be active. To silence
-     * it during a long pause, its owner can `stop()` and `start()` it again.
-     */
-    pause(timeoutSecs?: number): Promise<void>;
-    /**
-     * Resumes the operation of the autoscaled-pool by allowing more tasks to be run.
-     * Used together with {@link AutoscaledPool.pause}
-     *
-     * Tasks will automatically start running again in `options.maybeRunIntervalSecs`.
-     */
-    resume(): void;
-    /**
-     * Explicitly check the queue for new tasks. The AutoscaledPool checks the queue for new tasks periodically,
-     * every `maybeRunIntervalSecs` seconds. If you want to trigger the processing immediately, use this method.
-     */
-    notify(): Promise<void>;
-    /**
-     * Starts a new task
-     * if the number of running tasks (current concurrency) is lower than desired concurrency
-     * and the system is not currently overloaded
-     * and `isTaskReadyFunction()` returns true.
-     *
-     * It doesn't allow multiple concurrent runs of this method.
-     */
-    private maybeRunTask;
-    /**
-     * If there are no running tasks and `isFinishedFunction()` returns true then closes
-     * the pool and resolves the pool's promise returned by the run() method.
-     *
-     * It doesn't allow multiple concurrent runs of this method.
-     */
-    private maybeFinish;
-    /**
-     * Cleans up resources.
-     */
-    private destroy;
-}