@crawlee/core 4.0.0-beta.166 → 4.0.0-beta.168

package/crawlers/crawler_commons.d.ts

@@ -1,4 +1,4 @@
-import type { Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
+import type { Awaitable, Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
 import type { ReadonlyDeep } from 'type-fest';
 import type { EnqueueUrlsOptions } from '../enqueue_links/enqueue_links.js';
 import type { CrawleeLogger } from '../log.js';
@@ -136,6 +136,46 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
      * here can land more than once for a single request. Push results from the request handler itself.
      */
     registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
+    /**
+     * Registers `callback` to run once the request's storage writes have been committed, or once
+     * committing them has failed - the point where a write made in the handler can finally be reacted
+     * to where it was made. Two things belong here: side effects that must agree with what actually
+     * landed (result counters, progress reporting), and handling of a write that did not land at all.
+     *
+     * ```ts
+     * async requestHandler({ pushData, afterStorageCommit, useState }) {
+     *     const state = await useState({ itemCount: 0 });
+     *
+     *     await pushData(item);
+     *     afterStorageCommit((error) => {
+     *         if (error) {
+     *             throw new NonRetryableError('The page is too big to store', { cause: error });
+     *         }
+     *         state.itemCount++;
+     *     });
+     * }
+     * ```
+     *
+     * `useState()` is deliberately not transactional, which is what makes the counter case necessary:
+     * incrementing in the handler would keep the increment for a request that pushes an item and then
+     * fails, while the item itself is rolled back.
+     *
+     * Callbacks run in registration order, after the commit and before the request is marked as handled.
+     * The first one to throw propagates and the rest do not run; on a failed commit its error replaces
+     * the commit error, so a callback can decide what the request fails with. After a *successful*
+     * commit, a callback that throws fails the request **without a retry** whatever it throws - the
+     * writes are already durable and retrying would duplicate them, so anything that is not already a
+     * {@link NonRetryableError} is wrapped in an {@link AfterCommitError}.
+     *
+     * Callbacks are *not* run when the request handler itself fails - nothing was written, and
+     * `errorHandler` / `failedRequestHandler` cover that.
+     *
+     * Throws when storage is not transactional (`transactionalStorage: false`) - writes are then applied
+     * as they are made and throw at the call site on their own. In {@link AdaptivePlaywrightCrawler}
+     * the callback belongs to the current request handler attempt, so it runs only for the attempt whose
+     * writes are the ones being committed.
+     */
+    afterStorageCommit(callback: (error?: Error) => Awaitable<void>): void;
     /**
      * Gives the current request `secs` more seconds to finish, for when how long it needs is only apparent
      * once it is already running - a listing page that turns out to have far more to scroll through than

package/errors.d.ts

@@ -99,6 +99,16 @@ export declare class ContextPipelineCleanupError extends CriticalError {
 export declare class RequestHandlerError extends Error {
     constructor(error: unknown, options?: ErrorOptions);
 }
+/**
+ * Wraps the failure of a callback registered with {@link StorageTransaction.afterCommit|`afterCommit`}
+ * that ran after the request's writes had already been committed.
+ *
+ * Non-retryable by nature: the writes are durable, so re-running the request handler would duplicate
+ * them. A callback that throws a `NonRetryableError` of its own is left alone.
+ */
+export declare class AfterCommitError extends NonRetryableError {
+    constructor(cause: unknown);
+}
 /**
  * Thrown when attempting to set a different service instance after one has already been retrieved.
  */

package/errors.js

@@ -124,6 +124,18 @@ export class RequestHandlerError extends Error {
         super(undefined, { cause: error, ...options });
     }
 }
+/**
+ * Wraps the failure of a callback registered with {@link StorageTransaction.afterCommit|`afterCommit`}
+ * that ran after the request's writes had already been committed.
+ *
+ * Non-retryable by nature: the writes are durable, so re-running the request handler would duplicate
+ * them. A callback that throws a `NonRetryableError` of its own is left alone.
+ */
+export class AfterCommitError extends NonRetryableError {
+    constructor(cause) {
+        super(cause instanceof Error ? cause.message : String(cause), { cause });
+    }
+}
 /**
  * Thrown when attempting to set a different service instance after one has already been retrieved.
  */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.166",
+    "version": "4.0.0-beta.168",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,10 +52,10 @@
         "@apify/log": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/fs-storage": "4.0.0-beta.166",
-        "@crawlee/http-client": "4.0.0-beta.166",
-        "@crawlee/types": "4.0.0-beta.166",
-        "@crawlee/utils": "4.0.0-beta.166",
+        "@crawlee/fs-storage": "4.0.0-beta.168",
+        "@crawlee/http-client": "4.0.0-beta.168",
+        "@crawlee/types": "4.0.0-beta.168",
+        "@crawlee/utils": "4.0.0-beta.168",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "5d34cf3b9da170f22cf5b346014ba44060dad827"
+    "gitHead": "fc75a7073fc330331ff2d68d81a8e6815ab4edf2"
 }

package/storages/transaction.d.ts

@@ -162,6 +162,13 @@ export declare class StorageTransaction implements StorageTransactionView {
      * @internal
      */
     recordJournalEntry(entry: JournalEntry): void;
+    /**
+     * Registers a callback to run once a commit has been attempted, whether it succeeded (`error` is
+     * `undefined`) or failed - the only point at which a deferred write can be reacted to where it was
+     * made. Callbacks run in registration order; the first to throw propagates, the rest do not run, and
+     * on a failed commit its error replaces the commit error. Not run on rollback: nothing was written.
+     */
+    afterCommit(callback: (error?: Error) => Awaitable<void>): void;
     /**
      * Replays the journaled writes into real storage. A no-op unless the transaction is `open`.
      *
@@ -179,9 +186,10 @@ export declare class StorageTransaction implements StorageTransactionView {
      */
     rollback(): void;
     /**
-     * Releases the journal and the write-time snapshots it holds. Must be called for *every* terminal
-     * state, `failed` included. Idempotent, never throws, and does not change `state`. Any
-     * {@link StorageTransactionView} of this transaction is only valid until this is called.
+     * Releases the journal, the write-time snapshots it holds and any registered commit callbacks. Must
+     * be called for *every* terminal state, `failed` included. Idempotent, never throws, and does not
+     * change `state`. Any {@link StorageTransactionView} of this transaction is only valid until this
+     * is called.
      */
     dispose(): void;
     get datasetItems(): {

package/storages/transaction.js

@@ -1,6 +1,7 @@
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { addTimeoutToPromise, storage as timeoutStorage, tryCancel } from '@apify/timeout';
 import { serviceLocator } from '../service_locator.js';
+import { AfterCommitError, NonRetryableError } from '../errors.js';
 const DEFAULT_STORAGE_WRITE_POLICY = { requestQueue: 'writeThrough' };
 const DEFAULT_COMMIT_TIMEOUT_MILLIS = 300_000;
 const transactionStorage = new AsyncLocalStorage();
@@ -24,6 +25,8 @@ export class StorageTransaction {
     #commitTimeoutMillis;
     #state = 'open';
     #disposed = false;
+    /** Callbacks registered via `afterCommit()`, run in registration order once a commit has been attempted. */
+    #afterCommitCallbacks = [];
     /** @internal */
     constructor(options = {}) {
         this.policy = { ...DEFAULT_STORAGE_WRITE_POLICY, ...options.policy };
@@ -53,6 +56,18 @@ export class StorageTransaction {
         }
         this.journal.push(entry);
     }
+    /**
+     * Registers a callback to run once a commit has been attempted, whether it succeeded (`error` is
+     * `undefined`) or failed - the only point at which a deferred write can be reacted to where it was
+     * made. Callbacks run in registration order; the first to throw propagates, the rest do not run, and
+     * on a failed commit its error replaces the commit error. Not run on rollback: nothing was written.
+     */
+    afterCommit(callback) {
+        if (!this.isActive) {
+            throw new Error(`Cannot register a commit callback on a transaction in the '${this.#state}' state`);
+        }
+        this.#afterCommitCallbacks.push(callback);
+    }
     /**
      * Replays the journaled writes into real storage. A no-op unless the transaction is `open`.
      *
@@ -72,12 +87,31 @@ export class StorageTransaction {
             // frames, so a request-handler timeout that already fired would abort the commit of a
             // handler that succeeded. Hence a fresh timeout context, which also provides the time bound.
             await timeoutStorage.exit(async () => addTimeoutToPromise(async () => this.flush(), this.#commitTimeoutMillis, `Committing the storage transaction timed out after ${this.#commitTimeoutMillis / 1000} seconds.`));
-            this.#state = 'committed';
         }
         catch (error) {
             this.#state = 'failed';
-            throw error;
+            // Callbacks are handed an `Error`, so a backend that rejects with anything else gets wrapped.
+            const commitError = error instanceof Error ? error : new Error(String(error), { cause: error });
+            // A callback that throws here propagates instead: it was handed the failure and decided what
+            // the request should fail with.
+            await this.#runAfterCommitCallbacks(commitError);
+            throw commitError;
         }
+        this.#state = 'committed';
+        try {
+            await this.#runAfterCommitCallbacks();
+        }
+        catch (error) {
+            throw error instanceof NonRetryableError ? error : new AfterCommitError(error);
+        }
+    }
+    /** Fresh timeout context for the same reason as the flush. */
+    async #runAfterCommitCallbacks(error) {
+        await timeoutStorage.exit(async () => {
+            for (const callback of this.#afterCommitCallbacks) {
+                await callback(error);
+            }
+        });
     }
     async flush() {
         // Each participating frontend replays all of its buffered entries in one call. Frontends are
@@ -112,9 +146,10 @@ export class StorageTransaction {
         this.#state = 'rolledBack';
     }
     /**
-     * Releases the journal and the write-time snapshots it holds. Must be called for *every* terminal
-     * state, `failed` included. Idempotent, never throws, and does not change `state`. Any
-     * {@link StorageTransactionView} of this transaction is only valid until this is called.
+     * Releases the journal, the write-time snapshots it holds and any registered commit callbacks. Must
+     * be called for *every* terminal state, `failed` included. Idempotent, never throws, and does not
+     * change `state`. Any {@link StorageTransactionView} of this transaction is only valid until this
+     * is called.
      */
     dispose() {
         if (this.#disposed) {
@@ -134,6 +169,7 @@ export class StorageTransaction {
         }
         this.#disposed = true;
         this.journal.length = 0;
+        this.#afterCommitCallbacks.length = 0;
     }
     get datasetItems() {
         return this.journal.flatMap((entry) => entry.type === 'dataset' ? entry.items.map((item) => ({ item, datasetId: entry.storageId })) : []);