@hotmeshio/hotmesh 0.16.3 → 0.16.5

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.16.3",
+    "version": "0.16.5",
     "description": "Durable Workflow",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/activities/activity/process.js

@@ -72,13 +72,10 @@ async function processEvent(instance, status = stream_1.StreamStatus.SUCCESS, co
     }
     catch (error) {
         if (error instanceof errors_1.CollationError) {
-            //FORBIDDEN: Leg1 not complete — signal arrived in the window
-            //between registerHook (standalone) and Leg1 transaction commit.
-            //Rethrow so the stream message is retried with backoff; by then
-            //Leg1 will have committed and Leg2 processing will succeed.
-            //The GUID marker was already committed by notarizeLeg2Entry;
-            //on retry, collateLeg2Entry's SETNX is a no-op for the same
-            //GUID, and verifySyntheticInteger sees no steps done → allowed.
+            //FORBIDDEN: Leg1 not complete — should not occur after the fix
+            //that moved setHookSignal to post-commit. If seen, it indicates
+            //a new race window not covered by the fix. Rethrow so the inline
+            //retry in processWebHookEvent can attempt recovery.
             if (error.fault === collator_1.CollationFaultType.FORBIDDEN) {
                 instance.logger.warn('process-event-forbidden-retry', {
                     jid: instance.context.metadata.jid,

package/build/services/activities/hook.d.ts

@@ -160,6 +160,28 @@ declare class Hook extends Activity {
     private redeliverPendingSignal;
     doPassThrough(telemetry: TelemetryService): Promise<void>;
     getHookRule(topic: string): Promise<HookRule | undefined>;
+    /**
+     * Register the time hook (sleep) inside the Leg1 transaction.
+     * Time hooks don't participate in the signal race — they're
+     * purely internal timeout registrations.
+     */
+    registerTimeHook(transaction: ProviderTransaction): Promise<void>;
+    /**
+     * Register the web hook signal AFTER the Leg1 transaction commits.
+     * This ensures the hook signal is never visible before Leg1
+     * completion, eliminating the FORBIDDEN window where Leg2 could
+     * find the hook but fail on the collation check.
+     *
+     * If a pending signal was stored by an early-arriving Leg2,
+     * setHookSignal atomically detects and returns it.
+     */
+    registerWebHookSignal(): Promise<{
+        pending?: string;
+    } | void>;
+    /**
+     * @deprecated Use registerTimeHook + registerWebHookSignal instead.
+     * Kept for backward compatibility with tests that monkey-patch this method.
+     */
     registerHook(transaction?: ProviderTransaction): Promise<{
         jobId?: string;
         pending?: string;

package/build/services/activities/hook.js

@@ -6,6 +6,8 @@ const pipe_1 = require("../pipe");
 const task_1 = require("../task");
 const telemetry_1 = require("../telemetry");
 const stream_1 = require("../../types/stream");
+const errors_1 = require("../../modules/errors");
+const collator_2 = require("../../types/collator");
 const utils_1 = require("../../modules/utils");
 const activity_1 = require("./activity");
 /**
@@ -169,6 +171,17 @@ class Hook extends activity_1.Activity {
                 if (!isResume) {
                     await this.doHook(telemetry);
                 }
+                else if (this.config.hook?.topic) {
+                    //DUPLICATE: Leg1 completed previously but hook registration
+                    //may not have happened (crash between transaction.exec and
+                    //registerWebHookSignal). Attempt registration — setHookSignal
+                    //is idempotent (returns success:false if hook already exists).
+                    const hookResult = await this.registerWebHookSignal();
+                    const pending = hookResult && hookResult.pending;
+                    if (pending) {
+                        await this.redeliverPendingSignal(pending);
+                    }
+                }
             }
             else {
                 //Category B: passthrough with crash-safe step protocol + GUID ledger
@@ -204,7 +217,9 @@ class Hook extends activity_1.Activity {
     }
     async doHook(telemetry) {
         const transaction = this.store.transact();
-        const hookResult = await this.registerHook(transaction);
+        //register time hooks (sleep) inside the transaction — these
+        //don't participate in the signal race
+        await this.registerTimeHook(transaction);
         this.mapOutputData();
         this.mapJobData();
         await this.setState(transaction);
@@ -212,11 +227,15 @@ class Hook extends activity_1.Activity {
         await this.setStatus(0, transaction);
         await transaction.exec();
         telemetry.mapActivityAttributes();
-        //if a pending signal was detected (signal arrived before hook
-        //registered), re-publish the WEBHOOK so leg2 processes it
-        //now that the hook signal is committed and state is saved
-        if (hookResult && hookResult.pending) {
-            await this.redeliverPendingSignal(hookResult.pending);
+        //register the web hook signal AFTER the transaction commits.
+        //this eliminates the FORBIDDEN window: the hook signal is never
+        //visible before Leg1 completion. If Leg2 arrives before this
+        //point, getHookSignal finds no hook and stores $pending, which
+        //setHookSignal will detect and return for redelivery.
+        const hookResult = await this.registerWebHookSignal();
+        const pending = hookResult && hookResult.pending;
+        if (pending) {
+            await this.redeliverPendingSignal(pending);
         }
     }
     /**
@@ -257,12 +276,44 @@ class Hook extends activity_1.Activity {
         const rules = await this.store.getHookRules();
         return rules?.[topic]?.[0];
     }
+    /**
+     * Register the time hook (sleep) inside the Leg1 transaction.
+     * Time hooks don't participate in the signal race — they're
+     * purely internal timeout registrations.
+     */
+    async registerTimeHook(transaction) {
+        if (this.config.sleep) {
+            const duration = pipe_1.Pipe.resolve(this.config.sleep, this.context);
+            if (!isNaN(duration) && Number(duration) > 0) {
+                await this.engine.taskService.registerTimeHook(this.context.metadata.jid, this.context.metadata.gid, `${this.metadata.aid}${this.metadata.dad || ''}`, 'sleep', duration, this.metadata.dad || '', transaction);
+            }
+        }
+    }
+    /**
+     * Register the web hook signal AFTER the Leg1 transaction commits.
+     * This ensures the hook signal is never visible before Leg1
+     * completion, eliminating the FORBIDDEN window where Leg2 could
+     * find the hook but fail on the collation check.
+     *
+     * If a pending signal was stored by an early-arriving Leg2,
+     * setHookSignal atomically detects and returns it.
+     */
+    async registerWebHookSignal() {
+        if (this.config.hook?.topic) {
+            const hookResult = await this.engine.taskService.registerWebHook(this.config.hook.topic, this.context, this.resolveDad(), this.context.metadata.expire);
+            if (hookResult.pending) {
+                return { pending: hookResult.pending };
+            }
+        }
+    }
+    /**
+     * @deprecated Use registerTimeHook + registerWebHookSignal instead.
+     * Kept for backward compatibility with tests that monkey-patch this method.
+     */
     async registerHook(transaction) {
         let jobId;
         let pending;
         if (this.config.hook?.topic) {
-            //hook signal is set standalone (not in the transaction) so the
-            //single CTE query can atomically detect a pending signal collision
             const hookResult = await this.engine.taskService.registerWebHook(this.config.hook.topic, this.context, this.resolveDad(), this.context.metadata.expire);
             jobId = hookResult.jobId;
             pending = hookResult.pending;
@@ -293,9 +344,38 @@ class Hook extends activity_1.Activity {
             this.context.metadata.jid = jobId;
             this.context.metadata.gid = gId;
             this.context.metadata.dad = dad;
-            await this.processEvent(status, code, 'hook');
-            if (code === 200) {
-                await taskService.deleteWebHookSignal(this.config.hook.topic, data);
+            // Inline retry for FORBIDDEN: Leg2 arrived in the window between
+            // setHookSignal (standalone) and Leg1 transaction.exec(). The 100B
+            // ledger digit is not yet visible. Leg1 needs only milliseconds to
+            // commit — retry here, inside the message processing loop, before
+            // consumeOne's finally block acks the message. Stream-level retry
+            // won't help: ENGINE consumers have no retry policy, so shouldRetry
+            // returns [false, 0] and the message is ack'd with no retry.
+            const MAX_FORBIDDEN_RETRIES = 5;
+            const FORBIDDEN_RETRY_DELAY_MS = 50;
+            for (let attempt = 0; attempt <= MAX_FORBIDDEN_RETRIES; attempt++) {
+                try {
+                    await this.processEvent(status, code, 'hook');
+                    if (code === 200) {
+                        await taskService.deleteWebHookSignal(this.config.hook.topic, data);
+                    }
+                    return;
+                }
+                catch (error) {
+                    if (error instanceof errors_1.CollationError &&
+                        error.fault === collator_2.CollationFaultType.FORBIDDEN &&
+                        attempt < MAX_FORBIDDEN_RETRIES) {
+                        this.logger.warn('hook-webhook-forbidden-inline-retry', {
+                            attempt: attempt + 1,
+                            maxAttempts: MAX_FORBIDDEN_RETRIES,
+                            jid: this.context.metadata.jid,
+                            aid: this.metadata.aid,
+                        });
+                        await (0, utils_1.sleepFor)(FORBIDDEN_RETRY_DELAY_MS * (attempt + 1));
+                        continue;
+                    }
+                    throw error;
+                }
             }
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.16.3",
+  "version": "0.16.5",
   "description": "Durable Workflow",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",