@hotmeshio/hotmesh 0.16.1 → 0.16.2

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.16.1",
+    "version": "0.16.2",
     "description": "Durable Workflow",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/activities/activity/process.js

@@ -3,7 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.processEvent = void 0;
 const enums_1 = require("../../../modules/enums");
 const errors_1 = require("../../../modules/errors");
-const collator_1 = require("../../collator");
+const collator_1 = require("../../../types/collator");
+const collator_2 = require("../../collator");
 const telemetry_1 = require("../../telemetry");
 const stream_1 = require("../../../types/stream");
 // Per-instance collation error tracking for reservation timeout detection
@@ -34,7 +35,7 @@ async function processEvent(instance, status = stream_1.StreamStatus.SUCCESS, co
     try {
         const collationKey = await instance.verifyReentry();
         instance.adjacentIndex =
-            collator_1.CollatorService.getDimensionalIndex(collationKey);
+            collator_2.CollatorService.getDimensionalIndex(collationKey);
         telemetry = new telemetry_1.TelemetryService(instance.engine.appId, instance.config, instance.metadata, instance.context);
         telemetry.startActivitySpan(instance.leg);
         //bind data per status type
@@ -71,10 +72,27 @@ async function processEvent(instance, status = stream_1.StreamStatus.SUCCESS, co
     }
     catch (error) {
         if (error instanceof errors_1.CollationError) {
-            // INACTIVE is legitimate duplicate detection — the Postgres atomic
-            // CTE (collateLeg2Entry) serializes via row locks, so the GUID
-            // ledger value is correct. Silent ack is the right behavior:
-            // the work was already done by a prior delivery of this message.
+            //FORBIDDEN: Leg1 not complete — signal arrived in the window
+            //between registerHook (standalone) and Leg1 transaction commit.
+            //Rethrow so the stream message is retried with backoff; by then
+            //Leg1 will have committed and Leg2 processing will succeed.
+            //The GUID marker was already committed by notarizeLeg2Entry;
+            //on retry, collateLeg2Entry's SETNX is a no-op for the same
+            //GUID, and verifySyntheticInteger sees no steps done → allowed.
+            if (error.fault === collator_1.CollationFaultType.FORBIDDEN) {
+                instance.logger.warn('process-event-forbidden-retry', {
+                    jid: instance.context.metadata.jid,
+                    aid: instance.metadata.aid,
+                    message: 'Leg1 not committed yet; rethrowing for stream retry',
+                    error,
+                });
+                throw error;
+            }
+            // INACTIVE/DUPLICATE: legitimate duplicate detection — the
+            // Postgres atomic CTE (collateLeg2Entry) serializes via row
+            // locks, so the GUID ledger value is correct. Silent ack is
+            // the right behavior: the work was already done by a prior
+            // delivery of this message.
             const now = Date.now();
             if (now - collationWindowStart > COLLATION_WINDOW_MS) {
                 collationErrorCount = 0;

package/build/services/store/providers/postgres/postgres.d.ts

@@ -118,9 +118,14 @@ declare class PostgresStoreService extends StoreService<ProviderClient, Provider
     /**
      * Leg1: set hook signal, atomically detecting a pending signal.
      *
-     * Standalone (no transaction): single CTE query that reads any existing
-     * pending value, then inserts the hook signal (overwriting pending or
-     * expired entries). Returns `{success, pendingData}` in one round trip.
+     * Standalone (no transaction): acquires a per-key advisory lock to
+     * serialize with concurrent getHookSignal calls, then reads any
+     * existing pending value and inserts the hook signal.
+     *
+     * The advisory lock prevents a race where the CTE's read snapshot
+     * misses a concurrently inserted pending signal — under READ
+     * COMMITTED, ON CONFLICT sees committed writes but the SELECT CTE
+     * does not, causing the pending data to be silently overwritten.
      *
      * In a transaction: queues the setnxex; pending detection deferred.
      */
@@ -132,10 +137,13 @@ declare class PostgresStoreService extends StoreService<ProviderClient, Provider
      * Leg2: get hook signal OR atomically set a pending signal.
      *
      * When `pendingData` is provided and no hook signal exists, the
-     * pending value is inserted in the SAME SQL statement — no second
-     * round trip. This is the transactional edge that prevents the
-     * signal from being lost: by the time the query returns, the
-     * pending key is already visible to leg1's setnxex.
+     * pending value is stored so leg1's setHookSignal can detect it.
+     *
+     * Uses a per-key advisory lock to serialize with concurrent
+     * setHookSignal calls. Without the lock, a CTE race exists where
+     * the read snapshot misses a concurrently inserted hook signal AND
+     * the pending INSERT fails on conflict (the hook has valid expiry),
+     * silently losing the signal.
      *
      * When `pendingData` is omitted, behaves as a plain read.
      */

package/build/services/store/providers/postgres/postgres.js

@@ -755,9 +755,14 @@ class PostgresStoreService extends __1.StoreService {
     /**
      * Leg1: set hook signal, atomically detecting a pending signal.
      *
-     * Standalone (no transaction): single CTE query that reads any existing
-     * pending value, then inserts the hook signal (overwriting pending or
-     * expired entries). Returns `{success, pendingData}` in one round trip.
+     * Standalone (no transaction): acquires a per-key advisory lock to
+     * serialize with concurrent getHookSignal calls, then reads any
+     * existing pending value and inserts the hook signal.
+     *
+     * The advisory lock prevents a race where the CTE's read snapshot
+     * misses a concurrently inserted pending signal — under READ
+     * COMMITTED, ON CONFLICT sees committed writes but the SELECT CTE
+     * does not, causing the pending data to be silently overwritten.
      *
      * In a transaction: queues the setnxex; pending detection deferred.
      */
@@ -774,37 +779,30 @@ class PostgresStoreService extends __1.StoreService {
         const kv = this.kvsql();
         const tableName = kv.tableForKey(fullKey);
         const storedKey = kv.storageKey(fullKey);
-        const sql = `
-      WITH pre AS (
-        SELECT value FROM ${tableName}
-        WHERE key = $1 AND (expiry IS NULL OR expiry > NOW())
-      ),
-      ins AS (
-        INSERT INTO ${tableName} (key, value, expiry)
-        VALUES ($1, $2, NOW() + INTERVAL '${delay} seconds')
-        ON CONFLICT (key) DO UPDATE
-          SET value = EXCLUDED.value, expiry = EXCLUDED.expiry
-          WHERE ${tableName}.expiry IS NULL
-            OR ${tableName}.expiry <= NOW()
-            OR ${tableName}.value LIKE '$pending::%'
-        RETURNING true as success
-      )
-      SELECT
-        COALESCE((SELECT success FROM ins), false) as success,
-        (SELECT value FROM pre) as existing_value
-    `;
+        //acquire per-key advisory lock (session-level) to serialize
+        //with concurrent getHookSignal for the same signal key
+        await this.pgClient.query('SELECT pg_advisory_lock(901, hashtext($1))', [storedKey]);
         try {
-            const res = await this.pgClient.query(sql, [storedKey, jobId]);
-            const row = res.rows[0] || {};
-            const success = row.success === true;
-            const existing = row.existing_value;
-            if (success && existing?.startsWith('$pending::')) {
-                return {
-                    success: true,
-                    pendingData: existing.slice('$pending::'.length),
-                };
+            //read existing value under lock
+            const readRes = await this.pgClient.query(`SELECT value FROM ${tableName}
+         WHERE key = $1 AND (expiry IS NULL OR expiry > NOW())`, [storedKey]);
+            let pendingData;
+            if (readRes.rows.length > 0) {
+                const existing = readRes.rows[0].value;
+                if (existing?.startsWith('$pending::')) {
+                    pendingData = existing.slice('$pending::'.length);
+                }
+                else {
+                    //hook already set (retry) — no change needed
+                    return { success: false };
+                }
             }
-            return { success };
+            //insert hook value (or overwrite pending)
+            await this.pgClient.query(`INSERT INTO ${tableName} (key, value, expiry)
+         VALUES ($1, $2, NOW() + INTERVAL '${delay} seconds')
+         ON CONFLICT (key) DO UPDATE
+           SET value = EXCLUDED.value, expiry = EXCLUDED.expiry`, [storedKey, jobId]);
+            return { success: true, pendingData };
         }
         catch (error) {
             if (error?.message?.includes('closed') ||
@@ -813,15 +811,26 @@ class PostgresStoreService extends __1.StoreService {
             }
             throw error;
         }
+        finally {
+            try {
+                await this.pgClient.query('SELECT pg_advisory_unlock(901, hashtext($1))', [storedKey]);
+            }
+            catch {
+                //lock auto-releases on session close
+            }
+        }
     }
     /**
      * Leg2: get hook signal OR atomically set a pending signal.
      *
      * When `pendingData` is provided and no hook signal exists, the
-     * pending value is inserted in the SAME SQL statement — no second
-     * round trip. This is the transactional edge that prevents the
-     * signal from being lost: by the time the query returns, the
-     * pending key is already visible to leg1's setnxex.
+     * pending value is stored so leg1's setHookSignal can detect it.
+     *
+     * Uses a per-key advisory lock to serialize with concurrent
+     * setHookSignal calls. Without the lock, a CTE race exists where
+     * the read snapshot misses a concurrently inserted hook signal AND
+     * the pending INSERT fails on conflict (the hook has valid expiry),
+     * silently losing the signal.
      *
      * When `pendingData` is omitted, behaves as a plain read.
      */
@@ -838,38 +847,30 @@ class PostgresStoreService extends __1.StoreService {
                 return undefined;
             return value;
         }
-        //atomic get-or-set-pending: one round trip
         const kv = this.kvsql();
         const tableName = kv.tableForKey(fullKey);
         const storedKey = kv.storageKey(fullKey);
         const expire = pendingExpire || enums_1.HMSH_PENDING_SIGNAL_EXPIRE;
         const pendingValue = `$pending::${pendingData}`;
-        const sql = `
-      WITH existing AS (
-        SELECT value FROM ${tableName}
-        WHERE key = $1 AND (expiry IS NULL OR expiry > NOW())
-      ),
-      pending AS (
-        INSERT INTO ${tableName} (key, value, expiry)
-        SELECT $1, $2, NOW() + INTERVAL '${expire} seconds'
-        WHERE NOT EXISTS (SELECT 1 FROM existing)
-        ON CONFLICT (key) DO UPDATE
-          SET value = EXCLUDED.value, expiry = EXCLUDED.expiry
-          WHERE ${tableName}.expiry IS NULL OR ${tableName}.expiry <= NOW()
-        RETURNING true as inserted
-      )
-      SELECT
-        (SELECT value FROM existing) as hook_value,
-        (SELECT inserted FROM pending) as pending_inserted
-    `;
+        //acquire per-key advisory lock (session-level) to serialize
+        //with concurrent setHookSignal for the same signal key
+        await this.pgClient.query('SELECT pg_advisory_lock(901, hashtext($1))', [storedKey]);
         try {
-            const res = await this.pgClient.query(sql, [storedKey, pendingValue]);
-            const row = res.rows[0] || {};
-            const hookValue = row.hook_value;
-            if (hookValue && !hookValue.startsWith('$pending::')) {
-                return hookValue;
+            //read existing value under lock
+            const readRes = await this.pgClient.query(`SELECT value FROM ${tableName}
+         WHERE key = $1 AND (expiry IS NULL OR expiry > NOW())`, [storedKey]);
+            if (readRes.rows.length > 0) {
+                const value = readRes.rows[0].value;
+                if (value && !value.startsWith('$pending::')) {
+                    //hook found — return it
+                    return value;
+                }
             }
-            //no hook signal; pending was inserted (or already existed)
+            //no hook signal — store pending
+            await this.pgClient.query(`INSERT INTO ${tableName} (key, value, expiry)
+         VALUES ($1, $2, NOW() + INTERVAL '${expire} seconds')
+         ON CONFLICT (key) DO UPDATE
+           SET value = EXCLUDED.value, expiry = EXCLUDED.expiry`, [storedKey, pendingValue]);
             return undefined;
         }
         catch (error) {
@@ -879,6 +880,14 @@ class PostgresStoreService extends __1.StoreService {
             }
             throw error;
         }
+        finally {
+            try {
+                await this.pgClient.query('SELECT pg_advisory_unlock(901, hashtext($1))', [storedKey]);
+            }
+            catch {
+                //lock auto-releases on session close
+            }
+        }
     }
     async deleteHookSignal(topic, resolved) {
         const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.16.1",
+  "version": "0.16.2",
   "description": "Durable Workflow",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",