@hotmeshio/hotmesh 0.22.6 → 0.22.8

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.22.6",
+    "version": "0.22.8",
     "description": "Durable Workflow",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/durable/client.js

@@ -167,7 +167,7 @@ class ClientService {
                     pending: options?.pending,
                     entity: options?.entity,
                 });
-                return new handle_1.WorkflowHandleService(hotMeshClient, workflowTopic, jobId);
+                return new handle_1.WorkflowHandleService(hotMeshClient, workflowTopic, jobId, this.escalations);
             },
             /**
              * Sends a message payload to a running workflow that is paused and awaiting the signal.
@@ -267,7 +267,7 @@ class ClientService {
             getHandle: async (taskQueue, workflowName, workflowId, namespace) => {
                 const workflowTopic = `${taskQueue}-${workflowName}`;
                 const hotMeshClient = await this.getHotMeshClient(taskQueue, namespace);
-                return new handle_1.WorkflowHandleService(hotMeshClient, workflowTopic, workflowId);
+                return new handle_1.WorkflowHandleService(hotMeshClient, workflowTopic, workflowId, this.escalations);
             },
             /**
              * Provides direct access to the SEARCH backend

package/build/services/durable/handle.d.ts

@@ -3,6 +3,7 @@ import { DurableJobExport, ExportOptions, ExecutionExportOptions, WorkflowExecut
 import { JobInterruptOptions } from '../../types/job';
 import { StreamError } from '../../types/stream';
 import { ExporterService } from './exporter';
+import { EscalationClientService } from '../escalations/client';
 /**
  * Handle to a running or completed workflow execution. Returned by
  * `client.workflow.start()` and `client.workflow.getHandle()`.
@@ -32,10 +33,12 @@ export declare class WorkflowHandleService {
     hotMesh: HotMesh;
     workflowTopic: string;
     workflowId: string;
+    /** @private */
+    escalationClient?: EscalationClientService;
     /**
      * @private
      */
-    constructor(hotMesh: HotMesh, workflowTopic: string, workflowId: string);
+    constructor(hotMesh: HotMesh, workflowTopic: string, workflowId: string, escalationClient?: EscalationClientService);
     /**
      * Export the raw workflow state as a {@link DurableJobExport} with five sections:
      *
@@ -126,6 +129,12 @@ export declare class WorkflowHandleService {
      * subscribers are notified, and the job hash is expired. Unlike
      * {@link cancel}, this does **not** give the workflow a chance to
      * run cleanup code.
+     *
+     * Any pending escalations for this workflow are cancelled in the same
+     * Postgres transaction that decrements the job semaphore — one atomic
+     * write, no TOCTOU. A `system.escalation.*.cancelled` event is emitted
+     * locally for each cancelled row via the configured `events.publish`
+     * sink — instance-local only, never broadcast.
      */
     terminate(options?: JobInterruptOptions): Promise<string>;
     /**

package/build/services/durable/handle.js

@@ -27,10 +27,11 @@ class WorkflowHandleService {
     /**
      * @private
      */
-    constructor(hotMesh, workflowTopic, workflowId) {
+    constructor(hotMesh, workflowTopic, workflowId, escalationClient) {
         this.workflowTopic = workflowTopic;
         this.workflowId = workflowId;
         this.hotMesh = hotMesh;
+        this.escalationClient = escalationClient;
         this.exporter = new exporter_1.ExporterService(this.hotMesh.appId, this.hotMesh.engine.store, this.hotMesh.engine.logger);
     }
     /**
@@ -157,9 +158,23 @@ class WorkflowHandleService {
      * subscribers are notified, and the job hash is expired. Unlike
      * {@link cancel}, this does **not** give the workflow a chance to
      * run cleanup code.
+     *
+     * Any pending escalations for this workflow are cancelled in the same
+     * Postgres transaction that decrements the job semaphore — one atomic
+     * write, no TOCTOU. A `system.escalation.*.cancelled` event is emitted
+     * locally for each cancelled row via the configured `events.publish`
+     * sink — instance-local only, never broadcast.
      */
     async terminate(options) {
-        return await this.hotMesh.interrupt(`${this.hotMesh.appId}.execute`, this.workflowId, options);
+        let cancelledEntries = [];
+        const result = await this.hotMesh.interrupt(`${this.hotMesh.appId}.execute`, this.workflowId, {
+            ...options,
+            onEscalationsCancelled: (entries) => { cancelledEntries = entries; },
+        });
+        if (this.escalationClient && cancelledEntries.length > 0) {
+            this.escalationClient.emitCancelledBatch(cancelledEntries);
+        }
+        return result;
     }
     /**
      * Requests cooperative cancellation of the workflow. Unlike

package/build/services/durable/workflow/condition.d.ts

@@ -97,6 +97,7 @@ import { ConditionQueueConfig } from '../../../types/hmsh_escalations';
  *   {@link ConditionQueueConfig} that writes one row to `public.hmsh_escalations`
  *   atomically at suspension time. Cannot specify both; use the config object's
  *   `expiresAt` field for deadline enforcement when an escalation is involved.
- * @returns The signal payload, or `false` if a timeout string was given and it expired.
+ * @returns The signal payload, `false` if a timeout string was given and it expired,
+ *   or `null` if the escalation was cancelled via `client.escalations.cancel()`.
  */
-export declare function condition<T>(signalId: string, timeoutOrConfig?: string | ConditionQueueConfig): Promise<T | false>;
+export declare function condition<T>(signalId: string, timeoutOrConfig?: string | ConditionQueueConfig): Promise<T | false | null>;

package/build/services/durable/workflow/condition.js

@@ -102,7 +102,8 @@ const didRun_1 = require("./didRun");
  *   {@link ConditionQueueConfig} that writes one row to `public.hmsh_escalations`
  *   atomically at suspension time. Cannot specify both; use the config object's
  *   `expiresAt` field for deadline enforcement when an escalation is involved.
- * @returns The signal payload, or `false` if a timeout string was given and it expired.
+ * @returns The signal payload, `false` if a timeout string was given and it expired,
+ *   or `null` if the escalation was cancelled via `client.escalations.cancel()`.
  */
 async function condition(signalId, timeoutOrConfig) {
     const timeout = typeof timeoutOrConfig === 'string' ? timeoutOrConfig : undefined;
@@ -127,7 +128,13 @@ async function condition(signalId, timeoutOrConfig) {
         if (result?.timedOut) {
             return false;
         }
-        return result.data.data;
+        const signalData = result.data?.data;
+        // If the escalation was cancelled via cancel(), the signal carries this marker.
+        // Return null so the workflow can distinguish cancellation from a real resolution.
+        if (signalData && typeof signalData === 'object' && signalData.__escalation_cancelled === true) {
+            return null;
+        }
+        return signalData;
     }
     const store = common_1.asyncLocalStorage.getStore();
     // Emit DISPATCH span in debug mode

package/build/services/engine/version.js

@@ -49,10 +49,21 @@ async function getVID(instance, vid) {
         }
         return { id: instance.appId, version: app.version };
     }
-    else if (!instance.apps && vid) {
-        instance.apps = {};
-        instance.apps[instance.appId] = vid;
-        return vid;
+    else if (!instance.apps) {
+        // First call — always DB-refresh to avoid locking in a stale version.
+        // The `vid` parameter originates from store.getApp() (which may be
+        // cached from before the last activation).  If a worker missed the
+        // nocache NOTIFY (startup race: LISTEN not yet established when the
+        // NOTIFY fired), it would lock in the pre-activation version for its
+        // entire lifetime, silently loading the old schema on every request.
+        // One extra DB query here, once per engine lifetime, eliminates the
+        // race regardless of NOTIFY delivery.
+        const id = vid?.id ?? instance.appId;
+        const freshApp = await instance.store.getApp(id, true);
+        if (!instance.apps)
+            instance.apps = {};
+        instance.apps[instance.appId] = freshApp;
+        return { id: freshApp.id, version: freshApp.version };
     }
     else {
         return await fetchAndVerifyVID(instance, {

package/build/services/escalations/client.d.ts

@@ -120,10 +120,22 @@ export declare class EscalationClientService {
      */
     escalateToRole(params: EscalateToRoleParams): Promise<EscalationEntry | null>;
     /**
-     * Cancels a pending escalation without delivering a signal. Terminal rows
-     * return `already-terminal`.
+     * Cancels a pending escalation and delivers a cancellation signal to the
+     * waiting workflow so that `condition()` returns `null`. Terminal rows
+     * return `already-terminal`. Signal delivery is best-effort post-commit —
+     * the committed cancelled row is the durable record; any missed delivery
+     * can be detected via a sweep of rows with `status = 'cancelled'` and a
+     * non-null `signal_key`.
      */
     cancel(id: string, namespace?: string): Promise<CancelEscalationResult>;
+    /**
+     * Emits local `cancelled` events for a batch of already-cancelled escalation
+     * entries. Called by `WorkflowHandleService.terminate()` after the single
+     * atomic transaction that interrupts the workflow and cancels its escalations
+     * has committed. Fire-and-forget via the configured `events.publish` sink
+     * (e.g. NATS) — instance-local, never broadcast via Postgres LISTEN/NOTIFY.
+     */
+    emitCancelledBatch(entries: EscalationEntry[]): void;
     /**
      * Resolves a pending escalation by UUID. Uses an explicit Postgres transaction
      * with FOR UPDATE + WHERE guard: only one concurrent caller can commit the

package/build/services/escalations/client.js

@@ -242,16 +242,38 @@ class EscalationClientService {
         return entry;
     }
     /**
-     * Cancels a pending escalation without delivering a signal. Terminal rows
-     * return `already-terminal`.
+     * Cancels a pending escalation and delivers a cancellation signal to the
+     * waiting workflow so that `condition()` returns `null`. Terminal rows
+     * return `already-terminal`. Signal delivery is best-effort post-commit —
+     * the committed cancelled row is the durable record; any missed delivery
+     * can be detected via a sweep of rows with `status = 'cancelled'` and a
+     * non-null `signal_key`.
      */
     async cancel(id, namespace) {
-        const hm = await this._engine(null, namespace);
+        const ns = namespace ?? factory_1.APP_ID;
+        const hm = await this._engine(null, ns);
         const result = await hm.engine.store.cancelEscalation(id, namespace);
-        if (result.ok === true)
+        if (result.ok === true) {
             this._emit('cancelled', result.entry);
+            if (result.entry.signal_key) {
+                await this._deliverEscalationSignal(ns, result.entry.topic, {
+                    id: result.entry.signal_key,
+                    data: { __escalation_cancelled: true },
+                });
+            }
+        }
         return result;
     }
+    /**
+     * Emits local `cancelled` events for a batch of already-cancelled escalation
+     * entries. Called by `WorkflowHandleService.terminate()` after the single
+     * atomic transaction that interrupts the workflow and cancels its escalations
+     * has committed. Fire-and-forget via the configured `events.publish` sink
+     * (e.g. NATS) — instance-local, never broadcast via Postgres LISTEN/NOTIFY.
+     */
+    emitCancelledBatch(entries) {
+        this._emitMany('cancelled', entries);
+    }
     /**
      * Resolves a pending escalation by UUID. Uses an explicit Postgres transaction
      * with FOR UPDATE + WHERE guard: only one concurrent caller can commit the

package/build/services/store/providers/postgres/postgres.js

@@ -1009,32 +1009,23 @@ class PostgresStoreService extends __1.StoreService {
      */
     async interrupt(topic, jobId, options = {}) {
         try {
-            //verify job exists
+            //pre-flight: bail early if already inactive (optimization; hincrbyfloat is the real guard)
             const status = await this.getStatus(jobId, this.appId);
             if (status <= 0) {
-                //verify still active; job already completed
                 throw new Error(`Job ${jobId} already completed`);
             }
-            //decrement job status (:) by 1bil
             const amount = -1000000000;
-            const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
-                appId: this.appId,
-                jobId,
-            });
-            const result = await this.kvsql().hincrbyfloat(jobKey, ':', amount);
-            if (result <= amount) {
-                //verify active state; job already interrupted
-                throw new Error(`Job ${jobId} already completed`);
-            }
-            //persist the error unless specifically told not to
+            const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
+            //build error symbol BEFORE opening the transaction — symbol lookup is read-only
+            let errSymbol;
+            let err;
             if (options.throw !== false) {
-                const errKey = `metadata/err`; //job errors are stored at the path `metadata/err`
-                const symbolNames = [`$${topic}`]; //the symbol for `metadata/err` is in the backend
+                const errKey = `metadata/err`;
+                const symbolNames = [`$${topic}`];
                 const symKeys = await this.getSymbolKeys(symbolNames);
                 const symVals = await this.getSymbolValues();
                 this.serializer.resetSymbols(symKeys, symVals, {});
-                //persists the standard 410 error (job is `gone`)
-                const err = JSON.stringify({
+                err = JSON.stringify({
                     code: options.code ?? enums_1.HMSH_CODE_INTERRUPT,
                     message: options.reason ?? `job [${jobId}] interrupted`,
                     stack: options.stack ?? '',
@@ -1042,9 +1033,32 @@ class PostgresStoreService extends __1.StoreService {
                 });
                 const payload = { [errKey]: amount.toString() };
                 const hashData = this.serializer.package(payload, symbolNames);
-                const errSymbol = Object.keys(hashData)[0];
-                await this.kvsql().hset(jobKey, { [errSymbol]: err });
+                errSymbol = Object.keys(hashData)[0];
+            }
+            //single transaction: status decrement + optional error write + escalation cancel.
+            //WHERE guard on the escalation UPDATE prevents double-cancel;
+            //hincrbyfloat is the atomic idempotency proof checked post-commit.
+            const txn = this.kvsql(this.transact());
+            txn.hincrbyfloat(jobKey, ':', amount);
+            if (errSymbol && err) {
+                txn.hset(jobKey, { [errSymbol]: err });
+            }
+            txn.addCommand(`UPDATE public.hmsh_escalations
+         SET status = 'cancelled', updated_at = NOW()
+         WHERE workflow_id = $1
+           AND app_id = $2
+           AND status = 'pending'
+         RETURNING *`, [jobId, this.appId], 'array', (rows) => rows);
+            const results = await txn.exec();
+            //results[0] = new status after hincrbyfloat — the atomic idempotency guard
+            const newStatus = results[0];
+            if (newStatus <= amount) {
+                throw new Error(`Job ${jobId} already completed`);
             }
+            //fire the callback with any escalation rows cancelled in this same transaction;
+            //no second query, no second transaction
+            const cancelledEntries = (results[results.length - 1] || []);
+            options.onEscalationsCancelled?.(cancelledEntries);
         }
         catch (e) {
             if (!options.suppress) {

package/build/types/job.d.ts

@@ -130,7 +130,10 @@ type JobInterruptOptions = {
      */
     throw?: boolean;
     /**
-     * interrupt child/descendant jobs
+     * Reserved for future use: interrupt child/descendant jobs.
+     * This flag is parsed and threaded through the engine call chain
+     * but is **not yet implemented** — child workflows are not interrupted
+     * when this is set. Do not rely on it.
      * @default false
      */
     descend?: boolean;
@@ -154,6 +157,13 @@ type JobInterruptOptions = {
      * Optional stack trace
      */
     stack?: string;
+    /**
+     * Fires synchronously inside `store.interrupt()` after the single
+     * transaction that decrements the job semaphore AND cancels pending
+     * escalations commits. Only used by `WorkflowHandleService.terminate()`
+     * to emit local events without a second transaction or a separate query.
+     */
+    onEscalationsCancelled?: (entries: any[]) => void;
 };
 /**
  * format when publishing job meta/data on the wire when it completes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.22.6",
+  "version": "0.22.8",
   "description": "Durable Workflow",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",