@hotmeshio/hotmesh 0.25.2 → 0.25.5

package/ISSUE.md

@@ -0,0 +1,53 @@
+# ADDENDUM — lost condition wakes are NOT restart-dependent (registration-window race)
+
+Follow-up to the earlier "Durable wakes lost across process restart" report filed
+tonight (sleep timers + condition signals). New evidence sharpens the condition case.
+
+## Reproduced with no restart involved
+
+Same environment (hotmesh 0.25.4, Postgres, long-tail 0.7.3), process running
+continuously the whole time:
+
+- `13:59:36.204` — pill escalation created (conditionLT atomic park, child
+  `...-winddown-gluer-0`).
+- `13:59:36.237` — row **resolved**, 33 ms after creation (a hot consumer was
+  already scanning the pond and grabbed it instantly).
+- 15+ minutes later — the awaiting child is still `status > 0`, never woke;
+  its parent's `Promise.all` wedged with it.
+
+## The pattern across all three observed condition cases
+
+Every lost condition wake we have seen shares one property: **the resolve landed
+sub-second after the conditionLT park** (33 ms, 232 ms, ~600 ms). Slow resolves
+(seconds or minutes after park) have never lost a wake across hundreds of
+station-worker resolutions tonight.
+
+Hypothesis: a resolve that commits **inside the awaiting workflow's
+wake-registration window** — after the escalation row is visible to consumers
+but before the awaiter's subscription is fully registered — delivers to nobody,
+and nothing ever re-checks the persisted row against sleeping awaiters. The
+restart in the earlier case was likely incidental; the window is the bug.
+
+## Suggested repro (tighter than the original)
+
+1. Workflow A parks via `conditionLT` (the escalation-creating overload).
+2. A hot loop polls for the row and resolves it the instant it appears
+   (sub-100 ms). No process kill needed.
+3. Observe whether A ever wakes. In our runs, a same-process consumer beating
+   the registration window wedges A reliably enough to hit twice in one evening.
+
+## Suggested direction (unchanged, reinforced)
+
+Boot-time AND periodic reconciliation of resolved-rows-with-sleeping-awaiters
+would heal both the restart loss and this race — the row carries the awaiting
+`workflow_id`; the join is cheap. Making the wake itself a stream message
+(liveness-redelivered since 0.25.4) also closes the window, since the message
+persists until reserved and acked.
+
+## Our workaround (in production use as of tonight)
+
+Settlement guard in the watcher activity: after the row's outcome is known,
+poll `durable.jobs` for the awaiting child's status; if still running ~10s
+after its row resolved, terminate the child by id (terminate purges cleanly
+since 0.25.4) and treat the row's persisted resolver payload as authoritative.
+The parent catches the child's terminate rejection. Ugly but wedge-proof.

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.25.2",
+    "version": "0.25.5",
     "description": "Durable Workflow",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/engine/completion.js

@@ -68,6 +68,7 @@ function resolveError(metadata) {
 exports.resolveError = resolveError;
 async function interrupt(instance, topic, jobId, options = {}) {
     await instance.store.interrupt(topic, jobId, options);
+    await expireStreamMessages(instance, jobId);
     const context = (await instance.getState(topic, jobId));
     const completionOpts = {
         interrupt: options.descend,
@@ -77,9 +78,34 @@ async function interrupt(instance, topic, jobId, options = {}) {
 }
 exports.interrupt = interrupt;
 async function scrub(instance, jobId) {
+    await expireStreamMessages(instance, jobId);
     await instance.store.scrub(jobId);
 }
 exports.scrub = scrub;
+/**
+ * Soft-deletes the job's stream messages (queued, reserved, and
+ * scheduled retries) so no message belonging to a dead job is ever
+ * delivered again. Interrupting the job wins or throws before this
+ * runs; failures here are logged and absorbed because the delivery
+ * liveness guard drops any surviving message at claim time.
+ */
+async function expireStreamMessages(instance, jobId) {
+    try {
+        const expired = await instance.stream.expireJobMessages?.(jobId);
+        if (expired) {
+            instance.logger.info('engine-job-stream-messages-expired', {
+                jobId,
+                expired,
+            });
+        }
+    }
+    catch (error) {
+        instance.logger.error('engine-job-stream-expire-error', {
+            jobId,
+            error,
+        });
+    }
+}
 /**
  * Orchestrates all post-completion work for a finished job:
  * notify parent, publish to subscribers, schedule cleanup.

package/build/services/escalations/client.d.ts

@@ -72,6 +72,16 @@ export declare class EscalationClientService {
     private _makeEngineFactory;
     private _hashConnection;
     private _deliverEscalationSignal;
+    /**
+     * Builds the wake publish as a SQL command so the store can commit it
+     * INSIDE the resolve transaction — the wake becomes durable with the
+     * resolved row, closing the crash window between resolve commit and
+     * post-commit signal delivery. Mirrors `_deliverEscalationSignal`'s
+     * topic fallback chain; returns null when no hook rule is deployed
+     * for any candidate topic (the caller then keeps post-commit
+     * delivery as the only path, preserving prior behavior).
+     */
+    private _buildWakeCommand;
     /**
      * Returns all escalation rows matching the given filters. Each row includes
      * a computed `available` field (true = claimable). Supports `sortBy`,

package/build/services/escalations/client.js

@@ -3,8 +3,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.EscalationClientService = void 0;
 const enums_1 = require("../../modules/enums");
 const utils_1 = require("../../modules/utils");
+const key_1 = require("../../modules/key");
 const hotmesh_1 = require("../hotmesh");
 const types_1 = require("../../types");
+const stream_1 = require("../../types/stream");
 const factory_1 = require("../durable/schemas/factory");
 /**
  * Standalone client for the `public.hmsh_escalations` signal-pause surface.
@@ -145,6 +147,46 @@ class EscalationClientService {
         catch { }
         return delivered;
     }
+    /**
+     * Builds the wake publish as a SQL command so the store can commit it
+     * INSIDE the resolve transaction — the wake becomes durable with the
+     * resolved row, closing the crash window between resolve commit and
+     * post-commit signal delivery. Mirrors `_deliverEscalationSignal`'s
+     * topic fallback chain; returns null when no hook rule is deployed
+     * for any candidate topic (the caller then keeps post-commit
+     * delivery as the only path, preserving prior behavior).
+     */
+    async _buildWakeCommand(ns, topic, signalKey, data) {
+        const hm = await this._engine(null, ns);
+        const engine = hm.engine;
+        const candidates = [topic, `${ns}.wfs.signal`, `${ns}.wfs.wait`].filter(Boolean);
+        for (const candidate of candidates) {
+            try {
+                const hookRule = await engine.taskService.getHookRule(candidate);
+                if (!hookRule)
+                    continue;
+                const [aid] = await engine.getSchema(`.${hookRule.to}`);
+                const streamData = {
+                    type: stream_1.StreamDataType.WEBHOOK,
+                    status: types_1.StreamStatus.SUCCESS,
+                    code: 200,
+                    metadata: { guid: (0, utils_1.guid)(), aid, topic: candidate },
+                    data: { id: signalKey, data },
+                };
+                const streamKey = engine.stream.mintKey(key_1.KeyType.STREAMS, {
+                    topic: null,
+                });
+                const { sql, params } = engine.stream._publishMessages(streamKey, [
+                    JSON.stringify(streamData),
+                ]);
+                return { forSignalKey: signalKey, sql, params };
+            }
+            catch {
+                /* candidate not deployed — try the next topic */
+            }
+        }
+        return null;
+    }
     // ─── Public API ─────────────────────────────────────────────────────────────
     /**
      * Returns all escalation rows matching the given filters. Each row includes
@@ -292,10 +334,19 @@ class EscalationClientService {
         const ns = (params.namespace ?? namespace) ?? factory_1.APP_ID;
         const hm = await this._engine(null, ns);
         const store = hm.engine.store;
-        const dbResult = await store.resolveEscalation({ id: params.id, resolverPayload: params.resolverPayload, metadata: params.metadata });
+        //pre-build the wake so it commits INSIDE the resolve transaction; the
+        //row's signal routing (signal_key, topic) is immutable after creation
+        let wakeCommand = null;
+        const preview = await store.getEscalation(params.id, params.namespace);
+        if (preview?.signal_key) {
+            wakeCommand = await this._buildWakeCommand(ns, preview.topic, preview.signal_key, params.resolverPayload ?? {});
+        }
+        const dbResult = await store.resolveEscalation({ id: params.id, resolverPayload: params.resolverPayload, metadata: params.metadata }, wakeCommand ?? undefined);
         if (!dbResult.ok)
             return dbResult;
-        if (dbResult.signalKey) {
+        if (dbResult.signalKey && !dbResult.wakeEnqueued) {
+            //the wake was not part of the commit (no hook rule found, or the
+            //enqueue was rolled back to its savepoint) — deliver post-commit
             await this._deliverEscalationSignal(ns, dbResult.topic, {
                 id: dbResult.signalKey,
                 data: params.resolverPayload ?? {},
@@ -316,10 +367,22 @@ class EscalationClientService {
         const ns = (params.namespace ?? namespace) ?? factory_1.APP_ID;
         const hm = await this._engine(null, ns);
         const store = hm.engine.store;
-        const dbResult = await store.resolveEscalationByMetadata({ key: params.key, value: params.value, resolverPayload: params.resolverPayload, roles: params.roles, metadata: params.metadata });
+        //peek the row the resolve is expected to lock and pre-build its wake;
+        //the forSignalKey guard covers the race where a different row wins
+        let wakeCommand = null;
+        const preview = await store.peekEscalationByMetadata({
+            key: params.key,
+            value: params.value,
+            roles: params.roles,
+            namespace: params.namespace,
+        });
+        if (preview?.signalKey) {
+            wakeCommand = await this._buildWakeCommand(ns, preview.topic, preview.signalKey, params.resolverPayload ?? {});
+        }
+        const dbResult = await store.resolveEscalationByMetadata({ key: params.key, value: params.value, resolverPayload: params.resolverPayload, roles: params.roles, metadata: params.metadata }, wakeCommand ?? undefined);
         if (!dbResult.ok)
             return dbResult;
-        if (dbResult.signalKey) {
+        if (dbResult.signalKey && !dbResult.wakeEnqueued) {
             await this._deliverEscalationSignal(ns, dbResult.topic, {
                 id: dbResult.signalKey,
                 data: params.resolverPayload ?? {},

package/build/services/router/consumption/index.d.ts

@@ -30,6 +30,7 @@ export declare class ConsumptionManager<S extends StreamService<ProviderClient,
     private duressManager?;
     private onDuressChange?;
     private messagesSinceLastEval;
+    private canExtendReservations;
     private adaptiveReservationTimeout;
     private adaptiveBatchSize;
     private lastDepthCheckAt;
@@ -52,7 +53,12 @@ export declare class ConsumptionManager<S extends StreamService<ProviderClient,
     consumeMessages(stream: string, group: string, consumer: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
     private consumeWithNotifications;
     private consumeWithPolling;
-    consumeOne(stream: string, group: string, id: string, input: StreamData, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
+    /**
+     * Whether reservation heartbeats are available: the provider must
+     * implement extendReservation and advertise the capability.
+     */
+    private supportsHeartbeat;
+    consumeOne(stream: string, group: string, id: string, input: StreamData, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>, consumer?: string): Promise<void>;
     execStreamLeg(input: StreamData, stream: string, id: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<StreamDataResponse>;
     ackAndDelete(stream: string, group: string, id: string): Promise<void>;
     ackAndDeleteBatch(stream: string, group: string, ids: string[]): Promise<void>;

package/build/services/router/consumption/index.js

@@ -195,7 +195,7 @@ class ConsumptionManager {
                     if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                         return;
                     }
-                    return this.consumeOne(stream, group, message.id, message.data, callback);
+                    return this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                 });
                 // Process all messages in parallel
                 await Promise.allSettled(processingPromises);
@@ -212,7 +212,7 @@ class ConsumptionManager {
                     if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                         return;
                     }
-                    await this.consumeOne(stream, group, message.id, message.data, callback);
+                    await this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                 }
             }
             // Check for pending messages if supported and enough time has passed
@@ -231,7 +231,7 @@ class ConsumptionManager {
                         if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                             return;
                         }
-                        return this.consumeOne(stream, group, message.id, message.data, callback);
+                        return this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                     });
                     await Promise.allSettled(reclaimPromises);
                 }
@@ -240,7 +240,7 @@ class ConsumptionManager {
                         if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                             return;
                         }
-                        await this.consumeOne(stream, group, message.id, message.data, callback);
+                        await this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                     }
                 }
             }
@@ -349,7 +349,7 @@ class ConsumptionManager {
                             if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                                 return;
                             }
-                            return this.consumeOne(stream, group, message.id, message.data, callback);
+                            return this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                         });
                         // Process all messages in parallel
                         await Promise.allSettled(processingPromises);
@@ -366,7 +366,7 @@ class ConsumptionManager {
                             if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                                 return;
                             }
-                            await this.consumeOne(stream, group, message.id, message.data, callback);
+                            await this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                         }
                     }
                     // Check for pending messages if supported and enough time has passed
@@ -385,7 +385,7 @@ class ConsumptionManager {
                                 if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                                     return;
                                 }
-                                return this.consumeOne(stream, group, message.id, message.data, callback);
+                                return this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                             });
                             await Promise.allSettled(reclaimPromises);
                         }
@@ -394,7 +394,7 @@ class ConsumptionManager {
                                 if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                                     return;
                                 }
-                                await this.consumeOne(stream, group, message.id, message.data, callback);
+                                await this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                             }
                         }
                     }
@@ -434,7 +434,20 @@ class ConsumptionManager {
         };
         consume.call(this);
     }
-    async consumeOne(stream, group, id, input, callback) {
+    /**
+     * Whether reservation heartbeats are available: the provider must
+     * implement extendReservation and advertise the capability.
+     */
+    supportsHeartbeat() {
+        if (this.canExtendReservations === undefined) {
+            this.canExtendReservations =
+                typeof this.stream.extendReservation === 'function' &&
+                    this.stream.getProviderSpecificFeatures()
+                        .supportsReservationExtension === true;
+        }
+        return this.canExtendReservations;
+    }
+    async consumeOne(stream, group, id, input, callback, consumer) {
         this.logger.debug(`stream-read-one`, { group, stream, id });
         // Poison message circuit breaker. This is a SAFETY NET that sits above
         // the normal retry mechanism (ErrorHandler.handleRetry / shouldRetry).
@@ -498,20 +511,67 @@ class ConsumptionManager {
             }
             return;
         }
-        // Lease deadline: the full configured reservation timeout (N).
-        // The reclaim interval is N+5s, so the deadline always fires
-        // before a reclaimant can pick up the message. This preserves
-        // the user's contract — if they set 30s, the function gets 30s.
-        const deadlineMs = this.adaptiveReservationTimeout * 1000;
+        // Lease strategy. With heartbeat support (provider capability +
+        // consumer identity), the reservation is refreshed at half the base
+        // window while the callback runs, so an activity that outlives the
+        // window stays leased and executes exactly once; the deadline then
+        // acts as a runaway cap (HMSH_RESERVATION_TIMEOUT_MAX_S). A crashed
+        // consumer stops heartbeating and its message is rescued within one
+        // window through the normal claim path. Providers without extension
+        // keep the hard deadline at the reservation timeout (N), with
+        // reclaim at N+5s.
+        const canHeartbeat = Boolean(consumer) && this.supportsHeartbeat();
+        const deadlineS = canHeartbeat
+            ? config_1.HMSH_RESERVATION_TIMEOUT_MAX_S
+            : this.adaptiveReservationTimeout;
+        const deadlineMs = deadlineS * 1000;
         let output;
         const telemetry = new telemetry_1.RouterTelemetry(this.appId);
         const processingStart = Date.now();
         try {
             telemetry.startStreamSpan(input, this.role);
             let deadlineTimer;
+            let rejectLease;
             const deadlinePromise = new Promise((_, reject) => {
-                deadlineTimer = setTimeout(() => reject(new errors_1.LeaseExpiredError(deadlineMs, this.adaptiveReservationTimeout)), deadlineMs);
+                rejectLease = reject;
+                deadlineTimer = setTimeout(() => reject(new errors_1.LeaseExpiredError(deadlineMs, deadlineS)), deadlineMs);
             });
+            // Heartbeat at half the BASE window (adaptive scaling only grows
+            // the claim window, so the base cadence always outpaces reclaim
+            // eligibility). A beat that reports 0 means the lease is gone —
+            // reclaimed by another consumer, acked, or the job was
+            // interrupted — so this execution abandons without acking. A beat
+            // that errors is transient and retries on the next tick. Fast
+            // callbacks settle before the first beat ever fires.
+            let heartbeatTimer;
+            if (canHeartbeat) {
+                const intervalMs = Math.max(1000, Math.floor((config_1.HMSH_RESERVATION_TIMEOUT_S * 1000) / 2));
+                heartbeatTimer = setInterval(() => {
+                    this.stream
+                        .extendReservation(stream, id, consumer)
+                        .then((extended) => {
+                        if (extended === 0) {
+                            this.logger.warn('stream-lease-lost', {
+                                group,
+                                stream,
+                                id,
+                                consumer,
+                                topic: input.metadata?.topic,
+                                jobId: input.metadata?.jid,
+                            });
+                            rejectLease(new errors_1.LeaseExpiredError(deadlineMs, deadlineS));
+                        }
+                    })
+                        .catch((error) => {
+                        this.logger.warn('stream-lease-extend-error', {
+                            group,
+                            stream,
+                            id,
+                            error,
+                        });
+                    });
+                }, intervalMs);
+            }
             try {
                 output = await Promise.race([
                     this.execStreamLeg(input, stream, id, callback.bind(this)),
@@ -520,6 +580,9 @@ class ConsumptionManager {
             }
             finally {
                 clearTimeout(deadlineTimer);
+                if (heartbeatTimer) {
+                    clearInterval(heartbeatTimer);
+                }
             }
             telemetry.setStreamErrorFromOutput(output);
             this.errorCount = 0;
@@ -536,7 +599,7 @@ class ConsumptionManager {
                     stream,
                     id,
                     deadlineMs,
-                    reservationTimeoutS: this.adaptiveReservationTimeout,
+                    reservationTimeoutS: deadlineS,
                     topic: input.metadata?.topic,
                     activityId: input.metadata?.aid,
                     jobId: input.metadata?.jid,

package/build/services/store/index.d.ts

@@ -68,8 +68,17 @@ declare abstract class StoreService<Provider extends ProviderClient, Transaction
      * Leg1: Attempts to set the hook signal. If a pending signal occupies
      * the key (race condition), overwrites it and returns the pending data.
      * When called with a transaction, queues the setnxex (no pending detection).
+     *
+     * When `redelivery` is provided (the webhook routing for this hook),
+     * a consumed pending signal is republished as an engine stream message
+     * in the SAME transaction that overwrites the marker — the wake
+     * survives a crash at any instant. `pendingData` is then not returned,
+     * since the store already owns the redelivery.
      */
-    abstract setHookSignal(hook: HookSignal, transaction?: TransactionProvider): Promise<{
+    abstract setHookSignal(hook: HookSignal, transaction?: TransactionProvider, redelivery?: {
+        aid: string;
+        topic: string;
+    }): Promise<{
         success: boolean;
         pendingData?: string;
     }>;

package/build/services/store/providers/postgres/postgres.d.ts

@@ -128,7 +128,10 @@ declare class PostgresStoreService extends StoreService<ProviderClient, Provider
      *
      * In a transaction: queues the setnxex; pending detection deferred.
      */
-    setHookSignal(hook: HookSignal, transaction?: ProviderTransaction): Promise<{
+    setHookSignal(hook: HookSignal, transaction?: ProviderTransaction, redelivery?: {
+        aid: string;
+        topic: string;
+    }): Promise<{
         success: boolean;
         pendingData?: string;
     }>;
@@ -270,13 +273,34 @@ declare class PostgresStoreService extends StoreService<ProviderClient, Provider
     claimEscalation(params: import('../../../../types/hmsh_escalations').ClaimEscalationParams): Promise<import('../../../../types/hmsh_escalations').ClaimEscalationResult>;
     claimEscalationByMetadata(params: import('../../../../types/hmsh_escalations').ClaimByMetadataParams): Promise<import('../../../../types/hmsh_escalations').ClaimByMetadataResult>;
     releaseEscalation(params: import('../../../../types/hmsh_escalations').ReleaseEscalationParams): Promise<import('../../../../types/hmsh_escalations').ReleaseEscalationResult>;
-    resolveEscalation(params: import('../../../../types/hmsh_escalations').ResolveEscalationParams): Promise<import('../../../../types/hmsh_escalations').ResolveEscalationResult & {
+    /**
+     * Executes a pre-built wake publish inside the currently open resolve
+     * transaction, guarded by a SAVEPOINT so a wake failure can never roll
+     * back the resolve itself. Returns true when the wake row committed
+     * with the transaction; false means the caller should fall back to
+     * post-commit delivery.
+     */
+    private enqueueEscalationWake;
+    resolveEscalation(params: import('../../../../types/hmsh_escalations').ResolveEscalationParams, wakeCommand?: import('../../../../types/hmsh_escalations').EscalationWakeCommand): Promise<import('../../../../types/hmsh_escalations').ResolveEscalationResult & {
         signalKey?: string | null;
         topic?: string | null;
+        wakeEnqueued?: boolean;
     }>;
-    resolveEscalationByMetadata(params: import('../../../../types/hmsh_escalations').ResolveByMetadataParams): Promise<import('../../../../types/hmsh_escalations').ResolveEscalationResult & {
+    /**
+     * Non-locking preview of the row `resolveEscalationByMetadata` would
+     * select — used to pre-build the wake command before the resolve
+     * transaction opens. The `forSignalKey` guard on the wake command
+     * handles the race where a different row wins the lock.
+     */
+    peekEscalationByMetadata(params: import('../../../../types/hmsh_escalations').ResolveByMetadataParams): Promise<{
+        id: string;
+        signalKey: string | null;
+        topic: string | null;
+    } | null>;
+    resolveEscalationByMetadata(params: import('../../../../types/hmsh_escalations').ResolveByMetadataParams, wakeCommand?: import('../../../../types/hmsh_escalations').EscalationWakeCommand): Promise<import('../../../../types/hmsh_escalations').ResolveEscalationResult & {
         signalKey?: string | null;
         topic?: string | null;
+        wakeEnqueued?: boolean;
     }>;
     cancelEscalation(id: string, namespace?: string): Promise<import('../../../../types/hmsh_escalations').CancelEscalationResult>;
     escalateEscalationToRole(params: import('../../../../types/hmsh_escalations').EscalateToRoleParams): Promise<import('../../../../types/hmsh_escalations').EscalationEntry | null>;

package/build/services/store/providers/postgres/postgres.js

@@ -777,7 +777,7 @@ class PostgresStoreService extends __1.StoreService {
      *
      * In a transaction: queues the setnxex; pending detection deferred.
      */
-    async setHookSignal(hook, transaction) {
+    async setHookSignal(hook, transaction, redelivery) {
         const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
         const { topic, resolved, jobId } = hook;
         const signalKey = `${topic}:${resolved}`;
@@ -813,6 +813,45 @@ class PostgresStoreService extends __1.StoreService {
             const captured = lockResult.rows[0]?.value;
             const wasInsert = lockResult.rows[0]?.inserted;
             const isPending = captured?.startsWith('$pending::');
+            if (isPending && redelivery) {
+                //consume the marker and commit its redelivery as ONE unit: the
+                //pending wake becomes a durable engine stream message in the
+                //same transaction that destroys the marker, so the wake
+                //survives a crash at any instant. A crash before COMMIT leaves
+                //the marker intact for the resume path to consume again.
+                const pendingData = captured.slice('$pending::'.length);
+                const message = JSON.stringify({
+                    type: 'webhook',
+                    status: 'success',
+                    code: 200,
+                    metadata: {
+                        guid: (0, utils_1.guid)(),
+                        aid: redelivery.aid,
+                        topic: redelivery.topic,
+                    },
+                    data: JSON.parse(pendingData),
+                });
+                const schemaName = this.kvsql().safeName(this.appId);
+                await this.pgClient.query('BEGIN');
+                try {
+                    await this.pgClient.query(`UPDATE ${tableName}
+             SET value = $1, expiry = NOW() + INTERVAL '${delay} seconds'
+             WHERE key = $2`, [jobId, storedKey]);
+                    await this.pgClient.query(`INSERT INTO ${schemaName}.engine_streams
+             (stream_name, message, priority)
+             VALUES ($1, $2, 5)`, [this.appId, message]);
+                    await this.pgClient.query('COMMIT');
+                }
+                catch (redeliveryError) {
+                    await this.pgClient.query('ROLLBACK');
+                    throw redeliveryError;
+                }
+                this.logger.warn('hook-signal-pending-redelivered', {
+                    key: signalKey,
+                    topic: redelivery.topic,
+                });
+                return { success: true };
+            }
             if (!wasInsert) {
                 //step 2: row existed — overwrite with hook value
                 await this.pgClient.query(`UPDATE ${tableName}
@@ -1792,7 +1831,32 @@ class PostgresStoreService extends __1.StoreService {
             return { ok: false, reason: 'wrong-assignee' };
         return { ok: true, entry: row.entry_json };
     }
-    async resolveEscalation(params) {
+    /**
+     * Executes a pre-built wake publish inside the currently open resolve
+     * transaction, guarded by a SAVEPOINT so a wake failure can never roll
+     * back the resolve itself. Returns true when the wake row committed
+     * with the transaction; false means the caller should fall back to
+     * post-commit delivery.
+     */
+    async enqueueEscalationWake(wakeCommand, signalKey, escalationId) {
+        if (!wakeCommand || !signalKey || wakeCommand.forSignalKey !== signalKey) {
+            return false;
+        }
+        await this.pgClient.query('SAVEPOINT escalation_wake');
+        try {
+            await this.pgClient.query(wakeCommand.sql, wakeCommand.params);
+            return true;
+        }
+        catch (error) {
+            await this.pgClient.query('ROLLBACK TO SAVEPOINT escalation_wake');
+            this.logger.warn('escalation-wake-enqueue-error', {
+                escalationId,
+                error: error.message,
+            });
+            return false;
+        }
+    }
+    async resolveEscalation(params, wakeCommand) {
         const { id, namespace, resolverPayload, metadata } = params;
         const payloadJson = resolverPayload ? JSON.stringify(resolverPayload) : null;
         // metaJson is bound at a fixed index; the CASE no-ops when null so resolution
@@ -1835,15 +1899,41 @@ class PostgresStoreService extends __1.StoreService {
                 await this.pgClient.query('ROLLBACK');
                 return { ok: false, reason: 'already-resolved' };
             }
+            //the wake commits WITH the resolve: a crash after COMMIT leaves both
+            //the resolved row and its wake message durable; a crash before
+            //leaves neither. The engine_streams INSERT trigger emits the
+            //delivery NOTIFY on commit.
+            const wakeEnqueued = await this.enqueueEscalationWake(wakeCommand, signal_key, id);
             await this.pgClient.query('COMMIT');
-            return { ok: true, entry: updateResult.rows[0], signalKey: signal_key, topic };
+            return { ok: true, entry: updateResult.rows[0], signalKey: signal_key, topic, wakeEnqueued };
         }
         catch (e) {
             await this.pgClient.query('ROLLBACK');
             throw e;
         }
     }
-    async resolveEscalationByMetadata(params) {
+    /**
+     * Non-locking preview of the row `resolveEscalationByMetadata` would
+     * select — used to pre-build the wake command before the resolve
+     * transaction opens. The `forSignalKey` guard on the wake command
+     * handles the race where a different row wins the lock.
+     */
+    async peekEscalationByMetadata(params) {
+        const { key, value, namespace, roles } = params;
+        const filter = JSON.stringify({ [key]: value });
+        const result = await this.pgClient.query(`SELECT id, signal_key, topic FROM public.hmsh_escalations
+       WHERE ${namespace ? 'namespace = $3 AND' : ''}
+             metadata @> $1::jsonb
+         AND ($2::text[] IS NULL OR role = ANY($2::text[]))
+         AND status IN ('pending', 'cancelled')
+       ORDER BY priority ASC, created_at ASC
+       LIMIT 1`, namespace ? [filter, roles ?? null, namespace] : [filter, roles ?? null]);
+        const row = result.rows[0];
+        if (!row)
+            return null;
+        return { id: row.id, signalKey: row.signal_key, topic: row.topic };
+    }
+    async resolveEscalationByMetadata(params, wakeCommand) {
         const { key, value, namespace, resolverPayload, roles, metadata } = params;
         const filter = JSON.stringify({ [key]: value });
         const payloadJson = resolverPayload ? JSON.stringify(resolverPayload) : null;
@@ -1878,8 +1968,9 @@ class PostgresStoreService extends __1.StoreService {
                 await this.pgClient.query('ROLLBACK');
                 return { ok: false, reason: 'already-resolved' };
             }
+            const wakeEnqueued = await this.enqueueEscalationWake(wakeCommand, signal_key, id);
             await this.pgClient.query('COMMIT');
-            return { ok: true, entry: updateResult.rows[0], signalKey: signal_key, topic };
+            return { ok: true, entry: updateResult.rows[0], signalKey: signal_key, topic, wakeEnqueued };
         }
         catch (e) {
             await this.pgClient.query('ROLLBACK');

package/build/services/stream/index.d.ts

@@ -66,10 +66,13 @@ export declare abstract class StreamService<ClientProvider extends ProviderClien
         supportsRetry: boolean;
         supportsNotifications?: boolean;
         supportsParallelProcessing?: boolean;
+        supportsReservationExtension?: boolean;
         maxMessageSize: number;
         maxBatchSize: number;
     };
     deadLetterMessages?(streamName: string, groupName: string, messageIds: string[]): Promise<number>;
+    expireJobMessages?(jid: string): Promise<number>;
+    extendReservation?(streamName: string, messageId: string, consumerName: string): Promise<number>;
     stopNotificationConsumer?(streamName: string, groupName: string): Promise<void>;
     cleanup?(): Promise<void>;
 }

package/build/services/stream/providers/postgres/messages.d.ts

@@ -19,6 +19,31 @@ export declare function buildPublishSQL(tableName: string, streamName: string, i
     sql: string;
     params: any[];
 };
+/**
+ * Job-liveness context for the delivery guard. `keyPrefix` is the minted
+ * job-key prefix (`hmsh:<app>:j:`) so `keyPrefix || jid` addresses the
+ * jobs row. `enabled` is mutated to false (self-disable) when the jobs
+ * table is not visible from the stream connection.
+ */
+export interface JobLivenessContext {
+    jobsTable: string;
+    keyPrefix: string;
+    enabled: boolean;
+}
+/**
+ * Soft-delete every live stream row that belongs to a job. Called when a
+ * job is interrupted so its queued, reserved, and scheduled-retry
+ * messages are never delivered again. Uses the partial jid indexes
+ * (idx_*_streams_jid_created); runs once per interrupt.
+ */
+export declare function expireJobMessages(client: PostgresClientType & ProviderClient, tableNames: string[], jid: string, logger: ILogger): Promise<number>;
+/**
+ * Refresh an owned reservation (heartbeat). Scoped to the owning
+ * consumer and to live rows: a message that was reclaimed by another
+ * consumer, acked, or expired (job interrupted) reports 0 so the
+ * stale consumer can abandon its execution.
+ */
+export declare function extendReservation(client: PostgresClientType & ProviderClient, tableName: string, streamName: string, messageId: string, consumerName: string, logger: ILogger): Promise<number>;
 /**
  * Fetch messages from the stream with optional exponential backoff.
  * Uses SKIP LOCKED for high-concurrency consumption.
@@ -33,7 +58,7 @@ export declare function fetchMessages(client: PostgresClientType & ProviderClien
     initialBackoff?: number;
     maxBackoff?: number;
     maxRetries?: number;
-}, logger: ILogger): Promise<StreamMessage[]>;
+}, logger: ILogger, liveness?: JobLivenessContext): Promise<StreamMessage[]>;
 /**
  * Acknowledge messages (no-op for PostgreSQL - uses soft delete pattern).
  */

package/build/services/stream/providers/postgres/messages.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.retryMessages = exports.deadLetterMessages = exports.ackAndDelete = exports.deleteMessages = exports.acknowledgeMessages = exports.fetchMessages = exports.buildPublishSQL = exports.publishMessages = exports.getMessagePriority = void 0;
+exports.retryMessages = exports.deadLetterMessages = exports.ackAndDelete = exports.deleteMessages = exports.acknowledgeMessages = exports.fetchMessages = exports.extendReservation = exports.expireJobMessages = exports.buildPublishSQL = exports.publishMessages = exports.getMessagePriority = void 0;
 const enums_1 = require("../../../../modules/enums");
 const utils_1 = require("../../../../modules/utils");
 const stream_1 = require("../../../../types/stream");
@@ -203,12 +203,121 @@ function buildPublishSQL(tableName, streamName, isEngine, messages, options) {
     };
 }
 exports.buildPublishSQL = buildPublishSQL;
+/**
+ * Soft-delete every live stream row that belongs to a job. Called when a
+ * job is interrupted so its queued, reserved, and scheduled-retry
+ * messages are never delivered again. Uses the partial jid indexes
+ * (idx_*_streams_jid_created); runs once per interrupt.
+ */
+async function expireJobMessages(client, tableNames, jid, logger) {
+    if (!jid)
+        return 0;
+    let total = 0;
+    for (const tableName of tableNames) {
+        try {
+            const res = await client.query(`UPDATE ${tableName}
+         SET expired_at = NOW()
+         WHERE jid = $1 AND expired_at IS NULL`, [jid]);
+            total += res.rowCount ?? 0;
+        }
+        catch (error) {
+            logger.error(`postgres-stream-expire-job-error`, {
+                tableName,
+                jid,
+                error,
+            });
+            throw error;
+        }
+    }
+    return total;
+}
+exports.expireJobMessages = expireJobMessages;
+/**
+ * Refresh an owned reservation (heartbeat). Scoped to the owning
+ * consumer and to live rows: a message that was reclaimed by another
+ * consumer, acked, or expired (job interrupted) reports 0 so the
+ * stale consumer can abandon its execution.
+ */
+async function extendReservation(client, tableName, streamName, messageId, consumerName, logger) {
+    try {
+        const res = await client.query(`UPDATE ${tableName}
+       SET reserved_at = NOW()
+       WHERE stream_name = $1 AND id = $2
+         AND reserved_by = $3 AND expired_at IS NULL`, [streamName, parseInt(messageId), consumerName]);
+        return res.rowCount ?? 0;
+    }
+    catch (error) {
+        logger.error(`postgres-stream-extend-error-${streamName}`, {
+            messageId,
+            error,
+        });
+        throw error;
+    }
+}
+exports.extendReservation = extendReservation;
+/**
+ * Delivery liveness guard. Scoped to messages that are REDELIVERIES
+ * (a prior reservation lapsed) or RETRIES (retry_attempt > 0) — zombie
+ * messages of interrupted jobs only resurface through those paths, so
+ * first deliveries (the steady-state hot path) pay zero extra cost.
+ * A suspect whose job exists, is live, and has status <= 0 is expired
+ * in place and dropped from the batch. A missing job row still delivers
+ * (job-creating messages precede the row). Fails open on any error;
+ * self-disables when the jobs table is not visible (42P01).
+ */
+async function dropDeadJobMessages(client, tableName, streamName, rows, liveness, logger) {
+    const deadIds = new Set();
+    const suspects = rows.filter((row) => row.jid && (row.redelivered || row.retry_attempt > 0));
+    if (suspects.length === 0) {
+        return deadIds;
+    }
+    try {
+        const res = await client.query(`UPDATE ${tableName} t
+       SET expired_at = NOW()
+       FROM ${liveness.jobsTable} j
+       WHERE t.stream_name = $1
+         AND t.id = ANY($2::bigint[])
+         AND j.key = $3 || t.jid
+         AND j.is_live
+         AND j.status <= 0
+       RETURNING t.id`, [streamName, suspects.map((row) => row.id), liveness.keyPrefix]);
+        for (const row of res.rows) {
+            deadIds.add(row.id.toString());
+        }
+        if (deadIds.size > 0) {
+            logger.warn(`postgres-stream-zombie-dropped-${streamName}`, {
+                count: deadIds.size,
+                jids: [
+                    ...new Set(suspects
+                        .filter((row) => deadIds.has(row.id.toString()))
+                        .map((row) => row.jid)),
+                ],
+            });
+        }
+    }
+    catch (error) {
+        if (error?.code === '42P01') {
+            //jobs table is not visible from this connection; the guard cannot
+            //run here — interrupt-time purging (expireJobMessages) still applies
+            liveness.enabled = false;
+            logger.info('postgres-stream-liveness-guard-disabled', {
+                jobsTable: liveness.jobsTable,
+            });
+        }
+        else {
+            logger.error(`postgres-stream-liveness-guard-error-${streamName}`, {
+                error,
+            });
+        }
+    }
+    return deadIds;
+}
 /**
  * Fetch messages from the stream with optional exponential backoff.
  * Uses SKIP LOCKED for high-concurrency consumption.
  * No group_name filter needed - the table itself determines engine vs worker.
  */
-async function fetchMessages(client, tableName, streamName, isEngine, consumerName, options = {}, logger) {
+async function fetchMessages(client, tableName, streamName, isEngine, consumerName, options = {}, logger, liveness) {
     const enableBackoff = options?.enableBackoff ?? false;
     const initialBackoff = options?.initialBackoff ?? 100;
     const maxBackoff = options?.maxBackoff ?? 3000;
@@ -217,10 +326,11 @@ async function fetchMessages(client, tableName, streamName, isEngine, consumerNa
     let retries = 0;
     // Include workflow_name in RETURNING for worker streams. Columns are
     // qualified with the update target's alias because the claim UPDATE
-    // joins a CTE that also exposes an id column.
+    // joins a CTE that also exposes an id column. Worker streams also
+    // return jid and the pre-claim redelivery flag for the liveness guard.
     const returningClause = isEngine
         ? 't.id, t.message, t.max_retry_attempts, t.backoff_coefficient, t.maximum_interval_seconds, t.retry_attempt'
-        : 't.id, t.message, t.workflow_name, t.max_retry_attempts, t.backoff_coefficient, t.maximum_interval_seconds, t.retry_attempt';
+        : 't.id, t.message, t.workflow_name, t.max_retry_attempts, t.backoff_coefficient, t.maximum_interval_seconds, t.retry_attempt, t.jid, candidates.redelivered';
     try {
         while (retries < maxRetries) {
             retries++;
@@ -232,8 +342,11 @@ async function fetchMessages(client, tableName, streamName, isEngine, consumerNa
             // reserves MORE rows than LIMIT. The UPDATE repeats stream_name so
             // the planner prunes to a single hash partition and joins on the
             // (stream_name, id) primary key.
+            // candidates exposes the PRE-claim reservation state: a non-null
+            // reserved_at at claim time means a prior reservation lapsed
+            // (redelivery) — the trigger condition for the liveness guard.
             const res = await client.query(`WITH candidates AS MATERIALIZED (
-           SELECT id FROM ${tableName}
+           SELECT id, (reserved_at IS NOT NULL) AS redelivered FROM ${tableName}
            WHERE stream_name = $1
              AND (reserved_at IS NULL OR reserved_at < NOW() - INTERVAL '${reservationTimeout} seconds')
              AND expired_at IS NULL
@@ -247,7 +360,14 @@ async function fetchMessages(client, tableName, streamName, isEngine, consumerNa
          FROM candidates
          WHERE t.stream_name = $1 AND t.id = candidates.id
          RETURNING ${returningClause}`, [streamName, batchSize, consumerName]);
-            const messages = res.rows.map((row) => {
+            let rows = res.rows;
+            if (!isEngine && liveness?.enabled && rows.length > 0) {
+                const deadIds = await dropDeadJobMessages(client, tableName, streamName, rows, liveness, logger);
+                if (deadIds.size > 0) {
+                    rows = rows.filter((row) => !deadIds.has(row.id.toString()));
+                }
+            }
+            const messages = rows.map((row) => {
                 const data = (0, utils_1.parseStreamMessage)(row.message);
                 const hasDefaultRetryPolicy = (row.max_retry_attempts === 3 || row.max_retry_attempts === 5) &&
                     parseFloat(row.backoff_coefficient) === 10 &&

package/build/services/stream/providers/postgres/postgres.d.ts

@@ -31,6 +31,7 @@ declare class PostgresStreamService extends StreamService<PostgresClientType & P
      */
     securedMode: boolean;
     private scoutManager;
+    private liveness;
     private notificationManager;
     constructor(streamClient: PostgresClientType & ProviderClient, storeClient: ProviderClient, config?: StreamConfig);
     init(namespace: string, appId: string, logger: ILogger): Promise<void>;
@@ -91,6 +92,20 @@ declare class PostgresStreamService extends StreamService<PostgresClientType & P
     private setupNotificationConsumer;
     stopNotificationConsumer(streamName: string, groupName: string): Promise<void>;
     private fetchMessages;
+    /**
+     * Refreshes an owned reservation (heartbeat). Called by the consumer
+     * while an activity callback is still running so the message stays
+     * leased past the base reservation window. Returns 0 when the lease
+     * is no longer this consumer's to hold (reclaimed, acked, or expired).
+     */
+    extendReservation(streamName: string, messageId: string, consumerName: string): Promise<number>;
+    /**
+     * Soft-deletes every live stream row that belongs to a job, across the
+     * worker and engine stream tables. Called by the engine when a job is
+     * interrupted or scrubbed so its queued, reserved, and scheduled-retry
+     * messages are never delivered again.
+     */
+    expireJobMessages(jid: string): Promise<number>;
     ackAndDelete(streamName: string, groupName: string, messageIds: string[]): Promise<number>;
     deadLetterMessages(streamName: string, groupName: string, messageIds: string[]): Promise<number>;
     acknowledgeMessages(streamName: string, groupName: string, messageIds: string[], options?: StringAnyType): Promise<number>;
@@ -118,6 +133,7 @@ declare class PostgresStreamService extends StreamService<PostgresClientType & P
         exactLimit?: boolean;
     }): Promise<number>;
     getProviderSpecificFeatures(): {
+        supportsReservationExtension: boolean;
         supportsBatching: boolean;
         supportsDeadLetterQueue: boolean;
         supportsOrdering: boolean;

package/build/services/stream/providers/postgres/postgres.js

@@ -62,6 +62,14 @@ class PostgresStreamService extends index_1.StreamService {
         // and never need direct table access.
         if (!this.securedMode) {
             await (0, kvtables_1.deploySchema)(this.streamClient, this.appId, this.logger);
+            //delivery liveness guard: drops redelivered/retried worker messages
+            //whose job is dead. Secured workers cannot read the jobs table —
+            //interrupt-time purging (expireJobMessages) covers them instead.
+            this.liveness = {
+                jobsTable: `${this.safeName(this.appId)}.jobs`,
+                keyPrefix: this.mintKey(key_1.KeyType.JOB_STATE, { jobId: '' }),
+                enabled: true,
+            };
         }
         // Initialize scout manager (skipped in secured mode — roles table inaccessible)
         this.scoutManager = this.securedMode ? null : new scout_1.ScoutManager(this.streamClient, this.appId, this.getEngineTableName.bind(this), this.mintKey.bind(this), this.logger);
@@ -285,7 +293,32 @@ class PostgresStreamService extends index_1.StreamService {
             return Secured.fetchMessagesSecured(this.streamClient, this.safeName(this.appId), target.streamName, consumerName, options || {}, this.logger);
         }
         const target = this.resolveStreamTarget(streamName);
-        return Messages.fetchMessages(this.streamClient, target.tableName, target.streamName, target.isEngine, consumerName, options || {}, this.logger);
+        return Messages.fetchMessages(this.streamClient, target.tableName, target.streamName, target.isEngine, consumerName, options || {}, this.logger, this.liveness);
+    }
+    /**
+     * Refreshes an owned reservation (heartbeat). Called by the consumer
+     * while an activity callback is still running so the message stays
+     * leased past the base reservation window. Returns 0 when the lease
+     * is no longer this consumer's to hold (reclaimed, acked, or expired).
+     */
+    async extendReservation(streamName, messageId, consumerName) {
+        if (this.securedMode) {
+            return 0;
+        }
+        const target = this.resolveStreamTarget(streamName);
+        return Messages.extendReservation(this.streamClient, target.tableName, target.streamName, messageId, consumerName, this.logger);
+    }
+    /**
+     * Soft-deletes every live stream row that belongs to a job, across the
+     * worker and engine stream tables. Called by the engine when a job is
+     * interrupted or scrubbed so its queued, reserved, and scheduled-retry
+     * messages are never delivered again.
+     */
+    async expireJobMessages(jid) {
+        if (this.securedMode) {
+            return 0;
+        }
+        return Messages.expireJobMessages(this.streamClient, [this.getEngineTableName(), this.getWorkerTableName()], jid, this.logger);
     }
     async ackAndDelete(streamName, groupName, messageIds) {
         if (this.securedMode) {
@@ -355,7 +388,12 @@ class PostgresStreamService extends index_1.StreamService {
         return Stats.trimStream(this.streamClient, target.tableName, target.streamName, options, this.logger);
     }
     getProviderSpecificFeatures() {
-        return Stats.getProviderSpecificFeatures(this.config);
+        return {
+            ...Stats.getProviderSpecificFeatures(this.config),
+            //heartbeat lease extension uses direct table access; secured
+            //workers use stored procedures and keep hard-deadline leases
+            supportsReservationExtension: !this.securedMode,
+        };
     }
     async cleanup() {
         if (this.scoutManager) {

package/build/services/task/index.js

@@ -73,8 +73,13 @@ class TaskService {
                 expire,
             };
             //called standalone (no transaction) so the single CTE query can
-            //atomically detect and return pending signal data on collision
-            const result = await this.store.setHookSignal(hook);
+            //atomically detect and return pending signal data on collision.
+            //the redelivery routing lets the store republish a consumed
+            //pending signal durably, in the same transaction that consumes it
+            const result = await this.store.setHookSignal(hook, undefined, {
+                aid: hookRule.to,
+                topic,
+            });
             if (result.pendingData) {
                 this.logger.warn('task-signal-race-pending-consumed', {
                     topic,

package/build/types/hmsh_escalations.d.ts

@@ -105,6 +105,19 @@ export type ResolveEscalationResult = {
     ok: false;
     reason: 'not-found' | 'already-resolved' | 'already-cancelled' | 'already-expired';
 };
+/**
+ * A pre-built wake publish, executed INSIDE the resolve transaction so the
+ * awaiting workflow's wake commits atomically with `status='resolved'`. The
+ * SQL is a stream INSERT produced by the stream provider; `forSignalKey`
+ * pins the command to the row it was built for — the store executes it only
+ * when the locked row's `signal_key` matches (a mismatched row falls back
+ * to post-commit delivery).
+ */
+export interface EscalationWakeCommand {
+    forSignalKey: string;
+    sql: string;
+    params: unknown[];
+}
 export type ReleaseEscalationResult = {
     ok: true;
     entry: EscalationEntry;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.25.2",
+  "version": "0.25.5",
   "description": "Durable Workflow",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",