@hotmeshio/hotmesh 0.25.1 → 0.25.4

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.25.1",
+    "version": "0.25.4",
     "description": "Durable Workflow",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/activities/hook.js

@@ -496,7 +496,16 @@ class Hook extends activity_1.Activity {
     }
     async getHookRule(topic) {
         const rules = await this.store.getHookRules();
-        return rules?.[topic]?.[0];
+        const forTopic = rules?.[topic];
+        if (!forTopic?.length)
+            return undefined;
+        // A topic may carry one rule per waiter activity — the durable factory
+        // registers `waiter` and `signaler_waiter` on the same wait topic. Select
+        // the rule that targets THIS activity: the positional first rule addresses
+        // a sibling branch for every activity but its own, so its expected-pipe
+        // resolves against output that does not exist in this flow (which is how
+        // execHook-context waits were writing signal_key-less escalation rows).
+        return (forTopic.find((rule) => rule.to === this.metadata.aid) ?? forTopic[0]);
     }
     /**
      * Register the time hook (sleep) inside the Leg1 transaction.
@@ -654,10 +663,20 @@ class Hook extends activity_1.Activity {
         const store = this.store;
         if (typeof store.expireEscalationBySignalKey !== 'function')
             return;
+        // Hydration here is THROWAWAY. getState() REPLACES this.context with
+        // restored job state, which does not carry the per-message dimensional
+        // address (dad) — the stream message is its only carrier for this leg.
+        // processEvent must receive the pristine dispatch state (verifyReentry
+        // hydrates for itself, reading dad from context.metadata), so snapshot
+        // everything getState mutates — the context reference, metadata.dad,
+        // and leg — and restore them on every exit path. Without this, a wait
+        // executing below the root (any dimension) crashes Leg2 notarization.
+        const dispatchContext = this.context;
+        const dispatchDad = this.metadata.dad;
+        const dispatchLeg = this.leg;
         try {
-            // The TIMEHOOK dispatch context carries metadata only — hydrate job
-            // state so the signal-key pipe expression resolves to the same value
-            // Leg1 stored on the row.
+            // Hydrate job state so the signal-key pipe expression resolves to the
+            // same value Leg1 stored on the row.
             this.setLeg(2);
             await this.getState();
             const appId = this.engine.appId;
@@ -689,6 +708,11 @@ class Hook extends activity_1.Activity {
                 error: e.message,
             });
         }
+        finally {
+            this.context = dispatchContext;
+            this.metadata.dad = dispatchDad;
+            this.setLeg(dispatchLeg);
+        }
     }
 }
 exports.Hook = Hook;

package/build/services/durable/workflow/condition.d.ts

@@ -72,6 +72,15 @@ import { ConditionQueueConfig } from '../../../types/hmsh_escalations';
  * await client.escalations.resolve({ id: item.id, resolverPayload: { approved: true } });
  * ```
  *
+ * ## Placement: call escalation-bearing waits from main workflow code
+ *
+ * The resolve/signal delivery pipeline routes to the main flow's waiter.
+ * Inside a hook function (`execHook`), an escalation-bearing wait writes its
+ * row and honors `timeout` (the row expires and the hook resumes with
+ * `false`), while resolution delivery targets the main flow — so structure
+ * SLA-gated human waits in the workflow body and let hook functions report
+ * back via `signal()`.
+ *
  * ## Fan-in: wait for multiple signals in parallel
  *
  * ```typescript

package/build/services/durable/workflow/condition.js

@@ -77,6 +77,15 @@ const didRun_1 = require("./didRun");
  * await client.escalations.resolve({ id: item.id, resolverPayload: { approved: true } });
  * ```
  *
+ * ## Placement: call escalation-bearing waits from main workflow code
+ *
+ * The resolve/signal delivery pipeline routes to the main flow's waiter.
+ * Inside a hook function (`execHook`), an escalation-bearing wait writes its
+ * row and honors `timeout` (the row expires and the hook resumes with
+ * `false`), while resolution delivery targets the main flow — so structure
+ * SLA-gated human waits in the workflow body and let hook functions report
+ * back via `signal()`.
+ *
  * ## Fan-in: wait for multiple signals in parallel
  *
  * ```typescript

package/build/services/engine/completion.js

@@ -68,6 +68,7 @@ function resolveError(metadata) {
 exports.resolveError = resolveError;
 async function interrupt(instance, topic, jobId, options = {}) {
     await instance.store.interrupt(topic, jobId, options);
+    await expireStreamMessages(instance, jobId);
     const context = (await instance.getState(topic, jobId));
     const completionOpts = {
         interrupt: options.descend,
@@ -77,9 +78,34 @@ async function interrupt(instance, topic, jobId, options = {}) {
 }
 exports.interrupt = interrupt;
 async function scrub(instance, jobId) {
+    await expireStreamMessages(instance, jobId);
     await instance.store.scrub(jobId);
 }
 exports.scrub = scrub;
+/**
+ * Soft-deletes the job's stream messages (queued, reserved, and
+ * scheduled retries) so no message belonging to a dead job is ever
+ * delivered again. Interrupting the job wins or throws before this
+ * runs; failures here are logged and absorbed because the delivery
+ * liveness guard drops any surviving message at claim time.
+ */
+async function expireStreamMessages(instance, jobId) {
+    try {
+        const expired = await instance.stream.expireJobMessages?.(jobId);
+        if (expired) {
+            instance.logger.info('engine-job-stream-messages-expired', {
+                jobId,
+                expired,
+            });
+        }
+    }
+    catch (error) {
+        instance.logger.error('engine-job-stream-expire-error', {
+            jobId,
+            error,
+        });
+    }
+}
 /**
  * Orchestrates all post-completion work for a finished job:
  * notify parent, publish to subscribers, schedule cleanup.

package/build/services/router/consumption/index.d.ts

@@ -30,6 +30,7 @@ export declare class ConsumptionManager<S extends StreamService<ProviderClient,
     private duressManager?;
     private onDuressChange?;
     private messagesSinceLastEval;
+    private canExtendReservations;
     private adaptiveReservationTimeout;
     private adaptiveBatchSize;
     private lastDepthCheckAt;
@@ -52,7 +53,12 @@ export declare class ConsumptionManager<S extends StreamService<ProviderClient,
     consumeMessages(stream: string, group: string, consumer: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
     private consumeWithNotifications;
     private consumeWithPolling;
-    consumeOne(stream: string, group: string, id: string, input: StreamData, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
+    /**
+     * Whether reservation heartbeats are available: the provider must
+     * implement extendReservation and advertise the capability.
+     */
+    private supportsHeartbeat;
+    consumeOne(stream: string, group: string, id: string, input: StreamData, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>, consumer?: string): Promise<void>;
     execStreamLeg(input: StreamData, stream: string, id: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<StreamDataResponse>;
     ackAndDelete(stream: string, group: string, id: string): Promise<void>;
     ackAndDeleteBatch(stream: string, group: string, ids: string[]): Promise<void>;

package/build/services/router/consumption/index.js

@@ -195,7 +195,7 @@ class ConsumptionManager {
                     if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                         return;
                     }
-                    return this.consumeOne(stream, group, message.id, message.data, callback);
+                    return this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                 });
                 // Process all messages in parallel
                 await Promise.allSettled(processingPromises);
@@ -212,7 +212,7 @@ class ConsumptionManager {
                     if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                         return;
                     }
-                    await this.consumeOne(stream, group, message.id, message.data, callback);
+                    await this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                 }
             }
             // Check for pending messages if supported and enough time has passed
@@ -231,7 +231,7 @@ class ConsumptionManager {
                         if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                             return;
                         }
-                        return this.consumeOne(stream, group, message.id, message.data, callback);
+                        return this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                     });
                     await Promise.allSettled(reclaimPromises);
                 }
@@ -240,7 +240,7 @@ class ConsumptionManager {
                         if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                             return;
                         }
-                        await this.consumeOne(stream, group, message.id, message.data, callback);
+                        await this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                     }
                 }
             }
@@ -349,7 +349,7 @@ class ConsumptionManager {
                             if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                                 return;
                             }
-                            return this.consumeOne(stream, group, message.id, message.data, callback);
+                            return this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                         });
                         // Process all messages in parallel
                         await Promise.allSettled(processingPromises);
@@ -366,7 +366,7 @@ class ConsumptionManager {
                             if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                                 return;
                             }
-                            await this.consumeOne(stream, group, message.id, message.data, callback);
+                            await this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                         }
                     }
                     // Check for pending messages if supported and enough time has passed
@@ -385,7 +385,7 @@ class ConsumptionManager {
                                 if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                                     return;
                                 }
-                                return this.consumeOne(stream, group, message.id, message.data, callback);
+                                return this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                             });
                             await Promise.allSettled(reclaimPromises);
                         }
@@ -394,7 +394,7 @@ class ConsumptionManager {
                                 if (this.lifecycleManager.isStopped(group, consumer, stream)) {
                                     return;
                                 }
-                                await this.consumeOne(stream, group, message.id, message.data, callback);
+                                await this.consumeOne(stream, group, message.id, message.data, callback, consumer);
                             }
                         }
                     }
@@ -434,7 +434,20 @@ class ConsumptionManager {
         };
         consume.call(this);
     }
-    async consumeOne(stream, group, id, input, callback) {
+    /**
+     * Whether reservation heartbeats are available: the provider must
+     * implement extendReservation and advertise the capability.
+     */
+    supportsHeartbeat() {
+        if (this.canExtendReservations === undefined) {
+            this.canExtendReservations =
+                typeof this.stream.extendReservation === 'function' &&
+                    this.stream.getProviderSpecificFeatures()
+                        .supportsReservationExtension === true;
+        }
+        return this.canExtendReservations;
+    }
+    async consumeOne(stream, group, id, input, callback, consumer) {
         this.logger.debug(`stream-read-one`, { group, stream, id });
         // Poison message circuit breaker. This is a SAFETY NET that sits above
         // the normal retry mechanism (ErrorHandler.handleRetry / shouldRetry).
@@ -498,20 +511,67 @@ class ConsumptionManager {
             }
             return;
         }
-        // Lease deadline: the full configured reservation timeout (N).
-        // The reclaim interval is N+5s, so the deadline always fires
-        // before a reclaimant can pick up the message. This preserves
-        // the user's contract — if they set 30s, the function gets 30s.
-        const deadlineMs = this.adaptiveReservationTimeout * 1000;
+        // Lease strategy. With heartbeat support (provider capability +
+        // consumer identity), the reservation is refreshed at half the base
+        // window while the callback runs, so an activity that outlives the
+        // window stays leased and executes exactly once; the deadline then
+        // acts as a runaway cap (HMSH_RESERVATION_TIMEOUT_MAX_S). A crashed
+        // consumer stops heartbeating and its message is rescued within one
+        // window through the normal claim path. Providers without extension
+        // keep the hard deadline at the reservation timeout (N), with
+        // reclaim at N+5s.
+        const canHeartbeat = Boolean(consumer) && this.supportsHeartbeat();
+        const deadlineS = canHeartbeat
+            ? config_1.HMSH_RESERVATION_TIMEOUT_MAX_S
+            : this.adaptiveReservationTimeout;
+        const deadlineMs = deadlineS * 1000;
         let output;
         const telemetry = new telemetry_1.RouterTelemetry(this.appId);
         const processingStart = Date.now();
         try {
             telemetry.startStreamSpan(input, this.role);
             let deadlineTimer;
+            let rejectLease;
             const deadlinePromise = new Promise((_, reject) => {
-                deadlineTimer = setTimeout(() => reject(new errors_1.LeaseExpiredError(deadlineMs, this.adaptiveReservationTimeout)), deadlineMs);
+                rejectLease = reject;
+                deadlineTimer = setTimeout(() => reject(new errors_1.LeaseExpiredError(deadlineMs, deadlineS)), deadlineMs);
             });
+            // Heartbeat at half the BASE window (adaptive scaling only grows
+            // the claim window, so the base cadence always outpaces reclaim
+            // eligibility). A beat that reports 0 means the lease is gone —
+            // reclaimed by another consumer, acked, or the job was
+            // interrupted — so this execution abandons without acking. A beat
+            // that errors is transient and retries on the next tick. Fast
+            // callbacks settle before the first beat ever fires.
+            let heartbeatTimer;
+            if (canHeartbeat) {
+                const intervalMs = Math.max(1000, Math.floor((config_1.HMSH_RESERVATION_TIMEOUT_S * 1000) / 2));
+                heartbeatTimer = setInterval(() => {
+                    this.stream
+                        .extendReservation(stream, id, consumer)
+                        .then((extended) => {
+                        if (extended === 0) {
+                            this.logger.warn('stream-lease-lost', {
+                                group,
+                                stream,
+                                id,
+                                consumer,
+                                topic: input.metadata?.topic,
+                                jobId: input.metadata?.jid,
+                            });
+                            rejectLease(new errors_1.LeaseExpiredError(deadlineMs, deadlineS));
+                        }
+                    })
+                        .catch((error) => {
+                        this.logger.warn('stream-lease-extend-error', {
+                            group,
+                            stream,
+                            id,
+                            error,
+                        });
+                    });
+                }, intervalMs);
+            }
             try {
                 output = await Promise.race([
                     this.execStreamLeg(input, stream, id, callback.bind(this)),
@@ -520,6 +580,9 @@ class ConsumptionManager {
             }
             finally {
                 clearTimeout(deadlineTimer);
+                if (heartbeatTimer) {
+                    clearInterval(heartbeatTimer);
+                }
             }
             telemetry.setStreamErrorFromOutput(output);
             this.errorCount = 0;
@@ -536,7 +599,7 @@ class ConsumptionManager {
                     stream,
                     id,
                     deadlineMs,
-                    reservationTimeoutS: this.adaptiveReservationTimeout,
+                    reservationTimeoutS: deadlineS,
                     topic: input.metadata?.topic,
                     activityId: input.metadata?.aid,
                     jobId: input.metadata?.jid,

package/build/services/stream/index.d.ts

@@ -66,10 +66,13 @@ export declare abstract class StreamService<ClientProvider extends ProviderClien
         supportsRetry: boolean;
         supportsNotifications?: boolean;
         supportsParallelProcessing?: boolean;
+        supportsReservationExtension?: boolean;
         maxMessageSize: number;
         maxBatchSize: number;
     };
     deadLetterMessages?(streamName: string, groupName: string, messageIds: string[]): Promise<number>;
+    expireJobMessages?(jid: string): Promise<number>;
+    extendReservation?(streamName: string, messageId: string, consumerName: string): Promise<number>;
     stopNotificationConsumer?(streamName: string, groupName: string): Promise<void>;
     cleanup?(): Promise<void>;
 }

package/build/services/stream/providers/postgres/messages.d.ts

@@ -19,6 +19,31 @@ export declare function buildPublishSQL(tableName: string, streamName: string, i
     sql: string;
     params: any[];
 };
+/**
+ * Job-liveness context for the delivery guard. `keyPrefix` is the minted
+ * job-key prefix (`hmsh:<app>:j:`) so `keyPrefix || jid` addresses the
+ * jobs row. `enabled` is mutated to false (self-disable) when the jobs
+ * table is not visible from the stream connection.
+ */
+export interface JobLivenessContext {
+    jobsTable: string;
+    keyPrefix: string;
+    enabled: boolean;
+}
+/**
+ * Soft-delete every live stream row that belongs to a job. Called when a
+ * job is interrupted so its queued, reserved, and scheduled-retry
+ * messages are never delivered again. Uses the partial jid indexes
+ * (idx_*_streams_jid_created); runs once per interrupt.
+ */
+export declare function expireJobMessages(client: PostgresClientType & ProviderClient, tableNames: string[], jid: string, logger: ILogger): Promise<number>;
+/**
+ * Refresh an owned reservation (heartbeat). Scoped to the owning
+ * consumer and to live rows: a message that was reclaimed by another
+ * consumer, acked, or expired (job interrupted) reports 0 so the
+ * stale consumer can abandon its execution.
+ */
+export declare function extendReservation(client: PostgresClientType & ProviderClient, tableName: string, streamName: string, messageId: string, consumerName: string, logger: ILogger): Promise<number>;
 /**
  * Fetch messages from the stream with optional exponential backoff.
  * Uses SKIP LOCKED for high-concurrency consumption.
@@ -33,7 +58,7 @@ export declare function fetchMessages(client: PostgresClientType & ProviderClien
     initialBackoff?: number;
     maxBackoff?: number;
     maxRetries?: number;
-}, logger: ILogger): Promise<StreamMessage[]>;
+}, logger: ILogger, liveness?: JobLivenessContext): Promise<StreamMessage[]>;
 /**
  * Acknowledge messages (no-op for PostgreSQL - uses soft delete pattern).
  */

package/build/services/stream/providers/postgres/messages.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.retryMessages = exports.deadLetterMessages = exports.ackAndDelete = exports.deleteMessages = exports.acknowledgeMessages = exports.fetchMessages = exports.buildPublishSQL = exports.publishMessages = exports.getMessagePriority = void 0;
+exports.retryMessages = exports.deadLetterMessages = exports.ackAndDelete = exports.deleteMessages = exports.acknowledgeMessages = exports.fetchMessages = exports.extendReservation = exports.expireJobMessages = exports.buildPublishSQL = exports.publishMessages = exports.getMessagePriority = void 0;
 const enums_1 = require("../../../../modules/enums");
 const utils_1 = require("../../../../modules/utils");
 const stream_1 = require("../../../../types/stream");
@@ -203,12 +203,121 @@ function buildPublishSQL(tableName, streamName, isEngine, messages, options) {
     };
 }
 exports.buildPublishSQL = buildPublishSQL;
+/**
+ * Soft-delete every live stream row that belongs to a job. Called when a
+ * job is interrupted so its queued, reserved, and scheduled-retry
+ * messages are never delivered again. Uses the partial jid indexes
+ * (idx_*_streams_jid_created); runs once per interrupt.
+ */
+async function expireJobMessages(client, tableNames, jid, logger) {
+    if (!jid)
+        return 0;
+    let total = 0;
+    for (const tableName of tableNames) {
+        try {
+            const res = await client.query(`UPDATE ${tableName}
+         SET expired_at = NOW()
+         WHERE jid = $1 AND expired_at IS NULL`, [jid]);
+            total += res.rowCount ?? 0;
+        }
+        catch (error) {
+            logger.error(`postgres-stream-expire-job-error`, {
+                tableName,
+                jid,
+                error,
+            });
+            throw error;
+        }
+    }
+    return total;
+}
+exports.expireJobMessages = expireJobMessages;
+/**
+ * Refresh an owned reservation (heartbeat). Scoped to the owning
+ * consumer and to live rows: a message that was reclaimed by another
+ * consumer, acked, or expired (job interrupted) reports 0 so the
+ * stale consumer can abandon its execution.
+ */
+async function extendReservation(client, tableName, streamName, messageId, consumerName, logger) {
+    try {
+        const res = await client.query(`UPDATE ${tableName}
+       SET reserved_at = NOW()
+       WHERE stream_name = $1 AND id = $2
+         AND reserved_by = $3 AND expired_at IS NULL`, [streamName, parseInt(messageId), consumerName]);
+        return res.rowCount ?? 0;
+    }
+    catch (error) {
+        logger.error(`postgres-stream-extend-error-${streamName}`, {
+            messageId,
+            error,
+        });
+        throw error;
+    }
+}
+exports.extendReservation = extendReservation;
+/**
+ * Delivery liveness guard. Scoped to messages that are REDELIVERIES
+ * (a prior reservation lapsed) or RETRIES (retry_attempt > 0) — zombie
+ * messages of interrupted jobs only resurface through those paths, so
+ * first deliveries (the steady-state hot path) pay zero extra cost.
+ * A suspect whose job exists, is live, and has status <= 0 is expired
+ * in place and dropped from the batch. A missing job row still delivers
+ * (job-creating messages precede the row). Fails open on any error;
+ * self-disables when the jobs table is not visible (42P01).
+ */
+async function dropDeadJobMessages(client, tableName, streamName, rows, liveness, logger) {
+    const deadIds = new Set();
+    const suspects = rows.filter((row) => row.jid && (row.redelivered || row.retry_attempt > 0));
+    if (suspects.length === 0) {
+        return deadIds;
+    }
+    try {
+        const res = await client.query(`UPDATE ${tableName} t
+       SET expired_at = NOW()
+       FROM ${liveness.jobsTable} j
+       WHERE t.stream_name = $1
+         AND t.id = ANY($2::bigint[])
+         AND j.key = $3 || t.jid
+         AND j.is_live
+         AND j.status <= 0
+       RETURNING t.id`, [streamName, suspects.map((row) => row.id), liveness.keyPrefix]);
+        for (const row of res.rows) {
+            deadIds.add(row.id.toString());
+        }
+        if (deadIds.size > 0) {
+            logger.warn(`postgres-stream-zombie-dropped-${streamName}`, {
+                count: deadIds.size,
+                jids: [
+                    ...new Set(suspects
+                        .filter((row) => deadIds.has(row.id.toString()))
+                        .map((row) => row.jid)),
+                ],
+            });
+        }
+    }
+    catch (error) {
+        if (error?.code === '42P01') {
+            //jobs table is not visible from this connection; the guard cannot
+            //run here — interrupt-time purging (expireJobMessages) still applies
+            liveness.enabled = false;
+            logger.info('postgres-stream-liveness-guard-disabled', {
+                jobsTable: liveness.jobsTable,
+            });
+        }
+        else {
+            logger.error(`postgres-stream-liveness-guard-error-${streamName}`, {
+                error,
+            });
+        }
+    }
+    return deadIds;
+}
 /**
  * Fetch messages from the stream with optional exponential backoff.
  * Uses SKIP LOCKED for high-concurrency consumption.
  * No group_name filter needed - the table itself determines engine vs worker.
  */
-async function fetchMessages(client, tableName, streamName, isEngine, consumerName, options = {}, logger) {
+async function fetchMessages(client, tableName, streamName, isEngine, consumerName, options = {}, logger, liveness) {
     const enableBackoff = options?.enableBackoff ?? false;
     const initialBackoff = options?.initialBackoff ?? 100;
     const maxBackoff = options?.maxBackoff ?? 3000;
@@ -217,10 +326,11 @@ async function fetchMessages(client, tableName, streamName, isEngine, consumerNa
     let retries = 0;
     // Include workflow_name in RETURNING for worker streams. Columns are
     // qualified with the update target's alias because the claim UPDATE
-    // joins a CTE that also exposes an id column.
+    // joins a CTE that also exposes an id column. Worker streams also
+    // return jid and the pre-claim redelivery flag for the liveness guard.
     const returningClause = isEngine
         ? 't.id, t.message, t.max_retry_attempts, t.backoff_coefficient, t.maximum_interval_seconds, t.retry_attempt'
-        : 't.id, t.message, t.workflow_name, t.max_retry_attempts, t.backoff_coefficient, t.maximum_interval_seconds, t.retry_attempt';
+        : 't.id, t.message, t.workflow_name, t.max_retry_attempts, t.backoff_coefficient, t.maximum_interval_seconds, t.retry_attempt, t.jid, candidates.redelivered';
     try {
         while (retries < maxRetries) {
             retries++;
@@ -232,8 +342,11 @@ async function fetchMessages(client, tableName, streamName, isEngine, consumerNa
             // reserves MORE rows than LIMIT. The UPDATE repeats stream_name so
             // the planner prunes to a single hash partition and joins on the
             // (stream_name, id) primary key.
+            // candidates exposes the PRE-claim reservation state: a non-null
+            // reserved_at at claim time means a prior reservation lapsed
+            // (redelivery) — the trigger condition for the liveness guard.
             const res = await client.query(`WITH candidates AS MATERIALIZED (
-           SELECT id FROM ${tableName}
+           SELECT id, (reserved_at IS NOT NULL) AS redelivered FROM ${tableName}
            WHERE stream_name = $1
              AND (reserved_at IS NULL OR reserved_at < NOW() - INTERVAL '${reservationTimeout} seconds')
              AND expired_at IS NULL
@@ -247,7 +360,14 @@ async function fetchMessages(client, tableName, streamName, isEngine, consumerNa
          FROM candidates
          WHERE t.stream_name = $1 AND t.id = candidates.id
          RETURNING ${returningClause}`, [streamName, batchSize, consumerName]);
-            const messages = res.rows.map((row) => {
+            let rows = res.rows;
+            if (!isEngine && liveness?.enabled && rows.length > 0) {
+                const deadIds = await dropDeadJobMessages(client, tableName, streamName, rows, liveness, logger);
+                if (deadIds.size > 0) {
+                    rows = rows.filter((row) => !deadIds.has(row.id.toString()));
+                }
+            }
+            const messages = rows.map((row) => {
                 const data = (0, utils_1.parseStreamMessage)(row.message);
                 const hasDefaultRetryPolicy = (row.max_retry_attempts === 3 || row.max_retry_attempts === 5) &&
                     parseFloat(row.backoff_coefficient) === 10 &&

package/build/services/stream/providers/postgres/postgres.d.ts

@@ -31,6 +31,7 @@ declare class PostgresStreamService extends StreamService<PostgresClientType & P
      */
     securedMode: boolean;
     private scoutManager;
+    private liveness;
     private notificationManager;
     constructor(streamClient: PostgresClientType & ProviderClient, storeClient: ProviderClient, config?: StreamConfig);
     init(namespace: string, appId: string, logger: ILogger): Promise<void>;
@@ -91,6 +92,20 @@ declare class PostgresStreamService extends StreamService<PostgresClientType & P
     private setupNotificationConsumer;
     stopNotificationConsumer(streamName: string, groupName: string): Promise<void>;
     private fetchMessages;
+    /**
+     * Refreshes an owned reservation (heartbeat). Called by the consumer
+     * while an activity callback is still running so the message stays
+     * leased past the base reservation window. Returns 0 when the lease
+     * is no longer this consumer's to hold (reclaimed, acked, or expired).
+     */
+    extendReservation(streamName: string, messageId: string, consumerName: string): Promise<number>;
+    /**
+     * Soft-deletes every live stream row that belongs to a job, across the
+     * worker and engine stream tables. Called by the engine when a job is
+     * interrupted or scrubbed so its queued, reserved, and scheduled-retry
+     * messages are never delivered again.
+     */
+    expireJobMessages(jid: string): Promise<number>;
     ackAndDelete(streamName: string, groupName: string, messageIds: string[]): Promise<number>;
     deadLetterMessages(streamName: string, groupName: string, messageIds: string[]): Promise<number>;
     acknowledgeMessages(streamName: string, groupName: string, messageIds: string[], options?: StringAnyType): Promise<number>;
@@ -118,6 +133,7 @@ declare class PostgresStreamService extends StreamService<PostgresClientType & P
         exactLimit?: boolean;
     }): Promise<number>;
     getProviderSpecificFeatures(): {
+        supportsReservationExtension: boolean;
         supportsBatching: boolean;
         supportsDeadLetterQueue: boolean;
         supportsOrdering: boolean;

package/build/services/stream/providers/postgres/postgres.js

@@ -62,6 +62,14 @@ class PostgresStreamService extends index_1.StreamService {
         // and never need direct table access.
         if (!this.securedMode) {
             await (0, kvtables_1.deploySchema)(this.streamClient, this.appId, this.logger);
+            //delivery liveness guard: drops redelivered/retried worker messages
+            //whose job is dead. Secured workers cannot read the jobs table —
+            //interrupt-time purging (expireJobMessages) covers them instead.
+            this.liveness = {
+                jobsTable: `${this.safeName(this.appId)}.jobs`,
+                keyPrefix: this.mintKey(key_1.KeyType.JOB_STATE, { jobId: '' }),
+                enabled: true,
+            };
         }
         // Initialize scout manager (skipped in secured mode — roles table inaccessible)
         this.scoutManager = this.securedMode ? null : new scout_1.ScoutManager(this.streamClient, this.appId, this.getEngineTableName.bind(this), this.mintKey.bind(this), this.logger);
@@ -285,7 +293,32 @@ class PostgresStreamService extends index_1.StreamService {
             return Secured.fetchMessagesSecured(this.streamClient, this.safeName(this.appId), target.streamName, consumerName, options || {}, this.logger);
         }
         const target = this.resolveStreamTarget(streamName);
-        return Messages.fetchMessages(this.streamClient, target.tableName, target.streamName, target.isEngine, consumerName, options || {}, this.logger);
+        return Messages.fetchMessages(this.streamClient, target.tableName, target.streamName, target.isEngine, consumerName, options || {}, this.logger, this.liveness);
+    }
+    /**
+     * Refreshes an owned reservation (heartbeat). Called by the consumer
+     * while an activity callback is still running so the message stays
+     * leased past the base reservation window. Returns 0 when the lease
+     * is no longer this consumer's to hold (reclaimed, acked, or expired).
+     */
+    async extendReservation(streamName, messageId, consumerName) {
+        if (this.securedMode) {
+            return 0;
+        }
+        const target = this.resolveStreamTarget(streamName);
+        return Messages.extendReservation(this.streamClient, target.tableName, target.streamName, messageId, consumerName, this.logger);
+    }
+    /**
+     * Soft-deletes every live stream row that belongs to a job, across the
+     * worker and engine stream tables. Called by the engine when a job is
+     * interrupted or scrubbed so its queued, reserved, and scheduled-retry
+     * messages are never delivered again.
+     */
+    async expireJobMessages(jid) {
+        if (this.securedMode) {
+            return 0;
+        }
+        return Messages.expireJobMessages(this.streamClient, [this.getEngineTableName(), this.getWorkerTableName()], jid, this.logger);
     }
     async ackAndDelete(streamName, groupName, messageIds) {
         if (this.securedMode) {
@@ -355,7 +388,12 @@ class PostgresStreamService extends index_1.StreamService {
         return Stats.trimStream(this.streamClient, target.tableName, target.streamName, options, this.logger);
     }
     getProviderSpecificFeatures() {
-        return Stats.getProviderSpecificFeatures(this.config);
+        return {
+            ...Stats.getProviderSpecificFeatures(this.config),
+            //heartbeat lease extension uses direct table access; secured
+            //workers use stored procedures and keep hard-deadline leases
+            supportsReservationExtension: !this.securedMode,
+        };
     }
     async cleanup() {
         if (this.scoutManager) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.25.1",
+  "version": "0.25.4",
   "description": "Durable Workflow",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",