@hotmeshio/hotmesh 0.25.5 → 0.26.0

package/build/services/stream/providers/postgres/kvtables.js

@@ -35,6 +35,10 @@ async function deploySchema(streamClient, appId, logger) {
                 await ensureIndexes(client, schemaName);
                 await ensureProcedures(client, schemaName);
                 await ensureStatementLevelTriggers(client, schemaName);
+                // Re-deploy the fallback poller's discovery function so existing
+                // databases receive predicate changes (v0.25.6: stale-reservation
+                // reclaim — see getNotifyVisibleMessagesSQL)
+                await client.query(getNotifyVisibleMessagesSQL(schemaName));
             }
             finally {
                 await client.query('SELECT pg_advisory_unlock($1)', [lockId]);
@@ -177,6 +181,19 @@ async function ensureIndexes(client, schemaName) {
     CREATE INDEX IF NOT EXISTS idx_worker_streams_message_fetch
     ON ${workerTable} (stream_name, priority DESC, id)
     WHERE expired_at IS NULL;
+  `);
+    // v0.25.6: in-flight partial indexes — bounded to currently reserved
+    // rows (≈ concurrent executions) — serve the fallback poller's
+    // stale-reservation reclaim branch (see getNotifyVisibleMessagesSQL)
+    await client.query(`
+    CREATE INDEX IF NOT EXISTS idx_engine_streams_inflight
+    ON ${engineTable} (stream_name, reserved_at)
+    WHERE expired_at IS NULL AND reserved_at IS NOT NULL;
+  `);
+    await client.query(`
+    CREATE INDEX IF NOT EXISTS idx_worker_streams_inflight
+    ON ${workerTable} (stream_name, reserved_at)
+    WHERE expired_at IS NULL AND reserved_at IS NOT NULL;
   `);
     // v0.18.0: add jid column to engine_streams for job tracing
     await client.query(`ALTER TABLE ${engineTable} ADD COLUMN IF NOT EXISTS jid TEXT NOT NULL DEFAULT ''`);
@@ -461,7 +478,34 @@ async function createNotificationTriggers(client, schemaName) {
       EXECUTE FUNCTION ${schemaName}.notify_new_worker_stream_message();
   `);
     // ---- Visibility timeout notification function (queries both tables) ----
-    await client.query(`
+    await client.query(getNotifyVisibleMessagesSQL(schemaName));
+}
+/**
+ * The fallback poller's discovery function. Surfaces streams that have
+ * deliverable work: unreserved visible messages AND stale reservations
+ * whose holder died (heartbeats refresh reserved_at every ~15s while an
+ * activity runs, so any reservation older than 60s has a dead holder or
+ * is already claimable). Stale-reservation discovery is the ONLY path
+ * that reclaims orphaned in-flight work on a quiet stream — without it,
+ * a process death wedges its reserved messages forever. A premature
+ * surface (reservation window configured above 60s on a provider with
+ * static leases) costs one empty fetch; the claim query enforces the
+ * real window.
+ *
+ * Deployed on fresh schemas AND re-deployed on every boot (see
+ * deploySchema) so existing databases receive predicate changes.
+ */
+function getNotifyVisibleMessagesSQL(schemaName) {
+    const engineTable = `${schemaName}.engine_streams`;
+    const workerTable = `${schemaName}.worker_streams`;
+    //stale threshold tracks the configured base window: heartbeats refresh
+    //reserved_at every base/2 (15s default), so live holders never look
+    //stale; 2x base keeps static-lease holders (secured workers, one
+    //adaptive doubling) from surfacing as false positives. Baked at
+    //deploy time from the deploying node's env — a premature surface
+    //costs one empty fetch (the claim query enforces the real window).
+    const staleSeconds = Math.max(60, enums_1.HMSH_RESERVATION_TIMEOUT_S * 2);
+    return `
     CREATE OR REPLACE FUNCTION ${schemaName}.notify_visible_messages()
     RETURNS INTEGER AS $$
     DECLARE
@@ -470,13 +514,23 @@ async function createNotificationTriggers(client, schemaName) {
       payload JSON;
       notification_count INTEGER := 0;
     BEGIN
-      -- Engine streams
+      -- Engine streams: visible unreserved work (active_messages
+      -- partial index) UNION stale reservations whose holder died
+      -- (inflight partial index) — each branch index-matched
       FOR msg IN
-        SELECT DISTINCT stream_name
-        FROM ${engineTable}
-        WHERE visible_at <= NOW()
-          AND reserved_at IS NULL
-          AND expired_at IS NULL
+        SELECT DISTINCT u.stream_name FROM (
+          (SELECT stream_name FROM ${engineTable}
+           WHERE visible_at <= NOW()
+             AND reserved_at IS NULL
+             AND expired_at IS NULL
+           LIMIT 50)
+          UNION ALL
+          (SELECT stream_name FROM ${engineTable}
+           WHERE expired_at IS NULL
+             AND reserved_at IS NOT NULL
+             AND reserved_at < NOW() - INTERVAL '${staleSeconds} seconds'
+           LIMIT 50)
+        ) u
         LIMIT 50
       LOOP
         channel_name := 'eng_' || msg.stream_name;
@@ -493,13 +547,21 @@ async function createNotificationTriggers(client, schemaName) {
         notification_count := notification_count + 1;
       END LOOP;
 
-      -- Worker streams
+      -- Worker streams: same two index-matched branches
       FOR msg IN
-        SELECT DISTINCT stream_name
-        FROM ${workerTable}
-        WHERE visible_at <= NOW()
-          AND reserved_at IS NULL
-          AND expired_at IS NULL
+        SELECT DISTINCT u.stream_name FROM (
+          (SELECT stream_name FROM ${workerTable}
+           WHERE visible_at <= NOW()
+             AND reserved_at IS NULL
+             AND expired_at IS NULL
+           LIMIT 50)
+          UNION ALL
+          (SELECT stream_name FROM ${workerTable}
+           WHERE expired_at IS NULL
+             AND reserved_at IS NOT NULL
+             AND reserved_at < NOW() - INTERVAL '${staleSeconds} seconds'
+           LIMIT 50)
+        ) u
         LIMIT 50
       LOOP
         channel_name := 'wrk_' || msg.stream_name;
@@ -519,7 +581,7 @@ async function createNotificationTriggers(client, schemaName) {
       RETURN notification_count;
     END;
     $$ LANGUAGE plpgsql;
-  `);
+  `;
 }
 function getNotificationChannelName(streamName, isEngine) {
     const prefix = isEngine ? 'eng_' : 'wrk_';

package/build/services/stream/providers/postgres/messages.js

@@ -296,12 +296,15 @@ async function dropDeadJobMessages(client, tableName, streamName, rows, liveness
         }
     }
     catch (error) {
-        if (error?.code === '42P01') {
-            //jobs table is not visible from this connection; the guard cannot
-            //run here — interrupt-time purging (expireJobMessages) still applies
+        if (error?.code === '42P01' || error?.code === '42501') {
+            //jobs table is not visible (42P01) or not readable (42501) from
+            //this connection; the guard cannot run here — interrupt-time
+            //purging (expireJobMessages) still applies. Self-disable so the
+            //hot path stops issuing a failing cross-table query per fetch.
             liveness.enabled = false;
             logger.info('postgres-stream-liveness-guard-disabled', {
                 jobsTable: liveness.jobsTable,
+                code: error.code,
             });
         }
         else {

package/build/services/stream/providers/postgres/procedures.js

@@ -150,17 +150,19 @@ function getCreateProceduresSQL(schemaName) {
       -- Engine stream_name is the schema/appId name
       engine_stream_name := '${schemaName}';
 
+      -- jid stamped from the message metadata (parity with the raw
+      -- publish path) so job-scoped expiry can address these rows
       IF p_max_retry_attempts IS NOT NULL THEN
         INSERT INTO ${engineTable}
-          (stream_name, message, priority, max_retry_attempts, backoff_coefficient, maximum_interval_seconds, visible_at, retry_attempt)
+          (stream_name, jid, message, priority, max_retry_attempts, backoff_coefficient, maximum_interval_seconds, visible_at, retry_attempt)
         VALUES
-          (engine_stream_name, p_message, p_priority, p_max_retry_attempts, p_backoff_coefficient, p_maximum_interval_seconds, p_visible_at, p_retry_attempt)
+          (engine_stream_name, COALESCE(p_message::jsonb -> 'metadata' ->> 'jid', ''), p_message, p_priority, p_max_retry_attempts, p_backoff_coefficient, p_maximum_interval_seconds, p_visible_at, p_retry_attempt)
         RETURNING id INTO new_id;
       ELSE
         INSERT INTO ${engineTable}
-          (stream_name, message, priority, visible_at, retry_attempt)
+          (stream_name, jid, message, priority, visible_at, retry_attempt)
         VALUES
-          (engine_stream_name, p_message, p_priority, p_visible_at, p_retry_attempt)
+          (engine_stream_name, COALESCE(p_message::jsonb -> 'metadata' ->> 'jid', ''), p_message, p_priority, p_visible_at, p_retry_attempt)
         RETURNING id INTO new_id;
       END IF;
 

package/build/types/hmsh_escalations.d.ts

@@ -106,17 +106,16 @@ export type ResolveEscalationResult = {
     reason: 'not-found' | 'already-resolved' | 'already-cancelled' | 'already-expired';
 };
 /**
- * A pre-built wake publish, executed INSIDE the resolve transaction so the
- * awaiting workflow's wake commits atomically with `status='resolved'`. The
- * SQL is a stream INSERT produced by the stream provider; `forSignalKey`
- * pins the command to the row it was built for — the store executes it only
- * when the locked row's `signal_key` matches (a mismatched row falls back
- * to post-commit delivery).
+ * A pre-built wake message, committed INSIDE the resolve/cancel transaction
+ * so the awaiting workflow's wake is durable with the status change. The
+ * client composes the webhook message; the store owns the stream INSERT.
+ * `forSignalKey` pins the wake to the row it was built for — it is written
+ * only when the affected row's `signal_key` matches (a mismatched row falls
+ * back to post-commit delivery).
  */
 export interface EscalationWakeCommand {
     forSignalKey: string;
-    sql: string;
-    params: unknown[];
+    message: string;
 }
 export type ReleaseEscalationResult = {
     ok: true;
@@ -132,6 +131,29 @@ export type CancelEscalationResult = {
     ok: false;
     reason: 'not-found' | 'already-terminal';
 };
+/**
+ * Retention parameters for `pruneEscalations`. Prunes only terminal rows
+ * (`resolved`/`cancelled`/`expired`) — the statuses every engine state
+ * transition treats as final — older than the given horizon.
+ */
+export interface PruneEscalationsParams {
+    /**
+     * Age horizon as a Postgres interval string (e.g. `'90 days'`, `'12 hours'`).
+     * Rows qualify when `updated_at < NOW() - olderThan`.
+     */
+    olderThan: string;
+    /** Terminal statuses to prune. Defaults to all three; non-terminal values are ignored. */
+    statuses?: Array<'resolved' | 'cancelled' | 'expired'>;
+    namespace?: string;
+    /**
+     * Max rows deleted per call (bounds lock time and vacuum pressure).
+     * Default 10,000, capped at 100,000. Loop until `deleted` is 0 to drain.
+     */
+    limit?: number;
+}
+export interface PruneEscalationsResult {
+    deleted: number;
+}
 export interface ListEscalationsParams {
     namespace?: string;
     role?: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.25.5",
+  "version": "0.26.0",
   "description": "Durable Workflow",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",

package/ISSUE.md

@@ -1,53 +0,0 @@
-# ADDENDUM — lost condition wakes are NOT restart-dependent (registration-window race)
-
-Follow-up to the earlier "Durable wakes lost across process restart" report filed
-tonight (sleep timers + condition signals). New evidence sharpens the condition case.
-
-## Reproduced with no restart involved
-
-Same environment (hotmesh 0.25.4, Postgres, long-tail 0.7.3), process running
-continuously the whole time:
-
-- `13:59:36.204` — pill escalation created (conditionLT atomic park, child
-  `...-winddown-gluer-0`).
-- `13:59:36.237` — row **resolved**, 33 ms after creation (a hot consumer was
-  already scanning the pond and grabbed it instantly).
-- 15+ minutes later — the awaiting child is still `status > 0`, never woke;
-  its parent's `Promise.all` wedged with it.
-
-## The pattern across all three observed condition cases
-
-Every lost condition wake we have seen shares one property: **the resolve landed
-sub-second after the conditionLT park** (33 ms, 232 ms, ~600 ms). Slow resolves
-(seconds or minutes after park) have never lost a wake across hundreds of
-station-worker resolutions tonight.
-
-Hypothesis: a resolve that commits **inside the awaiting workflow's
-wake-registration window** — after the escalation row is visible to consumers
-but before the awaiter's subscription is fully registered — delivers to nobody,
-and nothing ever re-checks the persisted row against sleeping awaiters. The
-restart in the earlier case was likely incidental; the window is the bug.
-
-## Suggested repro (tighter than the original)
-
-1. Workflow A parks via `conditionLT` (the escalation-creating overload).
-2. A hot loop polls for the row and resolves it the instant it appears
-   (sub-100 ms). No process kill needed.
-3. Observe whether A ever wakes. In our runs, a same-process consumer beating
-   the registration window wedges A reliably enough to hit twice in one evening.
-
-## Suggested direction (unchanged, reinforced)
-
-Boot-time AND periodic reconciliation of resolved-rows-with-sleeping-awaiters
-would heal both the restart loss and this race — the row carries the awaiting
-`workflow_id`; the join is cheap. Making the wake itself a stream message
-(liveness-redelivered since 0.25.4) also closes the window, since the message
-persists until reserved and acked.
-
-## Our workaround (in production use as of tonight)
-
-Settlement guard in the watcher activity: after the row's outcome is known,
-poll `durable.jobs` for the awaiting child's status; if still running ~10s
-after its row resolved, terminate the child by id (terminate purges cleanly
-since 0.25.4) and treat the row's persisted resolver payload as authoritative.
-The parent catches the child's terminate rejection. Ugly but wedge-proof.