@hotmeshio/hotmesh 0.19.5 → 0.20.1

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.19.5",
+    "version": "0.20.1",
     "description": "Durable Workflow",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/stream/providers/postgres/kvtables.js

@@ -31,8 +31,10 @@ async function deploySchema(streamClient, appId, logger) {
                     }
                     await client.query('COMMIT');
                 }
-                // Always run index migrations under the lock
+                // Always run index, procedure, and trigger migrations under the lock
                 await ensureIndexes(client, schemaName);
+                await ensureProcedures(client, schemaName);
+                await ensureStatementLevelTriggers(client, schemaName);
             }
             finally {
                 await client.query('SELECT pg_advisory_unlock($1)', [lockId]);
@@ -129,7 +131,12 @@ async function waitForTablesCreation(streamClient, lockId, schemaName, logger) {
 async function ensureIndexes(client, schemaName) {
     const engineTable = `${schemaName}.engine_streams`;
     const workerTable = `${schemaName}.worker_streams`;
-    // Drop legacy indexes that don't include the priority column
+    // Drop legacy indexes that don't include the priority column, plus
+    // redundant ones: idx_*_expired_at duplicates the partial
+    // idx_*_processed_volume for the retention purge, and
+    // idx_*_stream_name_expired_at duplicates the leading column and
+    // predicate of idx_*_message_fetch. Every index here is maintained on
+    // each message's INSERT plus two non-HOT UPDATEs (reserve, ack).
     for (const idx of [
         'idx_engine_streams_dequeue',
         'idx_engine_streams_stale_reservations',
@@ -139,6 +146,10 @@ async function ensureIndexes(client, schemaName) {
         'idx_engine_streams_message_fetch',
         'idx_worker_streams_active_messages',
         'idx_worker_streams_message_fetch',
+        'idx_engine_streams_expired_at',
+        'idx_engine_stream_name_expired_at',
+        'idx_worker_streams_expired_at',
+        'idx_worker_stream_name_expired_at',
     ]) {
         await client.query(`DROP INDEX IF EXISTS ${schemaName}.${idx}`);
     }
@@ -148,9 +159,13 @@ async function ensureIndexes(client, schemaName) {
     ON ${engineTable} (stream_name, priority DESC, visible_at, id)
     WHERE reserved_at IS NULL AND expired_at IS NULL;
   `);
+    // message_fetch must match the dequeue ORDER BY (priority DESC, id)
+    // exactly — placing visible_at between them forces the claim query to
+    // fetch and sort the entire pending backlog instead of stopping at
+    // LIMIT. visible_at and stale-reservation checks are scan filters.
     await client.query(`
     CREATE INDEX IF NOT EXISTS idx_engine_streams_message_fetch
-    ON ${engineTable} (stream_name, priority DESC, visible_at, id)
+    ON ${engineTable} (stream_name, priority DESC, id)
     WHERE expired_at IS NULL;
   `);
     await client.query(`
@@ -160,7 +175,7 @@ async function ensureIndexes(client, schemaName) {
   `);
     await client.query(`
     CREATE INDEX IF NOT EXISTS idx_worker_streams_message_fetch
-    ON ${workerTable} (stream_name, priority DESC, visible_at, id)
+    ON ${workerTable} (stream_name, priority DESC, id)
     WHERE expired_at IS NULL;
   `);
     // v0.18.0: add jid column to engine_streams for job tracing
@@ -171,6 +186,50 @@ async function ensureIndexes(client, schemaName) {
     WHERE jid != '';
   `);
 }
+/**
+ * Re-deploy the SECURITY DEFINER stored procedures on existing
+ * databases so query changes (e.g., worker_dequeue) reach deployments
+ * created before the change. CREATE OR REPLACE preserves grants.
+ */
+async function ensureProcedures(client, schemaName) {
+    for (const sql of (0, procedures_1.getCreateProceduresSQL)(schemaName)) {
+        await client.query(sql);
+    }
+}
+/**
+ * Migrate pre-existing row-level notification triggers to the
+ * statement-level form. Recreating a trigger takes an ACCESS EXCLUSIVE
+ * lock on the table, so only do it when the installed trigger is still
+ * row-level (tgtype bit 0 set); subsequent boots are a no-op.
+ *
+ * The function replacement and trigger swap MUST commit atomically:
+ * between them, the still-installed row-level trigger would invoke the
+ * statement-level function body, whose transition-table reference
+ * (new_rows) errors under FOR EACH ROW — failing every concurrent
+ * INSERT until the swap lands (or indefinitely, if the migrating
+ * process dies between the two statements).
+ */
+async function ensureStatementLevelTriggers(client, schemaName) {
+    const result = await client.query(`SELECT count(*) AS row_level
+     FROM pg_trigger t
+     JOIN pg_class c ON c.oid = t.tgrelid
+     JOIN pg_namespace n ON n.oid = c.relnamespace
+     WHERE n.nspname = $1
+       AND c.relname IN ('engine_streams', 'worker_streams')
+       AND t.tgname IN ('notify_engine_stream_insert', 'notify_worker_stream_insert')
+       AND (t.tgtype & 1) = 1`, [schemaName]);
+    if (parseInt(result.rows[0].row_level, 10) > 0) {
+        await client.query('BEGIN');
+        try {
+            await createNotificationTriggers(client, schemaName);
+            await client.query('COMMIT');
+        }
+        catch (error) {
+            await client.query('ROLLBACK');
+            throw error;
+        }
+    }
+}
 async function createTables(client, schemaName) {
     await client.query(`CREATE SCHEMA IF NOT EXISTS ${schemaName};`);
     // ---- ENGINE_STREAMS table ----
@@ -210,16 +269,7 @@ async function createTables(client, schemaName) {
   `);
     await client.query(`
     CREATE INDEX IF NOT EXISTS idx_engine_streams_message_fetch
-    ON ${engineTable} (stream_name, priority DESC, visible_at, id)
-    WHERE expired_at IS NULL;
-  `);
-    await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_engine_streams_expired_at
-    ON ${engineTable} (expired_at);
-  `);
-    await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_engine_stream_name_expired_at
-    ON ${engineTable} (stream_name)
+    ON ${engineTable} (stream_name, priority DESC, id)
     WHERE expired_at IS NULL;
   `);
     await client.query(`
@@ -280,16 +330,7 @@ async function createTables(client, schemaName) {
   `);
     await client.query(`
     CREATE INDEX IF NOT EXISTS idx_worker_streams_message_fetch
-    ON ${workerTable} (stream_name, priority DESC, visible_at, id)
-    WHERE expired_at IS NULL;
-  `);
-    await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_worker_streams_expired_at
-    ON ${workerTable} (expired_at);
-  `);
-    await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_worker_stream_name_expired_at
-    ON ${workerTable} (stream_name)
+    ON ${workerTable} (stream_name, priority DESC, id)
     WHERE expired_at IS NULL;
   `);
     await client.query(`
@@ -342,28 +383,35 @@ async function createNotificationTriggers(client, schemaName) {
     const engineTable = `${schemaName}.engine_streams`;
     const workerTable = `${schemaName}.worker_streams`;
     // ---- ENGINE notification trigger ----
+    // Statement-level with a transition table: one pg_notify per distinct
+    // stream_name per INSERT statement. Row-level triggers fire pg_notify
+    // per message, which both multiplies trigger overhead and serializes
+    // commits on the global notification queue lock at high insert rates.
     await client.query(`
     CREATE OR REPLACE FUNCTION ${schemaName}.notify_new_engine_stream_message()
     RETURNS TRIGGER AS $$
     DECLARE
+      rec RECORD;
       channel_name TEXT;
       payload JSON;
     BEGIN
-      IF NEW.visible_at <= NOW() THEN
-        channel_name := 'eng_' || NEW.stream_name;
+      FOR rec IN
+        SELECT DISTINCT stream_name FROM new_rows WHERE visible_at <= NOW()
+      LOOP
+        channel_name := 'eng_' || rec.stream_name;
         IF length(channel_name) > 63 THEN
           channel_name := left(channel_name, 63);
         END IF;
 
         payload := json_build_object(
-          'stream_name', NEW.stream_name,
+          'stream_name', rec.stream_name,
           'table_type', 'engine'
         );
 
         PERFORM pg_notify(channel_name, payload::text);
-      END IF;
+      END LOOP;
 
-      RETURN NEW;
+      RETURN NULL;
     END;
     $$ LANGUAGE plpgsql;
   `);
@@ -371,7 +419,8 @@ async function createNotificationTriggers(client, schemaName) {
     DROP TRIGGER IF EXISTS notify_engine_stream_insert ON ${engineTable};
     CREATE TRIGGER notify_engine_stream_insert
       AFTER INSERT ON ${engineTable}
-      FOR EACH ROW
+      REFERENCING NEW TABLE AS new_rows
+      FOR EACH STATEMENT
       EXECUTE FUNCTION ${schemaName}.notify_new_engine_stream_message();
   `);
     // ---- WORKER notification trigger ----
@@ -379,24 +428,27 @@ async function createNotificationTriggers(client, schemaName) {
     CREATE OR REPLACE FUNCTION ${schemaName}.notify_new_worker_stream_message()
     RETURNS TRIGGER AS $$
     DECLARE
+      rec RECORD;
       channel_name TEXT;
       payload JSON;
     BEGIN
-      IF NEW.visible_at <= NOW() THEN
-        channel_name := 'wrk_' || NEW.stream_name;
+      FOR rec IN
+        SELECT DISTINCT stream_name FROM new_rows WHERE visible_at <= NOW()
+      LOOP
+        channel_name := 'wrk_' || rec.stream_name;
         IF length(channel_name) > 63 THEN
           channel_name := left(channel_name, 63);
         END IF;
 
         payload := json_build_object(
-          'stream_name', NEW.stream_name,
+          'stream_name', rec.stream_name,
           'table_type', 'worker'
         );
 
         PERFORM pg_notify(channel_name, payload::text);
-      END IF;
+      END LOOP;
 
-      RETURN NEW;
+      RETURN NULL;
     END;
     $$ LANGUAGE plpgsql;
   `);
@@ -404,7 +456,8 @@ async function createNotificationTriggers(client, schemaName) {
     DROP TRIGGER IF EXISTS notify_worker_stream_insert ON ${workerTable};
     CREATE TRIGGER notify_worker_stream_insert
       AFTER INSERT ON ${workerTable}
-      FOR EACH ROW
+      REFERENCING NEW TABLE AS new_rows
+      FOR EACH STATEMENT
       EXECUTE FUNCTION ${schemaName}.notify_new_worker_stream_message();
   `);
     // ---- Visibility timeout notification function (queries both tables) ----

package/build/services/stream/providers/postgres/messages.js

@@ -215,18 +215,24 @@ async function fetchMessages(client, tableName, streamName, isEngine, consumerNa
     const maxRetries = options?.maxRetries ?? 3;
     let backoff = initialBackoff;
     let retries = 0;
-    // Include workflow_name in RETURNING for worker streams
+    // Include workflow_name in RETURNING for worker streams. Columns are
+    // qualified with the update target's alias because the claim UPDATE
+    // joins a CTE that also exposes an id column.
     const returningClause = isEngine
-        ? 'id, message, max_retry_attempts, backoff_coefficient, maximum_interval_seconds, retry_attempt'
-        : 'id, message, workflow_name, max_retry_attempts, backoff_coefficient, maximum_interval_seconds, retry_attempt';
+        ? 't.id, t.message, t.max_retry_attempts, t.backoff_coefficient, t.maximum_interval_seconds, t.retry_attempt'
+        : 't.id, t.message, t.workflow_name, t.max_retry_attempts, t.backoff_coefficient, t.maximum_interval_seconds, t.retry_attempt';
     try {
         while (retries < maxRetries) {
             retries++;
             const batchSize = options?.batchSize || 1;
             const reservationTimeout = options?.reservationTimeout || (enums_1.HMSH_RESERVATION_TIMEOUT_S + 5);
-            const res = await client.query(`UPDATE ${tableName}
-         SET reserved_at = NOW(), reserved_by = $3
-         WHERE id IN (
+            // The locking SELECT must live in a MATERIALIZED CTE: as a plain IN
+            // subquery the planner may re-execute it per outer row (rows updated
+            // earlier in the same command are skipped as lock candidates), which
+            // reserves MORE rows than LIMIT. The UPDATE repeats stream_name so
+            // the planner prunes to a single hash partition and joins on the
+            // (stream_name, id) primary key.
+            const res = await client.query(`WITH candidates AS MATERIALIZED (
            SELECT id FROM ${tableName}
            WHERE stream_name = $1
              AND (reserved_at IS NULL OR reserved_at < NOW() - INTERVAL '${reservationTimeout} seconds')
@@ -236,6 +242,10 @@ async function fetchMessages(client, tableName, streamName, isEngine, consumerNa
            LIMIT $2
            FOR UPDATE SKIP LOCKED
          )
+         UPDATE ${tableName} t
+         SET reserved_at = NOW(), reserved_by = $3
+         FROM candidates
+         WHERE t.stream_name = $1 AND t.id = candidates.id
          RETURNING ${returningClause}`, [streamName, batchSize, consumerName]);
             const messages = res.rows.map((row) => {
                 const data = (0, utils_1.parseStreamMessage)(row.message);

package/build/services/stream/providers/postgres/postgres.d.ts

@@ -36,6 +36,13 @@ declare class PostgresStreamService extends StreamService<PostgresClientType & P
     init(namespace: string, appId: string, logger: ILogger): Promise<void>;
     private isNotificationsEnabled;
     private checkForMissedMessages;
+    /**
+     * Notification-driven fetch with coalescing. NOTIFYs that arrive while
+     * a fetch is in flight set fetchPending instead of issuing concurrent
+     * claim queries (a burst of N inserts otherwise triggers N claims per
+     * consumer, most returning empty). The drain loop re-fetches while the
+     * batch came back full or a NOTIFY arrived mid-fetch.
+     */
     private fetchAndDeliverMessages;
     private getConsumerKey;
     /**

package/build/services/stream/providers/postgres/postgres.js

@@ -82,11 +82,31 @@ class PostgresStreamService extends index_1.StreamService {
             return await instance.fetchMessages(consumer.streamName, consumer.groupName, consumer.consumerName, { batchSize: 10, reservationTimeout: instance.reservationTimeout, enableBackoff: false, maxRetries: 1 });
         });
     }
+    /**
+     * Notification-driven fetch with coalescing. NOTIFYs that arrive while
+     * a fetch is in flight set fetchPending instead of issuing concurrent
+     * claim queries (a burst of N inserts otherwise triggers N claims per
+     * consumer, most returning empty). The drain loop re-fetches while the
+     * batch came back full or a NOTIFY arrived mid-fetch.
+     */
     async fetchAndDeliverMessages(consumer) {
+        if (consumer.fetchInFlight) {
+            consumer.fetchPending = true;
+            return;
+        }
+        consumer.fetchInFlight = true;
+        const batchSize = 10;
         try {
-            const messages = await this.fetchMessages(consumer.streamName, consumer.groupName, consumer.consumerName, { batchSize: 10, reservationTimeout: this.reservationTimeout, enableBackoff: false, maxRetries: 1 });
-            if (messages.length > 0) {
-                consumer.callback(messages);
+            let drain = true;
+            while (drain && consumer.isListening !== false) {
+                consumer.fetchPending = false;
+                const messages = await this.fetchMessages(consumer.streamName, consumer.groupName, consumer.consumerName, { batchSize, reservationTimeout: this.reservationTimeout, enableBackoff: false, maxRetries: 1 });
+                if (messages.length > 0) {
+                    consumer.callback(messages);
+                }
+                // Boolean() rather than === true: fetchPending is mutated by the
+                // notification handler across the await, which TS narrowing misses
+                drain = messages.length === batchSize || Boolean(consumer.fetchPending);
             }
         }
         catch (error) {
@@ -96,6 +116,9 @@ class PostgresStreamService extends index_1.StreamService {
                 error,
             });
         }
+        finally {
+            consumer.fetchInFlight = false;
+        }
     }
     getConsumerKey(streamName, groupName) {
         return `${streamName}:${groupName}`;

package/build/services/stream/providers/postgres/procedures.js

@@ -54,10 +54,12 @@ function getCreateProceduresSQL(schemaName) {
     SET search_path = ${schemaName}, pg_temp
     AS $$
     ${STREAM_ACCESS_CHECK}
+      -- The locking SELECT must live in a MATERIALIZED CTE: as a plain IN
+      -- subquery the planner may re-execute it per outer row, reserving
+      -- MORE rows than p_batch_size. stream_name on the UPDATE prunes to
+      -- a single hash partition and joins on the primary key.
       RETURN QUERY
-      UPDATE ${workerTable} ws
-      SET reserved_at = NOW(), reserved_by = p_consumer_id
-      WHERE ws.id IN (
+      WITH candidates AS MATERIALIZED (
         SELECT ws2.id FROM ${workerTable} ws2
         WHERE ws2.stream_name = p_stream_name
           AND (ws2.reserved_at IS NULL OR ws2.reserved_at < NOW() - (p_reservation_timeout_sec || ' seconds')::INTERVAL)
@@ -67,6 +69,11 @@ function getCreateProceduresSQL(schemaName) {
         LIMIT p_batch_size
         FOR UPDATE SKIP LOCKED
       )
+      UPDATE ${workerTable} ws
+      SET reserved_at = NOW(), reserved_by = p_consumer_id
+      FROM candidates
+      WHERE ws.stream_name = p_stream_name
+        AND ws.id = candidates.id
       RETURNING ws.id, ws.message, ws.workflow_name, ws.max_retry_attempts,
                 ws.backoff_coefficient, ws.maximum_interval_seconds, ws.retry_attempt;
     END;

package/build/types/stream.d.ts

@@ -300,4 +300,8 @@ export interface NotificationConsumer {
     lastFallbackCheck: number;
     /** Service instance that owns this consumer (for fetchAndDeliverMessages dispatch) */
     serviceInstance?: any;
+    /** True while a notification-driven fetch is in flight (coalesces concurrent NOTIFYs) */
+    fetchInFlight?: boolean;
+    /** Set when a NOTIFY arrives mid-fetch; triggers one follow-up fetch */
+    fetchPending?: boolean;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.19.5",
+  "version": "0.20.1",
   "description": "Durable Workflow",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",