@hotmeshio/hotmesh 0.14.10 → 0.15.1

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.14.10",
+    "version": "0.15.1",
     "description": "Durable Workflow",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/stream/providers/postgres/kvtables.js

@@ -140,21 +140,31 @@ async function createTables(client, schemaName) {
     ) PARTITION BY HASH (stream_name);
   `);
     for (let i = 0; i < 8; i++) {
+        // fillfactor 70: reserves 30% page space for HOT updates (reserve/ack cycle)
         await client.query(`
       CREATE TABLE IF NOT EXISTS ${schemaName}.engine_streams_part_${i}
       PARTITION OF ${engineTable}
-      FOR VALUES WITH (modulus 8, remainder ${i});
+      FOR VALUES WITH (modulus 8, remainder ${i})
+      WITH (fillfactor = 70);
     `);
     }
+    // Dedicated dequeue index: columns match the hot-path query exactly
+    // (stream_name = $1, visible_at <= NOW(), ORDER BY id) with partial
+    // filter on reserved_at IS NULL AND expired_at IS NULL.
+    // Replaces the old active_messages index that had reserved_at as a
+    // column (redundant with the WHERE filter, displacing visible_at).
     await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_engine_streams_active_messages
-    ON ${engineTable} (stream_name, reserved_at, visible_at, id)
+    CREATE INDEX IF NOT EXISTS idx_engine_streams_dequeue
+    ON ${engineTable} (stream_name, visible_at, id)
     WHERE reserved_at IS NULL AND expired_at IS NULL;
   `);
+    // Stale-reservation recovery: covers the timed-out reservation path
+    // (reserved_at < NOW() - INTERVAL) separately from the hot path so
+    // the planner can use each index cleanly without an OR.
     await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_engine_streams_message_fetch
-    ON ${engineTable} (stream_name, visible_at, id)
-    WHERE expired_at IS NULL;
+    CREATE INDEX IF NOT EXISTS idx_engine_streams_stale_reservations
+    ON ${engineTable} (stream_name, reserved_at, visible_at, id)
+    WHERE reserved_at IS NOT NULL AND expired_at IS NULL;
   `);
     await client.query(`
     CREATE INDEX IF NOT EXISTS idx_engine_streams_expired_at
@@ -209,21 +219,25 @@ async function createTables(client, schemaName) {
     ) PARTITION BY HASH (stream_name);
   `);
     for (let i = 0; i < 8; i++) {
+        // fillfactor 70: reserves 30% page space for HOT updates (reserve/ack cycle)
         await client.query(`
       CREATE TABLE IF NOT EXISTS ${schemaName}.worker_streams_part_${i}
       PARTITION OF ${workerTable}
-      FOR VALUES WITH (modulus 8, remainder ${i});
+      FOR VALUES WITH (modulus 8, remainder ${i})
+      WITH (fillfactor = 70);
     `);
     }
+    // Dedicated dequeue index (see engine_streams comments above)
     await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_worker_streams_active_messages
-    ON ${workerTable} (stream_name, reserved_at, visible_at, id)
+    CREATE INDEX IF NOT EXISTS idx_worker_streams_dequeue
+    ON ${workerTable} (stream_name, visible_at, id)
     WHERE reserved_at IS NULL AND expired_at IS NULL;
   `);
+    // Stale-reservation recovery (see engine_streams comments above)
     await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_worker_streams_message_fetch
-    ON ${workerTable} (stream_name, visible_at, id)
-    WHERE expired_at IS NULL;
+    CREATE INDEX IF NOT EXISTS idx_worker_streams_stale_reservations
+    ON ${workerTable} (stream_name, reserved_at, visible_at, id)
+    WHERE reserved_at IS NOT NULL AND expired_at IS NULL;
   `);
     await client.query(`
     CREATE INDEX IF NOT EXISTS idx_worker_streams_expired_at

package/build/services/stream/providers/postgres/messages.js

@@ -207,12 +207,17 @@ async function fetchMessages(client, tableName, streamName, isEngine, consumerNa
             retries++;
             const batchSize = options?.batchSize || 1;
             const reservationTimeout = options?.reservationTimeout || enums_1.HMSH_RESERVATION_TIMEOUT_S;
-            const res = await client.query(`UPDATE ${tableName}
+            // Two-pass dequeue: stale reservations first (FIFO — they have
+            // lower ids and have waited longest), then fresh messages. Split
+            // avoids an OR that prevents the planner from using partial
+            // indexes cleanly. Stale check is a fast no-op (empty index scan)
+            // when there are no timed-out reservations.
+            let res = await client.query(`UPDATE ${tableName}
          SET reserved_at = NOW(), reserved_by = $3
          WHERE id IN (
            SELECT id FROM ${tableName}
            WHERE stream_name = $1
-             AND (reserved_at IS NULL OR reserved_at < NOW() - INTERVAL '${reservationTimeout} seconds')
+             AND reserved_at < NOW() - INTERVAL '${reservationTimeout} seconds'
              AND expired_at IS NULL
              AND visible_at <= NOW()
            ORDER BY id
@@ -220,6 +225,22 @@ async function fetchMessages(client, tableName, streamName, isEngine, consumerNa
            FOR UPDATE SKIP LOCKED
          )
          RETURNING ${returningClause}`, [streamName, batchSize, consumerName]);
+            // Fresh messages: unreserved, visible, not expired
+            if (res.rows.length === 0) {
+                res = await client.query(`UPDATE ${tableName}
+           SET reserved_at = NOW(), reserved_by = $3
+           WHERE id IN (
+             SELECT id FROM ${tableName}
+             WHERE stream_name = $1
+               AND reserved_at IS NULL
+               AND expired_at IS NULL
+               AND visible_at <= NOW()
+             ORDER BY id
+             LIMIT $2
+             FOR UPDATE SKIP LOCKED
+           )
+           RETURNING ${returningClause}`, [streamName, batchSize, consumerName]);
+            }
             const messages = res.rows.map((row) => {
                 const data = (0, utils_1.parseStreamMessage)(row.message);
                 const hasDefaultRetryPolicy = (row.max_retry_attempts === 3 || row.max_retry_attempts === 5) &&

package/build/services/stream/providers/postgres/procedures.js

@@ -54,13 +54,16 @@ function getCreateProceduresSQL(schemaName) {
     SET search_path = ${schemaName}, pg_temp
     AS $$
     ${STREAM_ACCESS_CHECK}
+      -- Two-pass dequeue: stale reservations first (FIFO — lower ids,
+      -- waited longest), then fresh. Split avoids OR that prevents
+      -- partial index usage. Stale check is a fast no-op when empty.
       RETURN QUERY
       UPDATE ${workerTable} ws
       SET reserved_at = NOW(), reserved_by = p_consumer_id
       WHERE ws.id IN (
         SELECT ws2.id FROM ${workerTable} ws2
         WHERE ws2.stream_name = p_stream_name
-          AND (ws2.reserved_at IS NULL OR ws2.reserved_at < NOW() - (p_reservation_timeout_sec || ' seconds')::INTERVAL)
+          AND ws2.reserved_at < NOW() - (p_reservation_timeout_sec || ' seconds')::INTERVAL
           AND ws2.expired_at IS NULL
           AND ws2.visible_at <= NOW()
         ORDER BY ws2.id
@@ -69,6 +72,25 @@ function getCreateProceduresSQL(schemaName) {
       )
       RETURNING ws.id, ws.message, ws.workflow_name, ws.max_retry_attempts,
                 ws.backoff_coefficient, ws.maximum_interval_seconds, ws.retry_attempt;
+
+      -- Fresh messages: unreserved, visible, not expired
+      IF NOT FOUND THEN
+        RETURN QUERY
+        UPDATE ${workerTable} ws
+        SET reserved_at = NOW(), reserved_by = p_consumer_id
+        WHERE ws.id IN (
+          SELECT ws2.id FROM ${workerTable} ws2
+          WHERE ws2.stream_name = p_stream_name
+            AND ws2.reserved_at IS NULL
+            AND ws2.expired_at IS NULL
+            AND ws2.visible_at <= NOW()
+          ORDER BY ws2.id
+          LIMIT p_batch_size
+          FOR UPDATE SKIP LOCKED
+        )
+        RETURNING ws.id, ws.message, ws.workflow_name, ws.max_retry_attempts,
+                  ws.backoff_coefficient, ws.maximum_interval_seconds, ws.retry_attempt;
+      END IF;
     END;
     $$;`,
         // -- worker_ack --

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.14.10",
+  "version": "0.15.1",
   "description": "Durable Workflow",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",