@hotmeshio/hotmesh 0.16.2 → 0.16.4

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.16.2",
+    "version": "0.16.4",
     "description": "Durable Workflow",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/activities/hook.js

@@ -6,6 +6,8 @@ const pipe_1 = require("../pipe");
 const task_1 = require("../task");
 const telemetry_1 = require("../telemetry");
 const stream_1 = require("../../types/stream");
+const errors_1 = require("../../modules/errors");
+const collator_2 = require("../../types/collator");
 const utils_1 = require("../../modules/utils");
 const activity_1 = require("./activity");
 /**
@@ -293,9 +295,38 @@ class Hook extends activity_1.Activity {
             this.context.metadata.jid = jobId;
             this.context.metadata.gid = gId;
             this.context.metadata.dad = dad;
-            await this.processEvent(status, code, 'hook');
-            if (code === 200) {
-                await taskService.deleteWebHookSignal(this.config.hook.topic, data);
+            // Inline retry for FORBIDDEN: Leg2 arrived in the window between
+            // setHookSignal (standalone) and Leg1 transaction.exec(). The 100B
+            // ledger digit is not yet visible. Leg1 needs only milliseconds to
+            // commit — retry here, inside the message processing loop, before
+            // consumeOne's finally block acks the message. Stream-level retry
+            // won't help: ENGINE consumers have no retry policy, so shouldRetry
+            // returns [false, 0] and the message is ack'd with no retry.
+            const MAX_FORBIDDEN_RETRIES = 5;
+            const FORBIDDEN_RETRY_DELAY_MS = 50;
+            for (let attempt = 0; attempt <= MAX_FORBIDDEN_RETRIES; attempt++) {
+                try {
+                    await this.processEvent(status, code, 'hook');
+                    if (code === 200) {
+                        await taskService.deleteWebHookSignal(this.config.hook.topic, data);
+                    }
+                    return;
+                }
+                catch (error) {
+                    if (error instanceof errors_1.CollationError &&
+                        error.fault === collator_2.CollationFaultType.FORBIDDEN &&
+                        attempt < MAX_FORBIDDEN_RETRIES) {
+                        this.logger.warn('hook-webhook-forbidden-inline-retry', {
+                            attempt: attempt + 1,
+                            maxAttempts: MAX_FORBIDDEN_RETRIES,
+                            jid: this.context.metadata.jid,
+                            aid: this.metadata.aid,
+                        });
+                        await (0, utils_1.sleepFor)(FORBIDDEN_RETRY_DELAY_MS * (attempt + 1));
+                        continue;
+                    }
+                    throw error;
+                }
             }
         }
     }

package/build/services/store/providers/postgres/kvtables.js

@@ -153,20 +153,6 @@ const KVTables = (context) => ({
         FOR EACH ROW EXECUTE FUNCTION ${schemaName}.update_jobs_updated_at();
       `);
         }
-        // v0.15.2: add dedicated is_live partial index for hot-path queries
-        const { rows: idxRows } = await client.query(`SELECT 1 FROM pg_indexes WHERE indexname = 'idx_jobs_key_live' AND schemaname = $1 LIMIT 1`, [schemaName]);
-        if (idxRows.length === 0) {
-            await client.query(`
-        CREATE INDEX IF NOT EXISTS idx_jobs_key_live
-        ON ${jobsTable} (key) WHERE is_live;
-      `);
-        }
-        // v0.15.2: partial index for sorted_set scheduler hot path
-        await client.query(`
-      CREATE INDEX IF NOT EXISTS idx_task_schedules_active
-      ON ${schemaName}.task_schedules (key, score)
-      WHERE expiry IS NULL;
-    `);
     },
     async createTables(client, appName) {
         try {
@@ -265,40 +251,27 @@ const KVTables = (context) => ({
               CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_context_gin 
               ON ${fullTableName} USING GIN (context);
             `);
-                        // Create partitions with fillfactor 70: reserves 30% page space
-                        // for HOT updates (status update cycle in setStatusAndCollateGuid)
+                        // Create partitions using a DO block
                         await client.query(`
               DO $$
               BEGIN
                 FOR i IN 0..7 LOOP
                   EXECUTE format(
                     'CREATE TABLE IF NOT EXISTS ${fullTableName}_part_%s PARTITION OF ${fullTableName}
-                    FOR VALUES WITH (modulus 8, remainder %s)
-                    WITH (fillfactor = 70)',
+                    FOR VALUES WITH (modulus 8, remainder %s)',
                     i, i
                   );
                 END LOOP;
               END$$;
             `);
-                        // Original index for queries that filter on expired_at directly
-                        // (e.g. _hmget's WHERE expired_at IS NULL OR expired_at > NOW())
+                        // Create optimized indexes
                         await client.query(`
               CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_key_expired_at
               ON ${fullTableName} (key, expired_at) INCLUDE (is_live);
             `);
-                        // Dedicated partial index for the hot-path (WHERE key = $1 AND is_live).
-                        // Covers the uniqueness trigger, hget, hgetall, hincrbyfloat, and
-                        // the two-pass upsert — all of which use AND is_live directly.
-                        await client.query(`
-              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_key_live
-              ON ${fullTableName} (key)
-              WHERE is_live;
-            `);
-                        // status in INCLUDE (not key) so semaphore increments don't
-                        // force index key updates — allows HOT on the hottest write path.
                         await client.query(`
               CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_entity_status
-              ON ${fullTableName} (entity) INCLUDE (status);
+              ON ${fullTableName} (entity, status);
             `);
                         await client.query(`
               CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_expired_at
@@ -330,9 +303,7 @@ const KVTables = (context) => ({
               BEFORE INSERT OR UPDATE ON ${fullTableName}
               FOR EACH ROW EXECUTE PROCEDURE ${schemaName}.update_is_live();
             `);
-                        // Enforce uniqueness of live jobs via trigger.
-                        // Uses is_live partial index for the EXISTS check (existing rows
-                        // have is_live maintained by trg_update_is_live).
+                        // Create function to enforce uniqueness of live jobs
                         await client.query(`
               CREATE OR REPLACE FUNCTION ${schemaName}.enforce_live_job_uniqueness()
               RETURNS TRIGGER AS $$
@@ -342,7 +313,7 @@ const KVTables = (context) => ({
                   IF EXISTS (
                     SELECT 1 FROM ${fullTableName}
                     WHERE key = NEW.key
-                    AND is_live
+                    AND (expired_at IS NULL OR expired_at > NOW())
                     AND id <> NEW.id
                   ) THEN
                     RAISE EXCEPTION 'A live job with key % already exists.', NEW.key;
@@ -413,16 +384,14 @@ const KVTables = (context) => ({
                   FOR EACH ROW
                   EXECUTE FUNCTION ${schemaName}.update_attributes_updated_at();
             `);
-                        // Create partitions with fillfactor 70: reserves 30% page space
-                        // for HOT updates (frequent upserts in hincrbyfloat/collateLeg2Entry)
+                        // Create partitions for attributes table
                         await client.query(`
               DO $$
               BEGIN
                 FOR i IN 0..7 LOOP
                   EXECUTE format(
                     'CREATE TABLE IF NOT EXISTS ${attributesTableName}_part_%s PARTITION OF ${attributesTableName}
-                    FOR VALUES WITH (modulus 8, remainder %s)
-                    WITH (fillfactor = 70)',
+                    FOR VALUES WITH (modulus 8, remainder %s)',
                     i, i
                   );
                 END LOOP;
@@ -466,14 +435,6 @@ const KVTables = (context) => ({
                         await client.query(`
               CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_key_score_member
               ON ${fullTableName} (key, score, member);
-            `);
-                        // Partial index for the scheduler hot path (zrangebyscore).
-                        // Most entries have no expiry; this avoids the OR filter that
-                        // prevents clean index-only scans.
-                        await client.query(`
-              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_active
-              ON ${fullTableName} (key, score)
-              WHERE expiry IS NULL;
             `);
                         break;
                     default:

package/build/services/store/providers/postgres/kvtypes/hash/basic.js

@@ -292,19 +292,11 @@ function _hset(context, key, fields, options) {
             params.push(key, fields[':'], options?.entity ?? null);
         }
         else {
-            // Two-pass upsert: UPDATE existing live job, INSERT only if no
-            // live job exists. Avoids ON CONFLICT seq_scan on partitioned
-            // tables that lack a unique partial index.
+            // Update existing job or insert new one
             sql = `
-        WITH existing AS (
-          UPDATE ${targetTable}
-          SET status = $2
-          WHERE key = $1 AND is_live
-          RETURNING 1
-        )
         INSERT INTO ${targetTable} (id, key, status, entity)
-        SELECT gen_random_uuid(), $1, $2, $3
-        WHERE NOT EXISTS (SELECT 1 FROM existing)
+        VALUES (gen_random_uuid(), $1, $2, $3)
+        ON CONFLICT (key) WHERE is_live DO UPDATE SET status = EXCLUDED.status
         RETURNING 1 as count
       `;
             params.push(key, fields[':'], options?.entity ?? null);
@@ -511,20 +503,31 @@ function _hgetall(context, key) {
     const tableName = context.tableForKey(key, 'hash');
     const isJobsTableResult = (0, utils_1.isJobsTable)(tableName);
     if (isJobsTableResult) {
-        // Single CTE: reads jobs row once, then joins attributes
         const sql = `
       WITH valid_job AS (
         SELECT id, status, context
         FROM ${tableName}
         WHERE key = $1 AND is_live
+      ),
+      job_data AS (
+        SELECT 'status' AS field, status::text AS value
+        FROM ${tableName}
+        WHERE key = $1 AND is_live
+
+        UNION ALL
+
+        SELECT 'context' AS field, context::text AS value
+        FROM ${tableName}
+        WHERE key = $1 AND is_live
+      ),
+      attribute_data AS (
+        SELECT symbol || dimension AS field, value
+        FROM ${tableName}_attributes
+        WHERE job_id IN (SELECT id FROM valid_job)
       )
-      SELECT 'status' AS field, status::text AS value FROM valid_job
-      UNION ALL
-      SELECT 'context' AS field, context::text AS value FROM valid_job
+      SELECT * FROM job_data
       UNION ALL
-      SELECT symbol || dimension AS field, value
-      FROM ${tableName}_attributes
-      WHERE job_id = (SELECT id FROM valid_job);
+      SELECT * FROM attribute_data;
     `;
         return { sql, params: [key] };
     }

package/build/services/store/providers/postgres/kvtypes/zset.js

@@ -83,17 +83,6 @@ const zsetModule = (context) => ({
     _zrange(key, start, stop, facet) {
         const tableName = context.tableForKey(key, 'sorted_set');
         const selectColumns = facet === 'WITHSCORES' ? 'member, score' : 'member';
-        // Fast path: (0, -1) means "get all" — skip COUNT/ROW_NUMBER entirely
-        if (start === 0 && stop === -1) {
-            const sql = `
-        SELECT ${selectColumns}
-        FROM ${tableName}
-        WHERE key = $1 AND (expiry IS NULL OR expiry > NOW())
-        ORDER BY score ASC, member ASC;
-      `;
-            return { sql, params: [context.storageKey(key)] };
-        }
-        // General case: negative indices require COUNT for resolution
         const sql = `
       WITH total_entries AS (
         SELECT COUNT(*) - 1 AS max_index FROM ${tableName}

package/build/services/store/providers/postgres/postgres.js

@@ -475,10 +475,12 @@ class PostgresStoreService extends __1.StoreService {
         const symVals = await this.getSymbolValues();
         this.serializer.resetSymbols(symKeys, symVals, dIds);
         const hashData = this.serializer.package(state, symbolNames);
-        // ':' (status) is NOT written to jobs_attributes — jobs.status is
-        // maintained by setStatus/setStatusAndCollateGuid. The attribute row
-        // was redundant, never read, and contended on every activity.
-        delete hashData[':'];
+        if (status !== null) {
+            hashData[':'] = status.toString();
+        }
+        else {
+            delete hashData[':'];
+        }
         await this.kvsql(transaction).hset(hashKey, hashData);
         return jobId;
     }

package/build/services/stream/providers/postgres/kvtables.js

@@ -140,31 +140,21 @@ async function createTables(client, schemaName) {
     ) PARTITION BY HASH (stream_name);
   `);
     for (let i = 0; i < 8; i++) {
-        // fillfactor 70: reserves 30% page space for HOT updates (reserve/ack cycle)
         await client.query(`
       CREATE TABLE IF NOT EXISTS ${schemaName}.engine_streams_part_${i}
       PARTITION OF ${engineTable}
-      FOR VALUES WITH (modulus 8, remainder ${i})
-      WITH (fillfactor = 70);
+      FOR VALUES WITH (modulus 8, remainder ${i});
     `);
     }
-    // Dedicated dequeue index: columns match the hot-path query exactly
-    // (stream_name = $1, visible_at <= NOW(), ORDER BY id) with partial
-    // filter on reserved_at IS NULL AND expired_at IS NULL.
-    // Replaces the old active_messages index that had reserved_at as a
-    // column (redundant with the WHERE filter, displacing visible_at).
     await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_engine_streams_dequeue
-    ON ${engineTable} (stream_name, visible_at, id)
+    CREATE INDEX IF NOT EXISTS idx_engine_streams_active_messages
+    ON ${engineTable} (stream_name, reserved_at, visible_at, id)
     WHERE reserved_at IS NULL AND expired_at IS NULL;
   `);
-    // Stale-reservation recovery: covers the timed-out reservation path
-    // (reserved_at < NOW() - INTERVAL) separately from the hot path so
-    // the planner can use each index cleanly without an OR.
     await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_engine_streams_stale_reservations
-    ON ${engineTable} (stream_name, reserved_at, visible_at, id)
-    WHERE reserved_at IS NOT NULL AND expired_at IS NULL;
+    CREATE INDEX IF NOT EXISTS idx_engine_streams_message_fetch
+    ON ${engineTable} (stream_name, visible_at, id)
+    WHERE expired_at IS NULL;
   `);
     await client.query(`
     CREATE INDEX IF NOT EXISTS idx_engine_streams_expired_at
@@ -219,25 +209,21 @@ async function createTables(client, schemaName) {
     ) PARTITION BY HASH (stream_name);
   `);
     for (let i = 0; i < 8; i++) {
-        // fillfactor 70: reserves 30% page space for HOT updates (reserve/ack cycle)
         await client.query(`
       CREATE TABLE IF NOT EXISTS ${schemaName}.worker_streams_part_${i}
       PARTITION OF ${workerTable}
-      FOR VALUES WITH (modulus 8, remainder ${i})
-      WITH (fillfactor = 70);
+      FOR VALUES WITH (modulus 8, remainder ${i});
     `);
     }
-    // Dedicated dequeue index (see engine_streams comments above)
     await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_worker_streams_dequeue
-    ON ${workerTable} (stream_name, visible_at, id)
+    CREATE INDEX IF NOT EXISTS idx_worker_streams_active_messages
+    ON ${workerTable} (stream_name, reserved_at, visible_at, id)
     WHERE reserved_at IS NULL AND expired_at IS NULL;
   `);
-    // Stale-reservation recovery (see engine_streams comments above)
     await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_worker_streams_stale_reservations
-    ON ${workerTable} (stream_name, reserved_at, visible_at, id)
-    WHERE reserved_at IS NOT NULL AND expired_at IS NULL;
+    CREATE INDEX IF NOT EXISTS idx_worker_streams_message_fetch
+    ON ${workerTable} (stream_name, visible_at, id)
+    WHERE expired_at IS NULL;
   `);
     await client.query(`
     CREATE INDEX IF NOT EXISTS idx_worker_streams_expired_at

package/build/services/stream/providers/postgres/messages.js

@@ -207,17 +207,12 @@ async function fetchMessages(client, tableName, streamName, isEngine, consumerNa
             retries++;
             const batchSize = options?.batchSize || 1;
             const reservationTimeout = options?.reservationTimeout || enums_1.HMSH_RESERVATION_TIMEOUT_S;
-            // Two-pass dequeue: stale reservations first (FIFO — they have
-            // lower ids and have waited longest), then fresh messages. Split
-            // avoids an OR that prevents the planner from using partial
-            // indexes cleanly. Stale check is a fast no-op (empty index scan)
-            // when there are no timed-out reservations.
-            let res = await client.query(`UPDATE ${tableName}
+            const res = await client.query(`UPDATE ${tableName}
          SET reserved_at = NOW(), reserved_by = $3
          WHERE id IN (
            SELECT id FROM ${tableName}
            WHERE stream_name = $1
-             AND reserved_at < NOW() - INTERVAL '${reservationTimeout} seconds'
+             AND (reserved_at IS NULL OR reserved_at < NOW() - INTERVAL '${reservationTimeout} seconds')
              AND expired_at IS NULL
              AND visible_at <= NOW()
            ORDER BY id
@@ -225,22 +220,6 @@ async function fetchMessages(client, tableName, streamName, isEngine, consumerNa
            FOR UPDATE SKIP LOCKED
          )
          RETURNING ${returningClause}`, [streamName, batchSize, consumerName]);
-            // Fresh messages: unreserved, visible, not expired
-            if (res.rows.length === 0) {
-                res = await client.query(`UPDATE ${tableName}
-           SET reserved_at = NOW(), reserved_by = $3
-           WHERE id IN (
-             SELECT id FROM ${tableName}
-             WHERE stream_name = $1
-               AND reserved_at IS NULL
-               AND expired_at IS NULL
-               AND visible_at <= NOW()
-             ORDER BY id
-             LIMIT $2
-             FOR UPDATE SKIP LOCKED
-           )
-           RETURNING ${returningClause}`, [streamName, batchSize, consumerName]);
-            }
             const messages = res.rows.map((row) => {
                 const data = (0, utils_1.parseStreamMessage)(row.message);
                 const hasDefaultRetryPolicy = (row.max_retry_attempts === 3 || row.max_retry_attempts === 5) &&

package/build/services/stream/providers/postgres/procedures.js

@@ -54,16 +54,13 @@ function getCreateProceduresSQL(schemaName) {
     SET search_path = ${schemaName}, pg_temp
     AS $$
     ${STREAM_ACCESS_CHECK}
-      -- Two-pass dequeue: stale reservations first (FIFO — lower ids,
-      -- waited longest), then fresh. Split avoids OR that prevents
-      -- partial index usage. Stale check is a fast no-op when empty.
       RETURN QUERY
       UPDATE ${workerTable} ws
       SET reserved_at = NOW(), reserved_by = p_consumer_id
       WHERE ws.id IN (
         SELECT ws2.id FROM ${workerTable} ws2
         WHERE ws2.stream_name = p_stream_name
-          AND ws2.reserved_at < NOW() - (p_reservation_timeout_sec || ' seconds')::INTERVAL
+          AND (ws2.reserved_at IS NULL OR ws2.reserved_at < NOW() - (p_reservation_timeout_sec || ' seconds')::INTERVAL)
           AND ws2.expired_at IS NULL
           AND ws2.visible_at <= NOW()
         ORDER BY ws2.id
@@ -72,25 +69,6 @@ function getCreateProceduresSQL(schemaName) {
       )
       RETURNING ws.id, ws.message, ws.workflow_name, ws.max_retry_attempts,
                 ws.backoff_coefficient, ws.maximum_interval_seconds, ws.retry_attempt;
-
-      -- Fresh messages: unreserved, visible, not expired
-      IF NOT FOUND THEN
-        RETURN QUERY
-        UPDATE ${workerTable} ws
-        SET reserved_at = NOW(), reserved_by = p_consumer_id
-        WHERE ws.id IN (
-          SELECT ws2.id FROM ${workerTable} ws2
-          WHERE ws2.stream_name = p_stream_name
-            AND ws2.reserved_at IS NULL
-            AND ws2.expired_at IS NULL
-            AND ws2.visible_at <= NOW()
-          ORDER BY ws2.id
-          LIMIT p_batch_size
-          FOR UPDATE SKIP LOCKED
-        )
-        RETURNING ws.id, ws.message, ws.workflow_name, ws.max_retry_attempts,
-                  ws.backoff_coefficient, ws.maximum_interval_seconds, ws.retry_attempt;
-      END IF;
     END;
     $$;`,
         // -- worker_ack --

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.16.2",
+  "version": "0.16.4",
   "description": "Durable Workflow",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",