@hotmeshio/hotmesh 0.14.10 → 0.16.0

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.14.10",
+    "version": "0.16.0",
     "description": "Durable Workflow",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/activities/activity/state.d.ts

@@ -30,7 +30,7 @@ interface StateContext {
 export declare function setState(instance: StateContext, transaction?: ProviderTransaction): Promise<string>;
 export declare function getState(instance: StateContext): Promise<void>;
 export declare function setStatus(instance: StateContext, amount: number, transaction?: ProviderTransaction): Promise<void | any>;
-export declare function bindJobMetadata(instance: StateContext): void;
+export declare function bindJobMetadata(_instance: StateContext): void;
 export declare function bindActivityMetadata(instance: StateContext): void;
 export declare function bindJobState(instance: StateContext, state: StringAnyType): Promise<void>;
 export declare function bindActivityState(instance: StateContext, state: StringAnyType): void;

package/build/services/activities/activity/state.js

@@ -68,8 +68,9 @@ async function setStatus(instance, amount, transaction) {
 }
 exports.setStatus = setStatus;
 //─── metadata binding ────────────────────────────────────────────────
-function bindJobMetadata(instance) {
-    instance.context.metadata.ju = (0, utils_1.formatISODate)(new Date());
+function bindJobMetadata(_instance) {
+    // ju (job_updated) is maintained by the jobs.updated_at trigger —
+    // no need to serialize it into jobs_attributes on every activity.
 }
 exports.bindJobMetadata = bindJobMetadata;
 function bindActivityMetadata(instance) {

package/build/services/durable/exporter.js

@@ -676,9 +676,23 @@ class ExporterService {
         const metadata = state?.output?.metadata ?? state?.metadata;
         const stateData = state?.output?.data ?? state?.data;
         const jobCreated = metadata?.jc ?? stateData?.jc ?? metadata?.ac;
-        const jobUpdated = metadata?.ju ?? stateData?.ju ?? metadata?.au;
         const startTime = parseTimestamp(jobCreated);
-        const closeTime = parseTimestamp(jobUpdated);
+        // Derive close time from the latest activity update (au) in the
+        // timeline, falling back to metadata.au. ju is no longer updated
+        // after creation — jobs.updated_at is the authoritative source,
+        // but it's not available in the DurableJobExport format.
+        let latestAu = null;
+        for (const entry of raw.timeline || []) {
+            const val = entry.value;
+            const au = val?.au;
+            if (au) {
+                const parsed = parseTimestamp(au);
+                if (parsed && (!latestAu || parsed > latestAu)) {
+                    latestAu = parsed;
+                }
+            }
+        }
+        const closeTime = latestAu ?? parseTimestamp(metadata?.au);
         // ── Synthetic workflow_execution_started ─────────────────────────
         if (startTime) {
             events.push(makeEvent(nextId++, 'workflow_execution_started', 'workflow', startTime, null, false, {

package/build/services/pipe/functions/cron.js

@@ -31,7 +31,7 @@ class CronHandler {
      */
     nextDelay(cronExpression) {
         if (!(0, utils_1.isValidCron)(cronExpression)) {
-            throw new Error(`Invalid cron expression: ${cronExpression}`);
+            return -1;
         }
         const interval = (0, cron_parser_1.parseExpression)(cronExpression, { utc: true });
         const nextDate = interval.next().toDate();

package/build/services/serializer/index.js

@@ -35,7 +35,7 @@ exports.MDATA_SYMBOLS = {
         ],
     },
     JOB_UPDATE: {
-        KEYS: ['ju', 'err'],
+        KEYS: ['err'],
     },
 };
 class SerializerService {

package/build/services/store/providers/postgres/kvtables.js

@@ -153,6 +153,20 @@ const KVTables = (context) => ({
         FOR EACH ROW EXECUTE FUNCTION ${schemaName}.update_jobs_updated_at();
       `);
         }
+        // v0.15.2: add dedicated is_live partial index for hot-path queries
+        const { rows: idxRows } = await client.query(`SELECT 1 FROM pg_indexes WHERE indexname = 'idx_jobs_key_live' AND schemaname = $1 LIMIT 1`, [schemaName]);
+        if (idxRows.length === 0) {
+            await client.query(`
+        CREATE INDEX IF NOT EXISTS idx_jobs_key_live
+        ON ${jobsTable} (key) WHERE is_live;
+      `);
+        }
+        // v0.15.2: partial index for sorted_set scheduler hot path
+        await client.query(`
+      CREATE INDEX IF NOT EXISTS idx_task_schedules_active
+      ON ${schemaName}.task_schedules (key, score)
+      WHERE expiry IS NULL;
+    `);
     },
     async createTables(client, appName) {
         try {
@@ -251,27 +265,40 @@ const KVTables = (context) => ({
               CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_context_gin 
               ON ${fullTableName} USING GIN (context);
             `);
-                        // Create partitions using a DO block
+                        // Create partitions with fillfactor 70: reserves 30% page space
+                        // for HOT updates (status update cycle in setStatusAndCollateGuid)
                         await client.query(`
               DO $$
               BEGIN
                 FOR i IN 0..7 LOOP
                   EXECUTE format(
-                    'CREATE TABLE IF NOT EXISTS ${fullTableName}_part_%s PARTITION OF ${fullTableName} 
-                    FOR VALUES WITH (modulus 8, remainder %s)',
+                    'CREATE TABLE IF NOT EXISTS ${fullTableName}_part_%s PARTITION OF ${fullTableName}
+                    FOR VALUES WITH (modulus 8, remainder %s)
+                    WITH (fillfactor = 70)',
                     i, i
                   );
                 END LOOP;
               END$$;
             `);
-                        // Create optimized indexes
+                        // Original index for queries that filter on expired_at directly
+                        // (e.g. _hmget's WHERE expired_at IS NULL OR expired_at > NOW())
                         await client.query(`
-              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_key_expired_at 
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_key_expired_at
               ON ${fullTableName} (key, expired_at) INCLUDE (is_live);
             `);
+                        // Dedicated partial index for the hot-path (WHERE key = $1 AND is_live).
+                        // Covers the uniqueness trigger, hget, hgetall, hincrbyfloat, and
+                        // the two-pass upsert — all of which use AND is_live directly.
                         await client.query(`
-              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_entity_status 
-              ON ${fullTableName} (entity, status);
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_key_live
+              ON ${fullTableName} (key)
+              WHERE is_live;
+            `);
+                        // status in INCLUDE (not key) so semaphore increments don't
+                        // force index key updates — allows HOT on the hottest write path.
+                        await client.query(`
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_entity_status
+              ON ${fullTableName} (entity) INCLUDE (status);
             `);
                         await client.query(`
               CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_expired_at
@@ -303,7 +330,9 @@ const KVTables = (context) => ({
               BEFORE INSERT OR UPDATE ON ${fullTableName}
               FOR EACH ROW EXECUTE PROCEDURE ${schemaName}.update_is_live();
             `);
-                        // Create function to enforce uniqueness of live jobs
+                        // Enforce uniqueness of live jobs via trigger.
+                        // Uses is_live partial index for the EXISTS check (existing rows
+                        // have is_live maintained by trg_update_is_live).
                         await client.query(`
               CREATE OR REPLACE FUNCTION ${schemaName}.enforce_live_job_uniqueness()
               RETURNS TRIGGER AS $$
@@ -312,8 +341,8 @@ const KVTables = (context) => ({
                   PERFORM pg_advisory_xact_lock(hashtextextended(NEW.key, 0));
                   IF EXISTS (
                     SELECT 1 FROM ${fullTableName}
-                    WHERE key = NEW.key 
-                    AND (expired_at IS NULL OR expired_at > NOW())
+                    WHERE key = NEW.key
+                    AND is_live
                     AND id <> NEW.id
                   ) THEN
                     RAISE EXCEPTION 'A live job with key % already exists.', NEW.key;
@@ -384,14 +413,16 @@ const KVTables = (context) => ({
                   FOR EACH ROW
                   EXECUTE FUNCTION ${schemaName}.update_attributes_updated_at();
             `);
-                        // Create partitions for attributes table
+                        // Create partitions with fillfactor 70: reserves 30% page space
+                        // for HOT updates (frequent upserts in hincrbyfloat/collateLeg2Entry)
                         await client.query(`
               DO $$
               BEGIN
                 FOR i IN 0..7 LOOP
                   EXECUTE format(
-                    'CREATE TABLE IF NOT EXISTS ${attributesTableName}_part_%s PARTITION OF ${attributesTableName} 
-                    FOR VALUES WITH (modulus 8, remainder %s)',
+                    'CREATE TABLE IF NOT EXISTS ${attributesTableName}_part_%s PARTITION OF ${attributesTableName}
+                    FOR VALUES WITH (modulus 8, remainder %s)
+                    WITH (fillfactor = 70)',
                     i, i
                   );
                 END LOOP;
@@ -433,8 +464,16 @@ const KVTables = (context) => ({
               );
             `);
                         await client.query(`
-              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_key_score_member 
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_key_score_member
               ON ${fullTableName} (key, score, member);
+            `);
+                        // Partial index for the scheduler hot path (zrangebyscore).
+                        // Most entries have no expiry; this avoids the OR filter that
+                        // prevents clean index-only scans.
+                        await client.query(`
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_active
+              ON ${fullTableName} (key, score)
+              WHERE expiry IS NULL;
             `);
                         break;
                     default:

package/build/services/store/providers/postgres/kvtypes/hash/basic.js

@@ -292,11 +292,19 @@ function _hset(context, key, fields, options) {
             params.push(key, fields[':'], options?.entity ?? null);
         }
         else {
-            // Update existing job or insert new one
+            // Two-pass upsert: UPDATE existing live job, INSERT only if no
+            // live job exists. Avoids ON CONFLICT seq_scan on partitioned
+            // tables that lack a unique partial index.
             sql = `
+        WITH existing AS (
+          UPDATE ${targetTable}
+          SET status = $2
+          WHERE key = $1 AND is_live
+          RETURNING 1
+        )
         INSERT INTO ${targetTable} (id, key, status, entity)
-        VALUES (gen_random_uuid(), $1, $2, $3)
-        ON CONFLICT (key) WHERE is_live DO UPDATE SET status = EXCLUDED.status
+        SELECT gen_random_uuid(), $1, $2, $3
+        WHERE NOT EXISTS (SELECT 1 FROM existing)
         RETURNING 1 as count
       `;
             params.push(key, fields[':'], options?.entity ?? null);
@@ -503,31 +511,20 @@ function _hgetall(context, key) {
     const tableName = context.tableForKey(key, 'hash');
     const isJobsTableResult = (0, utils_1.isJobsTable)(tableName);
     if (isJobsTableResult) {
+        // Single CTE: reads jobs row once, then joins attributes
         const sql = `
       WITH valid_job AS (
         SELECT id, status, context
         FROM ${tableName}
         WHERE key = $1 AND is_live
-      ),
-      job_data AS (
-        SELECT 'status' AS field, status::text AS value
-        FROM ${tableName}
-        WHERE key = $1 AND is_live
-
-        UNION ALL
-
-        SELECT 'context' AS field, context::text AS value
-        FROM ${tableName}
-        WHERE key = $1 AND is_live
-      ),
-      attribute_data AS (
-        SELECT symbol || dimension AS field, value
-        FROM ${tableName}_attributes
-        WHERE job_id IN (SELECT id FROM valid_job)
       )
-      SELECT * FROM job_data
+      SELECT 'status' AS field, status::text AS value FROM valid_job
+      UNION ALL
+      SELECT 'context' AS field, context::text AS value FROM valid_job
       UNION ALL
-      SELECT * FROM attribute_data;
+      SELECT symbol || dimension AS field, value
+      FROM ${tableName}_attributes
+      WHERE job_id = (SELECT id FROM valid_job);
     `;
         return { sql, params: [key] };
     }

package/build/services/store/providers/postgres/kvtypes/zset.js

@@ -83,6 +83,17 @@ const zsetModule = (context) => ({
     _zrange(key, start, stop, facet) {
         const tableName = context.tableForKey(key, 'sorted_set');
         const selectColumns = facet === 'WITHSCORES' ? 'member, score' : 'member';
+        // Fast path: (0, -1) means "get all" — skip COUNT/ROW_NUMBER entirely
+        if (start === 0 && stop === -1) {
+            const sql = `
+        SELECT ${selectColumns}
+        FROM ${tableName}
+        WHERE key = $1 AND (expiry IS NULL OR expiry > NOW())
+        ORDER BY score ASC, member ASC;
+      `;
+            return { sql, params: [context.storageKey(key)] };
+        }
+        // General case: negative indices require COUNT for resolution
         const sql = `
       WITH total_entries AS (
         SELECT COUNT(*) - 1 AS max_index FROM ${tableName}

package/build/services/store/providers/postgres/postgres.js

@@ -475,12 +475,10 @@ class PostgresStoreService extends __1.StoreService {
         const symVals = await this.getSymbolValues();
         this.serializer.resetSymbols(symKeys, symVals, dIds);
         const hashData = this.serializer.package(state, symbolNames);
-        if (status !== null) {
-            hashData[':'] = status.toString();
-        }
-        else {
-            delete hashData[':'];
-        }
+        // ':' (status) is NOT written to jobs_attributes — jobs.status is
+        // maintained by setStatus/setStatusAndCollateGuid. The attribute row
+        // was redundant, never read, and contended on every activity.
+        delete hashData[':'];
         await this.kvsql(transaction).hset(hashKey, hashData);
         return jobId;
     }

package/build/services/stream/providers/postgres/kvtables.js

@@ -140,21 +140,31 @@ async function createTables(client, schemaName) {
     ) PARTITION BY HASH (stream_name);
   `);
     for (let i = 0; i < 8; i++) {
+        // fillfactor 70: reserves 30% page space for HOT updates (reserve/ack cycle)
         await client.query(`
       CREATE TABLE IF NOT EXISTS ${schemaName}.engine_streams_part_${i}
       PARTITION OF ${engineTable}
-      FOR VALUES WITH (modulus 8, remainder ${i});
+      FOR VALUES WITH (modulus 8, remainder ${i})
+      WITH (fillfactor = 70);
     `);
     }
+    // Dedicated dequeue index: columns match the hot-path query exactly
+    // (stream_name = $1, visible_at <= NOW(), ORDER BY id) with partial
+    // filter on reserved_at IS NULL AND expired_at IS NULL.
+    // Replaces the old active_messages index that had reserved_at as a
+    // column (redundant with the WHERE filter, displacing visible_at).
     await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_engine_streams_active_messages
-    ON ${engineTable} (stream_name, reserved_at, visible_at, id)
+    CREATE INDEX IF NOT EXISTS idx_engine_streams_dequeue
+    ON ${engineTable} (stream_name, visible_at, id)
     WHERE reserved_at IS NULL AND expired_at IS NULL;
   `);
+    // Stale-reservation recovery: covers the timed-out reservation path
+    // (reserved_at < NOW() - INTERVAL) separately from the hot path so
+    // the planner can use each index cleanly without an OR.
     await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_engine_streams_message_fetch
-    ON ${engineTable} (stream_name, visible_at, id)
-    WHERE expired_at IS NULL;
+    CREATE INDEX IF NOT EXISTS idx_engine_streams_stale_reservations
+    ON ${engineTable} (stream_name, reserved_at, visible_at, id)
+    WHERE reserved_at IS NOT NULL AND expired_at IS NULL;
   `);
     await client.query(`
     CREATE INDEX IF NOT EXISTS idx_engine_streams_expired_at
@@ -209,21 +219,25 @@ async function createTables(client, schemaName) {
     ) PARTITION BY HASH (stream_name);
   `);
     for (let i = 0; i < 8; i++) {
+        // fillfactor 70: reserves 30% page space for HOT updates (reserve/ack cycle)
         await client.query(`
       CREATE TABLE IF NOT EXISTS ${schemaName}.worker_streams_part_${i}
       PARTITION OF ${workerTable}
-      FOR VALUES WITH (modulus 8, remainder ${i});
+      FOR VALUES WITH (modulus 8, remainder ${i})
+      WITH (fillfactor = 70);
     `);
     }
+    // Dedicated dequeue index (see engine_streams comments above)
     await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_worker_streams_active_messages
-    ON ${workerTable} (stream_name, reserved_at, visible_at, id)
+    CREATE INDEX IF NOT EXISTS idx_worker_streams_dequeue
+    ON ${workerTable} (stream_name, visible_at, id)
     WHERE reserved_at IS NULL AND expired_at IS NULL;
   `);
+    // Stale-reservation recovery (see engine_streams comments above)
     await client.query(`
-    CREATE INDEX IF NOT EXISTS idx_worker_streams_message_fetch
-    ON ${workerTable} (stream_name, visible_at, id)
-    WHERE expired_at IS NULL;
+    CREATE INDEX IF NOT EXISTS idx_worker_streams_stale_reservations
+    ON ${workerTable} (stream_name, reserved_at, visible_at, id)
+    WHERE reserved_at IS NOT NULL AND expired_at IS NULL;
   `);
     await client.query(`
     CREATE INDEX IF NOT EXISTS idx_worker_streams_expired_at

package/build/services/stream/providers/postgres/messages.js

@@ -207,12 +207,17 @@ async function fetchMessages(client, tableName, streamName, isEngine, consumerNa
             retries++;
             const batchSize = options?.batchSize || 1;
             const reservationTimeout = options?.reservationTimeout || enums_1.HMSH_RESERVATION_TIMEOUT_S;
-            const res = await client.query(`UPDATE ${tableName}
+            // Two-pass dequeue: stale reservations first (FIFO — they have
+            // lower ids and have waited longest), then fresh messages. Split
+            // avoids an OR that prevents the planner from using partial
+            // indexes cleanly. Stale check is a fast no-op (empty index scan)
+            // when there are no timed-out reservations.
+            let res = await client.query(`UPDATE ${tableName}
          SET reserved_at = NOW(), reserved_by = $3
          WHERE id IN (
            SELECT id FROM ${tableName}
            WHERE stream_name = $1
-             AND (reserved_at IS NULL OR reserved_at < NOW() - INTERVAL '${reservationTimeout} seconds')
+             AND reserved_at < NOW() - INTERVAL '${reservationTimeout} seconds'
              AND expired_at IS NULL
              AND visible_at <= NOW()
            ORDER BY id
@@ -220,6 +225,22 @@ async function fetchMessages(client, tableName, streamName, isEngine, consumerNa
            FOR UPDATE SKIP LOCKED
          )
          RETURNING ${returningClause}`, [streamName, batchSize, consumerName]);
+            // Fresh messages: unreserved, visible, not expired
+            if (res.rows.length === 0) {
+                res = await client.query(`UPDATE ${tableName}
+           SET reserved_at = NOW(), reserved_by = $3
+           WHERE id IN (
+             SELECT id FROM ${tableName}
+             WHERE stream_name = $1
+               AND reserved_at IS NULL
+               AND expired_at IS NULL
+               AND visible_at <= NOW()
+             ORDER BY id
+             LIMIT $2
+             FOR UPDATE SKIP LOCKED
+           )
+           RETURNING ${returningClause}`, [streamName, batchSize, consumerName]);
+            }
             const messages = res.rows.map((row) => {
                 const data = (0, utils_1.parseStreamMessage)(row.message);
                 const hasDefaultRetryPolicy = (row.max_retry_attempts === 3 || row.max_retry_attempts === 5) &&

package/build/services/stream/providers/postgres/procedures.js

@@ -54,13 +54,16 @@ function getCreateProceduresSQL(schemaName) {
     SET search_path = ${schemaName}, pg_temp
     AS $$
     ${STREAM_ACCESS_CHECK}
+      -- Two-pass dequeue: stale reservations first (FIFO — lower ids,
+      -- waited longest), then fresh. Split avoids OR that prevents
+      -- partial index usage. Stale check is a fast no-op when empty.
       RETURN QUERY
       UPDATE ${workerTable} ws
       SET reserved_at = NOW(), reserved_by = p_consumer_id
       WHERE ws.id IN (
         SELECT ws2.id FROM ${workerTable} ws2
         WHERE ws2.stream_name = p_stream_name
-          AND (ws2.reserved_at IS NULL OR ws2.reserved_at < NOW() - (p_reservation_timeout_sec || ' seconds')::INTERVAL)
+          AND ws2.reserved_at < NOW() - (p_reservation_timeout_sec || ' seconds')::INTERVAL
           AND ws2.expired_at IS NULL
           AND ws2.visible_at <= NOW()
         ORDER BY ws2.id
@@ -69,6 +72,25 @@ function getCreateProceduresSQL(schemaName) {
       )
       RETURNING ws.id, ws.message, ws.workflow_name, ws.max_retry_attempts,
                 ws.backoff_coefficient, ws.maximum_interval_seconds, ws.retry_attempt;
+
+      -- Fresh messages: unreserved, visible, not expired
+      IF NOT FOUND THEN
+        RETURN QUERY
+        UPDATE ${workerTable} ws
+        SET reserved_at = NOW(), reserved_by = p_consumer_id
+        WHERE ws.id IN (
+          SELECT ws2.id FROM ${workerTable} ws2
+          WHERE ws2.stream_name = p_stream_name
+            AND ws2.reserved_at IS NULL
+            AND ws2.expired_at IS NULL
+            AND ws2.visible_at <= NOW()
+          ORDER BY ws2.id
+          LIMIT p_batch_size
+          FOR UPDATE SKIP LOCKED
+        )
+        RETURNING ws.id, ws.message, ws.workflow_name, ws.max_retry_attempts,
+                  ws.backoff_coefficient, ws.maximum_interval_seconds, ws.retry_attempt;
+      END IF;
     END;
     $$;`,
         // -- worker_ack --

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.14.10",
+  "version": "0.16.0",
   "description": "Durable Workflow",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",