@hotmeshio/hotmesh 0.19.1 → 0.19.3

package/build/modules/errors.d.ts

@@ -1,6 +1,25 @@
 import { ActivityDuplex } from '../types/activity';
 import { CollationFaultType, CollationStage } from '../types/collator';
 import { DurableChildErrorType, DurableContinueAsNewErrorType, DurableProxyErrorType, DurableSleepErrorType, DurableWaitForAllErrorType, DurableWaitForErrorType } from '../types/error';
+/**
+ * Error classification for dispatcher logging.
+ *
+ * FATAL       — lease expired, invariant violation, corrupt state.
+ *               The activity must stop immediately; message is NOT acked.
+ * RETRYABLE   — transient infrastructure error (DB timeout, network).
+ *               Normal retry/backoff applies.
+ * TERMINAL    — permanent failure (user code threw, max retries exceeded).
+ *               Message is acked; job is marked failed.
+ * COLLATION   — duplicate delivery detected via GUID ledger.
+ *               Silent ack; no work needed.
+ */
+export declare enum ErrorCategory {
+    FATAL = "fatal",
+    RETRYABLE = "retryable",
+    TERMINAL = "terminal",
+    COLLATION = "collation"
+}
+export declare function classifyError(error: unknown): ErrorCategory;
 declare class GetStateError extends Error {
     jobId: string;
     code: number;
@@ -139,6 +158,13 @@ declare class GenerationalError extends Error {
 declare class ExecActivityError extends Error {
     constructor();
 }
+declare class LeaseExpiredError extends Error {
+    code: number;
+    type: string;
+    deadlineMs: number;
+    reservationTimeoutS: number;
+    constructor(deadlineMs: number, reservationTimeoutS: number);
+}
 declare class CollationError extends Error {
     status: number;
     leg: ActivityDuplex;
@@ -146,4 +172,4 @@ declare class CollationError extends Error {
     fault: CollationFaultType;
     constructor(status: number, leg: ActivityDuplex, stage: CollationStage, fault?: CollationFaultType);
 }
-export { CollationError, DurableChildError, DurableContinueAsNewError, DurableFatalError, DurableMaxedError, DurableProxyError, DurableRetryError, DurableSleepError, DurableTimeoutError, DurableWaitForAllError, DurableWaitForError, DuplicateJobError, ExecActivityError, GenerationalError, GetStateError, InactiveJobError, MapDataError, RegisterTimeoutError, SetStateError, };
+export { CollationError, DurableChildError, DurableContinueAsNewError, DurableFatalError, DurableMaxedError, DurableProxyError, DurableRetryError, DurableSleepError, DurableTimeoutError, DurableWaitForAllError, DurableWaitForError, DuplicateJobError, ExecActivityError, GenerationalError, GetStateError, InactiveJobError, LeaseExpiredError, MapDataError, RegisterTimeoutError, SetStateError, };

package/build/modules/errors.js

@@ -1,7 +1,53 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SetStateError = exports.RegisterTimeoutError = exports.MapDataError = exports.InactiveJobError = exports.GetStateError = exports.GenerationalError = exports.ExecActivityError = exports.DuplicateJobError = exports.DurableWaitForError = exports.DurableWaitForAllError = exports.DurableTimeoutError = exports.DurableSleepError = exports.DurableRetryError = exports.DurableProxyError = exports.DurableMaxedError = exports.DurableFatalError = exports.DurableContinueAsNewError = exports.DurableChildError = exports.CollationError = void 0;
+exports.SetStateError = exports.RegisterTimeoutError = exports.MapDataError = exports.LeaseExpiredError = exports.InactiveJobError = exports.GetStateError = exports.GenerationalError = exports.ExecActivityError = exports.DuplicateJobError = exports.DurableWaitForError = exports.DurableWaitForAllError = exports.DurableTimeoutError = exports.DurableSleepError = exports.DurableRetryError = exports.DurableProxyError = exports.DurableMaxedError = exports.DurableFatalError = exports.DurableContinueAsNewError = exports.DurableChildError = exports.CollationError = exports.classifyError = exports.ErrorCategory = void 0;
 const enums_1 = require("./enums");
+/**
+ * Error classification for dispatcher logging.
+ *
+ * FATAL       — lease expired, invariant violation, corrupt state.
+ *               The activity must stop immediately; message is NOT acked.
+ * RETRYABLE   — transient infrastructure error (DB timeout, network).
+ *               Normal retry/backoff applies.
+ * TERMINAL    — permanent failure (user code threw, max retries exceeded).
+ *               Message is acked; job is marked failed.
+ * COLLATION   — duplicate delivery detected via GUID ledger.
+ *               Silent ack; no work needed.
+ */
+var ErrorCategory;
+(function (ErrorCategory) {
+    ErrorCategory["FATAL"] = "fatal";
+    ErrorCategory["RETRYABLE"] = "retryable";
+    ErrorCategory["TERMINAL"] = "terminal";
+    ErrorCategory["COLLATION"] = "collation";
+})(ErrorCategory = exports.ErrorCategory || (exports.ErrorCategory = {}));
+function classifyError(error) {
+    if (error instanceof LeaseExpiredError) {
+        return ErrorCategory.FATAL;
+    }
+    if (error instanceof CollationError || error instanceof DuplicateJobError) {
+        return ErrorCategory.COLLATION;
+    }
+    if (error instanceof DurableRetryError) {
+        return ErrorCategory.RETRYABLE;
+    }
+    if (error instanceof DurableTimeoutError) {
+        return ErrorCategory.RETRYABLE;
+    }
+    if (error instanceof DurableFatalError ||
+        error instanceof DurableMaxedError) {
+        return ErrorCategory.TERMINAL;
+    }
+    if (error instanceof InactiveJobError ||
+        error instanceof GenerationalError ||
+        error instanceof GetStateError) {
+        return ErrorCategory.TERMINAL;
+    }
+    // Unknown errors default to retryable — the retry budget
+    // will promote them to terminal if they persist.
+    return ErrorCategory.RETRYABLE;
+}
+exports.classifyError = classifyError;
 class GetStateError extends Error {
     constructor(jobId) {
         super(`${jobId} Not Found`);
@@ -207,6 +253,17 @@ class ExecActivityError extends Error {
     }
 }
 exports.ExecActivityError = ExecActivityError;
+class LeaseExpiredError extends Error {
+    constructor(deadlineMs, reservationTimeoutS) {
+        super(`Activity exceeded lease deadline (${deadlineMs}ms of ${reservationTimeoutS}s reservation). ` +
+            `Aborting to prevent unauthorized writes after lease expiry.`);
+        this.type = 'LeaseExpiredError';
+        this.code = enums_1.HMSH_CODE_DURABLE_FATAL;
+        this.deadlineMs = deadlineMs;
+        this.reservationTimeoutS = reservationTimeoutS;
+    }
+}
+exports.LeaseExpiredError = LeaseExpiredError;
 class CollationError extends Error {
     constructor(status, leg, stage, fault) {
         super('collation-error');

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.19.1",
+    "version": "0.19.3",
     "description": "Durable Workflow",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/activities/activity/process.js

@@ -71,6 +71,7 @@ async function processEvent(instance, status = stream_1.StreamStatus.SUCCESS, co
         telemetry.setActivityAttributes({});
     }
     catch (error) {
+        const category = (0, errors_1.classifyError)(error);
         if (error instanceof errors_1.CollationError) {
             //FORBIDDEN: Leg1 not complete — should not occur after the fix
             //that moved setHookSignal to post-commit. If seen, it indicates
@@ -78,6 +79,7 @@ async function processEvent(instance, status = stream_1.StreamStatus.SUCCESS, co
             //retry in processWebHookEvent can attempt recovery.
             if (error.fault === collator_1.CollationFaultType.FORBIDDEN) {
                 instance.logger.warn('process-event-forbidden-retry', {
+                    category,
                     jid: instance.context.metadata.jid,
                     aid: instance.metadata.aid,
                     message: 'Leg1 not committed yet; rethrowing for stream retry',
@@ -98,6 +100,7 @@ async function processEvent(instance, status = stream_1.StreamStatus.SUCCESS, co
             collationErrorCount++;
             if (collationErrorCount === COLLATION_WARN_THRESHOLD) {
                 instance.logger.warn('process-event-collation-rate-exceeded', {
+                    category,
                     count: collationErrorCount,
                     windowMs: COLLATION_WINDOW_MS,
                     reservationTimeoutS: enums_1.HMSH_RESERVATION_TIMEOUT_S,
@@ -108,6 +111,7 @@ async function processEvent(instance, status = stream_1.StreamStatus.SUCCESS, co
                 });
             }
             instance.logger.warn(`process-event-${error.fault}-error`, {
+                category,
                 jid: instance.context.metadata.jid,
                 aid: instance.metadata.aid,
                 error,
@@ -115,20 +119,28 @@ async function processEvent(instance, status = stream_1.StreamStatus.SUCCESS, co
             return;
         }
         else if (error instanceof errors_1.InactiveJobError) {
-            instance.logger.info('process-event-inactive-job-error', { error });
+            instance.logger.info('process-event-inactive-job-error', {
+                category,
+                error,
+            });
             return;
         }
         else if (error instanceof errors_1.GenerationalError) {
             instance.logger.info('process-event-generational-job-error', {
+                category,
                 error,
             });
             return;
         }
         else if (error instanceof errors_1.GetStateError) {
-            instance.logger.info('process-event-get-job-error', { error });
+            instance.logger.info('process-event-get-job-error', {
+                category,
+                error,
+            });
             return;
         }
         instance.logger.error('activity-process-event-error', {
+            category,
             error,
             message: error.message,
             stack: error.stack,

package/build/services/engine/dispatch.js

@@ -97,6 +97,7 @@ async function dispatchAwait(instance, streamData, context) {
             // will deliver its RESULT back to the parent via the normal path.
             // Acknowledge the message so it doesn't loop.
             instance.logger.info('dispatch-await-child-exists', {
+                category: (0, errors_1.classifyError)(error),
                 childJobId: error.jobId,
                 parentJobId: streamData.metadata.jid,
                 parentDad: streamData.metadata.dad,

package/build/services/router/consumption/index.d.ts

@@ -32,6 +32,7 @@ export declare class ConsumptionManager<S extends StreamService<ProviderClient,
     private static readonly DEPTH_CHECK_INTERVAL_MS;
     private static readonly DEPTH_SCALE_UP_THRESHOLD;
     private static readonly DEPTH_SCALE_DOWN_THRESHOLD;
+    private static readonly LEASE_BUFFER_S;
     constructor(stream: S, logger: ILogger, throttleManager: ThrottleManager, errorHandler: ErrorHandler, lifecycleManager: LifecycleManager<S>, reclaimDelay: number, reclaimCount: number, appId: string, role: any, router: any, retry?: import('../../../types/stream').RetryPolicy);
     /**
      * Adjusts reservation timeout based on stream depth. Called periodically

package/build/services/router/consumption/index.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ConsumptionManager = void 0;
 const utils_1 = require("../../../modules/utils");
+const errors_1 = require("../../../modules/errors");
 const telemetry_1 = require("../telemetry");
 const config_1 = require("../config");
 const stream_1 = require("../../../types/stream");
@@ -65,7 +66,8 @@ class ConsumptionManager {
                 this.adaptiveBatchSize = Math.min(this.adaptiveBatchSize * 2, config_1.HMSH_BATCH_SIZE);
             }
             if (this.adaptiveReservationTimeout !== prevTimeout) {
-                this.stream.reservationTimeout = this.adaptiveReservationTimeout;
+                this.stream.reservationTimeout =
+                    this.adaptiveReservationTimeout + ConsumptionManager.LEASE_BUFFER_S;
                 this.logger.info('stream-reservation-timeout-adjusted', {
                     stream,
                     depth,
@@ -244,7 +246,7 @@ class ConsumptionManager {
                 enableNotifications: true,
                 notificationCallback,
                 blockTimeout: config_1.HMSH_BLOCK_TIME_MS,
-                reservationTimeout: config_1.HMSH_RESERVATION_TIMEOUT_S,
+                reservationTimeout: config_1.HMSH_RESERVATION_TIMEOUT_S + ConsumptionManager.LEASE_BUFFER_S,
             });
             // Don't block here - let the worker initialization complete
             // The notification system will handle message processing asynchronously
@@ -297,7 +299,7 @@ class ConsumptionManager {
                     messages = await this.stream.consumeMessages(stream, group, consumer, {
                         blockTimeout: streamDuration,
                         batchSize,
-                        reservationTimeout: this.adaptiveReservationTimeout,
+                        reservationTimeout: this.adaptiveReservationTimeout + ConsumptionManager.LEASE_BUFFER_S,
                         enableBackoff: true,
                         initialBackoff: config_1.INITIAL_STREAM_BACKOFF,
                         maxBackoff: config_1.MAX_STREAM_BACKOFF,
@@ -312,7 +314,7 @@ class ConsumptionManager {
                     messages = await this.stream.consumeMessages(stream, group, consumer, {
                         blockTimeout: streamDuration,
                         batchSize,
-                        reservationTimeout: this.adaptiveReservationTimeout,
+                        reservationTimeout: this.adaptiveReservationTimeout + ConsumptionManager.LEASE_BUFFER_S,
                         enableBackoff: false,
                         maxRetries: 1,
                     });
@@ -491,23 +493,63 @@ class ConsumptionManager {
             }
             return;
         }
+        // Lease deadline: the full configured reservation timeout (N).
+        // The reclaim interval is N+5s, so the deadline always fires
+        // before a reclaimant can pick up the message. This preserves
+        // the user's contract — if they set 30s, the function gets 30s.
+        const deadlineMs = this.adaptiveReservationTimeout * 1000;
         let output;
         const telemetry = new telemetry_1.RouterTelemetry(this.appId);
         try {
             telemetry.startStreamSpan(input, this.role);
-            output = await this.execStreamLeg(input, stream, id, callback.bind(this));
+            let deadlineTimer;
+            const deadlinePromise = new Promise((_, reject) => {
+                deadlineTimer = setTimeout(() => reject(new errors_1.LeaseExpiredError(deadlineMs, this.adaptiveReservationTimeout)), deadlineMs);
+            });
+            try {
+                output = await Promise.race([
+                    this.execStreamLeg(input, stream, id, callback.bind(this)),
+                    deadlinePromise,
+                ]);
+            }
+            finally {
+                clearTimeout(deadlineTimer);
+            }
             telemetry.setStreamErrorFromOutput(output);
             this.errorCount = 0;
         }
         catch (err) {
-            this.logger.error(`stream-read-one-error`, { group, stream, id, err });
+            const category = (0, errors_1.classifyError)(err);
+            if (err instanceof errors_1.LeaseExpiredError) {
+                // FATAL: lease expired — do NOT ack. The message remains in the
+                // stream for a reclaimant to pick up cleanly. Any partial writes
+                // from this consumer are idempotent via collation.
+                this.logger.error('stream-lease-expired', {
+                    category,
+                    group,
+                    stream,
+                    id,
+                    deadlineMs,
+                    reservationTimeoutS: this.adaptiveReservationTimeout,
+                    topic: input.metadata?.topic,
+                    activityId: input.metadata?.aid,
+                    jobId: input.metadata?.jid,
+                });
+                telemetry.setStreamErrorFromException(err);
+                telemetry.endStreamSpan();
+                return; // NO ack — leave for reclaimant
+            }
+            this.logger.error(`stream-read-one-error`, {
+                category,
+                group,
+                stream,
+                id,
+                err,
+            });
             telemetry.setStreamErrorFromException(err);
             output = this.errorHandler.structureUnhandledError(input, err instanceof Error ? err : new Error(String(err)));
         }
         try {
-            // When the ENGINE itself fails to process a message (e.g., schema not
-            // found, missing subscription), do NOT republish the error back to the
-            // engine stream — that creates an infinite poison loop. The engine
             // When the ENGINE encounters an infrastructure error (schema not found,
             // subscription missing — code 598), the message is permanently unprocessable.
             // Do NOT republish it — that creates an infinite poison loop. Only suppress
@@ -515,6 +557,7 @@ class ConsumptionManager {
             // duplicates, workflow failures) must still flow through normally.
             if (group === 'ENGINE' && output?.code === 598) {
                 this.logger.error(`stream-engine-dispatch-fatal`, {
+                    category: errors_1.ErrorCategory.FATAL,
                     stream, id, group,
                     aid: input.metadata?.aid,
                     jid: input.metadata?.jid,
@@ -530,6 +573,7 @@ class ConsumptionManager {
             // If publishResponse fails, still ack the message to prevent
             // infinite reprocessing. Log the error for debugging.
             this.logger.error(`stream-publish-response-error`, {
+                category: (0, errors_1.classifyError)(publishErr),
                 group, stream, id, error: publishErr,
             });
             this.errorCount++;
@@ -547,6 +591,7 @@ class ConsumptionManager {
         }
         catch (error) {
             this.logger.error(`stream-call-function-error`, {
+                category: (0, errors_1.classifyError)(error),
                 error,
                 input: input,
                 stack: error.stack,
@@ -620,4 +665,8 @@ class ConsumptionManager {
 ConsumptionManager.DEPTH_CHECK_INTERVAL_MS = 10000;
 ConsumptionManager.DEPTH_SCALE_UP_THRESHOLD = 100;
 ConsumptionManager.DEPTH_SCALE_DOWN_THRESHOLD = 10;
+// Buffer between the activity deadline (N) and the reclaim interval
+// (N+5). The function gets the full configured timeout; the extra 5s
+// ensures the deadline fires before a reclaimant can pick up the message.
+ConsumptionManager.LEASE_BUFFER_S = 5;
 exports.ConsumptionManager = ConsumptionManager;

package/build/services/stream/index.js

@@ -5,7 +5,9 @@ class StreamService {
     constructor(streamClient, storeClient, config = {}) {
         // Adaptive reservation timeout — set by the consumption manager
         // based on stream depth. Providers read this when reserving messages.
-        this.reservationTimeout = 30;
+        // Includes a +5s buffer over the activity deadline so the deadline
+        // always fires before reclaim (see ConsumptionManager.LEASE_BUFFER_S).
+        this.reservationTimeout = 35;
         this.streamClient = streamClient;
         this.storeClient = storeClient;
         this.config = config;

package/build/services/stream/providers/postgres/kvtables.js

@@ -161,17 +161,12 @@ async function ensureIndexes(client, schemaName) {
     WHERE expired_at IS NULL;
   `);
     // v0.18.0: add jid column to engine_streams for job tracing
-    const { rows: jidCol } = await client.query(`SELECT 1 FROM information_schema.columns
-     WHERE table_schema = $1 AND table_name = 'engine_streams' AND column_name = 'jid'
-     LIMIT 1`, [schemaName]);
-    if (jidCol.length === 0) {
-        await client.query(`ALTER TABLE ${engineTable} ADD COLUMN jid TEXT NOT NULL DEFAULT ''`);
-        await client.query(`
-      CREATE INDEX IF NOT EXISTS idx_engine_streams_jid_created
-      ON ${engineTable} (jid, created_at)
-      WHERE jid != '';
-    `);
-    }
+    await client.query(`ALTER TABLE ${engineTable} ADD COLUMN IF NOT EXISTS jid TEXT NOT NULL DEFAULT ''`);
+    await client.query(`
+    CREATE INDEX IF NOT EXISTS idx_engine_streams_jid_created
+    ON ${engineTable} (jid, created_at)
+    WHERE jid != '';
+  `);
 }
 async function createTables(client, schemaName) {
     await client.query(`CREATE SCHEMA IF NOT EXISTS ${schemaName};`);

package/build/services/stream/providers/postgres/messages.js

@@ -223,7 +223,7 @@ async function fetchMessages(client, tableName, streamName, isEngine, consumerNa
         while (retries < maxRetries) {
             retries++;
             const batchSize = options?.batchSize || 1;
-            const reservationTimeout = options?.reservationTimeout || enums_1.HMSH_RESERVATION_TIMEOUT_S;
+            const reservationTimeout = options?.reservationTimeout || (enums_1.HMSH_RESERVATION_TIMEOUT_S + 5);
             const res = await client.query(`UPDATE ${tableName}
          SET reserved_at = NOW(), reserved_by = $3
          WHERE id IN (

package/build/services/stream/providers/postgres/secured.js

@@ -21,7 +21,7 @@ async function fetchMessagesSecured(client, schema, streamName, consumerName, op
     const maxBackoff = options?.maxBackoff ?? 3000;
     const maxRetries = options?.maxRetries ?? 3;
     const batchSize = options?.batchSize || 1;
-    const reservationTimeout = options?.reservationTimeout || enums_1.HMSH_RESERVATION_TIMEOUT_S;
+    const reservationTimeout = options?.reservationTimeout || (enums_1.HMSH_RESERVATION_TIMEOUT_S + 5);
     let backoff = initialBackoff;
     let retries = 0;
     try {

package/build/types/postgres.d.ts

@@ -4,8 +4,10 @@ export interface PostgresClientOptions {
     user?: string;
     password?: string;
     database?: string;
+    connectionString?: string;
     max?: number;
     idleTimeoutMillis?: number;
+    ssl?: boolean | Record<string, unknown>;
 }
 export type PostgresJobEnumType = 'status' | 'jdata' | 'adata' | 'udata' | 'jmark' | 'hmark' | 'other';
 export type PostgresClassType = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.19.1",
+  "version": "0.19.3",
   "description": "Durable Workflow",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",