@hotmeshio/hotmesh 0.14.6 → 0.14.8

package/README.md

@@ -29,34 +29,17 @@ Install the package:
 npm install @hotmeshio/hotmesh
 ```
 
-The repo includes a `docker-compose.yml` that starts Postgres, NATS, and a development container:
+The repo includes a `docker-compose.yml` that starts Postgres and a development container:
 
 ```bash
 docker compose up -d
 ```
 
-Then follow the [Quick Start guide](https://github.com/hotmeshio/sdk-typescript/blob/main/docs/quickstart.md) for a progressive walkthrough — from a single trigger to conditional, parallel, and compositional workflows.
+See the [Durable API reference](https://docs.hotmesh.io/classes/services_durable.Durable.html) for the full API surface — workflows, activities, signals, child workflows, and more.
 
-## Two ways to write workflows
+## Writing workflows
 
-Both approaches reuse your activity functions:
-
-```typescript
-// activities.ts (shared between both approaches)
-export async function checkInventory(itemId: string): Promise<number> {
-  return getInventoryCount(itemId);
-}
-
-export async function reserveItem(itemId: string, quantity: number): Promise<string> {
-  return createReservation(itemId, quantity);
-}
-
-export async function notifyBackorder(itemId: string): Promise<void> {
-  await sendBackorderEmail(itemId);
-}
-```
-
-### Option 1: Code
+**Define the workflow** — plain TypeScript with branching, loops, and error handling. Activities are proxied so their results are checkpointed and replayed on restart.
 
 ```typescript
 // workflows.ts
@@ -76,124 +59,72 @@ export async function orderWorkflow(itemId: string, qty: number) {
     return 'backordered';
   }
 }
+```
 
-// main.ts
-import * as activities from './activities';
+**Start a worker** — connects to Postgres and begins processing workflows on the given task queue.
+
+```typescript
+// worker.ts
+import { Durable } from '@hotmeshio/hotmesh';
+import { Client as Postgres } from 'pg';
+import { orderWorkflow } from './workflows';
 
 const connection = {
   class: Postgres,
   options: { connectionString: 'postgresql://localhost:5432/mydb' }
 };
 
-await Durable.Worker.create({
+const worker = await Durable.Worker.create({
   connection,
   taskQueue: 'orders',
   workflow: orderWorkflow,
-  activities,
 });
 
+await worker.run();
+```
+
+**Run a workflow** — start an execution and await its result. The client can run in a different process, container, or server.
+
+```typescript
+// client.ts
+import { Durable } from '@hotmeshio/hotmesh';
+import { Client as Postgres } from 'pg';
+
+const connection = {
+  class: Postgres,
+  options: { connectionString: 'postgresql://localhost:5432/mydb' }
+};
+
 const client = new Durable.Client({ connection });
 const handle = await client.workflow.start({
   args: ['item-123', 5],
   taskQueue: 'orders',
   workflowName: 'orderWorkflow',
-  workflowId: 'order-456'
+  workflowId: 'order-456',
 });
 
 const result = await handle.result();
 ```
 
-### Option 2: YAML (functional approach)
-
-```yaml
-# order.yaml
-activities:
-  trigger:
-    type: trigger
-
-  checkInventory:
-    type: worker
-    topic: inventory.check
-
-  reserveItem:
-    type: worker
-    topic: inventory.reserve
-
-  notifyBackorder:
-    type: worker
-    topic: inventory.backorder.notify
-
-transitions:
-  trigger:
-    - to: checkInventory
-
-  checkInventory:
-    - to: reserveItem
-      conditions:
-        match:
-          - expected: true
-            actual:
-              '@pipe':
-                - ['{checkInventory.output.data.availableQty}', '{trigger.output.data.requestedQty}']
-                - ['{@conditional.gte}']
-
-    - to: notifyBackorder
-      conditions:
-        match:
-          - expected: false
-            actual:
-              '@pipe':
-                - ['{checkInventory.output.data.availableQty}', '{trigger.output.data.requestedQty}']
-                - ['{@conditional.gte}']
-```
+### Activities
 
-Deploy and run as follows:
-```typescript
-// main.ts (reuses same activities.ts)
-import * as activities from './activities';
+Activities are your side-effectful functions — database calls, API requests, anything non-deterministic. HotMesh checkpoints their results so they're never re-executed on replay.
 
-const hotMesh = await HotMesh.init({
-  appId: 'orders',
-  engine: { connection },
-  workers: [
-    {
-      topic: 'inventory.check',
-      connection,
-      callback: async (data) => {
-        const availableQty = await activities.checkInventory(data.data.itemId);
-        return { metadata: { ...data.metadata }, data: { availableQty } };
-      }
-    },
-    {
-      topic: 'inventory.reserve',
-      connection,
-      callback: async (data) => {
-        const reservationId = await activities.reserveItem(data.data.itemId, data.data.quantity);
-        return { metadata: { ...data.metadata }, data: { reservationId } };
-      }
-    },
-    {
-      topic: 'inventory.backorder.notify',
-      connection,
-      callback: async (data) => {
-        await activities.notifyBackorder(data.data.itemId);
-        return { metadata: { ...data.metadata } };
-      }
-    }
-  ]
-});
+```typescript
+// activities.ts
+export async function checkInventory(itemId: string): Promise<number> {
+  return getInventoryCount(itemId);
+}
 
-await hotMesh.deploy('./order.yaml');
-await hotMesh.activate('1');
+export async function reserveItem(itemId: string, quantity: number): Promise<string> {
+  return createReservation(itemId, quantity);
+}
 
-const result = await hotMesh.pubsub('order.requested', {
-  itemId: 'item-123',
-  requestedQty: 5
-});
+export async function notifyBackorder(itemId: string): Promise<void> {
+  await sendBackorderEmail(itemId);
+}
 ```
 
-Both compile to the same distributed execution model.
-
 ## Common patterns
 
 All snippets below run inside a workflow function (like `orderWorkflow` above). Durable methods are available as static imports:
@@ -321,6 +252,12 @@ There is no proprietary dashboard. Workflow state lives in Postgres, so use what
 - **Logging** — set `HMSH_LOGLEVEL` (`debug`, `info`, `warn`, `error`, `silent`) to control log verbosity.
 - **OpenTelemetry** — set `HMSH_TELEMETRY=true` to emit spans and metrics. Plug in any OTel-compatible collector.
 
+## YAML workflows
+
+HotMesh also supports a declarative YAML syntax. The same activities run in both modes — the difference is compilation speed. YAML workflows compile ~10x faster because the execution graph is declared upfront rather than discovered through replay. The tradeoff is expressiveness: YAML uses a functional pipe syntax for conditions and transformations instead of native TypeScript control flow.
+
+See the [Quick Start guide](https://github.com/hotmeshio/sdk-typescript/blob/main/docs/quickstart.md) for YAML examples and the `tests/functional/` directory for working implementations.
+
 ## Architecture
 
 For a deep dive into the transactional execution model — how every step is crash-safe, how the monotonic collation ledger guarantees exactly-once delivery, and how cycles and retries remain correct under arbitrary failure — see the [Collation Design Document](https://github.com/hotmeshio/sdk-typescript/blob/main/services/collator/README.md). The symbolic system (how to design workflows) and lifecycle details (how to deploy workflows) are covered in the [Architectural Overview](https://zenodo.org/records/12168558).

package/build/modules/enums.d.ts

@@ -162,6 +162,21 @@ export declare const HMSH_XCLAIM_DELAY_MS: number;
 export declare const HMSH_XCLAIM_COUNT: number;
 export declare const HMSH_XPENDING_COUNT: number;
 export declare const HMSH_BATCH_SIZE: number;
+/**
+ * Minimum batch size under adaptive scaling (default: 1).
+ *
+ * When stream depth is high, the adaptive logic reduces batch size
+ * to relieve back-pressure. This value is the floor — the smallest
+ * batch the system will fetch per consume cycle.
+ *
+ *   - 1 (default): fully serial under max stress, safest
+ *   - 2: retains some parallelism while limiting contention
+ *
+ * Both values produce equivalent throughput in practice (~233s for
+ * 1000 concurrent workflows). The reduction from the configured
+ * HMSH_BATCH_SIZE is what matters most — the floor is a safety net.
+ */
+export declare const HMSH_BATCH_SIZE_MIN: number;
 /**
  * Postgres stream reservation timeout in seconds (default: 30).
  *
@@ -197,6 +212,32 @@ export declare const HMSH_BATCH_SIZE: number;
  * HMSH_RESERVATION_TIMEOUT_S=30  (default)
  */
 export declare const HMSH_RESERVATION_TIMEOUT_S: number;
+/**
+ * Maximum reservation timeout in seconds for adaptive scaling (default: 1800).
+ *
+ * This is the ceiling for the adaptive reservation timeout — how far the
+ * system is allowed to stretch under sustained load. The adaptive logic
+ * only uses what it needs based on stream depth; this value defines the
+ * upper bound, not the steady state.
+ *
+ * The tradeoff is recovery time after a consumer crash: if a consumer
+ * reserves a message and dies, that message is unavailable until the
+ * timeout expires. A higher ceiling means longer recovery from crashes
+ * but prevents duplicate delivery under heavy sustained load.
+ *
+ * In practice, crashes are rare and the delay is bounded. The cost of
+ * a ceiling that is too low — duplicate delivery, collation errors,
+ * wasted CPU, workflow stalls — is far higher than a slightly longer
+ * recovery window after a crash.
+ *
+ * **Tuning guidance:**
+ *   - Dedicated infrastructure with ample CPU: lower ceiling is fine (600s)
+ *   - Shared/multi-tenant or CPU-constrained: use the default (1800s)
+ *   - Long-running batch imports or large workflow graphs: increase (3600s+)
+ *   - Cloud deployments without CPU contention: the adaptive logic will
+ *     naturally stay near the starting timeout and rarely approach the ceiling
+ */
+export declare const HMSH_RESERVATION_TIMEOUT_MAX_S: number;
 export declare const HMSH_EXPIRE_DURATION: number;
 export declare const HMSH_FIDELITY_SECONDS: number;
 export declare const HMSH_SCOUT_INTERVAL_SECONDS: number;

package/build/modules/enums.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.HMSH_SCOUT_INTERVAL_SECONDS = exports.HMSH_FIDELITY_SECONDS = exports.HMSH_EXPIRE_DURATION = exports.HMSH_RESERVATION_TIMEOUT_S = exports.HMSH_BATCH_SIZE = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_BLOCK_TIME_MS = exports.HMSH_DURABLE_INITIAL_INTERVAL = exports.HMSH_DURABLE_EXP_BACKOFF = exports.HMSH_DURABLE_MAX_INTERVAL = exports.HMSH_DURABLE_MAX_ATTEMPTS = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_POISON_MESSAGE_THRESHOLD = exports.HMSH_MAX_RETRIES = exports.MAX_DELAY = exports.MAX_STREAM_RETRIES = exports.INITIAL_STREAM_BACKOFF = exports.MAX_STREAM_BACKOFF = exports.HMSH_EXPIRE_JOB_SECONDS = exports.HMSH_OTT_WAIT_TIME = exports.HMSH_DEPLOYMENT_PAUSE = exports.HMSH_DEPLOYMENT_DELAY = exports.HMSH_ACTIVATION_MAX_RETRY = exports.HMSH_QUORUM_DELAY_MS = exports.HMSH_QUORUM_ROLLCALL_CYCLES = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_DURABLE_RETRYABLE = exports.HMSH_CODE_DURABLE_FATAL = exports.HMSH_CODE_DURABLE_MAXED = exports.HMSH_CODE_DURABLE_TIMEOUT = exports.HMSH_CODE_DURABLE_WAIT = exports.HMSH_CODE_DURABLE_CONTINUE = exports.HMSH_CODE_DURABLE_PROXY = exports.HMSH_CODE_DURABLE_CHILD = exports.HMSH_CODE_DURABLE_ALL = exports.HMSH_CODE_DURABLE_SLEEP = exports.HMSH_CODE_UNACKED = exports.HMSH_CODE_TIMEOUT = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_INTERRUPT = exports.HMSH_CODE_NOTFOUND = exports.HMSH_CODE_PENDING = exports.HMSH_CODE_SUCCESS = exports.HMSH_PENDING_SIGNAL_EXPIRE = exports.HMSH_SIGNAL_EXPIRE = exports.HMSH_TELEMETRY = exports.HMSH_LOGLEVEL = void 0;
-exports.HMSH_ROUTER_POLL_FALLBACK_INTERVAL = exports.HMSH_NOTIFY_PAYLOAD_LIMIT = exports.DEFAULT_TASK_QUEUE = exports.HMSH_GUID_SIZE = exports.HMSH_ROUTER_SCOUT_INTERVAL_MS = exports.HMSH_ROUTER_SCOUT_INTERVAL_SECONDS = void 0;
+exports.HMSH_EXPIRE_DURATION = exports.HMSH_RESERVATION_TIMEOUT_MAX_S = exports.HMSH_RESERVATION_TIMEOUT_S = exports.HMSH_BATCH_SIZE_MIN = exports.HMSH_BATCH_SIZE = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_BLOCK_TIME_MS = exports.HMSH_DURABLE_INITIAL_INTERVAL = exports.HMSH_DURABLE_EXP_BACKOFF = exports.HMSH_DURABLE_MAX_INTERVAL = exports.HMSH_DURABLE_MAX_ATTEMPTS = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_POISON_MESSAGE_THRESHOLD = exports.HMSH_MAX_RETRIES = exports.MAX_DELAY = exports.MAX_STREAM_RETRIES = exports.INITIAL_STREAM_BACKOFF = exports.MAX_STREAM_BACKOFF = exports.HMSH_EXPIRE_JOB_SECONDS = exports.HMSH_OTT_WAIT_TIME = exports.HMSH_DEPLOYMENT_PAUSE = exports.HMSH_DEPLOYMENT_DELAY = exports.HMSH_ACTIVATION_MAX_RETRY = exports.HMSH_QUORUM_DELAY_MS = exports.HMSH_QUORUM_ROLLCALL_CYCLES = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_DURABLE_RETRYABLE = exports.HMSH_CODE_DURABLE_FATAL = exports.HMSH_CODE_DURABLE_MAXED = exports.HMSH_CODE_DURABLE_TIMEOUT = exports.HMSH_CODE_DURABLE_WAIT = exports.HMSH_CODE_DURABLE_CONTINUE = exports.HMSH_CODE_DURABLE_PROXY = exports.HMSH_CODE_DURABLE_CHILD = exports.HMSH_CODE_DURABLE_ALL = exports.HMSH_CODE_DURABLE_SLEEP = exports.HMSH_CODE_UNACKED = exports.HMSH_CODE_TIMEOUT = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_INTERRUPT = exports.HMSH_CODE_NOTFOUND = exports.HMSH_CODE_PENDING = exports.HMSH_CODE_SUCCESS = exports.HMSH_PENDING_SIGNAL_EXPIRE = exports.HMSH_SIGNAL_EXPIRE = exports.HMSH_TELEMETRY = exports.HMSH_LOGLEVEL = void 0;
+exports.HMSH_ROUTER_POLL_FALLBACK_INTERVAL = exports.HMSH_NOTIFY_PAYLOAD_LIMIT = exports.DEFAULT_TASK_QUEUE = exports.HMSH_GUID_SIZE = exports.HMSH_ROUTER_SCOUT_INTERVAL_MS = exports.HMSH_ROUTER_SCOUT_INTERVAL_SECONDS = exports.HMSH_SCOUT_INTERVAL_SECONDS = exports.HMSH_FIDELITY_SECONDS = void 0;
 /**
  * Determines the log level for the application. The default is 'info'.
  */
@@ -179,6 +179,21 @@ exports.HMSH_XCLAIM_DELAY_MS = parseInt(process.env.HMSH_XCLAIM_DELAY_MS, 10) ||
 exports.HMSH_XCLAIM_COUNT = parseInt(process.env.HMSH_XCLAIM_COUNT, 10) || 3;
 exports.HMSH_XPENDING_COUNT = parseInt(process.env.HMSH_XPENDING_COUNT, 10) || 10;
 exports.HMSH_BATCH_SIZE = parseInt(process.env.HMSH_BATCH_SIZE, 10) || 10;
+/**
+ * Minimum batch size under adaptive scaling (default: 1).
+ *
+ * When stream depth is high, the adaptive logic reduces batch size
+ * to relieve back-pressure. This value is the floor — the smallest
+ * batch the system will fetch per consume cycle.
+ *
+ *   - 1 (default): fully serial under max stress, safest
+ *   - 2: retains some parallelism while limiting contention
+ *
+ * Both values produce equivalent throughput in practice (~233s for
+ * 1000 concurrent workflows). The reduction from the configured
+ * HMSH_BATCH_SIZE is what matters most — the floor is a safety net.
+ */
+exports.HMSH_BATCH_SIZE_MIN = parseInt(process.env.HMSH_BATCH_SIZE_MIN, 10) || 1;
 /**
  * Postgres stream reservation timeout in seconds (default: 30).
  *
@@ -214,6 +229,32 @@ exports.HMSH_BATCH_SIZE = parseInt(process.env.HMSH_BATCH_SIZE, 10) || 10;
  * HMSH_RESERVATION_TIMEOUT_S=30  (default)
  */
 exports.HMSH_RESERVATION_TIMEOUT_S = parseInt(process.env.HMSH_RESERVATION_TIMEOUT_S, 10) || 30;
+/**
+ * Maximum reservation timeout in seconds for adaptive scaling (default: 1800).
+ *
+ * This is the ceiling for the adaptive reservation timeout — how far the
+ * system is allowed to stretch under sustained load. The adaptive logic
+ * only uses what it needs based on stream depth; this value defines the
+ * upper bound, not the steady state.
+ *
+ * The tradeoff is recovery time after a consumer crash: if a consumer
+ * reserves a message and dies, that message is unavailable until the
+ * timeout expires. A higher ceiling means longer recovery from crashes
+ * but prevents duplicate delivery under heavy sustained load.
+ *
+ * In practice, crashes are rare and the delay is bounded. The cost of
+ * a ceiling that is too low — duplicate delivery, collation errors,
+ * wasted CPU, workflow stalls — is far higher than a slightly longer
+ * recovery window after a crash.
+ *
+ * **Tuning guidance:**
+ *   - Dedicated infrastructure with ample CPU: lower ceiling is fine (600s)
+ *   - Shared/multi-tenant or CPU-constrained: use the default (1800s)
+ *   - Long-running batch imports or large workflow graphs: increase (3600s+)
+ *   - Cloud deployments without CPU contention: the adaptive logic will
+ *     naturally stay near the starting timeout and rarely approach the ceiling
+ */
+exports.HMSH_RESERVATION_TIMEOUT_MAX_S = parseInt(process.env.HMSH_RESERVATION_TIMEOUT_MAX_S, 10) || 1800;
 // TASK WORKER
 exports.HMSH_EXPIRE_DURATION = parseInt(process.env.HMSH_EXPIRE_DURATION, 10) || 1;
 const BASE_FIDELITY_SECONDS = 5;

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.14.6",
+    "version": "0.14.8",
     "description": "Durable Workflow",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/activities/worker.js

@@ -181,6 +181,20 @@ class Worker extends activity_1.Activity {
                 retry: this.config.retry,
             };
         }
+        // Propagate per-activity retry config as _streamRetryConfig so
+        // the engine-level retry mechanism uses it for exponential backoff.
+        // The durable module's maximumAttempts means max retries
+        // (total executions = 1 + maximumAttempts), while the engine's
+        // max_retry_attempts means total attempts. Add 1 to align.
+        if (jobData?.maximumAttempts || jobData?.backoffCoefficient || jobData?.maximumInterval || jobData?.initialInterval) {
+            const durableMaxAttempts = jobData.maximumAttempts ?? 50;
+            streamData._streamRetryConfig = {
+                max_retry_attempts: durableMaxAttempts + 1,
+                backoff_coefficient: jobData.backoffCoefficient ?? 10,
+                maximum_interval_seconds: jobData.maximumInterval ?? 120,
+                initialInterval: jobData.initialInterval ?? 1,
+            };
+        }
         return (await this.engine.router?.publishMessage(topic, streamData, transaction));
     }
 }

package/build/services/durable/exporter.js

@@ -474,10 +474,10 @@ class ExporterService {
         const activityArgsField = this.resolveSymbolField(symbolSets, 'activity_trigger', 'activity_trigger/output/data/arguments');
         const workflowArgsField = this.resolveSymbolField(symbolSets, 'trigger', 'trigger/output/data/arguments');
         // ── 1. Enrich activity inputs ──
-        if (activityArgsField) {
+        {
             const activityEvents = execution.events.filter((e) => e.event_type === 'activity_task_scheduled' || e.event_type === 'activity_task_completed' || e.event_type === 'activity_task_failed');
             if (activityEvents.length > 0) {
-                const { byJobId, byNameIndex } = await this.store.getActivityInputs(workflowId, activityArgsField);
+                const { byJobId, byNameIndex } = await this.store.getActivityInputs(workflowId, activityArgsField || '');
                 for (const evt of activityEvents) {
                     const attrs = evt.attributes;
                     let input = attrs.timeline_key ? byJobId.get(attrs.timeline_key) : undefined;

package/build/services/durable/schemas/factory.d.ts

@@ -1,22 +1,4 @@
-/**
- *********** HOTMESH 'DURABLE' MODULE APPLICATION GRAPH **********
- *
- * This HotMesh application spec uses 50 activities and 25 transitions
- * to model a durable workflow engine using a pluggable backend.
- *
- * This YAML file can also serve as a useful starting point for building
- * Integration/BPM/Workflow servers in general (MuleSoft, etc) without the need
- * for a physical application server.
- *
- * Possible use cases include:
- * * Orchestration servers
- * * Integration servers
- * * BPMN engines
- * * Reentrant process servers
- * * Service Meshes
- * * Master Data Management systems
- */
-declare const APP_VERSION = "11";
+declare const APP_VERSION = "12";
 declare const APP_ID = "durable";
 /**
  * returns a new durable workflow schema