@hotmeshio/hotmesh 0.6.1 → 0.7.0

package/build/services/hotmesh/index.d.ts

@@ -10,30 +10,19 @@ import { StringAnyType, StringStringType } from '../../types/serializer';
 import { JobStatsInput, GetStatsOptions, IdsResponse, StatsResponse } from '../../types/stats';
 import { StreamCode, StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
 /**
- * HotMesh is a distributed, reentrant process orchestration engine that transforms
- * Postgres into a resilient service mesh capable of running
+ * HotMesh transforms Postgres into a durable workflow orchestration engine capable of running
  * fault-tolerant workflows across multiple services and systems.
  *
- * ## Core Concepts
- *
- * **Distributed Quorum Architecture**: HotMesh operates as a distributed quorum
- * where multiple engine and worker instances coordinate using CQRS principles.
- * Each member reads from assigned topic queues and writes results to other queues,
- * creating emergent workflow orchestration without a central controller.
- *
- * **Reentrant Process Engine**: Unlike traditional workflow engines, HotMesh
- * provides built-in retry logic, idempotency, and failure recovery. Your business
- * logic doesn't need to handle timeouts or retries - the engine manages all of that.
- *
  * ## Key Features
  *
- * - **Fault Tolerance**: Automatic retry, timeout, and failure recovery
+ * - **Fault Tolerance**: Automatic retry with exponential backoff and configurable policies
  * - **Distributed Execution**: No single point of failure
  * - **YAML-Driven**: Model-driven development with declarative workflow definitions
  * - **OpenTelemetry**: Built-in observability and tracing
  * - **Durable State**: Workflow state persists across system restarts
  * - **Pattern Matching**: Pub/sub with wildcard pattern support
  * - **Throttling**: Dynamic flow control and backpressure management
+ * - **Retry Policies**: PostgreSQL-native retry configuration with exponential backoff
  *
  * ## Architecture
  *
@@ -225,7 +214,12 @@ declare class HotMesh {
      * similarly to the engine, but as an array with
      * multiple worker objects.
      *
-     * @example
+     * ## Retry Policy Configuration
+     *
+     * HotMesh supports robust retry policies with exponential backoff for PostgreSQL.
+     * Configure retry behavior at the stream level for automatic fault tolerance.
+     *
+     * @example Basic Configuration
      * ```typescript
      * const config: HotMeshConfig = {
      *   appId: 'myapp',
@@ -241,6 +235,57 @@ declare class HotMesh {
      * };
      * const hotMesh = await HotMesh.init(config);
      * ```
+     *
+     * @example With Retry Policy (PostgreSQL)
+     * ```typescript
+     * import { HotMesh } from '@hotmeshio/hotmesh';
+     * import { Client as Postgres } from 'pg';
+     *
+     * const hotMesh = await HotMesh.init({
+     *   appId: 'my-app',
+     *   engine: {
+     *     connection: {
+     *       class: Postgres,
+     *       options: { connectionString: 'postgresql://...' }
+     *     },
+     *     // Default retry policy for engine streams
+     *     retryPolicy: {
+     *       maximumAttempts: 5,      // Retry up to 5 times
+     *       backoffCoefficient: 2,   // Exponential: 2^0, 2^1, 2^2...
+     *       maximumInterval: '300s'  // Cap delay at 5 minutes
+     *     }
+     *   },
+     *   workers: [{
+     *     topic: 'order.process',
+     *     connection: {
+     *       class: Postgres,
+     *       options: { connectionString: 'postgresql://...' }
+     *     },
+     *     // Worker-specific retry policy
+     *     retryPolicy: {
+     *       maximumAttempts: 10,
+     *       backoffCoefficient: 1.5,
+     *       maximumInterval: '600s'
+     *     },
+     *     callback: async (data) => {
+     *       // Your business logic here
+     *       // Failures will automatically retry with exponential backoff
+     *       return { status: 'success', data: processedData };
+     *     }
+     *   }]
+     * });
+     * ```
+     *
+     * **Retry Policy Options**:
+     * - `maximumAttempts` - Maximum retry attempts before failure (default: 3)
+     * - `backoffCoefficient` - Base for exponential backoff calculation (default: 10)
+     * - `maximumInterval` - Maximum delay between retries in seconds or duration string (default: '120s')
+     *
+     * **Retry Delays**: For `backoffCoefficient: 2`, delays are: 2s, 4s, 8s, 16s, 32s...
+     * capped at `maximumInterval`.
+     *
+     * **Note**: Retry policies are stored in PostgreSQL columns for efficient querying and
+     * observability. Each retry creates a new message, preserving message immutability.
      */
     static init(config: HotMeshConfig): Promise<HotMesh>;
     /**
@@ -272,6 +317,12 @@ declare class HotMesh {
      * @private
      */
     private initTaskQueue;
+    /**
+     * Apply retry policy to the stream connection within a ProviderConfig or ProvidersConfig.
+     * Handles both short-form (ProviderConfig) and long-form (ProvidersConfig) connection configs.
+     * @private
+     */
+    private applyRetryPolicy;
     /**
      * Starts a workflow
      * @example

package/build/services/hotmesh/index.js

@@ -11,30 +11,19 @@ const router_1 = require("../router");
 const worker_1 = require("../worker");
 const enums_1 = require("../../modules/enums");
 /**
- * HotMesh is a distributed, reentrant process orchestration engine that transforms
- * Postgres into a resilient service mesh capable of running
+ * HotMesh transforms Postgres into a durable workflow orchestration engine capable of running
  * fault-tolerant workflows across multiple services and systems.
  *
- * ## Core Concepts
- *
- * **Distributed Quorum Architecture**: HotMesh operates as a distributed quorum
- * where multiple engine and worker instances coordinate using CQRS principles.
- * Each member reads from assigned topic queues and writes results to other queues,
- * creating emergent workflow orchestration without a central controller.
- *
- * **Reentrant Process Engine**: Unlike traditional workflow engines, HotMesh
- * provides built-in retry logic, idempotency, and failure recovery. Your business
- * logic doesn't need to handle timeouts or retries - the engine manages all of that.
- *
  * ## Key Features
  *
- * - **Fault Tolerance**: Automatic retry, timeout, and failure recovery
+ * - **Fault Tolerance**: Automatic retry with exponential backoff and configurable policies
  * - **Distributed Execution**: No single point of failure
  * - **YAML-Driven**: Model-driven development with declarative workflow definitions
  * - **OpenTelemetry**: Built-in observability and tracing
  * - **Durable State**: Workflow state persists across system restarts
  * - **Pattern Matching**: Pub/sub with wildcard pattern support
  * - **Throttling**: Dynamic flow control and backpressure management
+ * - **Retry Policies**: PostgreSQL-native retry configuration with exponential backoff
  *
  * ## Architecture
  *
@@ -229,7 +218,12 @@ class HotMesh {
      * similarly to the engine, but as an array with
      * multiple worker objects.
      *
-     * @example
+     * ## Retry Policy Configuration
+     *
+     * HotMesh supports robust retry policies with exponential backoff for PostgreSQL.
+     * Configure retry behavior at the stream level for automatic fault tolerance.
+     *
+     * @example Basic Configuration
      * ```typescript
      * const config: HotMeshConfig = {
      *   appId: 'myapp',
@@ -245,6 +239,57 @@ class HotMesh {
      * };
      * const hotMesh = await HotMesh.init(config);
      * ```
+     *
+     * @example With Retry Policy (PostgreSQL)
+     * ```typescript
+     * import { HotMesh } from '@hotmeshio/hotmesh';
+     * import { Client as Postgres } from 'pg';
+     *
+     * const hotMesh = await HotMesh.init({
+     *   appId: 'my-app',
+     *   engine: {
+     *     connection: {
+     *       class: Postgres,
+     *       options: { connectionString: 'postgresql://...' }
+     *     },
+     *     // Default retry policy for engine streams
+     *     retryPolicy: {
+     *       maximumAttempts: 5,      // Retry up to 5 times
+     *       backoffCoefficient: 2,   // Exponential: 2^0, 2^1, 2^2...
+     *       maximumInterval: '300s'  // Cap delay at 5 minutes
+     *     }
+     *   },
+     *   workers: [{
+     *     topic: 'order.process',
+     *     connection: {
+     *       class: Postgres,
+     *       options: { connectionString: 'postgresql://...' }
+     *     },
+     *     // Worker-specific retry policy
+     *     retryPolicy: {
+     *       maximumAttempts: 10,
+     *       backoffCoefficient: 1.5,
+     *       maximumInterval: '600s'
+     *     },
+     *     callback: async (data) => {
+     *       // Your business logic here
+     *       // Failures will automatically retry with exponential backoff
+     *       return { status: 'success', data: processedData };
+     *     }
+     *   }]
+     * });
+     * ```
+     *
+     * **Retry Policy Options**:
+     * - `maximumAttempts` - Maximum retry attempts before failure (default: 3)
+     * - `backoffCoefficient` - Base for exponential backoff calculation (default: 10)
+     * - `maximumInterval` - Maximum delay between retries in seconds or duration string (default: '120s')
+     *
+     * **Retry Delays**: For `backoffCoefficient: 2`, delays are: 2s, 4s, 8s, 16s, 32s...
+     * capped at `maximumInterval`.
+     *
+     * **Note**: Retry policies are stored in PostgreSQL columns for efficient querying and
+     * observability. Each retry creates a new message, preserving message immutability.
      */
     static async init(config) {
         const instance = new HotMesh();
@@ -275,6 +320,10 @@ class HotMesh {
             if (config.engine.connection.readonly) {
                 config.engine.readonly = true;
             }
+            // Apply retry policy to stream connection if provided
+            if (config.engine.retryPolicy) {
+                this.applyRetryPolicy(config.engine.connection, config.engine.retryPolicy);
+            }
             // Initialize task queue for engine
             config.engine.taskQueue = this.initTaskQueue(config.engine.taskQueue, config.taskQueue);
             await factory_1.ConnectorService.initClients(config.engine);
@@ -313,6 +362,10 @@ class HotMesh {
         // Initialize task queues for workers
         if (config.workers) {
             for (const worker of config.workers) {
+                // Apply retry policy to stream connection if provided
+                if (worker.retryPolicy) {
+                    this.applyRetryPolicy(worker.connection, worker.retryPolicy);
+                }
                 worker.taskQueue = this.initTaskQueue(worker.taskQueue, config.taskQueue);
             }
         }
@@ -337,6 +390,22 @@ class HotMesh {
         // Default queue as fallback
         return enums_1.DEFAULT_TASK_QUEUE;
     }
+    /**
+     * Apply retry policy to the stream connection within a ProviderConfig or ProvidersConfig.
+     * Handles both short-form (ProviderConfig) and long-form (ProvidersConfig) connection configs.
+     * @private
+     */
+    applyRetryPolicy(connection, retryPolicy) {
+        // Check if this is ProvidersConfig (has 'stream' property)
+        if ('stream' in connection && connection.stream) {
+            // Long-form: apply to the stream sub-config
+            connection.stream.retryPolicy = retryPolicy;
+        }
+        else {
+            // Short-form: apply directly to the connection
+            connection.retryPolicy = retryPolicy;
+        }
+    }
     // ************* PUB/SUB METHODS *************
     /**
      * Starts a workflow

package/build/services/memflow/index.d.ts

@@ -1,4 +1,5 @@
 import { ContextType, WorkflowInterceptor } from '../../types/memflow';
+import { guid } from '../../modules/utils';
 import { ClientService } from './client';
 import { ConnectionService } from './connection';
 import { Search } from './search';
@@ -61,6 +62,7 @@ import { didInterrupt } from './workflow/interruption';
  * ### 3. Durable Activities & Proxies
  * Define and execute durable activities with automatic retry:
  * ```typescript
+ * // Default: activities use workflow's task queue
  * const activities = MemFlow.workflow.proxyActivities<{
  *   analyzeDocument: typeof analyzeDocument;
  *   validateFindings: typeof validateFindings;
@@ -77,7 +79,29 @@ import { didInterrupt } from './workflow/interruption';
  * const validation = await activities.validateFindings(analysis);
  * ```
  *
- * ### 4. Workflow Composition
+ * ### 4. Explicit Activity Registration
+ * Register activity workers explicitly before workflows start:
+ * ```typescript
+ * // Register shared activity pool for interceptors
+ * await MemFlow.registerActivityWorker({
+ *   connection: {
+ *     class: Postgres,
+ *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+ *   },
+ *   taskQueue: 'shared-activities'
+ * }, sharedActivities, 'shared-activities');
+ *
+ * // Register custom activity pool for specific use cases
+ * await MemFlow.registerActivityWorker({
+ *   connection: {
+ *     class: Postgres,
+ *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+ *   },
+ *   taskQueue: 'priority-activities'
+ * }, priorityActivities, 'priority-activities');
+ * ```
+ *
+ * ### 5. Workflow Composition
  * Build complex workflows through composition:
  * ```typescript
  * // Start a child workflow
@@ -100,29 +124,34 @@ import { didInterrupt } from './workflow/interruption';
  * });
  * ```
  *
- * ### 5. Workflow Interceptors
+ * ### 6. Workflow Interceptors
  * Add cross-cutting concerns through interceptors that run as durable functions:
  * ```typescript
- * // Add audit interceptor that uses MemFlow functions
+ * // First register shared activity worker for interceptors
+ * await MemFlow.registerActivityWorker({
+ *   connection: {
+ *     class: Postgres,
+ *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+ *   },
+ *   taskQueue: 'interceptor-activities'
+ * }, { auditLog }, 'interceptor-activities');
+ *
+ * // Add audit interceptor that uses activities with explicit taskQueue
  * MemFlow.registerInterceptor({
  *   async execute(ctx, next) {
  *     try {
- *       // Interceptors can use MemFlow functions and participate in replay
- *       const entity = await MemFlow.workflow.entity();
- *       await entity.append('auditLog', {
- *         action: 'started',
- *         timestamp: new Date().toISOString()
+ *       // Interceptors use explicit taskQueue to prevent per-workflow queues
+ *       const { auditLog } = MemFlow.workflow.proxyActivities<typeof activities>({
+ *         activities: { auditLog },
+ *         taskQueue: 'interceptor-activities', // Explicit shared queue
+ *         retryPolicy: { maximumAttempts: 3 }
  *       });
  *
- *       // Rate limiting with durable sleep
- *       await MemFlow.workflow.sleepFor('100 milliseconds');
+ *       await auditLog(ctx.get('workflowId'), 'started');
  *
  *       const result = await next();
  *
- *       await entity.append('auditLog', {
- *         action: 'completed',
- *         timestamp: new Date().toISOString()
- *       });
+ *       await auditLog(ctx.get('workflowId'), 'completed');
  *
  *       return result;
  *     } catch (err) {
@@ -141,6 +170,16 @@ import { didInterrupt } from './workflow/interruption';
  * ```typescript
  * import { Client, Worker, MemFlow } from '@hotmeshio/hotmesh';
  * import { Client as Postgres } from 'pg';
+ * import * as activities from './activities';
+ *
+ * // (Optional) Register shared activity workers for interceptors
+ * await MemFlow.registerActivityWorker({
+ *   connection: {
+ *     class: Postgres,
+ *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+ *   },
+ *   taskQueue: 'shared-activities'
+ * }, sharedActivities, 'shared-activities');
  *
  * // Initialize worker
  * await Worker.create({
@@ -210,6 +249,49 @@ declare class MemFlowClass {
      * equivalent to the Temporal `Worker` service.
      */
     static Worker: typeof WorkerService;
+    /**
+     * Register activity workers for a task queue. Activities execute via message queue
+     * and can run on different servers from workflows.
+     *
+     * @example
+     * ```typescript
+     * // Activity worker
+     * const activities = {
+     *   async processPayment(amount: number) { return `Processed $${amount}`; },
+     *   async sendEmail(to: string, msg: string) { /* ... *\/ }
+     * };
+     *
+     * await MemFlow.registerActivityWorker({
+     *   connection: {
+     *     class: Postgres,
+     *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+     *   },
+     *   taskQueue: 'payment'
+     * }, activities, 'payment');
+     *
+     * // Workflow worker (can be on different server)
+     * async function orderWorkflow(amount: number) {
+     *   const { processPayment, sendEmail } = MemFlow.workflow.proxyActivities<{
+     *     processPayment: (amount: number) => Promise<string>;
+     *     sendEmail: (to: string, msg: string) => Promise<void>;
+     *   }>({
+     *     taskQueue: 'payment',
+     *     retryPolicy: { maximumAttempts: 3 }
+     *   });
+     *
+     *   const result = await processPayment(amount);
+     *   await sendEmail('customer@example.com', result);
+     *   return result;
+     * }
+     *
+     * await MemFlow.Worker.create({
+     *   connection: { class: Postgres, options: { connectionString: '...' } },
+     *   taskQueue: 'orders',
+     *   workflow: orderWorkflow
+     * });
+     * ```
+     */
+    static registerActivityWorker: typeof WorkerService.registerActivityWorker;
     /**
      * The MemFlow `workflow` service is functionally
      * equivalent to the Temporal `Workflow` service
@@ -234,6 +316,10 @@ declare class MemFlowClass {
      * Clear all registered workflow interceptors
      */
     static clearInterceptors(): void;
+    /**
+     * Generate a unique identifier for workflow IDs
+     */
+    static guid: typeof guid;
     /**
      * Shutdown everything. All connections, workers, and clients will be closed.
      * Include in your signal handlers to ensure a clean shutdown.

package/build/services/memflow/index.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MemFlow = void 0;
 const hotmesh_1 = require("../hotmesh");
+const utils_1 = require("../../modules/utils");
 const client_1 = require("./client");
 const connection_1 = require("./connection");
 const search_1 = require("./search");
@@ -65,6 +66,7 @@ const interceptor_1 = require("./interceptor");
  * ### 3. Durable Activities & Proxies
  * Define and execute durable activities with automatic retry:
  * ```typescript
+ * // Default: activities use workflow's task queue
  * const activities = MemFlow.workflow.proxyActivities<{
  *   analyzeDocument: typeof analyzeDocument;
  *   validateFindings: typeof validateFindings;
@@ -81,7 +83,29 @@ const interceptor_1 = require("./interceptor");
  * const validation = await activities.validateFindings(analysis);
  * ```
  *
- * ### 4. Workflow Composition
+ * ### 4. Explicit Activity Registration
+ * Register activity workers explicitly before workflows start:
+ * ```typescript
+ * // Register shared activity pool for interceptors
+ * await MemFlow.registerActivityWorker({
+ *   connection: {
+ *     class: Postgres,
+ *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+ *   },
+ *   taskQueue: 'shared-activities'
+ * }, sharedActivities, 'shared-activities');
+ *
+ * // Register custom activity pool for specific use cases
+ * await MemFlow.registerActivityWorker({
+ *   connection: {
+ *     class: Postgres,
+ *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+ *   },
+ *   taskQueue: 'priority-activities'
+ * }, priorityActivities, 'priority-activities');
+ * ```
+ *
+ * ### 5. Workflow Composition
  * Build complex workflows through composition:
  * ```typescript
  * // Start a child workflow
@@ -104,29 +128,34 @@ const interceptor_1 = require("./interceptor");
  * });
  * ```
  *
- * ### 5. Workflow Interceptors
+ * ### 6. Workflow Interceptors
  * Add cross-cutting concerns through interceptors that run as durable functions:
  * ```typescript
- * // Add audit interceptor that uses MemFlow functions
+ * // First register shared activity worker for interceptors
+ * await MemFlow.registerActivityWorker({
+ *   connection: {
+ *     class: Postgres,
+ *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+ *   },
+ *   taskQueue: 'interceptor-activities'
+ * }, { auditLog }, 'interceptor-activities');
+ *
+ * // Add audit interceptor that uses activities with explicit taskQueue
  * MemFlow.registerInterceptor({
  *   async execute(ctx, next) {
  *     try {
- *       // Interceptors can use MemFlow functions and participate in replay
- *       const entity = await MemFlow.workflow.entity();
- *       await entity.append('auditLog', {
- *         action: 'started',
- *         timestamp: new Date().toISOString()
+ *       // Interceptors use explicit taskQueue to prevent per-workflow queues
+ *       const { auditLog } = MemFlow.workflow.proxyActivities<typeof activities>({
+ *         activities: { auditLog },
+ *         taskQueue: 'interceptor-activities', // Explicit shared queue
+ *         retryPolicy: { maximumAttempts: 3 }
  *       });
  *
- *       // Rate limiting with durable sleep
- *       await MemFlow.workflow.sleepFor('100 milliseconds');
+ *       await auditLog(ctx.get('workflowId'), 'started');
  *
  *       const result = await next();
  *
- *       await entity.append('auditLog', {
- *         action: 'completed',
- *         timestamp: new Date().toISOString()
- *       });
+ *       await auditLog(ctx.get('workflowId'), 'completed');
  *
  *       return result;
  *     } catch (err) {
@@ -145,6 +174,16 @@ const interceptor_1 = require("./interceptor");
  * ```typescript
  * import { Client, Worker, MemFlow } from '@hotmeshio/hotmesh';
  * import { Client as Postgres } from 'pg';
+ * import * as activities from './activities';
+ *
+ * // (Optional) Register shared activity workers for interceptors
+ * await MemFlow.registerActivityWorker({
+ *   connection: {
+ *     class: Postgres,
+ *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+ *   },
+ *   taskQueue: 'shared-activities'
+ * }, sharedActivities, 'shared-activities');
  *
  * // Initialize worker
  * await Worker.create({
@@ -245,6 +284,49 @@ MemFlowClass.Handle = handle_1.WorkflowHandleService;
  * equivalent to the Temporal `Worker` service.
  */
 MemFlowClass.Worker = worker_1.WorkerService;
+/**
+ * Register activity workers for a task queue. Activities execute via message queue
+ * and can run on different servers from workflows.
+ *
+ * @example
+ * ```typescript
+ * // Activity worker
+ * const activities = {
+ *   async processPayment(amount: number) { return `Processed $${amount}`; },
+ *   async sendEmail(to: string, msg: string) { /* ... *\/ }
+ * };
+ *
+ * await MemFlow.registerActivityWorker({
+ *   connection: {
+ *     class: Postgres,
+ *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+ *   },
+ *   taskQueue: 'payment'
+ * }, activities, 'payment');
+ *
+ * // Workflow worker (can be on different server)
+ * async function orderWorkflow(amount: number) {
+ *   const { processPayment, sendEmail } = MemFlow.workflow.proxyActivities<{
+ *     processPayment: (amount: number) => Promise<string>;
+ *     sendEmail: (to: string, msg: string) => Promise<void>;
+ *   }>({
+ *     taskQueue: 'payment',
+ *     retryPolicy: { maximumAttempts: 3 }
+ *   });
+ *
+ *   const result = await processPayment(amount);
+ *   await sendEmail('customer@example.com', result);
+ *   return result;
+ * }
+ *
+ * await MemFlow.Worker.create({
+ *   connection: { class: Postgres, options: { connectionString: '...' } },
+ *   taskQueue: 'orders',
+ *   workflow: orderWorkflow
+ * });
+ * ```
+ */
+MemFlowClass.registerActivityWorker = worker_1.WorkerService.registerActivityWorker;
 /**
  * The MemFlow `workflow` service is functionally
  * equivalent to the Temporal `Workflow` service
@@ -260,3 +342,7 @@ MemFlowClass.workflow = workflow_1.WorkflowService;
  */
 MemFlowClass.didInterrupt = interruption_1.didInterrupt;
 MemFlowClass.interceptorService = new interceptor_1.InterceptorService();
+/**
+ * Generate a unique identifier for workflow IDs
+ */
+MemFlowClass.guid = utils_1.guid;