@hotmeshio/hotmesh 0.4.0 → 0.4.2

package/build/services/hotmesh/index.js

@@ -0,0 +1,622 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HotMesh = void 0;
+const key_1 = require("../../modules/key");
+const utils_1 = require("../../modules/utils");
+const factory_1 = require("../connector/factory");
+const engine_1 = require("../engine");
+const logger_1 = require("../logger");
+const quorum_1 = require("../quorum");
+const router_1 = require("../router");
+const worker_1 = require("../worker");
+const enums_1 = require("../../modules/enums");
+/**
+ * HotMesh is a distributed, reentrant process orchestration engine that transforms
+ * Redis, Postgres, or NATS into a resilient service mesh capable of running
+ * fault-tolerant workflows across multiple services and systems.
+ *
+ * ## Core Concepts
+ *
+ * **Distributed Quorum Architecture**: HotMesh operates as a distributed quorum
+ * where multiple engine and worker instances coordinate using CQRS principles.
+ * Each member reads from assigned topic queues and writes results to other queues,
+ * creating emergent workflow orchestration without a central controller.
+ *
+ * **Reentrant Process Engine**: Unlike traditional workflow engines, HotMesh
+ * provides built-in retry logic, idempotency, and failure recovery. Your business
+ * logic doesn't need to handle timeouts or retries - the engine manages all of that.
+ *
+ * **Multi-Provider Support**: Supports Redis/ValKey, Postgres, and NATS as backend
+ * providers, allowing you to leverage existing infrastructure investments.
+ *
+ * ## Key Features
+ *
+ * - **Fault Tolerance**: Automatic retry, timeout, and failure recovery
+ * - **Distributed Execution**: No single point of failure
+ * - **Multi-Provider**: Redis, Postgres, NATS backend support
+ * - **YAML-Driven**: Model-driven development with declarative workflow definitions
+ * - **OpenTelemetry**: Built-in observability and tracing
+ * - **Durable State**: Workflow state persists across system restarts
+ * - **Pattern Matching**: Pub/sub with wildcard pattern support
+ * - **Throttling**: Dynamic flow control and backpressure management
+ *
+ * ## Architecture
+ *
+ * HotMesh consists of several specialized modules:
+ * - **HotMesh**: Core orchestration engine (this class)
+ * - **MemFlow**: Temporal.io-compatible workflow framework
+ * - **MeshCall**: Durable function execution (Temporal-like clone)
+ *
+ * ## Lifecycle Overview
+ *
+ * 1. **Initialize**: Create HotMesh instance with provider configuration
+ * 2. **Deploy**: Upload YAML workflow definitions to the backend
+ * 3. **Activate**: Coordinate quorum to enable the workflow version
+ * 4. **Execute**: Publish events to trigger workflow execution
+ * 5. **Monitor**: Track progress via OpenTelemetry and built-in observability
+ *
+ * ## Basic Usage
+ *
+ * @example
+ * ```typescript
+ * import { HotMesh } from '@hotmeshio/hotmesh';
+ * import Redis from 'ioredis';
+ *
+ * // Initialize with Redis backend
+ * const hotMesh = await HotMesh.init({
+ *   appId: 'my-app',
+ *   engine: {
+ *     connection: {
+ *       class: Redis,
+ *       options: { host: 'localhost', port: 6379 }
+ *     }
+ *   }
+ * });
+ *
+ * // Deploy workflow definition
+ * await hotMesh.deploy(`
+ * app:
+ *   id: my-app
+ *   version: '1'
+ *   graphs:
+ *     - subscribes: order.process
+ *       activities:
+ *         validate:
+ *           type: worker
+ *           topic: order.validate
+ *         approve:
+ *           type: hook
+ *           topic: order.approve
+ *         fulfill:
+ *           type: worker
+ *           topic: order.fulfill
+ *       transitions:
+ *         validate:
+ *           - to: approve
+ *         approve:
+ *           - to: fulfill
+ * `);
+ *
+ * // Activate the workflow version
+ * await hotMesh.activate('1');
+ *
+ * // Execute workflow (fire-and-forget)
+ * const jobId = await hotMesh.pub('order.process', {
+ *   orderId: '12345',
+ *   amount: 99.99
+ * });
+ *
+ * // Execute workflow and wait for result
+ * const result = await hotMesh.pubsub('order.process', {
+ *   orderId: '12345',
+ *   amount: 99.99
+ * });
+ * ```
+ *
+ * ## Postgres Backend Example
+ *
+ * @example
+ * ```typescript
+ * import { HotMesh } from '@hotmeshio/hotmesh';
+ * import { Client as Postgres } from 'pg';
+ *
+ * const hotMesh = await HotMesh.init({
+ *   appId: 'my-app',
+ *   engine: {
+ *     connection: {
+ *       class: Postgres,
+ *       options: {
+ *         connectionString: 'postgresql://user:pass@localhost:5432/db'
+ *       }
+ *     }
+ *   }
+ * });
+ * ```
+ *
+ * ## Advanced Features
+ *
+ * **Pattern Subscriptions**: Listen to multiple workflow topics
+ * ```typescript
+ * await hotMesh.psub('order.*', (topic, message) => {
+ *   console.log(`Received ${topic}:`, message);
+ * });
+ * ```
+ *
+ * **Throttling**: Control processing rates
+ * ```typescript
+ * // Pause all processing for 5 seconds
+ * await hotMesh.throttle({ throttle: 5000 });
+ *
+ * // Emergency stop (pause indefinitely)
+ * await hotMesh.throttle({ throttle: -1 });
+ * ```
+ *
+ * **Workflow Interruption**: Gracefully stop running workflows
+ * ```typescript
+ * await hotMesh.interrupt('order.process', jobId, {
+ *   reason: 'User cancellation'
+ * });
+ * ```
+ *
+ * **State Inspection**: Query workflow state and progress
+ * ```typescript
+ * const state = await hotMesh.getState('order.process', jobId);
+ * const status = await hotMesh.getStatus(jobId);
+ * ```
+ *
+ * ## Distributed Coordination
+ *
+ * HotMesh automatically handles distributed coordination through its quorum system:
+ *
+ * ```typescript
+ * // Check quorum health
+ * const members = await hotMesh.rollCall();
+ *
+ * // Coordinate version activation across all instances
+ * await hotMesh.activate('2', 1000); // 1 second delay for consensus
+ * ```
+ *
+ * ## Integration with Higher-Level Modules
+ *
+ * For most use cases, consider using the higher-level modules:
+ * - **MemFlow**: For Temporal.io-style workflows with TypeScript functions
+ * - **MeshCall**: For durable function calls and RPC patterns
+ *
+ * ## Cleanup
+ *
+ * Always clean up resources when shutting down:
+ * ```typescript
+ * // Stop this instance
+ * hotMesh.stop();
+ *
+ * // Stop all instances (typically in signal handlers)
+ * await HotMesh.stop();
+ * ```
+ *
+ * @see {@link https://hotmesh.io/docs} - Complete documentation
+ * @see {@link https://github.com/hotmeshio/samples-typescript} - Examples and tutorials
+ * @see {@link https://zenodo.org/records/12168558} - Academic paper on the architecture
+ */
+class HotMesh {
+    /**
+     * @private
+     */
+    verifyAndSetNamespace(namespace) {
+        if (!namespace) {
+            this.namespace = key_1.HMNS;
+        }
+        else if (!namespace.match(/^[A-Za-z0-9-]+$/)) {
+            throw new Error(`config.namespace [${namespace}] is invalid`);
+        }
+        else {
+            this.namespace = namespace;
+        }
+    }
+    /**
+     * @private
+     */
+    verifyAndSetAppId(appId) {
+        if (!appId?.match(/^[A-Za-z0-9-]+$/)) {
+            throw new Error(`config.appId [${appId}] is invalid`);
+        }
+        else if (appId === 'a') {
+            throw new Error(`config.appId [${appId}] is reserved`);
+        }
+        else {
+            this.appId = appId;
+        }
+    }
+    /**
+     * Instance initializer. Workers are configured
+     * similarly to the engine, but as an array with
+     * multiple worker objects.
+     *
+     * @example
+     * ```typescript
+     * const config: HotMeshConfig = {
+     *   appId: 'myapp',
+     *   engine: {
+     *     connection: {
+     *       class: Postgres,
+     *       options: {
+     *         connectionString: 'postgresql://usr:pwd@localhost:5432/db',
+     *       }
+     *     }
+     *   },
+     *   workers [...]
+     * };
+     * const hotMesh = await HotMesh.init(config);
+     * ```
+     */
+    static async init(config) {
+        const instance = new HotMesh();
+        instance.guid = config.guid ?? (0, utils_1.guid)();
+        instance.verifyAndSetNamespace(config.namespace);
+        instance.verifyAndSetAppId(config.appId);
+        instance.logger = new logger_1.LoggerService(config.appId, instance.guid, config.name || '', config.logLevel);
+        await instance.initEngine(config, instance.logger);
+        await instance.initQuorum(config, instance.engine, instance.logger);
+        await instance.doWork(config, instance.logger);
+        return instance;
+    }
+    /**
+     * returns a guid using the same core guid
+     * generator used by the HotMesh (nanoid)
+     */
+    static guid() {
+        return (0, utils_1.guid)();
+    }
+    /**
+     * @private
+     */
+    async initEngine(config, logger) {
+        if (config.engine) {
+            //connections that are 'readonly' transfer
+            //this property directly to the engine,
+            //and ALWAYS take precendence.
+            if (config.engine.connection.readonly) {
+                config.engine.readonly = true;
+            }
+            await factory_1.ConnectorService.initClients(config.engine);
+            this.engine = await engine_1.EngineService.init(this.namespace, this.appId, this.guid, config, logger);
+        }
+    }
+    /**
+     * @private
+     */
+    async initQuorum(config, engine, logger) {
+        if (engine) {
+            this.quorum = await quorum_1.QuorumService.init(this.namespace, this.appId, this.guid, config, engine, logger);
+        }
+    }
+    /**
+     * @private
+     */
+    constructor() {
+        /**
+         * @private
+         */
+        this.engine = null;
+        /**
+         * @private
+         */
+        this.quorum = null;
+        /**
+         * @private
+         */
+        this.workers = [];
+    }
+    /**
+     * @private
+     */
+    async doWork(config, logger) {
+        this.workers = await worker_1.WorkerService.init(this.namespace, this.appId, this.guid, config, logger);
+    }
+    // ************* PUB/SUB METHODS *************
+    /**
+     * Starts a workflow
+     * @example
+     * ```typescript
+     * await hotMesh.pub('a.b.c', { key: 'value' });
+     * ```
+     */
+    async pub(topic, data = {}, context, extended) {
+        return await this.engine?.pub(topic, data, context, extended);
+    }
+    /**
+     * Subscribe (listen) to all output and interim emissions of a single
+     * workflow topic. NOTE: Postgres does not support patterned
+     * unsubscription, so this method is not supported for Postgres.
+     *
+     * @example
+     * ```typescript
+     * await hotMesh.psub('a.b.c', (topic, message) => {
+     *  console.log(message);
+     * });
+     * ```
+     */
+    async sub(topic, callback) {
+        return await this.engine?.sub(topic, callback);
+    }
+    /**
+     * Stop listening in on a single workflow topic
+     */
+    async unsub(topic) {
+        return await this.engine?.unsub(topic);
+    }
+    /**
+     * Listen to all output and interim emissions of a workflow topic
+     * matching a wildcard pattern.
+     * @example
+     * ```typescript
+     * await hotMesh.psub('a.b.c*', (topic, message) => {
+     *  console.log(message);
+     * });
+     * ```
+     */
+    async psub(wild, callback) {
+        return await this.engine?.psub(wild, callback);
+    }
+    /**
+     * Patterned unsubscribe. NOTE: Postgres does not support patterned
+     * unsubscription, so this method is not supported for Postgres.
+     */
+    async punsub(wild) {
+        return await this.engine?.punsub(wild);
+    }
+    /**
+     * Starts a workflow and awaits the response
+     * @example
+     * ```typescript
+     * await hotMesh.pubsub('a.b.c', { key: 'value' });
+     * ```
+     */
+    async pubsub(topic, data = {}, context, timeout) {
+        return await this.engine?.pubsub(topic, data, context, timeout);
+    }
+    /**
+     * Add a transition message to the workstream, resuming leg 2 of a paused
+     * reentrant activity (e.g., await, worker, hook)
+     */
+    async add(streamData) {
+        return (await this.engine.add(streamData));
+    }
+    // ************* QUORUM METHODS *************
+    /**
+     * Request a roll call from the quorum (engine and workers)
+     */
+    async rollCall(delay) {
+        return await this.quorum?.rollCall(delay);
+    }
+    /**
+     * Sends a throttle message to the quorum (engine and/or workers)
+     * to limit the rate of processing. Pass `-1` to throttle indefinitely.
+     * The value must be a non-negative integer and not exceed `MAX_DELAY` ms.
+     *
+     * When throttling is set, the quorum will pause for the specified time
+     * before processing the next message. Target specific engines and
+     * workers by passing a `guid` and/or `topic`. Pass no arguments to
+     * throttle the entire quorum.
+     *
+     * In this example, all processing has been paused indefinitely for
+     * the entire quorum. This is equivalent to an emergency stop.
+     *
+     * HotMesh is a stateless sequence engine, so the throttle can be adjusted up
+     * and down with no loss of data.
+     *
+     *
+     * @example
+     * ```typescript
+     * await hotMesh.throttle({ throttle: -1 });
+     * ```
+     */
+    async throttle(options) {
+        let throttle;
+        if (options.throttle === -1) {
+            throttle = enums_1.MAX_DELAY;
+        }
+        else {
+            throttle = options.throttle;
+        }
+        if (!Number.isInteger(throttle) || throttle < 0 || throttle > enums_1.MAX_DELAY) {
+            throw new Error(`Throttle must be a non-negative integer and not exceed ${enums_1.MAX_DELAY} ms; send -1 to throttle indefinitely`);
+        }
+        const throttleMessage = {
+            type: 'throttle',
+            throttle: throttle,
+        };
+        if (options.guid) {
+            throttleMessage.guid = options.guid;
+        }
+        if (options.topic !== undefined) {
+            throttleMessage.topic = options.topic;
+        }
+        await this.engine.store.setThrottleRate(throttleMessage);
+        return await this.quorum?.pub(throttleMessage);
+    }
+    /**
+     * Publish a message to the quorum (engine and/or workers)
+     */
+    async pubQuorum(quorumMessage) {
+        return await this.quorum?.pub(quorumMessage);
+    }
+    /**
+     * Subscribe to quorum events (engine and workers)
+     */
+    async subQuorum(callback) {
+        return await this.quorum?.sub(callback);
+    }
+    /**
+     * Unsubscribe from quorum events (engine and workers)
+     */
+    async unsubQuorum(callback) {
+        return await this.quorum?.unsub(callback);
+    }
+    // ************* LIFECYCLE METHODS *************
+    /**
+     * Preview changes and provide an analysis of risk
+     * prior to deployment
+     * @private
+     */
+    async plan(path) {
+        return await this.engine?.plan(path);
+    }
+    /**
+     * When the app YAML descriptor file is ready, the `deploy` function can be called.
+     * This function is responsible for merging all referenced YAML source
+     * files and writing the JSON output to the file system and to the provider backend. It
+     * is also possible to embed the YAML in-line as a string.
+     *
+     * *The version will not be active until activation is explicitly called.*
+     */
+    async deploy(pathOrYAML) {
+        return await this.engine?.deploy(pathOrYAML);
+    }
+    /**
+     * Once the app YAML file is deployed to the provider backend, the `activate` function can be
+     * called to enable it for the entire quorum at the same moment.
+     *
+     * The approach is to establish the coordinated health of the system through series
+     * of call/response exchanges. Once it is established that the quorum is healthy,
+     * the quorum is instructed to run their engine in `no-cache` mode, ensuring
+     * that the provider backend is consulted for the active app version each time a
+     * call is processed. This ensures that all engines are running the same version
+     * of the app, switching over at the same moment and then enabling `cache` mode
+     * to improve performance.
+     *
+     * *Add a delay for the quorum to reach consensus if traffic is busy, but
+     * also consider throttling traffic flow to an acceptable level.*
+     */
+    async activate(version, delay) {
+        return await this.quorum?.activate(version, delay);
+    }
+    /**
+     * Returns the job state as a JSON object, useful
+     * for understanding dependency chains
+     */
+    async export(jobId) {
+        return await this.engine?.export(jobId);
+    }
+    /**
+     * Returns all data (HGETALL) for a job.
+     */
+    async getRaw(jobId) {
+        return await this.engine?.getRaw(jobId);
+    }
+    /**
+     * Reporter-related method to get the status of a job
+     * @private
+     */
+    async getStats(topic, query) {
+        return await this.engine?.getStats(topic, query);
+    }
+    /**
+     * Returns the status of a job. This is a numeric
+     * semaphore value that indicates the job's state.
+     * Any non-positive value indicates a completed job.
+     * Jobs with a value of `-1` are pending and will
+     * automatically be scrubbed after a set period.
+     * Jobs a value around -1billion have been interrupted
+     * and will be scrubbed after a set period. Jobs with
+     * a value of 0 completed normally. Jobs with a
+     * positive value are still running.
+     */
+    async getStatus(jobId) {
+        return this.engine?.getStatus(jobId);
+    }
+    /**
+     * Returns the job state (data and metadata) for a job.
+     */
+    async getState(topic, jobId) {
+        return this.engine?.getState(topic, jobId);
+    }
+    /**
+     * Returns searchable/queryable data for a job. In this
+     * example a literal field is also searched (the colon
+     * is used to track job status and is a reserved field;
+     * it can be read but not written).
+     *
+     * @example
+     * ```typescript
+     * const fields = ['fred', 'barney', '":"'];
+     * const queryState = await hotMesh.getQueryState('123', fields);
+     * //returns { fred: 'flintstone', barney: 'rubble', ':': '1' }
+     * ```
+     */
+    async getQueryState(jobId, fields) {
+        return await this.engine?.getQueryState(jobId, fields);
+    }
+    /**
+     * @private
+     */
+    async getIds(topic, query, queryFacets = []) {
+        return await this.engine?.getIds(topic, query, queryFacets);
+    }
+    /**
+     * @private
+     */
+    async resolveQuery(topic, query) {
+        return await this.engine?.resolveQuery(topic, query);
+    }
+    /**
+     * Interrupt an active job
+     */
+    async interrupt(topic, jobId, options = {}) {
+        return await this.engine?.interrupt(topic, jobId, options);
+    }
+    /**
+     * Immediately deletes (DEL) a completed job from the system.
+     *
+     * *Scrubbed jobs must be complete with a non-positive `status` value*
+     */
+    async scrub(jobId) {
+        await this.engine?.scrub(jobId);
+    }
+    /**
+     * Re/entry point for an active job. This is used to resume a paused job
+     * and close the reentry point or leave it open for subsequent reentry.
+     * Because `hooks` are public entry points, they include a `topic`
+     * which is established in the app YAML file.
+     *
+     * When this method is called, a hook rule will be located to establish
+     * the exact activity and activity dimension for reentry.
+     */
+    async hook(topic, data, status, code) {
+        return await this.engine?.hook(topic, data, status, code);
+    }
+    /**
+     * @private
+     */
+    async hookAll(hookTopic, data, query, queryFacets = []) {
+        return await this.engine?.hookAll(hookTopic, data, query, queryFacets);
+    }
+    /**
+     * Stop all points of presence, workers and engines
+     */
+    static async stop() {
+        if (!this.disconnecting) {
+            this.disconnecting = true;
+            await router_1.Router.stopConsuming();
+            await factory_1.ConnectorService.disconnectAll();
+        }
+    }
+    /**
+     * Stop this point of presence, workers and engines
+     */
+    stop() {
+        this.engine?.taskService.cancelCleanup();
+        this.quorum?.stop();
+        this.workers?.forEach((worker) => {
+            worker.stop();
+        });
+    }
+    /**
+     * @private
+     * @deprecated
+     */
+    async compress(terms) {
+        return await this.engine?.compress(terms);
+    }
+}
+exports.HotMesh = HotMesh;
+HotMesh.disconnecting = false;

package/build/services/logger/index.d.ts

@@ -0,0 +1,16 @@
+import { Logger } from 'winston';
+import { ILogger } from '../../types/logger';
+declare class LoggerService implements ILogger {
+    private appId;
+    private instanceId;
+    private name;
+    private logLevel;
+    private logger;
+    constructor(appId?: string, instanceId?: string, name?: string, logLevel?: string, customLogger?: Logger);
+    private createDefaultLogger;
+    info(message: string, ...meta: any[]): void;
+    error(message: string, ...meta: any[]): void;
+    warn(message: string, ...meta: any[]): void;
+    debug(message: string, ...meta: any[]): void;
+}
+export { LoggerService, ILogger };

package/build/services/logger/index.js

@@ -0,0 +1,54 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LoggerService = void 0;
+const winston_1 = require("winston");
+class LoggerService {
+    constructor(appId = 'appId', instanceId = 'instanceId', name = 'name', logLevel = 'info', customLogger) {
+        this.appId = appId;
+        this.instanceId = instanceId;
+        this.name = name;
+        this.logLevel = logLevel;
+        this.logger = customLogger || this.createDefaultLogger();
+    }
+    createDefaultLogger() {
+        // Custom format to ensure error objects include message and stack
+        const errorFormat = (0, winston_1.format)((info) => {
+            if (info.error instanceof Error) {
+                return {
+                    ...info,
+                    error: {
+                        message: info.error.message,
+                        stack: info.error.stack,
+                    },
+                };
+            }
+            return info;
+        });
+        return (0, winston_1.createLogger)({
+            level: this.logLevel,
+            format: winston_1.format.combine(winston_1.format.timestamp(), winston_1.format.errors({ stack: true }), errorFormat(), (0, winston_1.format)((info) => {
+                info.ts = info.timestamp;
+                delete info.timestamp;
+                return info;
+            })(), winston_1.format.json()),
+            transports: [new winston_1.transports.Console()],
+            defaultMeta: {
+                app: this.name || this.appId,
+                id: this.instanceId,
+            },
+        });
+    }
+    info(message, ...meta) {
+        this.logger.info(message, ...meta);
+    }
+    error(message, ...meta) {
+        this.logger.error(message, ...meta);
+    }
+    warn(message, ...meta) {
+        this.logger.warn(message, ...meta);
+    }
+    debug(message, ...meta) {
+        this.logger.debug(message, ...meta);
+    }
+}
+exports.LoggerService = LoggerService;