@hotmeshio/hotmesh 0.4.0 → 0.4.2

package/build/services/exporter/index.d.ts

@@ -0,0 +1,44 @@
+import { ILogger } from '../logger';
+import { StoreService } from '../store';
+import { DependencyExport, ExportOptions, JobActionExport, JobExport } from '../../types/exporter';
+import { ProviderClient, ProviderTransaction } from '../../types/provider';
+import { StringStringType, Symbols } from '../../types/serializer';
+/**
+ * Downloads job data from Redis (hscan, hmget, hgetall)
+ * Expands process data and includes dependency list
+ */
+declare class ExporterService {
+    appId: string;
+    logger: ILogger;
+    store: StoreService<ProviderClient, ProviderTransaction>;
+    symbols: Promise<Symbols> | Symbols;
+    constructor(appId: string, store: StoreService<ProviderClient, ProviderTransaction>, logger: ILogger);
+    /**
+     * Convert the job hash into a JobExport object.
+     * This object contains various facets that describe the interaction
+     * in terms relevant to narrative storytelling.
+     */
+    export(jobId: string, options?: ExportOptions): Promise<JobExport>;
+    /**
+     * Inflates the key from Redis, 3-character symbol
+     * into a human-readable JSON path, reflecting the
+     * tree-like structure of the unidimensional Hash
+     */
+    inflateKey(key: string): string;
+    /**
+     * Inflates the job data from Redis into a JobExport object
+     * @param jobHash - the job data from Redis
+     * @param dependencyList - the list of dependencies for the job
+     * @returns - the inflated job data
+     */
+    inflate(jobHash: StringStringType, dependencyList: string[]): JobExport;
+    /**
+     * Inflates the dependency data from Redis into a JobExport object by
+     * organizing the dimensional isolate in sch a way asto interleave
+     * into a story
+     * @param data - the dependency data from Redis
+     * @returns - the organized dependency data
+     */
+    inflateDependencyData(data: string[], actions: JobActionExport): DependencyExport[];
+}
+export { ExporterService };

package/build/services/exporter/index.js

@@ -0,0 +1,126 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExporterService = void 0;
+const key_1 = require("../../modules/key");
+const utils_1 = require("../../modules/utils");
+const serializer_1 = require("../serializer");
+/**
+ * Downloads job data from Redis (hscan, hmget, hgetall)
+ * Expands process data and includes dependency list
+ */
+class ExporterService {
+    constructor(appId, store, logger) {
+        this.appId = appId;
+        this.logger = logger;
+        this.store = store;
+    }
+    /**
+     * Convert the job hash into a JobExport object.
+     * This object contains various facets that describe the interaction
+     * in terms relevant to narrative storytelling.
+     */
+    async export(jobId, options = {}) {
+        if (!this.symbols) {
+            this.symbols = this.store.getAllSymbols();
+            this.symbols = await this.symbols;
+        }
+        const depData = []; // await this.store.getDependencies(jobId);
+        const jobData = await this.store.getRaw(jobId);
+        const jobExport = this.inflate(jobData, depData);
+        return jobExport;
+    }
+    /**
+     * Inflates the key from Redis, 3-character symbol
+     * into a human-readable JSON path, reflecting the
+     * tree-like structure of the unidimensional Hash
+     */
+    inflateKey(key) {
+        return key in this.symbols ? this.symbols[key] : key;
+    }
+    /**
+     * Inflates the job data from Redis into a JobExport object
+     * @param jobHash - the job data from Redis
+     * @param dependencyList - the list of dependencies for the job
+     * @returns - the inflated job data
+     */
+    inflate(jobHash, dependencyList) {
+        //the list of actions taken in the workflow and hook functions
+        const actions = {
+            hooks: {},
+            main: {
+                cursor: -1,
+                items: [],
+            },
+        };
+        const process = {};
+        const dependencies = this.inflateDependencyData(dependencyList, actions);
+        const regex = /^([a-zA-Z]{3}),(\d+(?:,\d+)*)/;
+        Object.entries(jobHash).forEach(([key, value]) => {
+            const match = key.match(regex);
+            if (match) {
+                //activity process state
+                const [_, letters, numbers] = match;
+                const path = this.inflateKey(letters);
+                const dimensions = `${numbers.replace(/,/g, '/')}`;
+                const resolved = serializer_1.SerializerService.fromString(value);
+                process[`${dimensions}/${path}`] = resolved;
+            }
+            else if (key.length === 3) {
+                //job state
+                process[this.inflateKey(key)] = serializer_1.SerializerService.fromString(value);
+            }
+        });
+        return {
+            dependencies,
+            process: (0, utils_1.restoreHierarchy)(process),
+            status: jobHash[':'],
+        };
+    }
+    /**
+     * Inflates the dependency data from Redis into a JobExport object by
+     * organizing the dimensional isolate in sch a way asto interleave
+     * into a story
+     * @param data - the dependency data from Redis
+     * @returns - the organized dependency data
+     */
+    inflateDependencyData(data, actions) {
+        const hookReg = /([0-9,]+)-(\d+)$/;
+        const flowReg = /-(\d+)$/;
+        return data.map((dependency, index) => {
+            const [action, topic, gid, _pd, ...jid] = dependency.split(key_1.VALSEP);
+            const jobId = jid.join(key_1.VALSEP);
+            const match = jobId.match(hookReg);
+            let prefix;
+            let type;
+            let dimensionKey = '';
+            if (match) {
+                //hook-originating dependency
+                const [_, dimension, counter] = match;
+                dimensionKey = dimension.split(',').join('/');
+                prefix = `${dimensionKey}[${counter}]`;
+                type = 'hook';
+            }
+            else {
+                const match = jobId.match(flowReg);
+                if (match) {
+                    //main workflow-originating dependency
+                    const [_, counter] = match;
+                    prefix = `[${counter}]`;
+                    type = 'flow';
+                }
+                else {
+                    //'other' types like signal cleanup
+                    prefix = '/';
+                    type = 'other';
+                }
+            }
+            return {
+                type: action,
+                topic,
+                gid,
+                jid: jobId,
+            };
+        });
+    }
+}
+exports.ExporterService = ExporterService;

package/build/services/hotmesh/index.d.ts

@@ -0,0 +1,483 @@
+import { EngineService } from '../engine';
+import { ILogger } from '../logger';
+import { QuorumService } from '../quorum';
+import { WorkerService } from '../worker';
+import { JobState, JobData, JobOutput, JobStatus, JobInterruptOptions, ExtensionType } from '../../types/job';
+import { HotMeshConfig, HotMeshManifest } from '../../types/hotmesh';
+import { JobExport } from '../../types/exporter';
+import { JobMessageCallback, QuorumMessage, QuorumMessageCallback, QuorumProfile, ThrottleOptions } from '../../types/quorum';
+import { StringAnyType, StringStringType } from '../../types/serializer';
+import { JobStatsInput, GetStatsOptions, IdsResponse, StatsResponse } from '../../types/stats';
+import { StreamCode, StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
+/**
+ * HotMesh is a distributed, reentrant process orchestration engine that transforms
+ * Redis, Postgres, or NATS into a resilient service mesh capable of running
+ * fault-tolerant workflows across multiple services and systems.
+ *
+ * ## Core Concepts
+ *
+ * **Distributed Quorum Architecture**: HotMesh operates as a distributed quorum
+ * where multiple engine and worker instances coordinate using CQRS principles.
+ * Each member reads from assigned topic queues and writes results to other queues,
+ * creating emergent workflow orchestration without a central controller.
+ *
+ * **Reentrant Process Engine**: Unlike traditional workflow engines, HotMesh
+ * provides built-in retry logic, idempotency, and failure recovery. Your business
+ * logic doesn't need to handle timeouts or retries - the engine manages all of that.
+ *
+ * **Multi-Provider Support**: Supports Redis/ValKey, Postgres, and NATS as backend
+ * providers, allowing you to leverage existing infrastructure investments.
+ *
+ * ## Key Features
+ *
+ * - **Fault Tolerance**: Automatic retry, timeout, and failure recovery
+ * - **Distributed Execution**: No single point of failure
+ * - **Multi-Provider**: Redis, Postgres, NATS backend support
+ * - **YAML-Driven**: Model-driven development with declarative workflow definitions
+ * - **OpenTelemetry**: Built-in observability and tracing
+ * - **Durable State**: Workflow state persists across system restarts
+ * - **Pattern Matching**: Pub/sub with wildcard pattern support
+ * - **Throttling**: Dynamic flow control and backpressure management
+ *
+ * ## Architecture
+ *
+ * HotMesh consists of several specialized modules:
+ * - **HotMesh**: Core orchestration engine (this class)
+ * - **MemFlow**: Temporal.io-compatible workflow framework
+ * - **MeshCall**: Durable function execution (Temporal-like clone)
+ *
+ * ## Lifecycle Overview
+ *
+ * 1. **Initialize**: Create HotMesh instance with provider configuration
+ * 2. **Deploy**: Upload YAML workflow definitions to the backend
+ * 3. **Activate**: Coordinate quorum to enable the workflow version
+ * 4. **Execute**: Publish events to trigger workflow execution
+ * 5. **Monitor**: Track progress via OpenTelemetry and built-in observability
+ *
+ * ## Basic Usage
+ *
+ * @example
+ * ```typescript
+ * import { HotMesh } from '@hotmeshio/hotmesh';
+ * import Redis from 'ioredis';
+ *
+ * // Initialize with Redis backend
+ * const hotMesh = await HotMesh.init({
+ *   appId: 'my-app',
+ *   engine: {
+ *     connection: {
+ *       class: Redis,
+ *       options: { host: 'localhost', port: 6379 }
+ *     }
+ *   }
+ * });
+ *
+ * // Deploy workflow definition
+ * await hotMesh.deploy(`
+ * app:
+ *   id: my-app
+ *   version: '1'
+ *   graphs:
+ *     - subscribes: order.process
+ *       activities:
+ *         validate:
+ *           type: worker
+ *           topic: order.validate
+ *         approve:
+ *           type: hook
+ *           topic: order.approve
+ *         fulfill:
+ *           type: worker
+ *           topic: order.fulfill
+ *       transitions:
+ *         validate:
+ *           - to: approve
+ *         approve:
+ *           - to: fulfill
+ * `);
+ *
+ * // Activate the workflow version
+ * await hotMesh.activate('1');
+ *
+ * // Execute workflow (fire-and-forget)
+ * const jobId = await hotMesh.pub('order.process', {
+ *   orderId: '12345',
+ *   amount: 99.99
+ * });
+ *
+ * // Execute workflow and wait for result
+ * const result = await hotMesh.pubsub('order.process', {
+ *   orderId: '12345',
+ *   amount: 99.99
+ * });
+ * ```
+ *
+ * ## Postgres Backend Example
+ *
+ * @example
+ * ```typescript
+ * import { HotMesh } from '@hotmeshio/hotmesh';
+ * import { Client as Postgres } from 'pg';
+ *
+ * const hotMesh = await HotMesh.init({
+ *   appId: 'my-app',
+ *   engine: {
+ *     connection: {
+ *       class: Postgres,
+ *       options: {
+ *         connectionString: 'postgresql://user:pass@localhost:5432/db'
+ *       }
+ *     }
+ *   }
+ * });
+ * ```
+ *
+ * ## Advanced Features
+ *
+ * **Pattern Subscriptions**: Listen to multiple workflow topics
+ * ```typescript
+ * await hotMesh.psub('order.*', (topic, message) => {
+ *   console.log(`Received ${topic}:`, message);
+ * });
+ * ```
+ *
+ * **Throttling**: Control processing rates
+ * ```typescript
+ * // Pause all processing for 5 seconds
+ * await hotMesh.throttle({ throttle: 5000 });
+ *
+ * // Emergency stop (pause indefinitely)
+ * await hotMesh.throttle({ throttle: -1 });
+ * ```
+ *
+ * **Workflow Interruption**: Gracefully stop running workflows
+ * ```typescript
+ * await hotMesh.interrupt('order.process', jobId, {
+ *   reason: 'User cancellation'
+ * });
+ * ```
+ *
+ * **State Inspection**: Query workflow state and progress
+ * ```typescript
+ * const state = await hotMesh.getState('order.process', jobId);
+ * const status = await hotMesh.getStatus(jobId);
+ * ```
+ *
+ * ## Distributed Coordination
+ *
+ * HotMesh automatically handles distributed coordination through its quorum system:
+ *
+ * ```typescript
+ * // Check quorum health
+ * const members = await hotMesh.rollCall();
+ *
+ * // Coordinate version activation across all instances
+ * await hotMesh.activate('2', 1000); // 1 second delay for consensus
+ * ```
+ *
+ * ## Integration with Higher-Level Modules
+ *
+ * For most use cases, consider using the higher-level modules:
+ * - **MemFlow**: For Temporal.io-style workflows with TypeScript functions
+ * - **MeshCall**: For durable function calls and RPC patterns
+ *
+ * ## Cleanup
+ *
+ * Always clean up resources when shutting down:
+ * ```typescript
+ * // Stop this instance
+ * hotMesh.stop();
+ *
+ * // Stop all instances (typically in signal handlers)
+ * await HotMesh.stop();
+ * ```
+ *
+ * @see {@link https://hotmesh.io/docs} - Complete documentation
+ * @see {@link https://github.com/hotmeshio/samples-typescript} - Examples and tutorials
+ * @see {@link https://zenodo.org/records/12168558} - Academic paper on the architecture
+ */
+declare class HotMesh {
+    namespace: string;
+    appId: string;
+    guid: string;
+    /**
+     * @private
+     */
+    engine: EngineService | null;
+    /**
+     * @private
+     */
+    quorum: QuorumService | null;
+    /**
+     * @private
+     */
+    workers: WorkerService[];
+    logger: ILogger;
+    static disconnecting: boolean;
+    /**
+     * @private
+     */
+    verifyAndSetNamespace(namespace?: string): void;
+    /**
+     * @private
+     */
+    verifyAndSetAppId(appId: string): void;
+    /**
+     * Instance initializer. Workers are configured
+     * similarly to the engine, but as an array with
+     * multiple worker objects.
+     *
+     * @example
+     * ```typescript
+     * const config: HotMeshConfig = {
+     *   appId: 'myapp',
+     *   engine: {
+     *     connection: {
+     *       class: Postgres,
+     *       options: {
+     *         connectionString: 'postgresql://usr:pwd@localhost:5432/db',
+     *       }
+     *     }
+     *   },
+     *   workers [...]
+     * };
+     * const hotMesh = await HotMesh.init(config);
+     * ```
+     */
+    static init(config: HotMeshConfig): Promise<HotMesh>;
+    /**
+     * returns a guid using the same core guid
+     * generator used by the HotMesh (nanoid)
+     */
+    static guid(): string;
+    /**
+     * @private
+     */
+    initEngine(config: HotMeshConfig, logger: ILogger): Promise<void>;
+    /**
+     * @private
+     */
+    initQuorum(config: HotMeshConfig, engine: EngineService, logger: ILogger): Promise<void>;
+    /**
+     * @private
+     */
+    constructor();
+    /**
+     * @private
+     */
+    doWork(config: HotMeshConfig, logger: ILogger): Promise<void>;
+    /**
+     * Starts a workflow
+     * @example
+     * ```typescript
+     * await hotMesh.pub('a.b.c', { key: 'value' });
+     * ```
+     */
+    pub(topic: string, data?: JobData, context?: JobState, extended?: ExtensionType): Promise<string>;
+    /**
+     * Subscribe (listen) to all output and interim emissions of a single
+     * workflow topic. NOTE: Postgres does not support patterned
+     * unsubscription, so this method is not supported for Postgres.
+     *
+     * @example
+     * ```typescript
+     * await hotMesh.psub('a.b.c', (topic, message) => {
+     *  console.log(message);
+     * });
+     * ```
+     */
+    sub(topic: string, callback: JobMessageCallback): Promise<void>;
+    /**
+     * Stop listening in on a single workflow topic
+     */
+    unsub(topic: string): Promise<void>;
+    /**
+     * Listen to all output and interim emissions of a workflow topic
+     * matching a wildcard pattern.
+     * @example
+     * ```typescript
+     * await hotMesh.psub('a.b.c*', (topic, message) => {
+     *  console.log(message);
+     * });
+     * ```
+     */
+    psub(wild: string, callback: JobMessageCallback): Promise<void>;
+    /**
+     * Patterned unsubscribe. NOTE: Postgres does not support patterned
+     * unsubscription, so this method is not supported for Postgres.
+     */
+    punsub(wild: string): Promise<void>;
+    /**
+     * Starts a workflow and awaits the response
+     * @example
+     * ```typescript
+     * await hotMesh.pubsub('a.b.c', { key: 'value' });
+     * ```
+     */
+    pubsub(topic: string, data?: JobData, context?: JobState | null, timeout?: number): Promise<JobOutput>;
+    /**
+     * Add a transition message to the workstream, resuming leg 2 of a paused
+     * reentrant activity (e.g., await, worker, hook)
+     */
+    add(streamData: StreamData | StreamDataResponse): Promise<string>;
+    /**
+     * Request a roll call from the quorum (engine and workers)
+     */
+    rollCall(delay?: number): Promise<QuorumProfile[]>;
+    /**
+     * Sends a throttle message to the quorum (engine and/or workers)
+     * to limit the rate of processing. Pass `-1` to throttle indefinitely.
+     * The value must be a non-negative integer and not exceed `MAX_DELAY` ms.
+     *
+     * When throttling is set, the quorum will pause for the specified time
+     * before processing the next message. Target specific engines and
+     * workers by passing a `guid` and/or `topic`. Pass no arguments to
+     * throttle the entire quorum.
+     *
+     * In this example, all processing has been paused indefinitely for
+     * the entire quorum. This is equivalent to an emergency stop.
+     *
+     * HotMesh is a stateless sequence engine, so the throttle can be adjusted up
+     * and down with no loss of data.
+     *
+     *
+     * @example
+     * ```typescript
+     * await hotMesh.throttle({ throttle: -1 });
+     * ```
+     */
+    throttle(options: ThrottleOptions): Promise<boolean>;
+    /**
+     * Publish a message to the quorum (engine and/or workers)
+     */
+    pubQuorum(quorumMessage: QuorumMessage): Promise<boolean>;
+    /**
+     * Subscribe to quorum events (engine and workers)
+     */
+    subQuorum(callback: QuorumMessageCallback): Promise<void>;
+    /**
+     * Unsubscribe from quorum events (engine and workers)
+     */
+    unsubQuorum(callback: QuorumMessageCallback): Promise<void>;
+    /**
+     * Preview changes and provide an analysis of risk
+     * prior to deployment
+     * @private
+     */
+    plan(path: string): Promise<HotMeshManifest>;
+    /**
+     * When the app YAML descriptor file is ready, the `deploy` function can be called.
+     * This function is responsible for merging all referenced YAML source
+     * files and writing the JSON output to the file system and to the provider backend. It
+     * is also possible to embed the YAML in-line as a string.
+     *
+     * *The version will not be active until activation is explicitly called.*
+     */
+    deploy(pathOrYAML: string): Promise<HotMeshManifest>;
+    /**
+     * Once the app YAML file is deployed to the provider backend, the `activate` function can be
+     * called to enable it for the entire quorum at the same moment.
+     *
+     * The approach is to establish the coordinated health of the system through series
+     * of call/response exchanges. Once it is established that the quorum is healthy,
+     * the quorum is instructed to run their engine in `no-cache` mode, ensuring
+     * that the provider backend is consulted for the active app version each time a
+     * call is processed. This ensures that all engines are running the same version
+     * of the app, switching over at the same moment and then enabling `cache` mode
+     * to improve performance.
+     *
+     * *Add a delay for the quorum to reach consensus if traffic is busy, but
+     * also consider throttling traffic flow to an acceptable level.*
+     */
+    activate(version: string, delay?: number): Promise<boolean>;
+    /**
+     * Returns the job state as a JSON object, useful
+     * for understanding dependency chains
+     */
+    export(jobId: string): Promise<JobExport>;
+    /**
+     * Returns all data (HGETALL) for a job.
+     */
+    getRaw(jobId: string): Promise<StringStringType>;
+    /**
+     * Reporter-related method to get the status of a job
+     * @private
+     */
+    getStats(topic: string, query: JobStatsInput): Promise<StatsResponse>;
+    /**
+     * Returns the status of a job. This is a numeric
+     * semaphore value that indicates the job's state.
+     * Any non-positive value indicates a completed job.
+     * Jobs with a value of `-1` are pending and will
+     * automatically be scrubbed after a set period.
+     * Jobs a value around -1billion have been interrupted
+     * and will be scrubbed after a set period. Jobs with
+     * a value of 0 completed normally. Jobs with a
+     * positive value are still running.
+     */
+    getStatus(jobId: string): Promise<JobStatus>;
+    /**
+     * Returns the job state (data and metadata) for a job.
+     */
+    getState(topic: string, jobId: string): Promise<JobOutput>;
+    /**
+     * Returns searchable/queryable data for a job. In this
+     * example a literal field is also searched (the colon
+     * is used to track job status and is a reserved field;
+     * it can be read but not written).
+     *
+     * @example
+     * ```typescript
+     * const fields = ['fred', 'barney', '":"'];
+     * const queryState = await hotMesh.getQueryState('123', fields);
+     * //returns { fred: 'flintstone', barney: 'rubble', ':': '1' }
+     * ```
+     */
+    getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
+    /**
+     * @private
+     */
+    getIds(topic: string, query: JobStatsInput, queryFacets?: any[]): Promise<IdsResponse>;
+    /**
+     * @private
+     */
+    resolveQuery(topic: string, query: JobStatsInput): Promise<GetStatsOptions>;
+    /**
+     * Interrupt an active job
+     */
+    interrupt(topic: string, jobId: string, options?: JobInterruptOptions): Promise<string>;
+    /**
+     * Immediately deletes (DEL) a completed job from the system.
+     *
+     * *Scrubbed jobs must be complete with a non-positive `status` value*
+     */
+    scrub(jobId: string): Promise<void>;
+    /**
+     * Re/entry point for an active job. This is used to resume a paused job
+     * and close the reentry point or leave it open for subsequent reentry.
+     * Because `hooks` are public entry points, they include a `topic`
+     * which is established in the app YAML file.
+     *
+     * When this method is called, a hook rule will be located to establish
+     * the exact activity and activity dimension for reentry.
+     */
+    hook(topic: string, data: JobData, status?: StreamStatus, code?: StreamCode): Promise<string>;
+    /**
+     * @private
+     */
+    hookAll(hookTopic: string, data: JobData, query: JobStatsInput, queryFacets?: string[]): Promise<string[]>;
+    /**
+     * Stop all points of presence, workers and engines
+     */
+    static stop(): Promise<void>;
+    /**
+     * Stop this point of presence, workers and engines
+     */
+    stop(): void;
+    /**
+     * @private
+     * @deprecated
+     */
+    compress(terms: string[]): Promise<boolean>;
+}
+export { HotMesh };