@hotmeshio/hotmesh 0.3.32 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +128 -823
- package/build/index.d.ts +9 -9
- package/build/index.js +10 -10
- package/build/modules/enums.d.ts +23 -23
- package/build/modules/enums.js +26 -26
- package/build/modules/errors.d.ts +16 -16
- package/build/modules/errors.js +28 -28
- package/build/modules/key.js +190 -1
- package/build/modules/utils.js +374 -1
- package/build/package.json +22 -21
- package/build/services/activities/activity.js +549 -1
- package/build/services/activities/await.js +114 -1
- package/build/services/activities/cycle.js +112 -1
- package/build/services/activities/hook.js +168 -1
- package/build/services/activities/index.js +20 -1
- package/build/services/activities/interrupt.js +158 -1
- package/build/services/activities/signal.js +134 -1
- package/build/services/activities/trigger.js +246 -1
- package/build/services/activities/worker.js +106 -1
- package/build/services/collator/index.js +293 -1
- package/build/services/compiler/deployer.js +488 -1
- package/build/services/compiler/index.js +112 -1
- package/build/services/compiler/validator.js +147 -1
- package/build/services/engine/index.js +761 -1
- package/build/services/exporter/index.js +126 -1
- package/build/services/hotmesh/index.d.ts +160 -17
- package/build/services/hotmesh/index.js +160 -17
- package/build/services/mapper/index.js +81 -1
- package/build/services/{meshflow → memflow}/client.d.ts +3 -3
- package/build/services/{meshflow → memflow}/client.js +17 -16
- package/build/services/{meshflow → memflow}/connection.d.ts +2 -2
- package/build/services/{meshflow → memflow}/connection.js +1 -1
- package/build/services/memflow/context.d.ts +143 -0
- package/build/services/memflow/context.js +299 -0
- package/build/services/{meshflow → memflow}/exporter.d.ts +6 -6
- package/build/services/memflow/exporter.js +215 -0
- package/build/services/{meshflow → memflow}/handle.d.ts +4 -4
- package/build/services/{meshflow → memflow}/handle.js +2 -2
- package/build/services/{meshflow → memflow}/index.d.ts +18 -13
- package/build/services/{meshflow → memflow}/index.js +26 -21
- package/build/services/{meshflow → memflow}/schemas/factory.d.ts +4 -4
- package/build/services/{meshflow → memflow}/schemas/factory.js +5 -5
- package/build/services/{meshflow → memflow}/search.d.ts +1 -1
- package/build/services/{meshflow → memflow}/search.js +4 -4
- package/build/services/{meshflow → memflow}/worker.d.ts +5 -5
- package/build/services/{meshflow → memflow}/worker.js +24 -24
- package/build/services/memflow/workflow/common.d.ts +20 -0
- package/build/services/memflow/workflow/common.js +47 -0
- package/build/services/memflow/workflow/contextMethods.d.ts +14 -0
- package/build/services/memflow/workflow/contextMethods.js +33 -0
- package/build/services/{meshflow → memflow}/workflow/execChild.js +12 -12
- package/build/services/memflow/workflow/execHook.d.ts +65 -0
- package/build/services/memflow/workflow/execHook.js +73 -0
- package/build/services/{meshflow → memflow}/workflow/hook.js +19 -3
- package/build/services/{meshflow → memflow}/workflow/index.d.ts +7 -3
- package/build/services/{meshflow → memflow}/workflow/index.js +7 -3
- package/build/services/{meshflow → memflow}/workflow/proxyActivities.d.ts +2 -2
- package/build/services/{meshflow → memflow}/workflow/proxyActivities.js +8 -8
- package/build/services/{meshflow → memflow}/workflow/sleepFor.js +2 -2
- package/build/services/{meshflow → memflow}/workflow/waitFor.js +2 -2
- package/build/services/meshdata/index.d.ts +24 -24
- package/build/services/meshdata/index.js +40 -40
- package/build/services/pipe/functions/array.js +74 -1
- package/build/services/pipe/functions/bitwise.js +24 -1
- package/build/services/pipe/functions/conditional.js +36 -1
- package/build/services/pipe/functions/cron.js +40 -1
- package/build/services/pipe/functions/date.js +171 -1
- package/build/services/pipe/functions/index.js +30 -1
- package/build/services/pipe/functions/json.js +12 -1
- package/build/services/pipe/functions/logical.js +12 -1
- package/build/services/pipe/functions/math.js +184 -1
- package/build/services/pipe/functions/number.js +60 -1
- package/build/services/pipe/functions/object.js +81 -1
- package/build/services/pipe/functions/string.js +69 -1
- package/build/services/pipe/functions/symbol.js +33 -1
- package/build/services/pipe/functions/unary.js +18 -1
- package/build/services/pipe/index.js +242 -1
- package/build/services/quorum/index.js +263 -1
- package/build/services/reporter/index.js +348 -1
- package/build/services/router/config/index.d.ts +11 -0
- package/build/services/router/config/index.js +36 -0
- package/build/services/router/consumption/index.d.ts +34 -0
- package/build/services/router/consumption/index.js +395 -0
- package/build/services/router/error-handling/index.d.ts +8 -0
- package/build/services/router/error-handling/index.js +98 -0
- package/build/services/router/index.d.ts +13 -16
- package/build/services/router/index.js +121 -1
- package/build/services/router/lifecycle/index.d.ts +27 -0
- package/build/services/router/lifecycle/index.js +80 -0
- package/build/services/router/telemetry/index.d.ts +11 -0
- package/build/services/router/telemetry/index.js +32 -0
- package/build/services/router/throttling/index.d.ts +23 -0
- package/build/services/router/throttling/index.js +76 -0
- package/build/services/search/index.d.ts +2 -1
- package/build/services/search/providers/postgres/postgres.d.ts +2 -1
- package/build/services/search/providers/postgres/postgres.js +149 -1
- package/build/services/search/providers/redis/ioredis.d.ts +1 -0
- package/build/services/search/providers/redis/ioredis.js +121 -1
- package/build/services/search/providers/redis/redis.d.ts +1 -0
- package/build/services/search/providers/redis/redis.js +134 -1
- package/build/services/serializer/index.js +282 -1
- package/build/services/store/providers/postgres/kvsql.d.ts +1 -1
- package/build/services/store/providers/postgres/kvsql.js +198 -1
- package/build/services/store/providers/postgres/kvtables.js +441 -1
- package/build/services/store/providers/postgres/kvtransaction.js +248 -1
- package/build/services/store/providers/postgres/kvtypes/hash.d.ts +1 -1
- package/build/services/store/providers/postgres/kvtypes/hash.js +1287 -1
- package/build/services/store/providers/postgres/kvtypes/list.js +194 -1
- package/build/services/store/providers/postgres/kvtypes/string.js +115 -1
- package/build/services/store/providers/postgres/kvtypes/zset.js +214 -1
- package/build/services/store/providers/postgres/postgres.js +1036 -1
- package/build/services/store/providers/redis/_base.js +980 -1
- package/build/services/store/providers/redis/ioredis.js +180 -1
- package/build/services/store/providers/redis/redis.js +199 -1
- package/build/services/store/providers/store-initializable.js +2 -1
- package/build/services/stream/index.d.ts +5 -0
- package/build/services/stream/providers/nats/nats.d.ts +1 -0
- package/build/services/stream/providers/nats/nats.js +225 -1
- package/build/services/stream/providers/postgres/kvtables.d.ts +1 -0
- package/build/services/stream/providers/postgres/kvtables.js +146 -1
- package/build/services/stream/providers/postgres/postgres.d.ts +19 -0
- package/build/services/stream/providers/postgres/postgres.js +519 -1
- package/build/services/stream/providers/redis/ioredis.d.ts +1 -0
- package/build/services/stream/providers/redis/ioredis.js +272 -1
- package/build/services/stream/providers/redis/redis.d.ts +1 -0
- package/build/services/stream/providers/redis/redis.js +305 -1
- package/build/services/stream/providers/stream-initializable.js +2 -1
- package/build/services/sub/providers/nats/nats.js +105 -1
- package/build/services/sub/providers/postgres/postgres.js +92 -1
- package/build/services/sub/providers/redis/ioredis.js +81 -1
- package/build/services/sub/providers/redis/redis.js +72 -1
- package/build/services/task/index.js +206 -1
- package/build/services/telemetry/index.js +306 -1
- package/build/services/worker/index.js +197 -1
- package/build/types/error.d.ts +5 -5
- package/build/types/exporter.d.ts +1 -1
- package/build/types/index.d.ts +3 -3
- package/build/types/manifest.d.ts +2 -2
- package/build/types/{meshflow.d.ts → memflow.d.ts} +15 -15
- package/build/types/meshdata.d.ts +3 -3
- package/build/types/postgres.d.ts +7 -0
- package/build/types/stream.d.ts +3 -0
- package/index.ts +11 -11
- package/package.json +22 -21
- package/.github/ISSUE_TEMPLATE/bug_report.md +0 -38
- package/.github/ISSUE_TEMPLATE/feature_request.md +0 -20
- package/build/services/meshflow/exporter.js +0 -1
- package/build/services/meshflow/workflow/common.d.ts +0 -18
- package/build/services/meshflow/workflow/common.js +0 -45
- package/typedoc.json +0 -46
- package/types/activity.ts +0 -268
- package/types/app.ts +0 -20
- package/types/async.ts +0 -6
- package/types/cache.ts +0 -1
- package/types/collator.ts +0 -9
- package/types/error.ts +0 -56
- package/types/exporter.ts +0 -102
- package/types/hook.ts +0 -44
- package/types/hotmesh.ts +0 -314
- package/types/index.ts +0 -306
- package/types/job.ts +0 -233
- package/types/logger.ts +0 -8
- package/types/manifest.ts +0 -70
- package/types/map.ts +0 -5
- package/types/meshcall.ts +0 -235
- package/types/meshdata.ts +0 -278
- package/types/meshflow.ts +0 -645
- package/types/ms.d.ts +0 -7
- package/types/nats.ts +0 -270
- package/types/pipe.ts +0 -90
- package/types/postgres.ts +0 -105
- package/types/provider.ts +0 -161
- package/types/quorum.ts +0 -167
- package/types/redis.ts +0 -404
- package/types/serializer.ts +0 -40
- package/types/stats.ts +0 -117
- package/types/stream.ts +0 -227
- package/types/task.ts +0 -7
- package/types/telemetry.ts +0 -16
- package/types/transition.ts +0 -20
- /package/build/services/{meshflow → memflow}/workflow/all.d.ts +0 -0
- /package/build/services/{meshflow → memflow}/workflow/all.js +0 -0
- /package/build/services/{meshflow → memflow}/workflow/context.d.ts +0 -0
- /package/build/services/{meshflow → memflow}/workflow/context.js +0 -0
- /package/build/services/{meshflow → memflow}/workflow/didRun.d.ts +0 -0
- /package/build/services/{meshflow → memflow}/workflow/didRun.js +0 -0
- /package/build/services/{meshflow → memflow}/workflow/emit.d.ts +0 -0
- /package/build/services/{meshflow → memflow}/workflow/emit.js +0 -0
- /package/build/services/{meshflow → memflow}/workflow/enrich.d.ts +0 -0
- /package/build/services/{meshflow → memflow}/workflow/enrich.js +0 -0
- /package/build/services/{meshflow → memflow}/workflow/execChild.d.ts +0 -0
- /package/build/services/{meshflow → memflow}/workflow/hook.d.ts +0 -0
- /package/build/services/{meshflow → memflow}/workflow/interrupt.d.ts +0 -0
- /package/build/services/{meshflow → memflow}/workflow/interrupt.js +0 -0
- /package/build/services/{meshflow → memflow}/workflow/isSideEffectAllowed.d.ts +0 -0
- /package/build/services/{meshflow → memflow}/workflow/isSideEffectAllowed.js +0 -0
- /package/build/services/{meshflow → memflow}/workflow/random.d.ts +0 -0
- /package/build/services/{meshflow → memflow}/workflow/random.js +0 -0
- /package/build/services/{meshflow → memflow}/workflow/searchMethods.d.ts +0 -0
- /package/build/services/{meshflow → memflow}/workflow/searchMethods.js +0 -0
- /package/build/services/{meshflow → memflow}/workflow/signal.d.ts +0 -0
- /package/build/services/{meshflow → memflow}/workflow/signal.js +0 -0
- /package/build/services/{meshflow → memflow}/workflow/sleepFor.d.ts +0 -0
- /package/build/services/{meshflow → memflow}/workflow/trace.d.ts +0 -0
- /package/build/services/{meshflow → memflow}/workflow/trace.js +0 -0
- /package/build/services/{meshflow → memflow}/workflow/waitFor.d.ts +0 -0
- /package/build/types/{meshflow.js → memflow.js} +0 -0
|
@@ -1 +1,126 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ExporterService = void 0;
|
|
4
|
+
const key_1 = require("../../modules/key");
|
|
5
|
+
const utils_1 = require("../../modules/utils");
|
|
6
|
+
const serializer_1 = require("../serializer");
|
|
7
|
+
/**
|
|
8
|
+
* Downloads job data from Redis (hscan, hmget, hgetall)
|
|
9
|
+
* Expands process data and includes dependency list
|
|
10
|
+
*/
|
|
11
|
+
class ExporterService {
|
|
12
|
+
constructor(appId, store, logger) {
|
|
13
|
+
this.appId = appId;
|
|
14
|
+
this.logger = logger;
|
|
15
|
+
this.store = store;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Convert the job hash into a JobExport object.
|
|
19
|
+
* This object contains various facets that describe the interaction
|
|
20
|
+
* in terms relevant to narrative storytelling.
|
|
21
|
+
*/
|
|
22
|
+
async export(jobId, options = {}) {
|
|
23
|
+
if (!this.symbols) {
|
|
24
|
+
this.symbols = this.store.getAllSymbols();
|
|
25
|
+
this.symbols = await this.symbols;
|
|
26
|
+
}
|
|
27
|
+
const depData = []; // await this.store.getDependencies(jobId);
|
|
28
|
+
const jobData = await this.store.getRaw(jobId);
|
|
29
|
+
const jobExport = this.inflate(jobData, depData);
|
|
30
|
+
return jobExport;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Inflates the key from Redis, 3-character symbol
|
|
34
|
+
* into a human-readable JSON path, reflecting the
|
|
35
|
+
* tree-like structure of the unidimensional Hash
|
|
36
|
+
*/
|
|
37
|
+
inflateKey(key) {
|
|
38
|
+
return key in this.symbols ? this.symbols[key] : key;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Inflates the job data from Redis into a JobExport object
|
|
42
|
+
* @param jobHash - the job data from Redis
|
|
43
|
+
* @param dependencyList - the list of dependencies for the job
|
|
44
|
+
* @returns - the inflated job data
|
|
45
|
+
*/
|
|
46
|
+
inflate(jobHash, dependencyList) {
|
|
47
|
+
//the list of actions taken in the workflow and hook functions
|
|
48
|
+
const actions = {
|
|
49
|
+
hooks: {},
|
|
50
|
+
main: {
|
|
51
|
+
cursor: -1,
|
|
52
|
+
items: [],
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
const process = {};
|
|
56
|
+
const dependencies = this.inflateDependencyData(dependencyList, actions);
|
|
57
|
+
const regex = /^([a-zA-Z]{3}),(\d+(?:,\d+)*)/;
|
|
58
|
+
Object.entries(jobHash).forEach(([key, value]) => {
|
|
59
|
+
const match = key.match(regex);
|
|
60
|
+
if (match) {
|
|
61
|
+
//activity process state
|
|
62
|
+
const [_, letters, numbers] = match;
|
|
63
|
+
const path = this.inflateKey(letters);
|
|
64
|
+
const dimensions = `${numbers.replace(/,/g, '/')}`;
|
|
65
|
+
const resolved = serializer_1.SerializerService.fromString(value);
|
|
66
|
+
process[`${dimensions}/${path}`] = resolved;
|
|
67
|
+
}
|
|
68
|
+
else if (key.length === 3) {
|
|
69
|
+
//job state
|
|
70
|
+
process[this.inflateKey(key)] = serializer_1.SerializerService.fromString(value);
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
return {
|
|
74
|
+
dependencies,
|
|
75
|
+
process: (0, utils_1.restoreHierarchy)(process),
|
|
76
|
+
status: jobHash[':'],
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Inflates the dependency data from Redis into a JobExport object by
|
|
81
|
+
* organizing the dimensional isolate in sch a way asto interleave
|
|
82
|
+
* into a story
|
|
83
|
+
* @param data - the dependency data from Redis
|
|
84
|
+
* @returns - the organized dependency data
|
|
85
|
+
*/
|
|
86
|
+
inflateDependencyData(data, actions) {
|
|
87
|
+
const hookReg = /([0-9,]+)-(\d+)$/;
|
|
88
|
+
const flowReg = /-(\d+)$/;
|
|
89
|
+
return data.map((dependency, index) => {
|
|
90
|
+
const [action, topic, gid, _pd, ...jid] = dependency.split(key_1.VALSEP);
|
|
91
|
+
const jobId = jid.join(key_1.VALSEP);
|
|
92
|
+
const match = jobId.match(hookReg);
|
|
93
|
+
let prefix;
|
|
94
|
+
let type;
|
|
95
|
+
let dimensionKey = '';
|
|
96
|
+
if (match) {
|
|
97
|
+
//hook-originating dependency
|
|
98
|
+
const [_, dimension, counter] = match;
|
|
99
|
+
dimensionKey = dimension.split(',').join('/');
|
|
100
|
+
prefix = `${dimensionKey}[${counter}]`;
|
|
101
|
+
type = 'hook';
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
const match = jobId.match(flowReg);
|
|
105
|
+
if (match) {
|
|
106
|
+
//main workflow-originating dependency
|
|
107
|
+
const [_, counter] = match;
|
|
108
|
+
prefix = `[${counter}]`;
|
|
109
|
+
type = 'flow';
|
|
110
|
+
}
|
|
111
|
+
else {
|
|
112
|
+
//'other' types like signal cleanup
|
|
113
|
+
prefix = '/';
|
|
114
|
+
type = 'other';
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return {
|
|
118
|
+
type: action,
|
|
119
|
+
topic,
|
|
120
|
+
gid,
|
|
121
|
+
jid: jobId,
|
|
122
|
+
};
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
exports.ExporterService = ExporterService;
|
|
@@ -10,48 +10,191 @@ import { StringAnyType, StringStringType } from '../../types/serializer';
|
|
|
10
10
|
import { JobStatsInput, GetStatsOptions, IdsResponse, StatsResponse } from '../../types/stats';
|
|
11
11
|
import { StreamCode, StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
|
|
12
12
|
/**
|
|
13
|
-
*
|
|
14
|
-
*
|
|
13
|
+
* HotMesh is a distributed, reentrant process orchestration engine that transforms
|
|
14
|
+
* Redis, Postgres, or NATS into a resilient service mesh capable of running
|
|
15
|
+
* fault-tolerant workflows across multiple services and systems.
|
|
15
16
|
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
17
|
+
* ## Core Concepts
|
|
18
|
+
*
|
|
19
|
+
* **Distributed Quorum Architecture**: HotMesh operates as a distributed quorum
|
|
20
|
+
* where multiple engine and worker instances coordinate using CQRS principles.
|
|
21
|
+
* Each member reads from assigned topic queues and writes results to other queues,
|
|
22
|
+
* creating emergent workflow orchestration without a central controller.
|
|
23
|
+
*
|
|
24
|
+
* **Reentrant Process Engine**: Unlike traditional workflow engines, HotMesh
|
|
25
|
+
* provides built-in retry logic, idempotency, and failure recovery. Your business
|
|
26
|
+
* logic doesn't need to handle timeouts or retries - the engine manages all of that.
|
|
27
|
+
*
|
|
28
|
+
* **Multi-Provider Support**: Supports Redis/ValKey, Postgres, and NATS as backend
|
|
29
|
+
* providers, allowing you to leverage existing infrastructure investments.
|
|
30
|
+
*
|
|
31
|
+
* ## Key Features
|
|
32
|
+
*
|
|
33
|
+
* - **Fault Tolerance**: Automatic retry, timeout, and failure recovery
|
|
34
|
+
* - **Distributed Execution**: No single point of failure
|
|
35
|
+
* - **Multi-Provider**: Redis, Postgres, NATS backend support
|
|
36
|
+
* - **YAML-Driven**: Model-driven development with declarative workflow definitions
|
|
37
|
+
* - **OpenTelemetry**: Built-in observability and tracing
|
|
38
|
+
* - **Durable State**: Workflow state persists across system restarts
|
|
39
|
+
* - **Pattern Matching**: Pub/sub with wildcard pattern support
|
|
40
|
+
* - **Throttling**: Dynamic flow control and backpressure management
|
|
41
|
+
*
|
|
42
|
+
* ## Architecture
|
|
43
|
+
*
|
|
44
|
+
* HotMesh consists of several specialized modules:
|
|
45
|
+
* - **HotMesh**: Core orchestration engine (this class)
|
|
46
|
+
* - **MemFlow**: Temporal.io-compatible workflow framework
|
|
47
|
+
* - **MeshCall**: Durable function execution (Temporal-like clone)
|
|
48
|
+
*
|
|
49
|
+
* ## Lifecycle Overview
|
|
50
|
+
*
|
|
51
|
+
* 1. **Initialize**: Create HotMesh instance with provider configuration
|
|
52
|
+
* 2. **Deploy**: Upload YAML workflow definitions to the backend
|
|
53
|
+
* 3. **Activate**: Coordinate quorum to enable the workflow version
|
|
54
|
+
* 4. **Execute**: Publish events to trigger workflow execution
|
|
55
|
+
* 5. **Monitor**: Track progress via OpenTelemetry and built-in observability
|
|
56
|
+
*
|
|
57
|
+
* ## Basic Usage
|
|
20
58
|
*
|
|
21
59
|
* @example
|
|
22
60
|
* ```typescript
|
|
23
|
-
* import { Client as Postgres } from 'pg';
|
|
24
61
|
* import { HotMesh } from '@hotmeshio/hotmesh';
|
|
62
|
+
* import Redis from 'ioredis';
|
|
25
63
|
*
|
|
64
|
+
* // Initialize with Redis backend
|
|
26
65
|
* const hotMesh = await HotMesh.init({
|
|
27
|
-
* appId: '
|
|
66
|
+
* appId: 'my-app',
|
|
28
67
|
* engine: {
|
|
29
68
|
* connection: {
|
|
30
|
-
* class:
|
|
31
|
-
* options: {
|
|
32
|
-
* connectionString: 'postgresql://usr:pwd@localhost:5432/db',
|
|
33
|
-
* }
|
|
69
|
+
* class: Redis,
|
|
70
|
+
* options: { host: 'localhost', port: 6379 }
|
|
34
71
|
* }
|
|
35
72
|
* }
|
|
36
73
|
* });
|
|
37
74
|
*
|
|
75
|
+
* // Deploy workflow definition
|
|
38
76
|
* await hotMesh.deploy(`
|
|
39
77
|
* app:
|
|
40
|
-
* id:
|
|
78
|
+
* id: my-app
|
|
41
79
|
* version: '1'
|
|
42
80
|
* graphs:
|
|
43
|
-
* - subscribes:
|
|
81
|
+
* - subscribes: order.process
|
|
44
82
|
* activities:
|
|
45
|
-
*
|
|
46
|
-
* type:
|
|
83
|
+
* validate:
|
|
84
|
+
* type: worker
|
|
85
|
+
* topic: order.validate
|
|
86
|
+
* approve:
|
|
87
|
+
* type: hook
|
|
88
|
+
* topic: order.approve
|
|
89
|
+
* fulfill:
|
|
90
|
+
* type: worker
|
|
91
|
+
* topic: order.fulfill
|
|
92
|
+
* transitions:
|
|
93
|
+
* validate:
|
|
94
|
+
* - to: approve
|
|
95
|
+
* approve:
|
|
96
|
+
* - to: fulfill
|
|
47
97
|
* `);
|
|
48
98
|
*
|
|
99
|
+
* // Activate the workflow version
|
|
49
100
|
* await hotMesh.activate('1');
|
|
50
101
|
*
|
|
51
|
-
*
|
|
102
|
+
* // Execute workflow (fire-and-forget)
|
|
103
|
+
* const jobId = await hotMesh.pub('order.process', {
|
|
104
|
+
* orderId: '12345',
|
|
105
|
+
* amount: 99.99
|
|
106
|
+
* });
|
|
107
|
+
*
|
|
108
|
+
* // Execute workflow and wait for result
|
|
109
|
+
* const result = await hotMesh.pubsub('order.process', {
|
|
110
|
+
* orderId: '12345',
|
|
111
|
+
* amount: 99.99
|
|
112
|
+
* });
|
|
113
|
+
* ```
|
|
114
|
+
*
|
|
115
|
+
* ## Postgres Backend Example
|
|
116
|
+
*
|
|
117
|
+
* @example
|
|
118
|
+
* ```typescript
|
|
119
|
+
* import { HotMesh } from '@hotmeshio/hotmesh';
|
|
120
|
+
* import { Client as Postgres } from 'pg';
|
|
52
121
|
*
|
|
122
|
+
* const hotMesh = await HotMesh.init({
|
|
123
|
+
* appId: 'my-app',
|
|
124
|
+
* engine: {
|
|
125
|
+
* connection: {
|
|
126
|
+
* class: Postgres,
|
|
127
|
+
* options: {
|
|
128
|
+
* connectionString: 'postgresql://user:pass@localhost:5432/db'
|
|
129
|
+
* }
|
|
130
|
+
* }
|
|
131
|
+
* }
|
|
132
|
+
* });
|
|
133
|
+
* ```
|
|
134
|
+
*
|
|
135
|
+
* ## Advanced Features
|
|
136
|
+
*
|
|
137
|
+
* **Pattern Subscriptions**: Listen to multiple workflow topics
|
|
138
|
+
* ```typescript
|
|
139
|
+
* await hotMesh.psub('order.*', (topic, message) => {
|
|
140
|
+
* console.log(`Received ${topic}:`, message);
|
|
141
|
+
* });
|
|
142
|
+
* ```
|
|
143
|
+
*
|
|
144
|
+
* **Throttling**: Control processing rates
|
|
145
|
+
* ```typescript
|
|
146
|
+
* // Pause all processing for 5 seconds
|
|
147
|
+
* await hotMesh.throttle({ throttle: 5000 });
|
|
148
|
+
*
|
|
149
|
+
* // Emergency stop (pause indefinitely)
|
|
150
|
+
* await hotMesh.throttle({ throttle: -1 });
|
|
151
|
+
* ```
|
|
152
|
+
*
|
|
153
|
+
* **Workflow Interruption**: Gracefully stop running workflows
|
|
154
|
+
* ```typescript
|
|
155
|
+
* await hotMesh.interrupt('order.process', jobId, {
|
|
156
|
+
* reason: 'User cancellation'
|
|
157
|
+
* });
|
|
158
|
+
* ```
|
|
159
|
+
*
|
|
160
|
+
* **State Inspection**: Query workflow state and progress
|
|
161
|
+
* ```typescript
|
|
162
|
+
* const state = await hotMesh.getState('order.process', jobId);
|
|
163
|
+
* const status = await hotMesh.getStatus(jobId);
|
|
164
|
+
* ```
|
|
165
|
+
*
|
|
166
|
+
* ## Distributed Coordination
|
|
167
|
+
*
|
|
168
|
+
* HotMesh automatically handles distributed coordination through its quorum system:
|
|
169
|
+
*
|
|
170
|
+
* ```typescript
|
|
171
|
+
* // Check quorum health
|
|
172
|
+
* const members = await hotMesh.rollCall();
|
|
173
|
+
*
|
|
174
|
+
* // Coordinate version activation across all instances
|
|
175
|
+
* await hotMesh.activate('2', 1000); // 1 second delay for consensus
|
|
176
|
+
* ```
|
|
177
|
+
*
|
|
178
|
+
* ## Integration with Higher-Level Modules
|
|
179
|
+
*
|
|
180
|
+
* For most use cases, consider using the higher-level modules:
|
|
181
|
+
* - **MemFlow**: For Temporal.io-style workflows with TypeScript functions
|
|
182
|
+
* - **MeshCall**: For durable function calls and RPC patterns
|
|
183
|
+
*
|
|
184
|
+
* ## Cleanup
|
|
185
|
+
*
|
|
186
|
+
* Always clean up resources when shutting down:
|
|
187
|
+
* ```typescript
|
|
188
|
+
* // Stop this instance
|
|
189
|
+
* hotMesh.stop();
|
|
190
|
+
*
|
|
191
|
+
* // Stop all instances (typically in signal handlers)
|
|
53
192
|
* await HotMesh.stop();
|
|
54
193
|
* ```
|
|
194
|
+
*
|
|
195
|
+
* @see {@link https://hotmesh.io/docs} - Complete documentation
|
|
196
|
+
* @see {@link https://github.com/hotmeshio/samples-typescript} - Examples and tutorials
|
|
197
|
+
* @see {@link https://zenodo.org/records/12168558} - Academic paper on the architecture
|
|
55
198
|
*/
|
|
56
199
|
declare class HotMesh {
|
|
57
200
|
namespace: string;
|
|
@@ -11,48 +11,191 @@ const router_1 = require("../router");
|
|
|
11
11
|
const worker_1 = require("../worker");
|
|
12
12
|
const enums_1 = require("../../modules/enums");
|
|
13
13
|
/**
|
|
14
|
-
*
|
|
15
|
-
*
|
|
14
|
+
* HotMesh is a distributed, reentrant process orchestration engine that transforms
|
|
15
|
+
* Redis, Postgres, or NATS into a resilient service mesh capable of running
|
|
16
|
+
* fault-tolerant workflows across multiple services and systems.
|
|
16
17
|
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
18
|
+
* ## Core Concepts
|
|
19
|
+
*
|
|
20
|
+
* **Distributed Quorum Architecture**: HotMesh operates as a distributed quorum
|
|
21
|
+
* where multiple engine and worker instances coordinate using CQRS principles.
|
|
22
|
+
* Each member reads from assigned topic queues and writes results to other queues,
|
|
23
|
+
* creating emergent workflow orchestration without a central controller.
|
|
24
|
+
*
|
|
25
|
+
* **Reentrant Process Engine**: Unlike traditional workflow engines, HotMesh
|
|
26
|
+
* provides built-in retry logic, idempotency, and failure recovery. Your business
|
|
27
|
+
* logic doesn't need to handle timeouts or retries - the engine manages all of that.
|
|
28
|
+
*
|
|
29
|
+
* **Multi-Provider Support**: Supports Redis/ValKey, Postgres, and NATS as backend
|
|
30
|
+
* providers, allowing you to leverage existing infrastructure investments.
|
|
31
|
+
*
|
|
32
|
+
* ## Key Features
|
|
33
|
+
*
|
|
34
|
+
* - **Fault Tolerance**: Automatic retry, timeout, and failure recovery
|
|
35
|
+
* - **Distributed Execution**: No single point of failure
|
|
36
|
+
* - **Multi-Provider**: Redis, Postgres, NATS backend support
|
|
37
|
+
* - **YAML-Driven**: Model-driven development with declarative workflow definitions
|
|
38
|
+
* - **OpenTelemetry**: Built-in observability and tracing
|
|
39
|
+
* - **Durable State**: Workflow state persists across system restarts
|
|
40
|
+
* - **Pattern Matching**: Pub/sub with wildcard pattern support
|
|
41
|
+
* - **Throttling**: Dynamic flow control and backpressure management
|
|
42
|
+
*
|
|
43
|
+
* ## Architecture
|
|
44
|
+
*
|
|
45
|
+
* HotMesh consists of several specialized modules:
|
|
46
|
+
* - **HotMesh**: Core orchestration engine (this class)
|
|
47
|
+
* - **MemFlow**: Temporal.io-compatible workflow framework
|
|
48
|
+
* - **MeshCall**: Durable function execution (Temporal-like clone)
|
|
49
|
+
*
|
|
50
|
+
* ## Lifecycle Overview
|
|
51
|
+
*
|
|
52
|
+
* 1. **Initialize**: Create HotMesh instance with provider configuration
|
|
53
|
+
* 2. **Deploy**: Upload YAML workflow definitions to the backend
|
|
54
|
+
* 3. **Activate**: Coordinate quorum to enable the workflow version
|
|
55
|
+
* 4. **Execute**: Publish events to trigger workflow execution
|
|
56
|
+
* 5. **Monitor**: Track progress via OpenTelemetry and built-in observability
|
|
57
|
+
*
|
|
58
|
+
* ## Basic Usage
|
|
21
59
|
*
|
|
22
60
|
* @example
|
|
23
61
|
* ```typescript
|
|
24
|
-
* import { Client as Postgres } from 'pg';
|
|
25
62
|
* import { HotMesh } from '@hotmeshio/hotmesh';
|
|
63
|
+
* import Redis from 'ioredis';
|
|
26
64
|
*
|
|
65
|
+
* // Initialize with Redis backend
|
|
27
66
|
* const hotMesh = await HotMesh.init({
|
|
28
|
-
* appId: '
|
|
67
|
+
* appId: 'my-app',
|
|
29
68
|
* engine: {
|
|
30
69
|
* connection: {
|
|
31
|
-
* class:
|
|
32
|
-
* options: {
|
|
33
|
-
* connectionString: 'postgresql://usr:pwd@localhost:5432/db',
|
|
34
|
-
* }
|
|
70
|
+
* class: Redis,
|
|
71
|
+
* options: { host: 'localhost', port: 6379 }
|
|
35
72
|
* }
|
|
36
73
|
* }
|
|
37
74
|
* });
|
|
38
75
|
*
|
|
76
|
+
* // Deploy workflow definition
|
|
39
77
|
* await hotMesh.deploy(`
|
|
40
78
|
* app:
|
|
41
|
-
* id:
|
|
79
|
+
* id: my-app
|
|
42
80
|
* version: '1'
|
|
43
81
|
* graphs:
|
|
44
|
-
* - subscribes:
|
|
82
|
+
* - subscribes: order.process
|
|
45
83
|
* activities:
|
|
46
|
-
*
|
|
47
|
-
* type:
|
|
84
|
+
* validate:
|
|
85
|
+
* type: worker
|
|
86
|
+
* topic: order.validate
|
|
87
|
+
* approve:
|
|
88
|
+
* type: hook
|
|
89
|
+
* topic: order.approve
|
|
90
|
+
* fulfill:
|
|
91
|
+
* type: worker
|
|
92
|
+
* topic: order.fulfill
|
|
93
|
+
* transitions:
|
|
94
|
+
* validate:
|
|
95
|
+
* - to: approve
|
|
96
|
+
* approve:
|
|
97
|
+
* - to: fulfill
|
|
48
98
|
* `);
|
|
49
99
|
*
|
|
100
|
+
* // Activate the workflow version
|
|
50
101
|
* await hotMesh.activate('1');
|
|
51
102
|
*
|
|
52
|
-
*
|
|
103
|
+
* // Execute workflow (fire-and-forget)
|
|
104
|
+
* const jobId = await hotMesh.pub('order.process', {
|
|
105
|
+
* orderId: '12345',
|
|
106
|
+
* amount: 99.99
|
|
107
|
+
* });
|
|
108
|
+
*
|
|
109
|
+
* // Execute workflow and wait for result
|
|
110
|
+
* const result = await hotMesh.pubsub('order.process', {
|
|
111
|
+
* orderId: '12345',
|
|
112
|
+
* amount: 99.99
|
|
113
|
+
* });
|
|
114
|
+
* ```
|
|
115
|
+
*
|
|
116
|
+
* ## Postgres Backend Example
|
|
117
|
+
*
|
|
118
|
+
* @example
|
|
119
|
+
* ```typescript
|
|
120
|
+
* import { HotMesh } from '@hotmeshio/hotmesh';
|
|
121
|
+
* import { Client as Postgres } from 'pg';
|
|
53
122
|
*
|
|
123
|
+
* const hotMesh = await HotMesh.init({
|
|
124
|
+
* appId: 'my-app',
|
|
125
|
+
* engine: {
|
|
126
|
+
* connection: {
|
|
127
|
+
* class: Postgres,
|
|
128
|
+
* options: {
|
|
129
|
+
* connectionString: 'postgresql://user:pass@localhost:5432/db'
|
|
130
|
+
* }
|
|
131
|
+
* }
|
|
132
|
+
* }
|
|
133
|
+
* });
|
|
134
|
+
* ```
|
|
135
|
+
*
|
|
136
|
+
* ## Advanced Features
|
|
137
|
+
*
|
|
138
|
+
* **Pattern Subscriptions**: Listen to multiple workflow topics
|
|
139
|
+
* ```typescript
|
|
140
|
+
* await hotMesh.psub('order.*', (topic, message) => {
|
|
141
|
+
* console.log(`Received ${topic}:`, message);
|
|
142
|
+
* });
|
|
143
|
+
* ```
|
|
144
|
+
*
|
|
145
|
+
* **Throttling**: Control processing rates
|
|
146
|
+
* ```typescript
|
|
147
|
+
* // Pause all processing for 5 seconds
|
|
148
|
+
* await hotMesh.throttle({ throttle: 5000 });
|
|
149
|
+
*
|
|
150
|
+
* // Emergency stop (pause indefinitely)
|
|
151
|
+
* await hotMesh.throttle({ throttle: -1 });
|
|
152
|
+
* ```
|
|
153
|
+
*
|
|
154
|
+
* **Workflow Interruption**: Gracefully stop running workflows
|
|
155
|
+
* ```typescript
|
|
156
|
+
* await hotMesh.interrupt('order.process', jobId, {
|
|
157
|
+
* reason: 'User cancellation'
|
|
158
|
+
* });
|
|
159
|
+
* ```
|
|
160
|
+
*
|
|
161
|
+
* **State Inspection**: Query workflow state and progress
|
|
162
|
+
* ```typescript
|
|
163
|
+
* const state = await hotMesh.getState('order.process', jobId);
|
|
164
|
+
* const status = await hotMesh.getStatus(jobId);
|
|
165
|
+
* ```
|
|
166
|
+
*
|
|
167
|
+
* ## Distributed Coordination
|
|
168
|
+
*
|
|
169
|
+
* HotMesh automatically handles distributed coordination through its quorum system:
|
|
170
|
+
*
|
|
171
|
+
* ```typescript
|
|
172
|
+
* // Check quorum health
|
|
173
|
+
* const members = await hotMesh.rollCall();
|
|
174
|
+
*
|
|
175
|
+
* // Coordinate version activation across all instances
|
|
176
|
+
* await hotMesh.activate('2', 1000); // 1 second delay for consensus
|
|
177
|
+
* ```
|
|
178
|
+
*
|
|
179
|
+
* ## Integration with Higher-Level Modules
|
|
180
|
+
*
|
|
181
|
+
* For most use cases, consider using the higher-level modules:
|
|
182
|
+
* - **MemFlow**: For Temporal.io-style workflows with TypeScript functions
|
|
183
|
+
* - **MeshCall**: For durable function calls and RPC patterns
|
|
184
|
+
*
|
|
185
|
+
* ## Cleanup
|
|
186
|
+
*
|
|
187
|
+
* Always clean up resources when shutting down:
|
|
188
|
+
* ```typescript
|
|
189
|
+
* // Stop this instance
|
|
190
|
+
* hotMesh.stop();
|
|
191
|
+
*
|
|
192
|
+
* // Stop all instances (typically in signal handlers)
|
|
54
193
|
* await HotMesh.stop();
|
|
55
194
|
* ```
|
|
195
|
+
*
|
|
196
|
+
* @see {@link https://hotmesh.io/docs} - Complete documentation
|
|
197
|
+
* @see {@link https://github.com/hotmeshio/samples-typescript} - Examples and tutorials
|
|
198
|
+
* @see {@link https://zenodo.org/records/12168558} - Academic paper on the architecture
|
|
56
199
|
*/
|
|
57
200
|
class HotMesh {
|
|
58
201
|
/**
|
|
@@ -1 +1,81 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MapperService = void 0;
|
|
4
|
+
const pipe_1 = require("../pipe");
|
|
5
|
+
class MapperService {
|
|
6
|
+
constructor(rules, data) {
|
|
7
|
+
this.rules = rules;
|
|
8
|
+
this.data = data;
|
|
9
|
+
}
|
|
10
|
+
mapRules() {
|
|
11
|
+
return this.traverseRules(this.rules);
|
|
12
|
+
}
|
|
13
|
+
traverseRules(rules) {
|
|
14
|
+
if (typeof rules === 'object' && '@pipe' in rules) {
|
|
15
|
+
return this.pipe(rules['@pipe']);
|
|
16
|
+
}
|
|
17
|
+
if (typeof rules === 'object' && rules !== null) {
|
|
18
|
+
const mappedRules = {};
|
|
19
|
+
for (const key in rules) {
|
|
20
|
+
if (Object.prototype.hasOwnProperty.call(rules, key)) {
|
|
21
|
+
mappedRules[key] = this.traverseRules(rules[key]);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return mappedRules;
|
|
25
|
+
}
|
|
26
|
+
else {
|
|
27
|
+
return this.resolve(rules);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* resolves a pipe expression of the form: { @pipe: [["{data.foo.bar}", 2, false, "hello world"]] }
|
|
32
|
+
* @param value
|
|
33
|
+
* @returns
|
|
34
|
+
*/
|
|
35
|
+
pipe(value) {
|
|
36
|
+
const pipe = new pipe_1.Pipe(value, this.data);
|
|
37
|
+
return pipe.process();
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* resolves a mapping expression in the form: "{data.foo.bar}" or 2 or false or "hello world"
|
|
41
|
+
* @param value
|
|
42
|
+
* @returns
|
|
43
|
+
*/
|
|
44
|
+
resolve(value) {
|
|
45
|
+
const pipe = new pipe_1.Pipe([[value]], this.data);
|
|
46
|
+
return pipe.process();
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Evaluates a transition rule against the current job state and incoming Stream message
|
|
50
|
+
* to determine which (if any) transition should be taken.
|
|
51
|
+
*/
|
|
52
|
+
static evaluate(transitionRule, context, code) {
|
|
53
|
+
if (typeof transitionRule === 'boolean') {
|
|
54
|
+
return transitionRule;
|
|
55
|
+
}
|
|
56
|
+
if ((Array.isArray(transitionRule.code) &&
|
|
57
|
+
transitionRule.code.includes(code || 200)) ||
|
|
58
|
+
code.toString() === (transitionRule.code || 200).toString()) {
|
|
59
|
+
if (!transitionRule.match) {
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
62
|
+
const orGate = transitionRule.gate === 'or';
|
|
63
|
+
let allAreTrue = true;
|
|
64
|
+
let someAreTrue = false;
|
|
65
|
+
transitionRule.match.forEach(({ expected, actual }) => {
|
|
66
|
+
if ((orGate && !someAreTrue) || (!orGate && allAreTrue)) {
|
|
67
|
+
const result = pipe_1.Pipe.resolve(actual, context) === expected;
|
|
68
|
+
if (orGate && result) {
|
|
69
|
+
someAreTrue = true;
|
|
70
|
+
}
|
|
71
|
+
else if (!orGate && !result) {
|
|
72
|
+
allAreTrue = false;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
return orGate ? someAreTrue : allAreTrue;
|
|
77
|
+
}
|
|
78
|
+
return false;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
exports.MapperService = MapperService;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { HotMesh } from '../hotmesh';
|
|
2
|
-
import { ClientConfig, ClientWorkflow, Connection, WorkflowOptions } from '../../types/
|
|
2
|
+
import { ClientConfig, ClientWorkflow, Connection, WorkflowOptions } from '../../types/memflow';
|
|
3
3
|
/**
|
|
4
|
-
* The
|
|
4
|
+
* The MemFlow `Client` service is functionally
|
|
5
5
|
* equivalent to the Temporal `Client` service.
|
|
6
6
|
* Start a new workflow execution by calling
|
|
7
7
|
* `workflow.start`. Note the direct connection to
|
|
@@ -81,7 +81,7 @@ export declare class ClientService {
|
|
|
81
81
|
*/
|
|
82
82
|
search: (hotMeshClient: HotMesh, index: string, query: string[]) => Promise<string[]>;
|
|
83
83
|
/**
|
|
84
|
-
* The
|
|
84
|
+
* The MemFlow `Client` service is functionally
|
|
85
85
|
* equivalent to the Temporal `Client` service.
|
|
86
86
|
* Starting a workflow is the primary use case and
|
|
87
87
|
* is accessed by calling workflow.start().
|