@hotmeshio/hotmesh 0.0.55 → 0.0.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/modules/enums.js +1 -10
- package/build/modules/key.d.ts +0 -38
- package/build/modules/key.js +4 -46
- package/build/modules/utils.d.ts +0 -8
- package/build/modules/utils.js +0 -14
- package/build/package.json +11 -4
- package/build/services/activities/activity.d.ts +0 -28
- package/build/services/activities/activity.js +1 -46
- package/build/services/activities/await.js +0 -4
- package/build/services/activities/cycle.d.ts +0 -7
- package/build/services/activities/cycle.js +1 -16
- package/build/services/activities/hook.d.ts +0 -6
- package/build/services/activities/hook.js +2 -12
- package/build/services/activities/interrupt.js +0 -8
- package/build/services/activities/signal.d.ts +0 -6
- package/build/services/activities/signal.js +0 -15
- package/build/services/activities/trigger.d.ts +0 -4
- package/build/services/activities/trigger.js +1 -7
- package/build/services/activities/worker.js +0 -4
- package/build/services/collator/index.d.ts +0 -70
- package/build/services/collator/index.js +1 -91
- package/build/services/compiler/deployer.js +6 -38
- package/build/services/compiler/index.d.ts +0 -15
- package/build/services/compiler/index.js +0 -20
- package/build/services/compiler/validator.d.ts +0 -3
- package/build/services/compiler/validator.js +0 -25
- package/build/services/connector/clients/ioredis.d.ts +2 -2
- package/build/services/connector/clients/ioredis.js +0 -2
- package/build/services/connector/clients/redis.d.ts +4 -4
- package/build/services/connector/clients/redis.js +1 -3
- package/build/services/connector/index.d.ts +1 -1
- package/build/services/connector/index.js +0 -2
- package/build/services/durable/client.d.ts +1 -26
- package/build/services/durable/client.js +0 -56
- package/build/services/durable/exporter.d.ts +0 -22
- package/build/services/durable/exporter.js +1 -30
- package/build/services/durable/handle.d.ts +0 -36
- package/build/services/durable/handle.js +0 -46
- package/build/services/durable/index.d.ts +0 -4
- package/build/services/durable/index.js +0 -4
- package/build/services/durable/schemas/factory.d.ts +0 -29
- package/build/services/durable/schemas/factory.js +0 -29
- package/build/services/durable/search.d.ts +1 -36
- package/build/services/durable/search.js +56 -56
- package/build/services/durable/worker.js +2 -22
- package/build/services/durable/workflow.d.ts +0 -114
- package/build/services/durable/workflow.js +1 -141
- package/build/services/engine/index.d.ts +1 -6
- package/build/services/engine/index.js +1 -43
- package/build/services/exporter/index.d.ts +0 -27
- package/build/services/exporter/index.js +0 -33
- package/build/services/hotmesh/index.d.ts +2 -2
- package/build/services/hotmesh/index.js +1 -9
- package/build/services/logger/index.js +0 -2
- package/build/services/mapper/index.d.ts +0 -14
- package/build/services/mapper/index.js +0 -14
- package/build/services/pipe/functions/date.d.ts +0 -7
- package/build/services/pipe/functions/date.js +0 -7
- package/build/services/pipe/functions/math.js +0 -2
- package/build/services/pipe/index.d.ts +0 -15
- package/build/services/pipe/index.js +2 -23
- package/build/services/quorum/index.d.ts +0 -7
- package/build/services/quorum/index.js +0 -21
- package/build/services/reporter/index.d.ts +0 -5
- package/build/services/reporter/index.js +0 -9
- package/build/services/router/index.d.ts +0 -9
- package/build/services/router/index.js +2 -38
- package/build/services/serializer/index.js +7 -26
- package/build/services/store/cache.d.ts +0 -18
- package/build/services/store/cache.js +0 -18
- package/build/services/store/clients/ioredis.d.ts +1 -1
- package/build/services/store/clients/ioredis.js +0 -1
- package/build/services/store/clients/redis.d.ts +1 -1
- package/build/services/store/index.d.ts +0 -55
- package/build/services/store/index.js +5 -81
- package/build/services/stream/clients/ioredis.d.ts +1 -1
- package/build/services/stream/clients/ioredis.js +1 -4
- package/build/services/stream/clients/redis.d.ts +1 -1
- package/build/services/sub/clients/ioredis.d.ts +1 -1
- package/build/services/sub/clients/redis.d.ts +1 -1
- package/build/services/task/index.d.ts +0 -9
- package/build/services/task/index.js +0 -31
- package/build/services/telemetry/index.d.ts +0 -7
- package/build/services/telemetry/index.js +1 -13
- package/build/services/worker/index.d.ts +0 -4
- package/build/services/worker/index.js +2 -6
- package/build/types/activity.d.ts +0 -81
- package/build/types/durable.d.ts +26 -177
- package/build/types/exporter.d.ts +0 -13
- package/build/types/hotmesh.d.ts +4 -16
- package/build/types/hotmesh.js +0 -3
- package/build/types/index.d.ts +4 -6
- package/build/types/index.js +4 -3
- package/build/types/job.d.ts +1 -86
- package/build/types/pipe.d.ts +0 -65
- package/build/types/quorum.d.ts +15 -10
- package/build/types/redis.d.ts +225 -7
- package/build/types/redis.js +9 -0
- package/build/types/stream.d.ts +0 -58
- package/build/types/stream.js +0 -4
- package/package.json +11 -4
- package/types/durable.ts +131 -4
- package/types/hotmesh.ts +3 -6
- package/types/index.ts +23 -10
- package/types/job.ts +1 -1
- package/types/quorum.ts +22 -0
- package/types/redis.ts +267 -18
- package/build/types/ioredisclient.d.ts +0 -5
- package/build/types/ioredisclient.js +0 -5
- package/build/types/redisclient.d.ts +0 -26
- package/build/types/redisclient.js +0 -2
- package/modules/enums.ts +0 -62
- package/modules/errors.ts +0 -280
- package/modules/key.ts +0 -101
- package/modules/storage.ts +0 -3
- package/modules/utils.ts +0 -242
- package/services/activities/activity.ts +0 -589
- package/services/activities/await.ts +0 -113
- package/services/activities/cycle.ts +0 -115
- package/services/activities/hook.ts +0 -197
- package/services/activities/index.ts +0 -19
- package/services/activities/interrupt.ts +0 -172
- package/services/activities/signal.ts +0 -148
- package/services/activities/trigger.ts +0 -295
- package/services/activities/worker.ts +0 -107
- package/services/collator/README.md +0 -102
- package/services/collator/index.ts +0 -291
- package/services/compiler/deployer.ts +0 -504
- package/services/compiler/index.ts +0 -98
- package/services/compiler/validator.ts +0 -158
- package/services/connector/clients/ioredis.ts +0 -57
- package/services/connector/clients/redis.ts +0 -72
- package/services/connector/index.ts +0 -42
- package/services/durable/client.ts +0 -266
- package/services/durable/connection.ts +0 -10
- package/services/durable/exporter.ts +0 -232
- package/services/durable/handle.ts +0 -160
- package/services/durable/index.ts +0 -27
- package/services/durable/schemas/factory.ts +0 -2358
- package/services/durable/search.ts +0 -196
- package/services/durable/worker.ts +0 -401
- package/services/durable/workflow.ts +0 -557
- package/services/engine/index.ts +0 -761
- package/services/exporter/index.ts +0 -146
- package/services/hotmesh/index.ts +0 -237
- package/services/logger/index.ts +0 -79
- package/services/mapper/index.ts +0 -89
- package/services/pipe/functions/array.ts +0 -78
- package/services/pipe/functions/bitwise.ts +0 -27
- package/services/pipe/functions/conditional.ts +0 -35
- package/services/pipe/functions/date.ts +0 -220
- package/services/pipe/functions/index.ts +0 -27
- package/services/pipe/functions/json.ts +0 -11
- package/services/pipe/functions/logical.ts +0 -11
- package/services/pipe/functions/math.ts +0 -217
- package/services/pipe/functions/number.ts +0 -75
- package/services/pipe/functions/object.ts +0 -98
- package/services/pipe/functions/string.ts +0 -86
- package/services/pipe/functions/symbol.ts +0 -39
- package/services/pipe/functions/unary.ts +0 -19
- package/services/pipe/index.ts +0 -216
- package/services/quorum/index.ts +0 -319
- package/services/reporter/index.ts +0 -387
- package/services/router/index.ts +0 -426
- package/services/serializer/README.md +0 -10
- package/services/serializer/index.ts +0 -285
- package/services/store/cache.ts +0 -172
- package/services/store/clients/ioredis.ts +0 -145
- package/services/store/clients/redis.ts +0 -191
- package/services/store/index.ts +0 -1091
- package/services/stream/clients/ioredis.ts +0 -157
- package/services/stream/clients/redis.ts +0 -158
- package/services/stream/index.ts +0 -58
- package/services/sub/clients/ioredis.ts +0 -83
- package/services/sub/clients/redis.ts +0 -74
- package/services/sub/index.ts +0 -25
- package/services/task/index.ts +0 -250
- package/services/telemetry/index.ts +0 -273
- package/services/worker/index.ts +0 -248
- package/types/ioredisclient.ts +0 -10
- package/types/redisclient.ts +0 -30
|
@@ -7,8 +7,8 @@ const collator_1 = require("../collator");
|
|
|
7
7
|
const serializer_1 = require("../serializer");
|
|
8
8
|
const pipe_1 = require("../pipe");
|
|
9
9
|
const validator_1 = require("./validator");
|
|
10
|
-
const DEFAULT_METADATA_RANGE_SIZE = 26;
|
|
11
|
-
const DEFAULT_DATA_RANGE_SIZE = 260;
|
|
10
|
+
const DEFAULT_METADATA_RANGE_SIZE = 26;
|
|
11
|
+
const DEFAULT_DATA_RANGE_SIZE = 260;
|
|
12
12
|
const DEFAULT_RANGE_SIZE = DEFAULT_METADATA_RANGE_SIZE + DEFAULT_DATA_RANGE_SIZE;
|
|
13
13
|
class Deployer {
|
|
14
14
|
constructor(manifest) {
|
|
@@ -41,22 +41,19 @@ class Deployer {
|
|
|
41
41
|
};
|
|
42
42
|
}
|
|
43
43
|
async generateSymKeys() {
|
|
44
|
-
//note: symbol ranges are additive (per version); path assignments are immutable
|
|
45
44
|
const appId = this.manifest.app.id;
|
|
46
45
|
for (const graph of this.manifest.app.graphs) {
|
|
47
|
-
//generate JOB symbols
|
|
48
46
|
const [, trigger] = this.findTrigger(graph);
|
|
49
47
|
const topic = trigger.subscribes;
|
|
50
48
|
const [lower, upper, symbols] = await this.store.reserveSymbolRange(`$${topic}`, DEFAULT_RANGE_SIZE, 'JOB');
|
|
51
|
-
const prefix = '';
|
|
49
|
+
const prefix = '';
|
|
52
50
|
const newSymbols = this.bindSymbols(lower, upper, symbols, prefix, trigger.PRODUCES);
|
|
53
51
|
if (Object.keys(newSymbols).length) {
|
|
54
52
|
await this.store.addSymbols(`$${topic}`, newSymbols);
|
|
55
53
|
}
|
|
56
|
-
//generate ACTIVITY symbols
|
|
57
54
|
for (const [activityId, activity] of Object.entries(graph.activities)) {
|
|
58
55
|
const [lower, upper, symbols] = await this.store.reserveSymbolRange(activityId, DEFAULT_RANGE_SIZE, 'ACTIVITY');
|
|
59
|
-
const prefix = `${activityId}/`;
|
|
56
|
+
const prefix = `${activityId}/`;
|
|
60
57
|
this.bindSelf(activity.consumes, activity.produces, activityId);
|
|
61
58
|
const newSymbols = this.bindSymbols(lower, upper, symbols, prefix, activity.produces);
|
|
62
59
|
if (Object.keys(newSymbols).length) {
|
|
@@ -66,7 +63,6 @@ class Deployer {
|
|
|
66
63
|
}
|
|
67
64
|
}
|
|
68
65
|
bindSelf(consumes, produces, activityId) {
|
|
69
|
-
//bind self-referential mappings
|
|
70
66
|
for (const selfId of [activityId, '$self']) {
|
|
71
67
|
const selfConsumes = consumes[selfId];
|
|
72
68
|
if (selfConsumes) {
|
|
@@ -90,7 +86,7 @@ class Deployer {
|
|
|
90
86
|
const symbol = (0, utils_1.getSymKey)(startIndex);
|
|
91
87
|
startIndex++;
|
|
92
88
|
newSymbols[fullPath] = symbol;
|
|
93
|
-
currentSymbols[fullPath] = symbol;
|
|
89
|
+
currentSymbols[fullPath] = symbol;
|
|
94
90
|
}
|
|
95
91
|
}
|
|
96
92
|
return newSymbols;
|
|
@@ -103,20 +99,16 @@ class Deployer {
|
|
|
103
99
|
if (!jobSchema && !outputSchema)
|
|
104
100
|
continue;
|
|
105
101
|
const activities = graph.activities;
|
|
106
|
-
// Find the trigger activity and bind the job schema to it
|
|
107
|
-
// at execution time, the trigger is a standin for the job
|
|
108
102
|
for (const activityKey in activities) {
|
|
109
103
|
if (activities[activityKey].type === 'trigger') {
|
|
110
104
|
const trigger = activities[activityKey];
|
|
111
105
|
if (jobSchema) {
|
|
112
|
-
//possible for trigger to have job mappings
|
|
113
106
|
if (!trigger.job) {
|
|
114
107
|
trigger.job = {};
|
|
115
108
|
}
|
|
116
109
|
trigger.job.schema = jobSchema;
|
|
117
110
|
}
|
|
118
111
|
if (outputSchema) {
|
|
119
|
-
//impossible for trigger to have output mappings.
|
|
120
112
|
trigger.output = { schema: outputSchema };
|
|
121
113
|
}
|
|
122
114
|
}
|
|
@@ -137,8 +129,6 @@ class Deployer {
|
|
|
137
129
|
}
|
|
138
130
|
}
|
|
139
131
|
}
|
|
140
|
-
//the cycle/goto activity includes and ancestor target;
|
|
141
|
-
//update with the cycle flag, so it can be rerun
|
|
142
132
|
bindCycleTarget() {
|
|
143
133
|
for (const graph of this.manifest.app.graphs) {
|
|
144
134
|
const activities = graph.activities;
|
|
@@ -150,8 +140,6 @@ class Deployer {
|
|
|
150
140
|
}
|
|
151
141
|
}
|
|
152
142
|
}
|
|
153
|
-
//it's more intuitive for SDK users to use 'topic',
|
|
154
|
-
//but the compiler is desiged to be generic and uses the attribute, 'subtypes'
|
|
155
143
|
convertTopicsToTypes() {
|
|
156
144
|
for (const graph of this.manifest.app.graphs) {
|
|
157
145
|
const activities = graph.activities;
|
|
@@ -163,7 +151,6 @@ class Deployer {
|
|
|
163
151
|
}
|
|
164
152
|
}
|
|
165
153
|
}
|
|
166
|
-
//legacy; remove at beta (assume no legacy refs to 'activity' at that point)
|
|
167
154
|
convertActivitiesToHooks() {
|
|
168
155
|
for (const graph of this.manifest.app.graphs) {
|
|
169
156
|
const activities = graph.activities;
|
|
@@ -183,11 +170,8 @@ class Deployer {
|
|
|
183
170
|
const toTransitions = graph.transitions[fromActivity];
|
|
184
171
|
for (const transition of toTransitions) {
|
|
185
172
|
const to = transition.to;
|
|
186
|
-
//DAGs have one parent; easy to optimize for
|
|
187
173
|
graph.activities[to].parent = fromActivity;
|
|
188
174
|
}
|
|
189
|
-
//temporarily bind the transitions to the parent activity,
|
|
190
|
-
// so the consumer/producer registrar picks up the bindings
|
|
191
175
|
graph.activities[fromActivity].transitions = toTransitions;
|
|
192
176
|
}
|
|
193
177
|
}
|
|
@@ -245,10 +229,6 @@ class Deployer {
|
|
|
245
229
|
traverse(obj[key], newPath);
|
|
246
230
|
}
|
|
247
231
|
else {
|
|
248
|
-
//wildcard mapping (e.g., 'friends[25]')
|
|
249
|
-
//when this is resolved, it will be expanded to
|
|
250
|
-
//`'friends/0', ..., 'friends/24'`, providing 25 dynamic
|
|
251
|
-
//slots in the flow's output data
|
|
252
232
|
const pathName = [...path, key].join('/');
|
|
253
233
|
if (!pathName.includes('[')) {
|
|
254
234
|
const finalPath = `data/${pathName}`;
|
|
@@ -258,17 +238,15 @@ class Deployer {
|
|
|
258
238
|
}
|
|
259
239
|
else {
|
|
260
240
|
const [left, right] = pathName.split('[');
|
|
261
|
-
//check if this variable isLiteralKeyType (#, -, or _)
|
|
262
241
|
const [amount, _] = right.split(']');
|
|
263
242
|
if (!isNaN(parseInt(amount))) {
|
|
264
|
-
//loop to create all possible paths (0 to amount)
|
|
265
243
|
for (let i = 0; i < parseInt(amount); i++) {
|
|
266
244
|
const finalPath = `data/${left}/${i}`;
|
|
267
245
|
if (!result.includes(finalPath)) {
|
|
268
246
|
result.push(finalPath);
|
|
269
247
|
}
|
|
270
248
|
}
|
|
271
|
-
}
|
|
249
|
+
}
|
|
272
250
|
}
|
|
273
251
|
}
|
|
274
252
|
}
|
|
@@ -290,7 +268,6 @@ class Deployer {
|
|
|
290
268
|
}
|
|
291
269
|
resolveMappingDependencies() {
|
|
292
270
|
const dynamicMappingRules = [];
|
|
293
|
-
//recursive function to descend into the object and find all dynamic mapping rules
|
|
294
271
|
function traverse(obj, consumes) {
|
|
295
272
|
for (const key in obj) {
|
|
296
273
|
if (typeof obj[key] === 'string') {
|
|
@@ -319,7 +296,6 @@ class Deployer {
|
|
|
319
296
|
}
|
|
320
297
|
}
|
|
321
298
|
const groupedRules = this.groupMappingRules(dynamicMappingRules);
|
|
322
|
-
// Iterate through the graph and add 'produces' field to each activity
|
|
323
299
|
for (const graph of graphs) {
|
|
324
300
|
const activities = graph.activities;
|
|
325
301
|
for (const activityId in activities) {
|
|
@@ -330,7 +306,6 @@ class Deployer {
|
|
|
330
306
|
}
|
|
331
307
|
groupMappingRules(rules) {
|
|
332
308
|
rules = Array.from(new Set(rules)).sort();
|
|
333
|
-
// Group by the first symbol before the period (this is the activity name)
|
|
334
309
|
const groupedRules = {};
|
|
335
310
|
for (const rule of rules) {
|
|
336
311
|
const [group, resolved] = this.resolveMappableValue(rule);
|
|
@@ -349,7 +324,6 @@ class Deployer {
|
|
|
349
324
|
return [group, path.join('/')];
|
|
350
325
|
}
|
|
351
326
|
else {
|
|
352
|
-
//normalize paths to be relative to the activity
|
|
353
327
|
const [group, type, subtype, ...path] = parts;
|
|
354
328
|
const prefix = {
|
|
355
329
|
hook: 'hook/data',
|
|
@@ -366,7 +340,6 @@ class Deployer {
|
|
|
366
340
|
const activities = graph.activities;
|
|
367
341
|
for (const activityKey in activities) {
|
|
368
342
|
const target = activities[activityKey];
|
|
369
|
-
//remove transitions; no longer necessary for runtime
|
|
370
343
|
delete target.transitions;
|
|
371
344
|
activitySchemas[activityKey] = target;
|
|
372
345
|
}
|
|
@@ -379,7 +352,6 @@ class Deployer {
|
|
|
379
352
|
for (const graph of graphs) {
|
|
380
353
|
const activities = graph.activities;
|
|
381
354
|
const subscribesTopic = graph.subscribes;
|
|
382
|
-
// Find the activity ID associated with the subscribes topic
|
|
383
355
|
for (const activityKey in activities) {
|
|
384
356
|
if (activities[activityKey].type === 'trigger') {
|
|
385
357
|
publicSubscriptions[subscribesTopic] = activityKey;
|
|
@@ -442,7 +414,6 @@ class Deployer {
|
|
|
442
414
|
if (!targetActivity.hook) {
|
|
443
415
|
targetActivity.hook = {};
|
|
444
416
|
}
|
|
445
|
-
//create back-reference to the hook topic
|
|
446
417
|
targetActivity.hook.topic = topic;
|
|
447
418
|
}
|
|
448
419
|
}
|
|
@@ -451,7 +422,6 @@ class Deployer {
|
|
|
451
422
|
await this.store.setHookRules(hookRules);
|
|
452
423
|
}
|
|
453
424
|
async deployConsumerGroups() {
|
|
454
|
-
//create one engine group
|
|
455
425
|
const params = { appId: this.manifest.app.id };
|
|
456
426
|
const key = this.store.mintKey(key_1.KeyType.STREAMS, params);
|
|
457
427
|
await this.deployConsumerGroup(key, 'ENGINE');
|
|
@@ -459,11 +429,9 @@ class Deployer {
|
|
|
459
429
|
const activities = graph.activities;
|
|
460
430
|
for (const activityKey in activities) {
|
|
461
431
|
const activity = activities[activityKey];
|
|
462
|
-
//only precreate if the topic is concrete and not `mappable`
|
|
463
432
|
if (activity.type === 'worker' && pipe_1.Pipe.resolve(activity.subtype, {}) === activity.subtype) {
|
|
464
433
|
params.topic = activity.subtype;
|
|
465
434
|
const key = this.store.mintKey(key_1.KeyType.STREAMS, params);
|
|
466
|
-
//create one worker group per unique activity subtype (the topic)
|
|
467
435
|
await this.deployConsumerGroup(key, 'WORKER');
|
|
468
436
|
}
|
|
469
437
|
}
|
|
@@ -2,28 +2,13 @@ import { ILogger } from '../logger';
|
|
|
2
2
|
import { StoreService } from '../store';
|
|
3
3
|
import { HotMeshManifest } from '../../types/hotmesh';
|
|
4
4
|
import { RedisClient, RedisMulti } from '../../types/redis';
|
|
5
|
-
/**
|
|
6
|
-
* The compiler service converts a graph into a executable program.
|
|
7
|
-
*/
|
|
8
5
|
declare class CompilerService {
|
|
9
6
|
store: StoreService<RedisClient, RedisMulti> | null;
|
|
10
7
|
logger: ILogger;
|
|
11
8
|
constructor(store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
|
|
12
|
-
/**
|
|
13
|
-
* verifies and plans the deployment of an app to Redis; the app is not deployed yet
|
|
14
|
-
* @param path
|
|
15
|
-
*/
|
|
16
9
|
plan(mySchemaOrPath: string): Promise<HotMeshManifest>;
|
|
17
10
|
isPath(input: string): boolean;
|
|
18
|
-
/**
|
|
19
|
-
* deploys an app to Redis but does NOT activate it.
|
|
20
|
-
*/
|
|
21
11
|
deploy(mySchemaOrPath: string): Promise<HotMeshManifest>;
|
|
22
|
-
/**
|
|
23
|
-
* activates a deployed version of an app;
|
|
24
|
-
* @param appId
|
|
25
|
-
* @param appVersion
|
|
26
|
-
*/
|
|
27
12
|
activate(appId: string, appVersion: string): Promise<boolean>;
|
|
28
13
|
saveAsJSON(originalPath: string, schema: HotMeshManifest): Promise<void>;
|
|
29
14
|
}
|
|
@@ -33,18 +33,11 @@ const fs = __importStar(require("fs/promises"));
|
|
|
33
33
|
const path = __importStar(require("path"));
|
|
34
34
|
const deployer_1 = require("./deployer");
|
|
35
35
|
const validator_1 = require("./validator");
|
|
36
|
-
/**
|
|
37
|
-
* The compiler service converts a graph into a executable program.
|
|
38
|
-
*/
|
|
39
36
|
class CompilerService {
|
|
40
37
|
constructor(store, logger) {
|
|
41
38
|
this.store = store;
|
|
42
39
|
this.logger = logger;
|
|
43
40
|
}
|
|
44
|
-
/**
|
|
45
|
-
* verifies and plans the deployment of an app to Redis; the app is not deployed yet
|
|
46
|
-
* @param path
|
|
47
|
-
*/
|
|
48
41
|
async plan(mySchemaOrPath) {
|
|
49
42
|
try {
|
|
50
43
|
let schema;
|
|
@@ -54,10 +47,8 @@ class CompilerService {
|
|
|
54
47
|
else {
|
|
55
48
|
schema = js_yaml_1.default.load(mySchemaOrPath);
|
|
56
49
|
}
|
|
57
|
-
// 1) validate the manifest file
|
|
58
50
|
const validator = new validator_1.Validator(schema);
|
|
59
51
|
validator.validate(this.store);
|
|
60
|
-
// 2) todo: add a PlannerService module that will plan the deployment (what might break, drift, etc)
|
|
61
52
|
return schema;
|
|
62
53
|
}
|
|
63
54
|
catch (err) {
|
|
@@ -67,9 +58,6 @@ class CompilerService {
|
|
|
67
58
|
isPath(input) {
|
|
68
59
|
return !input.trim().startsWith('app:');
|
|
69
60
|
}
|
|
70
|
-
/**
|
|
71
|
-
* deploys an app to Redis but does NOT activate it.
|
|
72
|
-
*/
|
|
73
61
|
async deploy(mySchemaOrPath) {
|
|
74
62
|
try {
|
|
75
63
|
let schema;
|
|
@@ -80,13 +68,10 @@ class CompilerService {
|
|
|
80
68
|
else {
|
|
81
69
|
schema = js_yaml_1.default.load(mySchemaOrPath);
|
|
82
70
|
}
|
|
83
|
-
// 2) validate the manifest file (synchronous operation...no callbacks)
|
|
84
71
|
const validator = new validator_1.Validator(schema);
|
|
85
72
|
validator.validate(this.store);
|
|
86
|
-
// 3) deploy the schema (segment, optimize, etc; save to Redis)
|
|
87
73
|
const deployer = new deployer_1.Deployer(schema);
|
|
88
74
|
await deployer.deploy(this.store);
|
|
89
|
-
// 4) save the app version to Redis (so it can be activated later)
|
|
90
75
|
await this.store.setApp(schema.app.id, schema.app.version);
|
|
91
76
|
return schema;
|
|
92
77
|
}
|
|
@@ -94,11 +79,6 @@ class CompilerService {
|
|
|
94
79
|
this.logger.error('compiler-deploy-error', err);
|
|
95
80
|
}
|
|
96
81
|
}
|
|
97
|
-
/**
|
|
98
|
-
* activates a deployed version of an app;
|
|
99
|
-
* @param appId
|
|
100
|
-
* @param appVersion
|
|
101
|
-
*/
|
|
102
82
|
async activate(appId, appVersion) {
|
|
103
83
|
return await this.store.activateAppVersion(appId, appVersion);
|
|
104
84
|
}
|
|
@@ -10,9 +10,6 @@ declare class Validator {
|
|
|
10
10
|
static SYS_VARS: string[];
|
|
11
11
|
static CONTEXT_VARS: string[];
|
|
12
12
|
constructor(manifest: HotMeshManifest);
|
|
13
|
-
/**
|
|
14
|
-
* validate the manifest file
|
|
15
|
-
*/
|
|
16
13
|
validate(store: StoreService<RedisClient, RedisMulti>): Promise<void>;
|
|
17
14
|
validateActivityIds(): void;
|
|
18
15
|
isMappingStatement(value: string): boolean;
|
|
@@ -10,9 +10,6 @@ class Validator {
|
|
|
10
10
|
this.store = null;
|
|
11
11
|
this.manifest = manifest;
|
|
12
12
|
}
|
|
13
|
-
/**
|
|
14
|
-
* validate the manifest file
|
|
15
|
-
*/
|
|
16
13
|
async validate(store) {
|
|
17
14
|
this.store = store;
|
|
18
15
|
this.getMappingStatements();
|
|
@@ -28,12 +25,10 @@ class Validator {
|
|
|
28
25
|
this.validateHooks();
|
|
29
26
|
this.validateConditionalStatements();
|
|
30
27
|
}
|
|
31
|
-
// 1.1) Validate the manifest file activity ids are unique (no duplicates)
|
|
32
28
|
validateActivityIds() {
|
|
33
29
|
const activityIdsSet = new Set();
|
|
34
30
|
this.manifest.app.graphs.forEach((graph) => {
|
|
35
31
|
const ids = Object.keys(graph.activities);
|
|
36
|
-
// Check for duplicates and add ids to the set
|
|
37
32
|
ids.forEach((id) => {
|
|
38
33
|
if (activityIdsSet.has(id)) {
|
|
39
34
|
throw new Error(`Duplicate activity id found: ${id}`);
|
|
@@ -72,9 +67,7 @@ class Validator {
|
|
|
72
67
|
});
|
|
73
68
|
this.mappingStatements = mappingStatements;
|
|
74
69
|
}
|
|
75
|
-
// 1.2) Validate no activity ids are referenced that don't exist
|
|
76
70
|
validateReferencedActivityIds() {
|
|
77
|
-
// get list of all mapping statements and validate
|
|
78
71
|
const mappingStatements = this.mappingStatements;
|
|
79
72
|
const activityIds = this.activityIds;
|
|
80
73
|
for (const activity in mappingStatements) {
|
|
@@ -96,41 +89,23 @@ class Validator {
|
|
|
96
89
|
isContextVariable(value) {
|
|
97
90
|
return ['{$input}', '{$output}', '{$item}', '{$key}', '{$index}'].includes(value);
|
|
98
91
|
}
|
|
99
|
-
// 1.3) Validate the mapping/@pipe statements are valid
|
|
100
92
|
validateMappingStatements() {
|
|
101
|
-
// Implement the method content
|
|
102
93
|
}
|
|
103
|
-
// 1.4) Validate the transitions are valid
|
|
104
94
|
validateTransitions() {
|
|
105
|
-
// Implement the method content
|
|
106
95
|
}
|
|
107
|
-
// 1.5) Validate the transition conditions are valid
|
|
108
96
|
validateTransitionConditions() {
|
|
109
|
-
// Implement the method content
|
|
110
97
|
}
|
|
111
|
-
// 1.6) Validate the stats
|
|
112
98
|
validateStats() {
|
|
113
|
-
// Implement the method content
|
|
114
99
|
}
|
|
115
|
-
// 1.7) Validate the schemas
|
|
116
100
|
validateSchemas() {
|
|
117
|
-
// Implement the method content
|
|
118
101
|
}
|
|
119
|
-
// 1.8) Validate the topics are unique and handled
|
|
120
102
|
validateUniqueHandledTopics() {
|
|
121
|
-
// Implement the method content
|
|
122
103
|
}
|
|
123
|
-
// 1.9) Validate that every graph has publishes and subscribes
|
|
124
104
|
validateGraphPublishSubscribe() {
|
|
125
|
-
// Implement the method content
|
|
126
105
|
}
|
|
127
|
-
// 1.10) Validate hooks, including mapping statements
|
|
128
106
|
validateHooks() {
|
|
129
|
-
// Implement the method content
|
|
130
107
|
}
|
|
131
|
-
// 1.11) Validate conditional statements
|
|
132
108
|
validateConditionalStatements() {
|
|
133
|
-
// Implement the method content
|
|
134
109
|
}
|
|
135
110
|
}
|
|
136
111
|
exports.Validator = Validator;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RedisClientOptions, RedisClassType, RedisClientType } from '../../../types/
|
|
1
|
+
import { IORedisClientOptions as RedisClientOptions, IORedisClassType as RedisClassType, IORedisClientType as RedisClientType } from '../../../types/redis';
|
|
2
2
|
declare class RedisConnection {
|
|
3
3
|
private connection;
|
|
4
4
|
private static instances;
|
|
@@ -7,7 +7,7 @@ declare class RedisConnection {
|
|
|
7
7
|
private createConnection;
|
|
8
8
|
getClient(): RedisClientType;
|
|
9
9
|
disconnect(): Promise<void>;
|
|
10
|
-
static connect(id: string, Redis: RedisClassType
|
|
10
|
+
static connect(id: string, Redis: Partial<RedisClassType>, options?: RedisClientOptions): Promise<RedisConnection>;
|
|
11
11
|
static disconnectAll(): Promise<void>;
|
|
12
12
|
}
|
|
13
13
|
export { RedisConnection };
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { RedisRedisClassType, RedisRedisClientType, RedisRedisClientOptions } from '../../../types/redis';
|
|
2
2
|
declare class RedisConnection {
|
|
3
3
|
private connection;
|
|
4
4
|
private static instances;
|
|
5
5
|
private id;
|
|
6
6
|
private static clientOptions;
|
|
7
7
|
private createConnection;
|
|
8
|
-
getClient():
|
|
8
|
+
getClient(): RedisRedisClientType;
|
|
9
9
|
disconnect(): Promise<void>;
|
|
10
|
-
static connect(id: string, Redis:
|
|
10
|
+
static connect(id: string, Redis: Partial<RedisRedisClassType>, options?: RedisRedisClientOptions): Promise<RedisConnection>;
|
|
11
11
|
static disconnectAll(): Promise<void>;
|
|
12
12
|
}
|
|
13
|
-
export { RedisConnection,
|
|
13
|
+
export { RedisConnection, RedisRedisClientType };
|
|
@@ -39,7 +39,7 @@ class RedisConnection {
|
|
|
39
39
|
}
|
|
40
40
|
const instance = new RedisConnection();
|
|
41
41
|
const opts = options ? { ...options } : { ...this.clientOptions };
|
|
42
|
-
instance.connection = await instance.createConnection(Redis, opts);
|
|
42
|
+
instance.connection = (await instance.createConnection(Redis, opts));
|
|
43
43
|
instance.id = id;
|
|
44
44
|
this.instances.set(id, instance);
|
|
45
45
|
return instance;
|
|
@@ -57,6 +57,4 @@ RedisConnection.clientOptions = {
|
|
|
57
57
|
port: 6379,
|
|
58
58
|
tls: false,
|
|
59
59
|
},
|
|
60
|
-
//password: config.REDIS_PASSWORD,
|
|
61
|
-
//database: config.REDIS_DATABASE,
|
|
62
60
|
};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { HotMeshEngine, HotMeshWorker } from '../../types/hotmesh';
|
|
2
2
|
import { RedisClass, RedisOptions } from '../../types/redis';
|
|
3
3
|
export declare class ConnectorService {
|
|
4
|
-
static initRedisClients(Redis: RedisClass
|
|
4
|
+
static initRedisClients(Redis: Partial<RedisClass>, options: Partial<RedisOptions>, target: HotMeshEngine | HotMeshWorker): Promise<void>;
|
|
5
5
|
}
|
|
@@ -5,8 +5,6 @@ const utils_1 = require("../../modules/utils");
|
|
|
5
5
|
const ioredis_1 = require("../connector/clients/ioredis");
|
|
6
6
|
const redis_1 = require("../connector/clients/redis");
|
|
7
7
|
class ConnectorService {
|
|
8
|
-
//1) Initialize `store`, `stream`, and `subscription` Redis clients.
|
|
9
|
-
//2) Bind to the target if not already present
|
|
10
8
|
static async initRedisClients(Redis, options, target) {
|
|
11
9
|
if (!target.store || !target.stream || !target.sub) {
|
|
12
10
|
const instances = [];
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { WorkflowHandleService } from './handle';
|
|
2
2
|
import { HotMeshService as HotMesh } from '../hotmesh';
|
|
3
|
-
import { ClientConfig, Connection, HookOptions, WorkflowOptions
|
|
3
|
+
import { ClientConfig, Connection, HookOptions, WorkflowOptions } from '../../types/durable';
|
|
4
4
|
export declare class ClientService {
|
|
5
5
|
connection: Connection;
|
|
6
6
|
options: WorkflowOptions;
|
|
@@ -8,37 +8,12 @@ export declare class ClientService {
|
|
|
8
8
|
static instances: Map<string, HotMesh | Promise<HotMesh>>;
|
|
9
9
|
constructor(config: ClientConfig);
|
|
10
10
|
getHotMeshClient: (workflowTopic: string, namespace?: string) => Promise<HotMesh>;
|
|
11
|
-
/**
|
|
12
|
-
* Creates a stream (Redis `XGROUP.CREATE`) where events can be published (XADD).
|
|
13
|
-
* It is possible that the worker that will read from this stream channel
|
|
14
|
-
* has not yet been initialized, so this call ensures that the channel
|
|
15
|
-
* exists and is ready to serve as a container for events.
|
|
16
|
-
*/
|
|
17
11
|
static createStream: (hotMeshClient: HotMesh, workflowTopic: string, namespace?: string) => Promise<void>;
|
|
18
|
-
/**
|
|
19
|
-
* It is possible for a client to invoke a workflow without first
|
|
20
|
-
* creating the stream. This method will verify that the stream
|
|
21
|
-
* exists and if not, create it.
|
|
22
|
-
*/
|
|
23
12
|
static verifyStream: (workflowTopic: string, namespace?: string) => Promise<HotMesh>;
|
|
24
|
-
/**
|
|
25
|
-
* For those deployments with a redis stack backend (with the FT module),
|
|
26
|
-
* this method will configure the search index for the workflow.
|
|
27
|
-
*/
|
|
28
|
-
configureSearchIndex: (hotMeshClient: HotMesh, search?: WorkflowSearchOptions) => Promise<void>;
|
|
29
13
|
search: (hotMeshClient: HotMesh, index: string, query: string[]) => Promise<string[]>;
|
|
30
14
|
workflow: {
|
|
31
15
|
start: (options: WorkflowOptions) => Promise<WorkflowHandleService>;
|
|
32
|
-
/**
|
|
33
|
-
* send a message to a running workflow that is paused and awaiting the signal
|
|
34
|
-
*/
|
|
35
16
|
signal: (signalId: string, data: Record<any, any>, namespace?: string) => Promise<string>;
|
|
36
|
-
/**
|
|
37
|
-
* send a message to spawn an parallel in-process thread of execution
|
|
38
|
-
* with the same job state as the main thread but bound to a different
|
|
39
|
-
* handler function. All job state will be journaled to the same hash
|
|
40
|
-
* as is used by the main thread.
|
|
41
|
-
*/
|
|
42
17
|
hook: (options: HookOptions) => Promise<string>;
|
|
43
18
|
getHandle: (taskQueue: string, workflowName: string, workflowId: string, namespace?: string) => Promise<WorkflowHandleService>;
|
|
44
19
|
search: (taskQueue: string, workflowName: string, namespace: null | string, index: string, ...query: string[]) => Promise<string[]>;
|
|
@@ -27,7 +27,6 @@ class ClientService {
|
|
|
27
27
|
}
|
|
28
28
|
return hotMeshClient;
|
|
29
29
|
}
|
|
30
|
-
//create and cache an instance
|
|
31
30
|
const hotMeshClient = hotmesh_1.HotMeshService.init({
|
|
32
31
|
appId: targetNS,
|
|
33
32
|
logLevel: enums_1.HMSH_LOGLEVEL,
|
|
@@ -43,36 +42,6 @@ class ClientService {
|
|
|
43
42
|
await this.activateWorkflow(await hotMeshClient, targetNS);
|
|
44
43
|
return hotMeshClient;
|
|
45
44
|
};
|
|
46
|
-
/**
|
|
47
|
-
* For those deployments with a redis stack backend (with the FT module),
|
|
48
|
-
* this method will configure the search index for the workflow.
|
|
49
|
-
*/
|
|
50
|
-
this.configureSearchIndex = async (hotMeshClient, search) => {
|
|
51
|
-
if (search?.schema) {
|
|
52
|
-
const store = hotMeshClient.engine.store;
|
|
53
|
-
const schema = [];
|
|
54
|
-
for (const [key, value] of Object.entries(search.schema)) {
|
|
55
|
-
//prefix with an underscore (avoids collisions with hotmesh reserved symbols)
|
|
56
|
-
schema.push(`_${key}`);
|
|
57
|
-
schema.push(value.type);
|
|
58
|
-
if (value.sortable) {
|
|
59
|
-
schema.push('SORTABLE');
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
try {
|
|
63
|
-
const keyParams = {
|
|
64
|
-
appId: hotMeshClient.appId,
|
|
65
|
-
jobId: ''
|
|
66
|
-
};
|
|
67
|
-
const hotMeshPrefix = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
|
|
68
|
-
const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
|
|
69
|
-
await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length, ...prefixes, 'SCHEMA', ...schema);
|
|
70
|
-
}
|
|
71
|
-
catch (error) {
|
|
72
|
-
hotMeshClient.engine.logger.info('durable-client-search-err', { ...error });
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
};
|
|
76
45
|
this.search = async (hotMeshClient, index, query) => {
|
|
77
46
|
const store = hotMeshClient.engine.store;
|
|
78
47
|
if (query[0]?.startsWith('FT.')) {
|
|
@@ -86,11 +55,8 @@ class ClientService {
|
|
|
86
55
|
const workflowName = options.entity ?? options.workflowName;
|
|
87
56
|
const trc = options.workflowTrace;
|
|
88
57
|
const spn = options.workflowSpan;
|
|
89
|
-
//NOTE: HotMesh 'workflowTopic' is a created by concatenating
|
|
90
|
-
// the taskQueue and workflowName used by the Durable module
|
|
91
58
|
const workflowTopic = `${taskQueueName}-${workflowName}`;
|
|
92
59
|
const hotMeshClient = await this.getHotMeshClient(workflowTopic, options.namespace);
|
|
93
|
-
this.configureSearchIndex(hotMeshClient, options.search);
|
|
94
60
|
const payload = {
|
|
95
61
|
arguments: [...options.args],
|
|
96
62
|
originJobId: options.originJobId,
|
|
@@ -106,19 +72,10 @@ class ClientService {
|
|
|
106
72
|
const jobId = await hotMeshClient.pub(`${options.namespace ?? factory_1.APP_ID}.execute`, payload, context, { search: options?.search?.data, marker: options?.marker });
|
|
107
73
|
return new handle_1.WorkflowHandleService(hotMeshClient, workflowTopic, jobId);
|
|
108
74
|
},
|
|
109
|
-
/**
|
|
110
|
-
* send a message to a running workflow that is paused and awaiting the signal
|
|
111
|
-
*/
|
|
112
75
|
signal: async (signalId, data, namespace) => {
|
|
113
76
|
const topic = `${namespace ?? factory_1.APP_ID}.wfs.signal`;
|
|
114
77
|
return await (await this.getHotMeshClient(topic, namespace)).hook(topic, { id: signalId, data });
|
|
115
78
|
},
|
|
116
|
-
/**
|
|
117
|
-
* send a message to spawn an parallel in-process thread of execution
|
|
118
|
-
* with the same job state as the main thread but bound to a different
|
|
119
|
-
* handler function. All job state will be journaled to the same hash
|
|
120
|
-
* as is used by the main thread.
|
|
121
|
-
*/
|
|
122
79
|
hook: async (options) => {
|
|
123
80
|
const workflowTopic = `${options.taskQueue}-${options.workflowName}`;
|
|
124
81
|
const payload = {
|
|
@@ -129,7 +86,6 @@ class ClientService {
|
|
|
129
86
|
maximumAttempts: options.config?.maximumAttempts || enums_1.HMSH_DURABLE_MAX_ATTEMPTS,
|
|
130
87
|
maximumInterval: (0, ms_1.default)(options.config?.maximumInterval || enums_1.HMSH_DURABLE_MAX_INTERVAL) / 1000,
|
|
131
88
|
};
|
|
132
|
-
//seed search data if presentthe hook before entering
|
|
133
89
|
const hotMeshClient = await this.getHotMeshClient(workflowTopic, options.namespace);
|
|
134
90
|
const msgId = await hotMeshClient.hook(`${hotMeshClient.appId}.flow.signal`, payload, types_1.StreamStatus.PENDING, 202);
|
|
135
91
|
if (options.search?.data) {
|
|
@@ -203,12 +159,6 @@ class ClientService {
|
|
|
203
159
|
_a = ClientService;
|
|
204
160
|
ClientService.topics = [];
|
|
205
161
|
ClientService.instances = new Map();
|
|
206
|
-
/**
|
|
207
|
-
* Creates a stream (Redis `XGROUP.CREATE`) where events can be published (XADD).
|
|
208
|
-
* It is possible that the worker that will read from this stream channel
|
|
209
|
-
* has not yet been initialized, so this call ensures that the channel
|
|
210
|
-
* exists and is ready to serve as a container for events.
|
|
211
|
-
*/
|
|
212
162
|
ClientService.createStream = async (hotMeshClient, workflowTopic, namespace) => {
|
|
213
163
|
const store = hotMeshClient.engine.store;
|
|
214
164
|
const params = { appId: namespace ?? factory_1.APP_ID, topic: workflowTopic };
|
|
@@ -217,14 +167,8 @@ ClientService.createStream = async (hotMeshClient, workflowTopic, namespace) =>
|
|
|
217
167
|
await store.xgroup('CREATE', streamKey, 'WORKER', '$', 'MKSTREAM');
|
|
218
168
|
}
|
|
219
169
|
catch (err) {
|
|
220
|
-
//ignore if already exists
|
|
221
170
|
}
|
|
222
171
|
};
|
|
223
|
-
/**
|
|
224
|
-
* It is possible for a client to invoke a workflow without first
|
|
225
|
-
* creating the stream. This method will verify that the stream
|
|
226
|
-
* exists and if not, create it.
|
|
227
|
-
*/
|
|
228
172
|
ClientService.verifyStream = async (workflowTopic, namespace) => {
|
|
229
173
|
const targetNS = namespace ?? factory_1.APP_ID;
|
|
230
174
|
if (ClientService.instances.has(targetNS)) {
|