@hotmeshio/hotmesh 0.0.57 → 0.0.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/modules/enums.js +10 -1
- package/build/modules/key.d.ts +38 -0
- package/build/modules/key.js +46 -4
- package/build/modules/utils.d.ts +9 -0
- package/build/modules/utils.js +19 -1
- package/build/package.json +2 -1
- package/build/services/activities/activity.d.ts +28 -0
- package/build/services/activities/activity.js +46 -1
- package/build/services/activities/await.js +4 -0
- package/build/services/activities/cycle.d.ts +7 -0
- package/build/services/activities/cycle.js +16 -1
- package/build/services/activities/hook.d.ts +6 -0
- package/build/services/activities/hook.js +12 -2
- package/build/services/activities/interrupt.js +8 -0
- package/build/services/activities/signal.d.ts +6 -0
- package/build/services/activities/signal.js +15 -0
- package/build/services/activities/trigger.d.ts +4 -0
- package/build/services/activities/trigger.js +7 -1
- package/build/services/activities/worker.js +4 -0
- package/build/services/collator/index.d.ts +70 -0
- package/build/services/collator/index.js +91 -1
- package/build/services/compiler/deployer.js +38 -6
- package/build/services/compiler/index.d.ts +15 -0
- package/build/services/compiler/index.js +20 -0
- package/build/services/compiler/validator.d.ts +3 -0
- package/build/services/compiler/validator.js +25 -0
- package/build/services/connector/clients/ioredis.js +2 -0
- package/build/services/connector/clients/redis.js +2 -0
- package/build/services/connector/index.js +2 -0
- package/build/services/durable/client.d.ts +20 -0
- package/build/services/durable/client.js +25 -0
- package/build/services/durable/exporter.d.ts +22 -0
- package/build/services/durable/exporter.js +30 -1
- package/build/services/durable/handle.d.ts +36 -0
- package/build/services/durable/handle.js +46 -0
- package/build/services/durable/index.d.ts +4 -0
- package/build/services/durable/index.js +4 -0
- package/build/services/durable/schemas/factory.d.ts +29 -0
- package/build/services/durable/schemas/factory.js +29 -0
- package/build/services/durable/search.d.ts +97 -0
- package/build/services/durable/search.js +99 -0
- package/build/services/durable/worker.js +35 -6
- package/build/services/durable/workflow.d.ts +118 -0
- package/build/services/durable/workflow.js +153 -6
- package/build/services/engine/index.d.ts +5 -0
- package/build/services/engine/index.js +43 -1
- package/build/services/exporter/index.d.ts +27 -0
- package/build/services/exporter/index.js +33 -0
- package/build/services/hotmesh/index.js +8 -0
- package/build/services/logger/index.js +2 -0
- package/build/services/mapper/index.d.ts +14 -0
- package/build/services/mapper/index.js +14 -0
- package/build/services/pipe/functions/date.d.ts +7 -0
- package/build/services/pipe/functions/date.js +7 -0
- package/build/services/pipe/functions/math.js +2 -0
- package/build/services/pipe/index.d.ts +16 -0
- package/build/services/pipe/index.js +45 -3
- package/build/services/quorum/index.d.ts +7 -0
- package/build/services/quorum/index.js +21 -0
- package/build/services/reporter/index.d.ts +5 -0
- package/build/services/reporter/index.js +9 -0
- package/build/services/router/index.d.ts +9 -0
- package/build/services/router/index.js +30 -2
- package/build/services/serializer/index.js +23 -6
- package/build/services/store/cache.d.ts +19 -0
- package/build/services/store/cache.js +19 -0
- package/build/services/store/clients/ioredis.js +1 -0
- package/build/services/store/index.d.ts +55 -0
- package/build/services/store/index.js +81 -5
- package/build/services/stream/clients/ioredis.js +4 -1
- package/build/services/task/index.d.ts +9 -0
- package/build/services/task/index.js +31 -0
- package/build/services/telemetry/index.d.ts +7 -0
- package/build/services/telemetry/index.js +13 -1
- package/build/services/worker/index.d.ts +4 -0
- package/build/services/worker/index.js +6 -2
- package/build/types/activity.d.ts +81 -0
- package/build/types/durable.d.ts +255 -0
- package/build/types/exporter.d.ts +13 -0
- package/build/types/hotmesh.d.ts +10 -1
- package/build/types/hotmesh.js +3 -0
- package/build/types/index.js +1 -1
- package/build/types/job.d.ts +85 -0
- package/build/types/pipe.d.ts +65 -0
- package/build/types/quorum.d.ts +14 -0
- package/build/types/redis.d.ts +6 -0
- package/build/types/stream.d.ts +58 -0
- package/build/types/stream.js +4 -0
- package/package.json +2 -1
|
@@ -96,6 +96,7 @@ class EngineService {
|
|
|
96
96
|
if (this.cacheMode === 'nocache') {
|
|
97
97
|
const app = await this.store.getApp(this.appId, true);
|
|
98
98
|
if (app.version.toString() === this.untilVersion.toString()) {
|
|
99
|
+
//new version is deployed; OK to cache again
|
|
99
100
|
if (!this.apps)
|
|
100
101
|
this.apps = {};
|
|
101
102
|
this.apps[this.appId] = app;
|
|
@@ -133,6 +134,7 @@ class EngineService {
|
|
|
133
134
|
async throttle(delayInMillis) {
|
|
134
135
|
this.router.setThrottle(delayInMillis);
|
|
135
136
|
}
|
|
137
|
+
// ************* METADATA/MODEL METHODS *************
|
|
136
138
|
async initActivity(topic, data = {}, context) {
|
|
137
139
|
const [activityId, schema] = await this.getSchema(topic);
|
|
138
140
|
const ActivityHandler = activities_1.default[utils_1.polyfill.resolveActivityType(schema.type)];
|
|
@@ -158,11 +160,13 @@ class EngineService {
|
|
|
158
160
|
throw new Error(`no app found for id ${this.appId}`);
|
|
159
161
|
}
|
|
160
162
|
if (this.isPrivate(topic)) {
|
|
163
|
+
//private subscriptions use the schema id (.activityId)
|
|
161
164
|
const activityId = topic.substring(1);
|
|
162
165
|
const schema = await this.store.getSchema(activityId, await this.getVID(app));
|
|
163
166
|
return [activityId, schema];
|
|
164
167
|
}
|
|
165
168
|
else {
|
|
169
|
+
//public subscriptions use a topic (a.b.c) that is associated with a schema id
|
|
166
170
|
const activityId = await this.store.getSubscription(topic, await this.getVID(app));
|
|
167
171
|
if (activityId) {
|
|
168
172
|
const schema = await this.store.getSchema(activityId, await this.getVID(app));
|
|
@@ -177,6 +181,7 @@ class EngineService {
|
|
|
177
181
|
isPrivate(topic) {
|
|
178
182
|
return topic.startsWith('.');
|
|
179
183
|
}
|
|
184
|
+
// ************* COMPILER METHODS *************
|
|
180
185
|
async plan(pathOrYAML) {
|
|
181
186
|
const compiler = new compiler_1.CompilerService(this.store, this.logger);
|
|
182
187
|
return await compiler.plan(pathOrYAML);
|
|
@@ -185,6 +190,7 @@ class EngineService {
|
|
|
185
190
|
const compiler = new compiler_1.CompilerService(this.store, this.logger);
|
|
186
191
|
return await compiler.deploy(pathOrYAML);
|
|
187
192
|
}
|
|
193
|
+
// ************* REPORTER METHODS *************
|
|
188
194
|
async getStats(topic, query) {
|
|
189
195
|
const { id, version } = await this.getVID();
|
|
190
196
|
const reporter = new reporter_1.ReporterService({ id, version }, this.store, this.logger);
|
|
@@ -209,6 +215,7 @@ class EngineService {
|
|
|
209
215
|
sparse: query.sparse,
|
|
210
216
|
};
|
|
211
217
|
}
|
|
218
|
+
// ****************** STREAM RE-ENTRY POINT *****************
|
|
212
219
|
async processStreamMessage(streamData) {
|
|
213
220
|
this.logger.debug('engine-process-stream-message', {
|
|
214
221
|
jid: streamData.metadata.jid,
|
|
@@ -230,18 +237,22 @@ class EngineService {
|
|
|
230
237
|
data: streamData.data,
|
|
231
238
|
};
|
|
232
239
|
if (streamData.type === stream_1.StreamDataType.TIMEHOOK) {
|
|
240
|
+
//TIMEHOOK AWAKEN
|
|
233
241
|
const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context);
|
|
234
242
|
await activityHandler.processTimeHookEvent(streamData.metadata.jid);
|
|
235
243
|
}
|
|
236
244
|
else if (streamData.type === stream_1.StreamDataType.WEBHOOK) {
|
|
245
|
+
//WEBHOOK AWAKEN (SIGNAL IN)
|
|
237
246
|
const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context);
|
|
238
247
|
await activityHandler.processWebHookEvent(streamData.status, streamData.code);
|
|
239
248
|
}
|
|
240
249
|
else if (streamData.type === stream_1.StreamDataType.TRANSITION) {
|
|
241
|
-
|
|
250
|
+
//TRANSITION (ADJACENT ACTIVITY)
|
|
251
|
+
const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context); //todo: `as Activity` (type is more generic)
|
|
242
252
|
await activityHandler.process();
|
|
243
253
|
}
|
|
244
254
|
else if (streamData.type === stream_1.StreamDataType.AWAIT) {
|
|
255
|
+
//TRIGGER JOB
|
|
245
256
|
context.metadata = {
|
|
246
257
|
...context.metadata,
|
|
247
258
|
pj: streamData.metadata.jid,
|
|
@@ -256,10 +267,12 @@ class EngineService {
|
|
|
256
267
|
await activityHandler.process();
|
|
257
268
|
}
|
|
258
269
|
else if (streamData.type === stream_1.StreamDataType.RESULT) {
|
|
270
|
+
//AWAIT RESULT
|
|
259
271
|
const activityHandler = await this.initActivity(`.${context.metadata.aid}`, streamData.data, context);
|
|
260
272
|
await activityHandler.processEvent(streamData.status, streamData.code);
|
|
261
273
|
}
|
|
262
274
|
else {
|
|
275
|
+
//WORKER RESULT
|
|
263
276
|
const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, streamData.data, context);
|
|
264
277
|
await activityHandler.processEvent(streamData.status, streamData.code, 'output');
|
|
265
278
|
}
|
|
@@ -269,8 +282,10 @@ class EngineService {
|
|
|
269
282
|
aid: streamData.metadata.aid
|
|
270
283
|
});
|
|
271
284
|
}
|
|
285
|
+
// ***************** `AWAIT` ACTIVITY RETURN RESPONSE ****************
|
|
272
286
|
async execAdjacentParent(context, jobOutput, emit = false) {
|
|
273
287
|
if (this.hasParentJob(context)) {
|
|
288
|
+
//errors are stringified `StreamError` objects
|
|
274
289
|
const error = this.resolveError(jobOutput.metadata);
|
|
275
290
|
const spn = context['$self']?.output?.metadata?.l2s || context['$self']?.output?.metadata?.l1s;
|
|
276
291
|
const streamData = {
|
|
@@ -314,8 +329,11 @@ class EngineService {
|
|
|
314
329
|
return JSON.parse(metadata.err);
|
|
315
330
|
}
|
|
316
331
|
}
|
|
332
|
+
// ****************** `INTERRUPT` ACTIVE JOBS *****************
|
|
317
333
|
async interrupt(topic, jobId, options = {}) {
|
|
334
|
+
//immediately interrupt the job, going directly to the data source
|
|
318
335
|
await this.store.interrupt(topic, jobId, options);
|
|
336
|
+
//now that the job is interrupted, we can clean up
|
|
319
337
|
const context = await this.getState(topic, jobId);
|
|
320
338
|
const completionOpts = {
|
|
321
339
|
interrupt: options.descend,
|
|
@@ -323,9 +341,12 @@ class EngineService {
|
|
|
323
341
|
};
|
|
324
342
|
return await this.runJobCompletionTasks(context, completionOpts);
|
|
325
343
|
}
|
|
344
|
+
// ****************** `SCRUB` CLEAN COMPLETED JOBS *****************
|
|
326
345
|
async scrub(jobId) {
|
|
346
|
+
//todo: do not allow scrubbing of non-existent or actively running job
|
|
327
347
|
await this.store.scrub(jobId);
|
|
328
348
|
}
|
|
349
|
+
// ****************** `HOOK` ACTIVITY RE-ENTRY POINT *****************
|
|
329
350
|
async hook(topic, data, status = stream_1.StreamStatus.SUCCESS, code = 200) {
|
|
330
351
|
const hookRule = await this.taskService.getHookRule(topic);
|
|
331
352
|
const [aid] = await this.getSchema(`.${hookRule.to}`);
|
|
@@ -385,6 +406,8 @@ class EngineService {
|
|
|
385
406
|
throw new Error(`unable to find hook rule for topic ${hookTopic}`);
|
|
386
407
|
}
|
|
387
408
|
}
|
|
409
|
+
// ********************** PUB/SUB ENTRY POINT **********************
|
|
410
|
+
//publish (returns just the job id)
|
|
388
411
|
async pub(topic, data, context, extended) {
|
|
389
412
|
const activityHandler = await this.initActivity(topic, data, context);
|
|
390
413
|
if (activityHandler) {
|
|
@@ -394,24 +417,29 @@ class EngineService {
|
|
|
394
417
|
throw new Error(`unable to process activity for topic ${topic}`);
|
|
395
418
|
}
|
|
396
419
|
}
|
|
420
|
+
//subscribe to all jobs for a topic
|
|
397
421
|
async sub(topic, callback) {
|
|
398
422
|
const subscriptionCallback = async (topic, message) => {
|
|
399
423
|
callback(message.topic, message.job);
|
|
400
424
|
};
|
|
401
425
|
return await this.subscribe.subscribe(key_1.KeyType.QUORUM, subscriptionCallback, this.appId, topic);
|
|
402
426
|
}
|
|
427
|
+
//unsubscribe to all jobs for a topic
|
|
403
428
|
async unsub(topic) {
|
|
404
429
|
return await this.subscribe.unsubscribe(key_1.KeyType.QUORUM, this.appId, topic);
|
|
405
430
|
}
|
|
431
|
+
//subscribe to all jobs for a wildcard topic
|
|
406
432
|
async psub(wild, callback) {
|
|
407
433
|
const subscriptionCallback = async (topic, message) => {
|
|
408
434
|
callback(message.topic, message.job);
|
|
409
435
|
};
|
|
410
436
|
return await this.subscribe.psubscribe(key_1.KeyType.QUORUM, subscriptionCallback, this.appId, wild);
|
|
411
437
|
}
|
|
438
|
+
//unsubscribe to all jobs for a wildcard topic
|
|
412
439
|
async punsub(wild) {
|
|
413
440
|
return await this.subscribe.punsubscribe(key_1.KeyType.QUORUM, this.appId, wild);
|
|
414
441
|
}
|
|
442
|
+
//publish and await (returns the job and data (if ready)); throws error with jobid if not
|
|
415
443
|
async pubsub(topic, data, context, timeout = enums_1.HMSH_OTT_WAIT_TIME) {
|
|
416
444
|
context = {
|
|
417
445
|
metadata: {
|
|
@@ -435,6 +463,7 @@ class EngineService {
|
|
|
435
463
|
}
|
|
436
464
|
});
|
|
437
465
|
setTimeout(() => {
|
|
466
|
+
//note: job is still active (the subscriber timed out)
|
|
438
467
|
this.delistJobCallback(jobId);
|
|
439
468
|
reject({
|
|
440
469
|
code: enums_1.HMSH_CODE_TIMEOUT,
|
|
@@ -445,6 +474,7 @@ class EngineService {
|
|
|
445
474
|
});
|
|
446
475
|
}
|
|
447
476
|
async pubOneTimeSubs(context, jobOutput, emit = false) {
|
|
477
|
+
//todo: subscriber should query for the job...only publish minimum context needed
|
|
448
478
|
if (this.hasOneTimeSubscription(context)) {
|
|
449
479
|
const message = {
|
|
450
480
|
type: 'job',
|
|
@@ -483,7 +513,9 @@ class EngineService {
|
|
|
483
513
|
hasOneTimeSubscription(context) {
|
|
484
514
|
return Boolean(context.metadata.ngn);
|
|
485
515
|
}
|
|
516
|
+
// ********** JOB COMPLETION/CLEANUP (AND JOB EMIT) ***********
|
|
486
517
|
async runJobCompletionTasks(context, options = {}) {
|
|
518
|
+
//'emit' indicates the job is still active
|
|
487
519
|
const isAwait = this.hasParentJob(context, true);
|
|
488
520
|
const isOneTimeSub = this.hasOneTimeSubscription(context);
|
|
489
521
|
const topic = await this.getPublishesTopic(context);
|
|
@@ -499,9 +531,15 @@ class EngineService {
|
|
|
499
531
|
}
|
|
500
532
|
return msgId;
|
|
501
533
|
}
|
|
534
|
+
/**
|
|
535
|
+
* Job hash expiration is typically reliant on the metadata field
|
|
536
|
+
* if the activity concludes normally. However, if the job is `interrupted`,
|
|
537
|
+
* it will be expired immediately.
|
|
538
|
+
*/
|
|
502
539
|
resolveExpires(context, options) {
|
|
503
540
|
return options.expire ?? context.metadata.expire ?? enums_1.HMSH_EXPIRE_JOB_SECONDS;
|
|
504
541
|
}
|
|
542
|
+
// ****** GET JOB STATE/COLLATION STATUS BY ID *********
|
|
505
543
|
async export(jobId) {
|
|
506
544
|
return await this.exporter.export(jobId);
|
|
507
545
|
}
|
|
@@ -512,11 +550,15 @@ class EngineService {
|
|
|
512
550
|
const { id: appId } = await this.getVID();
|
|
513
551
|
return await this.store.getStatus(jobId, appId);
|
|
514
552
|
}
|
|
553
|
+
//todo: add 'options' parameter;
|
|
554
|
+
// (e.g, if {dimensions:true}, use hscan to deliver
|
|
555
|
+
// the full set of dimensional job data)
|
|
515
556
|
async getState(topic, jobId) {
|
|
516
557
|
const jobSymbols = await this.store.getSymbols(`$${topic}`);
|
|
517
558
|
const consumes = {
|
|
518
559
|
[`$${topic}`]: Object.keys(jobSymbols)
|
|
519
560
|
};
|
|
561
|
+
//job data exists at the 'zero' dimension; pass an empty object
|
|
520
562
|
const dIds = {};
|
|
521
563
|
const output = await this.store.getState(jobId, consumes, dIds);
|
|
522
564
|
if (!output) {
|
|
@@ -3,15 +3,42 @@ import { StoreService } from '../store';
|
|
|
3
3
|
import { DependencyExport, ExportOptions, JobActionExport, JobExport } from '../../types/exporter';
|
|
4
4
|
import { RedisClient, RedisMulti } from '../../types/redis';
|
|
5
5
|
import { StringStringType, Symbols } from "../../types/serializer";
|
|
6
|
+
/**
|
|
7
|
+
* Downloads job data from Redis (hscan, hmget, hgetall)
|
|
8
|
+
* Expands process data and includes dependency list
|
|
9
|
+
*/
|
|
6
10
|
declare class ExporterService {
|
|
7
11
|
appId: string;
|
|
8
12
|
logger: ILogger;
|
|
9
13
|
store: StoreService<RedisClient, RedisMulti>;
|
|
10
14
|
symbols: Promise<Symbols> | Symbols;
|
|
11
15
|
constructor(appId: string, store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
|
|
16
|
+
/**
|
|
17
|
+
* Convert the job hash and dependency list into a JobExport object.
|
|
18
|
+
* This object contains various facets that describe the interaction
|
|
19
|
+
* in terms relevant to narrative storytelling.
|
|
20
|
+
*/
|
|
12
21
|
export(jobId: string, options?: ExportOptions): Promise<JobExport>;
|
|
22
|
+
/**
|
|
23
|
+
* Inflates the key from Redis, 3-character symbol
|
|
24
|
+
* into a human-readable JSON path, reflecting the
|
|
25
|
+
* tree-like structure of the unidimensional Hash
|
|
26
|
+
*/
|
|
13
27
|
inflateKey(key: string): string;
|
|
28
|
+
/**
|
|
29
|
+
* Inflates the job data from Redis into a JobExport object
|
|
30
|
+
* @param jobHash - the job data from Redis
|
|
31
|
+
* @param dependencyList - the list of dependencies for the job
|
|
32
|
+
* @returns - the inflated job data
|
|
33
|
+
*/
|
|
14
34
|
inflate(jobHash: StringStringType, dependencyList: string[]): JobExport;
|
|
35
|
+
/**
|
|
36
|
+
* Inflates the dependency data from Redis into a JobExport object by
|
|
37
|
+
* organizing the dimensional isolate in sch a way asto interleave
|
|
38
|
+
* into a story
|
|
39
|
+
* @param data - the dependency data from Redis
|
|
40
|
+
* @returns - the organized dependency data
|
|
41
|
+
*/
|
|
15
42
|
inflateDependencyData(data: string[], actions: JobActionExport): DependencyExport[];
|
|
16
43
|
}
|
|
17
44
|
export { ExporterService };
|
|
@@ -4,12 +4,21 @@ exports.ExporterService = void 0;
|
|
|
4
4
|
const key_1 = require("../../modules/key");
|
|
5
5
|
const utils_1 = require("../../modules/utils");
|
|
6
6
|
const serializer_1 = require("../serializer");
|
|
7
|
+
/**
|
|
8
|
+
* Downloads job data from Redis (hscan, hmget, hgetall)
|
|
9
|
+
* Expands process data and includes dependency list
|
|
10
|
+
*/
|
|
7
11
|
class ExporterService {
|
|
8
12
|
constructor(appId, store, logger) {
|
|
9
13
|
this.appId = appId;
|
|
10
14
|
this.logger = logger;
|
|
11
15
|
this.store = store;
|
|
12
16
|
}
|
|
17
|
+
/**
|
|
18
|
+
* Convert the job hash and dependency list into a JobExport object.
|
|
19
|
+
* This object contains various facets that describe the interaction
|
|
20
|
+
* in terms relevant to narrative storytelling.
|
|
21
|
+
*/
|
|
13
22
|
async export(jobId, options = {}) {
|
|
14
23
|
if (!this.symbols) {
|
|
15
24
|
this.symbols = this.store.getAllSymbols();
|
|
@@ -20,10 +29,22 @@ class ExporterService {
|
|
|
20
29
|
const jobExport = this.inflate(jobData, depData);
|
|
21
30
|
return jobExport;
|
|
22
31
|
}
|
|
32
|
+
/**
|
|
33
|
+
* Inflates the key from Redis, 3-character symbol
|
|
34
|
+
* into a human-readable JSON path, reflecting the
|
|
35
|
+
* tree-like structure of the unidimensional Hash
|
|
36
|
+
*/
|
|
23
37
|
inflateKey(key) {
|
|
24
38
|
return (key in this.symbols) ? this.symbols[key] : key;
|
|
25
39
|
}
|
|
40
|
+
/**
|
|
41
|
+
* Inflates the job data from Redis into a JobExport object
|
|
42
|
+
* @param jobHash - the job data from Redis
|
|
43
|
+
* @param dependencyList - the list of dependencies for the job
|
|
44
|
+
* @returns - the inflated job data
|
|
45
|
+
*/
|
|
26
46
|
inflate(jobHash, dependencyList) {
|
|
47
|
+
//the list of actions taken in the workflow and hook functions
|
|
27
48
|
const actions = {
|
|
28
49
|
hooks: {},
|
|
29
50
|
main: {
|
|
@@ -37,6 +58,7 @@ class ExporterService {
|
|
|
37
58
|
Object.entries(jobHash).forEach(([key, value]) => {
|
|
38
59
|
const match = key.match(regex);
|
|
39
60
|
if (match) {
|
|
61
|
+
//activity process state
|
|
40
62
|
const [_, letters, numbers] = match;
|
|
41
63
|
const path = this.inflateKey(letters);
|
|
42
64
|
const dimensions = `${numbers.replace(/,/g, '/')}`;
|
|
@@ -44,6 +66,7 @@ class ExporterService {
|
|
|
44
66
|
process[`${dimensions}/${path}`] = resolved;
|
|
45
67
|
}
|
|
46
68
|
else if (key.length === 3) {
|
|
69
|
+
//job state
|
|
47
70
|
process[this.inflateKey(key)] = serializer_1.SerializerService.fromString(value);
|
|
48
71
|
}
|
|
49
72
|
});
|
|
@@ -53,6 +76,13 @@ class ExporterService {
|
|
|
53
76
|
status: jobHash[':'],
|
|
54
77
|
};
|
|
55
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* Inflates the dependency data from Redis into a JobExport object by
|
|
81
|
+
* organizing the dimensional isolate in sch a way asto interleave
|
|
82
|
+
* into a story
|
|
83
|
+
* @param data - the dependency data from Redis
|
|
84
|
+
* @returns - the organized dependency data
|
|
85
|
+
*/
|
|
56
86
|
inflateDependencyData(data, actions) {
|
|
57
87
|
const hookReg = /([0-9,]+)-(\d+)$/;
|
|
58
88
|
const flowReg = /-(\d+)$/;
|
|
@@ -64,6 +94,7 @@ class ExporterService {
|
|
|
64
94
|
let type;
|
|
65
95
|
let dimensionKey = '';
|
|
66
96
|
if (match) {
|
|
97
|
+
//hook-originating dependency
|
|
67
98
|
const [_, dimension, counter] = match;
|
|
68
99
|
dimensionKey = dimension.split(',').join('/');
|
|
69
100
|
prefix = `${dimensionKey}[${counter}]`;
|
|
@@ -72,11 +103,13 @@ class ExporterService {
|
|
|
72
103
|
else {
|
|
73
104
|
const match = jobId.match(flowReg);
|
|
74
105
|
if (match) {
|
|
106
|
+
//main workflow-originating dependency
|
|
75
107
|
const [_, counter] = match;
|
|
76
108
|
prefix = `[${counter}]`;
|
|
77
109
|
type = 'flow';
|
|
78
110
|
}
|
|
79
111
|
else {
|
|
112
|
+
//'other' types like signal cleanup
|
|
80
113
|
prefix = '/';
|
|
81
114
|
type = 'other';
|
|
82
115
|
}
|
|
@@ -67,6 +67,7 @@ class HotMeshService {
|
|
|
67
67
|
async doWork(config, logger) {
|
|
68
68
|
this.workers = await worker_1.WorkerService.init(this.namespace, this.appId, this.guid, config, logger);
|
|
69
69
|
}
|
|
70
|
+
// ************* PUB/SUB METHODS *************
|
|
70
71
|
async pub(topic, data = {}, context, extended) {
|
|
71
72
|
return await this.engine?.pub(topic, data, context, extended);
|
|
72
73
|
}
|
|
@@ -88,6 +89,7 @@ class HotMeshService {
|
|
|
88
89
|
async add(streamData) {
|
|
89
90
|
return await this.engine.add(streamData);
|
|
90
91
|
}
|
|
92
|
+
// ************* QUORUM METHODS *************
|
|
91
93
|
async rollCall(delay) {
|
|
92
94
|
return await this.quorum?.rollCall(delay);
|
|
93
95
|
}
|
|
@@ -104,6 +106,7 @@ class HotMeshService {
|
|
|
104
106
|
}
|
|
105
107
|
return await this.quorum?.pub(throttleMessage);
|
|
106
108
|
}
|
|
109
|
+
// ************* COMPILER METHODS *************
|
|
107
110
|
async plan(path) {
|
|
108
111
|
return await this.engine?.plan(path);
|
|
109
112
|
}
|
|
@@ -111,8 +114,10 @@ class HotMeshService {
|
|
|
111
114
|
return await this.engine?.deploy(pathOrYAML);
|
|
112
115
|
}
|
|
113
116
|
async activate(version, delay) {
|
|
117
|
+
//activation is a quorum operation
|
|
114
118
|
return await this.quorum?.activate(version, delay);
|
|
115
119
|
}
|
|
120
|
+
// ************* REPORTER METHODS *************
|
|
116
121
|
async export(jobId) {
|
|
117
122
|
return await this.engine?.export(jobId);
|
|
118
123
|
}
|
|
@@ -137,12 +142,15 @@ class HotMeshService {
|
|
|
137
142
|
async resolveQuery(topic, query) {
|
|
138
143
|
return await this.engine?.resolveQuery(topic, query);
|
|
139
144
|
}
|
|
145
|
+
// ****************** `INTERRUPT` ACTIVE JOBS *****************
|
|
140
146
|
async interrupt(topic, jobId, options = {}) {
|
|
141
147
|
return await this.engine?.interrupt(topic, jobId, options);
|
|
142
148
|
}
|
|
149
|
+
// ****************** `SCRUB` CLEAN COMPLETED JOBS *****************
|
|
143
150
|
async scrub(jobId) {
|
|
144
151
|
await this.engine?.scrub(jobId);
|
|
145
152
|
}
|
|
153
|
+
// ****** `HOOK` ACTIVITY RE-ENTRY POINT ******
|
|
146
154
|
async hook(topic, data, status, code) {
|
|
147
155
|
return await this.engine?.hook(topic, data, status, code);
|
|
148
156
|
}
|
|
@@ -15,12 +15,14 @@ class LoggerService {
|
|
|
15
15
|
level: this.logLevel,
|
|
16
16
|
format: winston_1.format.combine(winston_1.format.colorize(), winston_1.format.timestamp(), winston_1.format.printf((info) => {
|
|
17
17
|
const { timestamp, level, message } = info;
|
|
18
|
+
// Extract the object from the `info` object's `Symbol(splat)` field
|
|
18
19
|
const symbols = Object.getOwnPropertySymbols(info);
|
|
19
20
|
const splatSymbol = symbols.find(symbol => symbol.toString() === 'Symbol(splat)');
|
|
20
21
|
let splatData = {};
|
|
21
22
|
if (splatSymbol) {
|
|
22
23
|
splatData = info[splatSymbol][0] || {};
|
|
23
24
|
}
|
|
25
|
+
// Pass it to the `tagify` method
|
|
24
26
|
const tags = this.tagify(splatData);
|
|
25
27
|
return `${timestamp} [${level}] [${this.name || this.appId}:${this.instanceId}] ${message} ${tags}`;
|
|
26
28
|
})),
|
|
@@ -7,8 +7,22 @@ declare class MapperService {
|
|
|
7
7
|
constructor(rules: Record<string, unknown>, data: JobState);
|
|
8
8
|
mapRules(): Record<string, unknown>;
|
|
9
9
|
private traverseRules;
|
|
10
|
+
/**
|
|
11
|
+
* resolves a pipe expression of the form: { @pipe: [["{data.foo.bar}", 2, false, "hello world"]] }
|
|
12
|
+
* @param value
|
|
13
|
+
* @returns
|
|
14
|
+
*/
|
|
10
15
|
private pipe;
|
|
16
|
+
/**
|
|
17
|
+
* resolves a mapping expression in the form: "{data.foo.bar}" or 2 or false or "hello world"
|
|
18
|
+
* @param value
|
|
19
|
+
* @returns
|
|
20
|
+
*/
|
|
11
21
|
private resolve;
|
|
22
|
+
/**
|
|
23
|
+
* Evaluates a transition rule against the current job state and incoming Stream message
|
|
24
|
+
* to determine which (if any) transition should be taken.
|
|
25
|
+
*/
|
|
12
26
|
static evaluate(transitionRule: TransitionRule | boolean, context: JobState, code: StreamCode): boolean;
|
|
13
27
|
}
|
|
14
28
|
export { MapperService };
|
|
@@ -27,14 +27,28 @@ class MapperService {
|
|
|
27
27
|
return this.resolve(rules);
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
|
+
/**
|
|
31
|
+
* resolves a pipe expression of the form: { @pipe: [["{data.foo.bar}", 2, false, "hello world"]] }
|
|
32
|
+
* @param value
|
|
33
|
+
* @returns
|
|
34
|
+
*/
|
|
30
35
|
pipe(value) {
|
|
31
36
|
const pipe = new pipe_1.Pipe(value, this.data);
|
|
32
37
|
return pipe.process();
|
|
33
38
|
}
|
|
39
|
+
/**
|
|
40
|
+
* resolves a mapping expression in the form: "{data.foo.bar}" or 2 or false or "hello world"
|
|
41
|
+
* @param value
|
|
42
|
+
* @returns
|
|
43
|
+
*/
|
|
34
44
|
resolve(value) {
|
|
35
45
|
const pipe = new pipe_1.Pipe([[value]], this.data);
|
|
36
46
|
return pipe.process();
|
|
37
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* Evaluates a transition rule against the current job state and incoming Stream message
|
|
50
|
+
* to determine which (if any) transition should be taken.
|
|
51
|
+
*/
|
|
38
52
|
static evaluate(transitionRule, context, code) {
|
|
39
53
|
if (typeof transitionRule === 'boolean') {
|
|
40
54
|
return transitionRule;
|
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
type DateInput = Date | string | number;
|
|
2
2
|
declare class DateHandler {
|
|
3
|
+
/**
|
|
4
|
+
* It is so common in mapping operations to use a string (ISO) date as input. This helper
|
|
5
|
+
* method allows for a more-concise mapping ruleset by avoiding date initialization boilerplate
|
|
6
|
+
* code and instead handles the ISO, Milliseconds, and ECMAScript Date input types.
|
|
7
|
+
* @param input
|
|
8
|
+
* @returns
|
|
9
|
+
*/
|
|
3
10
|
static getDateInstance(input: DateInput): Date;
|
|
4
11
|
fromISOString(isoString: string): Date;
|
|
5
12
|
now(): number;
|
|
@@ -3,6 +3,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.DateHandler = void 0;
|
|
4
4
|
const utils_1 = require("../../../modules/utils");
|
|
5
5
|
class DateHandler {
|
|
6
|
+
/**
|
|
7
|
+
* It is so common in mapping operations to use a string (ISO) date as input. This helper
|
|
8
|
+
* method allows for a more-concise mapping ruleset by avoiding date initialization boilerplate
|
|
9
|
+
* code and instead handles the ISO, Milliseconds, and ECMAScript Date input types.
|
|
10
|
+
* @param input
|
|
11
|
+
* @returns
|
|
12
|
+
*/
|
|
6
13
|
static getDateInstance(input) {
|
|
7
14
|
const ISO_REGEX = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z)?$/;
|
|
8
15
|
if (typeof input === 'string') {
|
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.MathHandler = void 0;
|
|
4
4
|
class MathHandler {
|
|
5
5
|
add(...operands) {
|
|
6
|
+
// @ts-ignore
|
|
6
7
|
return operands.reduce((a, b) => {
|
|
7
8
|
if (Array.isArray(b)) {
|
|
8
9
|
return a + this.add(...b);
|
|
@@ -37,6 +38,7 @@ class MathHandler {
|
|
|
37
38
|
if (operands.length === 0) {
|
|
38
39
|
throw new Error('At least one operand is required.');
|
|
39
40
|
}
|
|
41
|
+
// @ts-ignore
|
|
40
42
|
return operands.reduce((a, b) => {
|
|
41
43
|
if (Array.isArray(b)) {
|
|
42
44
|
return a * this.multiply(...b);
|
|
@@ -13,7 +13,23 @@ declare class Pipe {
|
|
|
13
13
|
static resolve(unresolved: {
|
|
14
14
|
[key: string]: unknown;
|
|
15
15
|
} | PipeItem, context: Partial<JobState>): any;
|
|
16
|
+
/**
|
|
17
|
+
* loop through each PipeItem row in this Pipe, resolving and transforming line by line
|
|
18
|
+
* @returns {any} the result of the pipe
|
|
19
|
+
*/
|
|
16
20
|
process(resolved?: unknown[] | null): any;
|
|
21
|
+
cloneUnknown<T>(value: T): T;
|
|
22
|
+
/**
|
|
23
|
+
* Transforms iterable `input` into a single value. Vars $output, $item, $key
|
|
24
|
+
* and $input are available. The final statement in the iterator (the reduction)
|
|
25
|
+
* is assumed to be the return value. A default $output object may be provided
|
|
26
|
+
* to the iterator by placing the the second cell of the preceding row. Otherwise,
|
|
27
|
+
* construct the object during first run and ensure it is the first cell of the
|
|
28
|
+
* last row of the iterator, so it is returned as the $output for the next cycle
|
|
29
|
+
* @param {unknown[]} input
|
|
30
|
+
* @returns {unknown}
|
|
31
|
+
* @private
|
|
32
|
+
*/
|
|
17
33
|
reduce(input: Array<unknown[]>): unknown;
|
|
18
34
|
private processRow;
|
|
19
35
|
static resolveFunction(functionName: string): any;
|
|
@@ -30,10 +30,14 @@ class Pipe {
|
|
|
30
30
|
}
|
|
31
31
|
return pipe.process();
|
|
32
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* loop through each PipeItem row in this Pipe, resolving and transforming line by line
|
|
35
|
+
* @returns {any} the result of the pipe
|
|
36
|
+
*/
|
|
33
37
|
process(resolved = null) {
|
|
34
38
|
let index = 0;
|
|
35
39
|
if (!(resolved || this.isPipeType(this.rules[0]) || this.isreduceType(this.rules[0]))) {
|
|
36
|
-
resolved = this.processCells(this.rules[0]);
|
|
40
|
+
resolved = this.processCells(this.rules[0]); // Add type assertion
|
|
37
41
|
index = 1;
|
|
38
42
|
}
|
|
39
43
|
const len = this.rules.length;
|
|
@@ -43,8 +47,40 @@ class Pipe {
|
|
|
43
47
|
}
|
|
44
48
|
return resolved[0];
|
|
45
49
|
}
|
|
50
|
+
cloneUnknown(value) {
|
|
51
|
+
if (value === null || typeof value !== 'object') {
|
|
52
|
+
return value;
|
|
53
|
+
}
|
|
54
|
+
if (value instanceof Date) {
|
|
55
|
+
return new Date(value.getTime());
|
|
56
|
+
}
|
|
57
|
+
if (value instanceof RegExp) {
|
|
58
|
+
return new RegExp(value);
|
|
59
|
+
}
|
|
60
|
+
if (Array.isArray(value)) {
|
|
61
|
+
return value.map(item => this.cloneUnknown(item));
|
|
62
|
+
}
|
|
63
|
+
const clonedObj = {};
|
|
64
|
+
for (const key in value) {
|
|
65
|
+
if (Object.prototype.hasOwnProperty.call(value, key)) {
|
|
66
|
+
clonedObj[key] = this.cloneUnknown(value[key]);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return clonedObj;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Transforms iterable `input` into a single value. Vars $output, $item, $key
|
|
73
|
+
* and $input are available. The final statement in the iterator (the reduction)
|
|
74
|
+
* is assumed to be the return value. A default $output object may be provided
|
|
75
|
+
* to the iterator by placing the the second cell of the preceding row. Otherwise,
|
|
76
|
+
* construct the object during first run and ensure it is the first cell of the
|
|
77
|
+
* last row of the iterator, so it is returned as the $output for the next cycle
|
|
78
|
+
* @param {unknown[]} input
|
|
79
|
+
* @returns {unknown}
|
|
80
|
+
* @private
|
|
81
|
+
*/
|
|
46
82
|
reduce(input) {
|
|
47
|
-
let resolved = input[1] ?? null;
|
|
83
|
+
let resolved = this.cloneUnknown(input[1] ?? null);
|
|
48
84
|
if (Array.isArray(input[0])) {
|
|
49
85
|
for (let index = 0; index < input[0].length; index++) {
|
|
50
86
|
this.context = { $input: input[0], $output: resolved, $item: input[0][index], $key: index.toString(), $index: index };
|
|
@@ -63,25 +99,30 @@ class Pipe {
|
|
|
63
99
|
}
|
|
64
100
|
processRow(currentRow, resolvedPriorRow, subPipeQueue) {
|
|
65
101
|
if (resolvedPriorRow && this.isreduceType(currentRow)) {
|
|
102
|
+
//reduce the resolvedPriorRow and return the output from the reducer
|
|
66
103
|
const subPipe = new Pipe(currentRow['@reduce'], this.jobData);
|
|
67
104
|
const reduced = subPipe.reduce(resolvedPriorRow);
|
|
68
105
|
return reduced;
|
|
69
106
|
}
|
|
70
107
|
else if (this.isPipeType(currentRow)) {
|
|
108
|
+
//process subPipe and push to subPipeQueue; echo resolvedPriorRow
|
|
71
109
|
const subPipe = new Pipe(currentRow['@pipe'], this.jobData, this.context);
|
|
72
110
|
subPipeQueue.push(subPipe.process());
|
|
73
111
|
return resolvedPriorRow;
|
|
74
112
|
}
|
|
75
113
|
else {
|
|
114
|
+
//pivot the subPipeQueue into the arguments array (resolvedPriorRow)
|
|
76
115
|
if (subPipeQueue.length > 0) {
|
|
77
116
|
resolvedPriorRow = [...subPipeQueue];
|
|
78
117
|
subPipeQueue.length = 0;
|
|
79
118
|
}
|
|
80
119
|
if (!resolvedPriorRow) {
|
|
120
|
+
//if no prior row, use current row as prior row
|
|
81
121
|
return [].concat(this.processCells(Array.isArray(currentRow) ? [...currentRow] : []));
|
|
82
122
|
}
|
|
83
123
|
else {
|
|
84
|
-
const [functionName, ...params] = currentRow;
|
|
124
|
+
const [functionName, ...params] = currentRow; // Add type assertion
|
|
125
|
+
//use resolved values from prior row (n - 1) as input params to cell 1 function
|
|
85
126
|
let resolvedValue;
|
|
86
127
|
if (this.isContextVariable(functionName)) {
|
|
87
128
|
resolvedValue = this.resolveContextValue(functionName);
|
|
@@ -89,6 +130,7 @@ class Pipe {
|
|
|
89
130
|
else {
|
|
90
131
|
resolvedValue = Pipe.resolveFunction(functionName)(...resolvedPriorRow);
|
|
91
132
|
}
|
|
133
|
+
//resolve remaining cells in row and return concatenated with resolvedValue
|
|
92
134
|
return [resolvedValue].concat(this.processCells([...params]));
|
|
93
135
|
}
|
|
94
136
|
}
|
|
@@ -28,6 +28,10 @@ declare class QuorumService {
|
|
|
28
28
|
subscriptionHandler(): SubscriptionCallback;
|
|
29
29
|
sayPong(appId: string, guid: string, originator: string, details?: boolean): Promise<void>;
|
|
30
30
|
requestQuorum(delay?: number, details?: boolean): Promise<number>;
|
|
31
|
+
/**
|
|
32
|
+
* A quorum-wide command to broadcaset system details.
|
|
33
|
+
*
|
|
34
|
+
*/
|
|
31
35
|
doRollCall(message: RollCallMessage): Promise<void>;
|
|
32
36
|
cancelRollCall(): void;
|
|
33
37
|
stop(): void;
|
|
@@ -35,6 +39,9 @@ declare class QuorumService {
|
|
|
35
39
|
sub(callback: QuorumMessageCallback): Promise<void>;
|
|
36
40
|
unsub(callback: QuorumMessageCallback): Promise<void>;
|
|
37
41
|
rollCall(delay?: number): Promise<QuorumProfile[]>;
|
|
42
|
+
/**
|
|
43
|
+
* request a quorum; if successful activate the app version
|
|
44
|
+
*/
|
|
38
45
|
activate(version: string, delay?: number, count?: number): Promise<boolean>;
|
|
39
46
|
}
|
|
40
47
|
export { QuorumService };
|