@hotmeshio/hotmesh 0.0.57 → 0.0.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -10
- package/build/modules/enums.js +10 -1
- package/build/modules/key.d.ts +38 -0
- package/build/modules/key.js +46 -4
- package/build/modules/utils.d.ts +9 -0
- package/build/modules/utils.js +19 -1
- package/build/package.json +1 -1
- package/build/services/activities/activity.d.ts +28 -0
- package/build/services/activities/activity.js +46 -1
- package/build/services/activities/await.js +4 -0
- package/build/services/activities/cycle.d.ts +7 -0
- package/build/services/activities/cycle.js +16 -1
- package/build/services/activities/hook.d.ts +6 -0
- package/build/services/activities/hook.js +12 -2
- package/build/services/activities/interrupt.js +8 -0
- package/build/services/activities/signal.d.ts +6 -0
- package/build/services/activities/signal.js +15 -0
- package/build/services/activities/trigger.d.ts +4 -0
- package/build/services/activities/trigger.js +7 -1
- package/build/services/activities/worker.js +4 -0
- package/build/services/collator/index.d.ts +70 -0
- package/build/services/collator/index.js +91 -1
- package/build/services/compiler/deployer.js +38 -6
- package/build/services/compiler/index.d.ts +15 -0
- package/build/services/compiler/index.js +20 -0
- package/build/services/compiler/validator.d.ts +3 -0
- package/build/services/compiler/validator.js +25 -0
- package/build/services/connector/clients/ioredis.js +2 -0
- package/build/services/connector/clients/redis.js +2 -0
- package/build/services/connector/index.js +2 -0
- package/build/services/durable/client.d.ts +20 -0
- package/build/services/durable/client.js +25 -0
- package/build/services/durable/exporter.d.ts +22 -0
- package/build/services/durable/exporter.js +30 -1
- package/build/services/durable/handle.d.ts +36 -0
- package/build/services/durable/handle.js +46 -0
- package/build/services/durable/index.d.ts +4 -0
- package/build/services/durable/index.js +4 -0
- package/build/services/durable/schemas/factory.d.ts +29 -0
- package/build/services/durable/schemas/factory.js +29 -0
- package/build/services/durable/search.d.ts +97 -0
- package/build/services/durable/search.js +99 -0
- package/build/services/durable/worker.js +35 -6
- package/build/services/durable/workflow.d.ts +118 -0
- package/build/services/durable/workflow.js +153 -6
- package/build/services/engine/index.d.ts +5 -0
- package/build/services/engine/index.js +43 -1
- package/build/services/exporter/index.d.ts +27 -0
- package/build/services/exporter/index.js +33 -0
- package/build/services/hotmesh/index.js +8 -0
- package/build/services/logger/index.js +2 -0
- package/build/services/mapper/index.d.ts +14 -0
- package/build/services/mapper/index.js +14 -0
- package/build/services/pipe/functions/date.d.ts +7 -0
- package/build/services/pipe/functions/date.js +7 -0
- package/build/services/pipe/functions/math.js +2 -0
- package/build/services/pipe/index.d.ts +15 -0
- package/build/services/pipe/index.js +23 -2
- package/build/services/quorum/index.d.ts +7 -0
- package/build/services/quorum/index.js +21 -0
- package/build/services/reporter/index.d.ts +5 -0
- package/build/services/reporter/index.js +9 -0
- package/build/services/router/index.d.ts +9 -0
- package/build/services/router/index.js +30 -2
- package/build/services/serializer/index.js +23 -6
- package/build/services/store/cache.d.ts +19 -0
- package/build/services/store/cache.js +19 -0
- package/build/services/store/clients/ioredis.js +1 -0
- package/build/services/store/index.d.ts +55 -0
- package/build/services/store/index.js +81 -5
- package/build/services/stream/clients/ioredis.js +4 -1
- package/build/services/task/index.d.ts +9 -0
- package/build/services/task/index.js +31 -0
- package/build/services/telemetry/index.d.ts +7 -0
- package/build/services/telemetry/index.js +13 -1
- package/build/services/worker/index.d.ts +4 -0
- package/build/services/worker/index.js +6 -2
- package/build/types/activity.d.ts +81 -0
- package/build/types/durable.d.ts +255 -0
- package/build/types/exporter.d.ts +13 -0
- package/build/types/hotmesh.d.ts +10 -1
- package/build/types/hotmesh.js +3 -0
- package/build/types/index.js +1 -1
- package/build/types/job.d.ts +85 -0
- package/build/types/pipe.d.ts +65 -0
- package/build/types/quorum.d.ts +14 -0
- package/build/types/redis.d.ts +6 -0
- package/build/types/stream.d.ts +58 -0
- package/build/types/stream.js +4 -0
- package/package.json +1 -1
|
@@ -10,6 +10,7 @@ class Cycle extends activity_1.Activity {
|
|
|
10
10
|
constructor(config, data, metadata, hook, engine, context) {
|
|
11
11
|
super(config, data, metadata, hook, engine, context);
|
|
12
12
|
}
|
|
13
|
+
//******** LEG 1 ENTRY ********//
|
|
13
14
|
async process() {
|
|
14
15
|
this.logger.debug('cycle-process', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
|
|
15
16
|
let telemetry;
|
|
@@ -18,18 +19,21 @@ class Cycle extends activity_1.Activity {
|
|
|
18
19
|
telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
19
20
|
telemetry.startActivitySpan(this.leg);
|
|
20
21
|
this.mapInputData();
|
|
22
|
+
//set state/status
|
|
21
23
|
let multi = this.store.getMulti();
|
|
22
24
|
await this.setState(multi);
|
|
23
|
-
await this.setStatus(0, multi);
|
|
25
|
+
await this.setStatus(0, multi); //leg 1 never changes job status
|
|
24
26
|
const multiResponse = await multi.exec();
|
|
25
27
|
telemetry.mapActivityAttributes();
|
|
26
28
|
const jobStatus = this.resolveStatus(multiResponse);
|
|
29
|
+
//cycle the target ancestor
|
|
27
30
|
multi = this.store.getMulti();
|
|
28
31
|
const messageId = await this.cycleAncestorActivity(multi);
|
|
29
32
|
telemetry.setActivityAttributes({
|
|
30
33
|
'app.activity.mid': messageId,
|
|
31
34
|
'app.job.jss': jobStatus
|
|
32
35
|
});
|
|
36
|
+
//exit early (`Cycle` activities only execute Leg 1)
|
|
33
37
|
await collator_1.CollatorService.notarizeEarlyExit(this, multi);
|
|
34
38
|
await multi.exec();
|
|
35
39
|
return this.context.metadata.aid;
|
|
@@ -58,7 +62,18 @@ class Cycle extends activity_1.Activity {
|
|
|
58
62
|
this.logger.debug('cycle-process-end', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
|
|
59
63
|
}
|
|
60
64
|
}
|
|
65
|
+
/**
|
|
66
|
+
* Trigger the target ancestor to execute in a cycle,
|
|
67
|
+
* without violating the constraints of the DAG. Immutable
|
|
68
|
+
* `individual activity state` will execute in a new dimensional
|
|
69
|
+
* thread while `shared job state` can change. This
|
|
70
|
+
* pattern allows for retries without violating the DAG.
|
|
71
|
+
*/
|
|
61
72
|
async cycleAncestorActivity(multi) {
|
|
73
|
+
//Cycle activity L1 is a standin for the target ancestor L1.
|
|
74
|
+
//Input data mapping (mapInputData) allows for the
|
|
75
|
+
//next dimensonal thread to execute with different
|
|
76
|
+
//input data than the current dimensional thread
|
|
62
77
|
this.mapInputData();
|
|
63
78
|
const streamData = {
|
|
64
79
|
metadata: {
|
|
@@ -6,10 +6,16 @@ import { HookRule } from '../../types/hook';
|
|
|
6
6
|
import { JobState, JobStatus } from '../../types/job';
|
|
7
7
|
import { RedisMulti } from '../../types/redis';
|
|
8
8
|
import { StreamCode, StreamStatus } from '../../types/stream';
|
|
9
|
+
/**
|
|
10
|
+
* Supports `signal hook`, `time hook`, and `cycle hook` patterns
|
|
11
|
+
*/
|
|
9
12
|
declare class Hook extends Activity {
|
|
10
13
|
config: HookActivity;
|
|
11
14
|
constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
|
|
12
15
|
process(): Promise<string>;
|
|
16
|
+
/**
|
|
17
|
+
* does this activity use a time-hook or web-hook
|
|
18
|
+
*/
|
|
13
19
|
doesHook(): boolean;
|
|
14
20
|
doHook(telemetry: TelemetryService): Promise<void>;
|
|
15
21
|
doPassThrough(telemetry: TelemetryService): Promise<void>;
|
|
@@ -8,10 +8,14 @@ const pipe_1 = require("../pipe");
|
|
|
8
8
|
const task_1 = require("../task");
|
|
9
9
|
const telemetry_1 = require("../telemetry");
|
|
10
10
|
const stream_1 = require("../../types/stream");
|
|
11
|
+
/**
|
|
12
|
+
* Supports `signal hook`, `time hook`, and `cycle hook` patterns
|
|
13
|
+
*/
|
|
11
14
|
class Hook extends activity_1.Activity {
|
|
12
15
|
constructor(config, data, metadata, hook, engine, context) {
|
|
13
16
|
super(config, data, metadata, hook, engine, context);
|
|
14
17
|
}
|
|
18
|
+
//******** INITIAL ENTRY POINT (A) ********//
|
|
15
19
|
async process() {
|
|
16
20
|
this.logger.debug('hook-process', {
|
|
17
21
|
jid: this.context.metadata.jid,
|
|
@@ -24,9 +28,11 @@ class Hook extends activity_1.Activity {
|
|
|
24
28
|
telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
25
29
|
telemetry.startActivitySpan(this.leg);
|
|
26
30
|
if (this.doesHook()) {
|
|
31
|
+
//sleep and wait to awaken upon a signal
|
|
27
32
|
await this.doHook(telemetry);
|
|
28
33
|
}
|
|
29
34
|
else {
|
|
35
|
+
//end the activity and transition to its children
|
|
30
36
|
await this.doPassThrough(telemetry);
|
|
31
37
|
}
|
|
32
38
|
return this.context.metadata.aid;
|
|
@@ -55,6 +61,9 @@ class Hook extends activity_1.Activity {
|
|
|
55
61
|
this.logger.debug('hook-process-end', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
|
|
56
62
|
}
|
|
57
63
|
}
|
|
64
|
+
/**
|
|
65
|
+
* does this activity use a time-hook or web-hook
|
|
66
|
+
*/
|
|
58
67
|
doesHook() {
|
|
59
68
|
if (this.config.sleep) {
|
|
60
69
|
const duration = pipe_1.Pipe.resolve(this.config.sleep, this.context);
|
|
@@ -106,6 +115,7 @@ class Hook extends activity_1.Activity {
|
|
|
106
115
|
return this.context.metadata.jid;
|
|
107
116
|
}
|
|
108
117
|
}
|
|
118
|
+
//******** SIGNAL RE-ENTRY POINT ********//
|
|
109
119
|
async processWebHookEvent(status = stream_1.StreamStatus.SUCCESS, code = 200) {
|
|
110
120
|
this.logger.debug('hook-process-web-hook-event', {
|
|
111
121
|
topic: this.config.hook.topic,
|
|
@@ -124,8 +134,8 @@ class Hook extends activity_1.Activity {
|
|
|
124
134
|
await this.processEvent(status, code, 'hook');
|
|
125
135
|
if (code === 200) {
|
|
126
136
|
await taskService.deleteWebHookSignal(this.config.hook.topic, data);
|
|
127
|
-
}
|
|
128
|
-
}
|
|
137
|
+
} //else => 202/keep alive
|
|
138
|
+
} //else => already resolved
|
|
129
139
|
}
|
|
130
140
|
async processTimeHookEvent(jobId) {
|
|
131
141
|
this.logger.debug('hook-process-time-hook-event', {
|
|
@@ -10,6 +10,7 @@ class Interrupt extends activity_1.Activity {
|
|
|
10
10
|
constructor(config, data, metadata, hook, engine, context) {
|
|
11
11
|
super(config, data, metadata, hook, engine, context);
|
|
12
12
|
}
|
|
13
|
+
//******** LEG 1 ENTRY ********//
|
|
13
14
|
async process() {
|
|
14
15
|
this.logger.debug('interrupt-process', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
|
|
15
16
|
let telemetry;
|
|
@@ -49,11 +50,14 @@ class Interrupt extends activity_1.Activity {
|
|
|
49
50
|
}
|
|
50
51
|
}
|
|
51
52
|
async interruptSelf(telemetry) {
|
|
53
|
+
// Apply final updates to THIS job's state
|
|
52
54
|
if (this.config.job?.maps) {
|
|
53
55
|
this.mapJobData();
|
|
54
56
|
await this.setState();
|
|
55
57
|
}
|
|
58
|
+
// Interrupt THIS job
|
|
56
59
|
const messageId = await this.interrupt();
|
|
60
|
+
// Notarize completion and log
|
|
57
61
|
telemetry.mapActivityAttributes();
|
|
58
62
|
const multi = this.store.getMulti();
|
|
59
63
|
await collator_1.CollatorService.notarizeEarlyCompletion(this, multi);
|
|
@@ -67,8 +71,10 @@ class Interrupt extends activity_1.Activity {
|
|
|
67
71
|
return this.context.metadata.aid;
|
|
68
72
|
}
|
|
69
73
|
async interruptAnother(telemetry) {
|
|
74
|
+
// Interrupt ANOTHER job
|
|
70
75
|
const messageId = await this.interrupt();
|
|
71
76
|
const attrs = { 'app.activity.mid': messageId };
|
|
77
|
+
// Apply updates to THIS job's state
|
|
72
78
|
telemetry.mapActivityAttributes();
|
|
73
79
|
this.adjacencyList = await this.filterAdjacent();
|
|
74
80
|
if (this.config.job?.maps || this.config.output?.maps) {
|
|
@@ -77,12 +83,14 @@ class Interrupt extends activity_1.Activity {
|
|
|
77
83
|
const multi = this.store.getMulti();
|
|
78
84
|
await this.setState(multi);
|
|
79
85
|
}
|
|
86
|
+
// Notarize completion
|
|
80
87
|
const multi = this.store.getMulti();
|
|
81
88
|
await collator_1.CollatorService.notarizeEarlyCompletion(this, multi);
|
|
82
89
|
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
83
90
|
const multiResponse = await multi.exec();
|
|
84
91
|
const jobStatus = this.resolveStatus(multiResponse);
|
|
85
92
|
attrs['app.job.jss'] = jobStatus;
|
|
93
|
+
// Transition next generation and log
|
|
86
94
|
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
87
95
|
if (messageIds.length) {
|
|
88
96
|
attrs['app.activity.mids'] = messageIds.join(',');
|
|
@@ -8,7 +8,13 @@ declare class Signal extends Activity {
|
|
|
8
8
|
process(): Promise<string>;
|
|
9
9
|
mapSignalData(): Record<string, any>;
|
|
10
10
|
mapResolverData(): Record<string, any>;
|
|
11
|
+
/**
|
|
12
|
+
* The signal activity will hook one
|
|
13
|
+
*/
|
|
11
14
|
hookOne(): Promise<string>;
|
|
15
|
+
/**
|
|
16
|
+
* The signal activity will hook all paused jobs that share the same job key.
|
|
17
|
+
*/
|
|
12
18
|
hookAll(): Promise<string[]>;
|
|
13
19
|
}
|
|
14
20
|
export { Signal };
|
|
@@ -11,6 +11,7 @@ class Signal extends activity_1.Activity {
|
|
|
11
11
|
constructor(config, data, metadata, hook, engine, context) {
|
|
12
12
|
super(config, data, metadata, hook, engine, context);
|
|
13
13
|
}
|
|
14
|
+
//******** LEG 1 ENTRY ********//
|
|
14
15
|
async process() {
|
|
15
16
|
this.logger.debug('signal-process', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
|
|
16
17
|
let telemetry;
|
|
@@ -18,6 +19,7 @@ class Signal extends activity_1.Activity {
|
|
|
18
19
|
await this.verifyEntry();
|
|
19
20
|
telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
20
21
|
telemetry.startActivitySpan(this.leg);
|
|
22
|
+
//save state and notarize early completion (signals only run leg1)
|
|
21
23
|
const multi = this.store.getMulti();
|
|
22
24
|
this.adjacencyList = await this.filterAdjacent();
|
|
23
25
|
this.mapOutputData();
|
|
@@ -26,12 +28,14 @@ class Signal extends activity_1.Activity {
|
|
|
26
28
|
await collator_1.CollatorService.notarizeEarlyCompletion(this, multi);
|
|
27
29
|
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
28
30
|
const multiResponse = await multi.exec();
|
|
31
|
+
//todo: this should execute BEFORE the status is decremented
|
|
29
32
|
if (this.config.subtype === 'all') {
|
|
30
33
|
await this.hookAll();
|
|
31
34
|
}
|
|
32
35
|
else {
|
|
33
36
|
await this.hookOne();
|
|
34
37
|
}
|
|
38
|
+
//transition to adjacent activities
|
|
35
39
|
const jobStatus = this.resolveStatus(multiResponse);
|
|
36
40
|
const attrs = { 'app.job.jss': jobStatus };
|
|
37
41
|
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
@@ -78,6 +82,9 @@ class Signal extends activity_1.Activity {
|
|
|
78
82
|
return mapper.mapRules();
|
|
79
83
|
}
|
|
80
84
|
}
|
|
85
|
+
/**
|
|
86
|
+
* The signal activity will hook one
|
|
87
|
+
*/
|
|
81
88
|
async hookOne() {
|
|
82
89
|
const topic = pipe_1.Pipe.resolve(this.config.topic, this.context);
|
|
83
90
|
const signalInputData = this.mapSignalData();
|
|
@@ -85,15 +92,23 @@ class Signal extends activity_1.Activity {
|
|
|
85
92
|
const code = pipe_1.Pipe.resolve(this.config.code, this.context);
|
|
86
93
|
return await this.engine.hook(topic, signalInputData, status, code);
|
|
87
94
|
}
|
|
95
|
+
/**
|
|
96
|
+
* The signal activity will hook all paused jobs that share the same job key.
|
|
97
|
+
*/
|
|
88
98
|
async hookAll() {
|
|
99
|
+
//prep 1) generate `input signal data` (essentially the webhook payload)
|
|
89
100
|
const signalInputData = this.mapSignalData();
|
|
101
|
+
//prep 2) generate data that resolves the job key (per the YAML config)
|
|
90
102
|
const keyResolverData = this.mapResolverData();
|
|
91
103
|
if (this.config.scrub) {
|
|
104
|
+
//self-clean the indexes upon use if configured
|
|
92
105
|
keyResolverData.scrub = true;
|
|
93
106
|
}
|
|
107
|
+
//prep 3) jobKeys can contain multiple indexes (per the YAML config)
|
|
94
108
|
const key_name = pipe_1.Pipe.resolve(this.config.key_name, this.context);
|
|
95
109
|
const key_value = pipe_1.Pipe.resolve(this.config.key_value, this.context);
|
|
96
110
|
const indexQueryFacets = [`${key_name}:${key_value}`];
|
|
111
|
+
//execute: `hookAll` will now resume all paused jobs that share the same job key
|
|
97
112
|
return await this.engine.hookAll(this.config.topic, signalInputData, keyResolverData, indexQueryFacets);
|
|
98
113
|
}
|
|
99
114
|
}
|
|
@@ -21,6 +21,10 @@ declare class Trigger extends Activity {
|
|
|
21
21
|
resolveJobId(context: Partial<JobState>): string;
|
|
22
22
|
resolveJobKey(context: Partial<JobState>): string;
|
|
23
23
|
setStateNX(): Promise<void>;
|
|
24
|
+
/**
|
|
25
|
+
* Registers this job as a dependent of the parent job; when the
|
|
26
|
+
* parent job is interrupted, this job will be interrupted
|
|
27
|
+
*/
|
|
24
28
|
registerJobDependency(multi?: RedisMulti): Promise<void>;
|
|
25
29
|
setStats(multi?: RedisMulti): Promise<void>;
|
|
26
30
|
}
|
|
@@ -33,6 +33,8 @@ class Trigger extends activity_1.Activity {
|
|
|
33
33
|
await this.setStats(multi);
|
|
34
34
|
await this.registerJobDependency(multi);
|
|
35
35
|
await multi.exec();
|
|
36
|
+
//if the parent (spawner) chose not to await,
|
|
37
|
+
// emit the job_id as the data payload { job_id }
|
|
36
38
|
this.execAdjacentParent();
|
|
37
39
|
telemetry.mapActivityAttributes();
|
|
38
40
|
const jobStatus = Number(this.context.metadata.js);
|
|
@@ -160,7 +162,7 @@ class Trigger extends activity_1.Activity {
|
|
|
160
162
|
},
|
|
161
163
|
};
|
|
162
164
|
this.context['$self'] = this.context[this.metadata.aid];
|
|
163
|
-
this.context['$job'] = this.context;
|
|
165
|
+
this.context['$job'] = this.context; //NEVER call STRINGIFY! (circular)
|
|
164
166
|
}
|
|
165
167
|
bindJobMetadataPaths() {
|
|
166
168
|
return serializer_1.MDATA_SYMBOLS.JOB.KEYS.map((key) => `metadata/${key}`);
|
|
@@ -188,6 +190,10 @@ class Trigger extends activity_1.Activity {
|
|
|
188
190
|
throw new errors_1.DuplicateJobError(jobId);
|
|
189
191
|
}
|
|
190
192
|
}
|
|
193
|
+
/**
|
|
194
|
+
* Registers this job as a dependent of the parent job; when the
|
|
195
|
+
* parent job is interrupted, this job will be interrupted
|
|
196
|
+
*/
|
|
191
197
|
async registerJobDependency(multi) {
|
|
192
198
|
const depKey = this.config.stats?.parent ?? this.context.metadata.pj;
|
|
193
199
|
let resolvedDepKey = depKey ? pipe_1.Pipe.resolve(depKey, this.context) : '';
|
|
@@ -11,6 +11,7 @@ class Worker extends activity_1.Activity {
|
|
|
11
11
|
constructor(config, data, metadata, hook, engine, context) {
|
|
12
12
|
super(config, data, metadata, hook, engine, context);
|
|
13
13
|
}
|
|
14
|
+
//******** INITIAL ENTRY POINT (A) ********//
|
|
14
15
|
async process() {
|
|
15
16
|
this.logger.debug('worker-process', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
|
|
16
17
|
let telemetry;
|
|
@@ -19,12 +20,15 @@ class Worker extends activity_1.Activity {
|
|
|
19
20
|
telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
20
21
|
telemetry.startActivitySpan(this.leg);
|
|
21
22
|
this.mapInputData();
|
|
23
|
+
//save state and authorize reentry
|
|
22
24
|
const multi = this.store.getMulti();
|
|
25
|
+
//todo: await this.registerTimeout();
|
|
23
26
|
const messageId = await this.execActivity(multi);
|
|
24
27
|
await collator_1.CollatorService.authorizeReentry(this, multi);
|
|
25
28
|
await this.setState(multi);
|
|
26
29
|
await this.setStatus(0, multi);
|
|
27
30
|
const multiResponse = await multi.exec();
|
|
31
|
+
//telemetry
|
|
28
32
|
telemetry.mapActivityAttributes();
|
|
29
33
|
const jobStatus = this.resolveStatus(multiResponse);
|
|
30
34
|
telemetry.setActivityAttributes({
|
|
@@ -6,13 +6,40 @@ import { Activity } from '../activities/activity';
|
|
|
6
6
|
import { Cycle } from '../activities/cycle';
|
|
7
7
|
declare class CollatorService {
|
|
8
8
|
static targetLength: number;
|
|
9
|
+
/**
|
|
10
|
+
* Upon re/entry, verify that the job status is active
|
|
11
|
+
*/
|
|
9
12
|
static assertJobActive(status: number, jobId: string, activityId: string): void;
|
|
13
|
+
/**
|
|
14
|
+
* returns the dimensional address (dad) for the target; due
|
|
15
|
+
* to the nature of the notary system, the dad for leg 2 entry
|
|
16
|
+
* must target the `0` index while leg 2 exit must target the
|
|
17
|
+
* current index (0)
|
|
18
|
+
*/
|
|
10
19
|
static getDimensionalAddress(activity: Activity, isEntry?: boolean): Record<string, string>;
|
|
20
|
+
/**
|
|
21
|
+
* resolves the dimensional address for the
|
|
22
|
+
* ancestor in the graph to go back to. this address
|
|
23
|
+
* is determined by trimming the last digits from
|
|
24
|
+
* the `dad` (including the target).
|
|
25
|
+
* the target activity index is then set to `0`, so that
|
|
26
|
+
* the origin node can be queried for approval/entry.
|
|
27
|
+
*/
|
|
11
28
|
static resolveReentryDimension(activity: Cycle): string;
|
|
12
29
|
static notarizeEntry(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
13
30
|
static authorizeReentry(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
14
31
|
static notarizeEarlyExit(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
15
32
|
static notarizeEarlyCompletion(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
33
|
+
/**
|
|
34
|
+
* verifies both the concrete and synthetic keys for the activity; concrete keys
|
|
35
|
+
* exist in the original model and are effectively the 'real' keys. In reality,
|
|
36
|
+
* hook activities are atomized during compilation to create a synthetic DAG that
|
|
37
|
+
* is used to track the status of the graph in a distributed environment. The
|
|
38
|
+
* synthetic key represents different dimensional realities and is used to
|
|
39
|
+
* track re-entry overages (it distinguishes between the original and re-entry).
|
|
40
|
+
* The essential challenge is: is this a re-entry that is purposeful in
|
|
41
|
+
* order to induce cycles, or is the re-entry due to a failure in the system?
|
|
42
|
+
*/
|
|
16
43
|
static notarizeReentry(activity: Activity, guid: string, multi?: RedisMulti): Promise<number>;
|
|
17
44
|
static notarizeContinuation(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
18
45
|
static notarizeCompletion(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
@@ -21,13 +48,56 @@ declare class CollatorService {
|
|
|
21
48
|
static isDuplicate(num: number, targetDigitIndex: number): boolean;
|
|
22
49
|
static isInactive(num: number): boolean;
|
|
23
50
|
static isPrimed(amount: number, leg: ActivityDuplex): boolean;
|
|
51
|
+
/**
|
|
52
|
+
* During compilation, the graphs are compiled into structures necessary
|
|
53
|
+
* for distributed processing; these are referred to as 'synthetic DAGs',
|
|
54
|
+
* because they are not part of the original graph, but are used to track
|
|
55
|
+
* the status of the graph in a distributed environment. This check ensures
|
|
56
|
+
* that the 'synthetic key' is not a duplicate. (which is different than
|
|
57
|
+
* saying the 'key' is not a duplicate)
|
|
58
|
+
*/
|
|
24
59
|
static verifySyntheticInteger(amount: number): void;
|
|
25
60
|
static verifyInteger(amount: number, leg: ActivityDuplex, stage: CollationStage): void;
|
|
26
61
|
static getDimensionsById(ancestors: string[], dad: string): Record<string, string>;
|
|
62
|
+
/**
|
|
63
|
+
* All non-trigger activities are assigned a status seed by their parent
|
|
64
|
+
*/
|
|
27
65
|
static getSeed(): string;
|
|
66
|
+
/**
|
|
67
|
+
* All trigger activities are assigned a status seed in a completed state
|
|
68
|
+
*/
|
|
28
69
|
static getTriggerSeed(): string;
|
|
70
|
+
/**
|
|
71
|
+
* entry point for compiler-type activities. This is called by the compiler
|
|
72
|
+
* to bind the sorted activity IDs to the trigger activity. These are then used
|
|
73
|
+
* at runtime by the activities to track job/activity status.
|
|
74
|
+
* @param graphs
|
|
75
|
+
*/
|
|
29
76
|
static compile(graphs: HotMeshGraph[]): void;
|
|
77
|
+
/**
|
|
78
|
+
* binds the ancestor array to each activity.
|
|
79
|
+
* Used in conjunction with the dimensional
|
|
80
|
+
* address (dad). If dad is `,0,1,0,0` and the
|
|
81
|
+
* ancestor array is `['t1', 'a1', 'a2']` for
|
|
82
|
+
* activity 'a3', then the SAVED DAD
|
|
83
|
+
* will always have the trailing
|
|
84
|
+
* 0's removed. This ensures that the addressing
|
|
85
|
+
* remains consistent even if the graph changes.
|
|
86
|
+
* id DAD SAVED DAD
|
|
87
|
+
* * t1 => ,0 => [empty]
|
|
88
|
+
* * a1 => ,0,1 => ,0,1
|
|
89
|
+
* * a2 => ,0,1,0 => ,0,1
|
|
90
|
+
* * a3 => ,0,1,0,0 => ,0,1
|
|
91
|
+
*
|
|
92
|
+
*/
|
|
30
93
|
static bindAncestorArray(graphs: HotMeshGraph[]): void;
|
|
94
|
+
/**
|
|
95
|
+
* All activities exist on a dimensional plane. Zero
|
|
96
|
+
* is the default. A value of
|
|
97
|
+
* `AxY,0,0,0,0,1,0,0` would reflect that
|
|
98
|
+
* an ancestor activity was dimensionalized beyond
|
|
99
|
+
* the default.
|
|
100
|
+
*/
|
|
31
101
|
static getDimensionalSeed(index?: number): string;
|
|
32
102
|
}
|
|
33
103
|
export { CollatorService };
|
|
@@ -4,11 +4,20 @@ exports.CollatorService = void 0;
|
|
|
4
4
|
const errors_1 = require("../../modules/errors");
|
|
5
5
|
const collator_1 = require("../../types/collator");
|
|
6
6
|
class CollatorService {
|
|
7
|
+
/**
|
|
8
|
+
* Upon re/entry, verify that the job status is active
|
|
9
|
+
*/
|
|
7
10
|
static assertJobActive(status, jobId, activityId) {
|
|
8
11
|
if (status <= 0) {
|
|
9
12
|
throw new errors_1.InactiveJobError(jobId, status, activityId);
|
|
10
13
|
}
|
|
11
14
|
}
|
|
15
|
+
/**
|
|
16
|
+
* returns the dimensional address (dad) for the target; due
|
|
17
|
+
* to the nature of the notary system, the dad for leg 2 entry
|
|
18
|
+
* must target the `0` index while leg 2 exit must target the
|
|
19
|
+
* current index (0)
|
|
20
|
+
*/
|
|
12
21
|
static getDimensionalAddress(activity, isEntry = false) {
|
|
13
22
|
let dad = activity.context.metadata.dad || activity.metadata.dad;
|
|
14
23
|
if (isEntry && dad && activity.leg === 2) {
|
|
@@ -16,37 +25,62 @@ class CollatorService {
|
|
|
16
25
|
}
|
|
17
26
|
return CollatorService.getDimensionsById([...activity.config.ancestors, activity.metadata.aid], dad);
|
|
18
27
|
}
|
|
28
|
+
/**
|
|
29
|
+
* resolves the dimensional address for the
|
|
30
|
+
* ancestor in the graph to go back to. this address
|
|
31
|
+
* is determined by trimming the last digits from
|
|
32
|
+
* the `dad` (including the target).
|
|
33
|
+
* the target activity index is then set to `0`, so that
|
|
34
|
+
* the origin node can be queried for approval/entry.
|
|
35
|
+
*/
|
|
19
36
|
static resolveReentryDimension(activity) {
|
|
20
37
|
const targetActivityId = activity.config.ancestor;
|
|
21
38
|
const ancestors = activity.config.ancestors;
|
|
22
39
|
const ancestorIndex = ancestors.indexOf(targetActivityId);
|
|
23
|
-
const dimensions = activity.metadata.dad.split(',');
|
|
40
|
+
const dimensions = activity.metadata.dad.split(','); //e.g., `,0,0,1,0`
|
|
24
41
|
dimensions.length = ancestorIndex + 1;
|
|
25
42
|
dimensions.push('0');
|
|
26
43
|
return dimensions.join(',');
|
|
27
44
|
}
|
|
28
45
|
static async notarizeEntry(activity, multi) {
|
|
46
|
+
//decrement by -100_000_000_000_000
|
|
29
47
|
const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -100000000000000, this.getDimensionalAddress(activity), multi);
|
|
30
48
|
this.verifyInteger(amount, 1, 'enter');
|
|
31
49
|
return amount;
|
|
32
50
|
}
|
|
33
51
|
;
|
|
34
52
|
static async authorizeReentry(activity, multi) {
|
|
53
|
+
//set second digit to 8, allowing for re-entry
|
|
54
|
+
//decrement by -10_000_000_000_000
|
|
35
55
|
const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -10000000000000, this.getDimensionalAddress(activity), multi);
|
|
36
56
|
return amount;
|
|
37
57
|
}
|
|
38
58
|
static async notarizeEarlyExit(activity, multi) {
|
|
59
|
+
//decrement the 2nd and 3rd digits to fully deactivate (`cycle` activities use this command to fully exit after leg 1) (should result in `888000000000000`)
|
|
39
60
|
return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -11000000000000, this.getDimensionalAddress(activity), multi);
|
|
40
61
|
}
|
|
41
62
|
;
|
|
42
63
|
static async notarizeEarlyCompletion(activity, multi) {
|
|
64
|
+
//initialize both `possible` (1m) and `actualized` (1) zero dimension, while decrementing the 2nd
|
|
65
|
+
//3rd digit is optionally kept open if the activity might be used in a cycle
|
|
43
66
|
const decrement = activity.config.cycle ? 10000000000000 : 11000000000000;
|
|
44
67
|
return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1000001 - decrement, this.getDimensionalAddress(activity), multi);
|
|
45
68
|
}
|
|
46
69
|
;
|
|
70
|
+
/**
|
|
71
|
+
* verifies both the concrete and synthetic keys for the activity; concrete keys
|
|
72
|
+
* exist in the original model and are effectively the 'real' keys. In reality,
|
|
73
|
+
* hook activities are atomized during compilation to create a synthetic DAG that
|
|
74
|
+
* is used to track the status of the graph in a distributed environment. The
|
|
75
|
+
* synthetic key represents different dimensional realities and is used to
|
|
76
|
+
* track re-entry overages (it distinguishes between the original and re-entry).
|
|
77
|
+
* The essential challenge is: is this a re-entry that is purposeful in
|
|
78
|
+
* order to induce cycles, or is the re-entry due to a failure in the system?
|
|
79
|
+
*/
|
|
47
80
|
static async notarizeReentry(activity, guid, multi) {
|
|
48
81
|
const jid = activity.context.metadata.jid;
|
|
49
82
|
const localMulti = multi || activity.store.getMulti();
|
|
83
|
+
//increment by 1_000_000 (indicates re-entry and is used to drive the 'dimensional address' for adjacent activities (minus 1))
|
|
50
84
|
await activity.store.collate(jid, activity.metadata.aid, 1000000, this.getDimensionalAddress(activity, true), localMulti);
|
|
51
85
|
await activity.store.collateSynthetic(jid, guid, 1000000, localMulti);
|
|
52
86
|
const [_amountConcrete, _amountSynthetic] = await localMulti.exec();
|
|
@@ -58,10 +92,13 @@ class CollatorService {
|
|
|
58
92
|
}
|
|
59
93
|
;
|
|
60
94
|
static async notarizeContinuation(activity, multi) {
|
|
95
|
+
//keep open; actualize the leg2 dimension (+1)
|
|
61
96
|
return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1, this.getDimensionalAddress(activity), multi);
|
|
62
97
|
}
|
|
63
98
|
;
|
|
64
99
|
static async notarizeCompletion(activity, multi) {
|
|
100
|
+
//1) ALWAYS actualize leg2 dimension (+1)
|
|
101
|
+
//2) IF the activity is used in a cycle, don't close leg 2!
|
|
65
102
|
const decrement = activity.config.cycle ? 0 : 1000000000000;
|
|
66
103
|
return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1 - decrement, this.getDimensionalAddress(activity), multi);
|
|
67
104
|
}
|
|
@@ -90,6 +127,7 @@ class CollatorService {
|
|
|
90
127
|
return this.getDigitAtIndex(num, 2) < 9;
|
|
91
128
|
}
|
|
92
129
|
static isPrimed(amount, leg) {
|
|
130
|
+
//activity entry is not allowed if paths not properly pre-set
|
|
93
131
|
if (leg == 1) {
|
|
94
132
|
return amount != -100000000000000;
|
|
95
133
|
}
|
|
@@ -98,13 +136,23 @@ class CollatorService {
|
|
|
98
136
|
this.getDigitAtIndex(amount, 1) < 9;
|
|
99
137
|
}
|
|
100
138
|
}
|
|
139
|
+
/**
|
|
140
|
+
* During compilation, the graphs are compiled into structures necessary
|
|
141
|
+
* for distributed processing; these are referred to as 'synthetic DAGs',
|
|
142
|
+
* because they are not part of the original graph, but are used to track
|
|
143
|
+
* the status of the graph in a distributed environment. This check ensures
|
|
144
|
+
* that the 'synthetic key' is not a duplicate. (which is different than
|
|
145
|
+
* saying the 'key' is not a duplicate)
|
|
146
|
+
*/
|
|
101
147
|
static verifySyntheticInteger(amount) {
|
|
102
148
|
const samount = amount.toString();
|
|
103
149
|
const isCompletedValue = parseInt(samount[samount.length - 1], 10);
|
|
104
150
|
if (isCompletedValue > 0) {
|
|
151
|
+
//already done error (ack/delete clearly failed; this is a duplicate)
|
|
105
152
|
throw new errors_1.CollationError(amount, 2, 'enter', collator_1.CollationFaultType.INACTIVE);
|
|
106
153
|
}
|
|
107
154
|
else if (amount >= 2000000) {
|
|
155
|
+
//duplicate synthetic key (todo: need to resolve/fix this!!)
|
|
108
156
|
throw new errors_1.CollationError(amount, 2, 'enter', collator_1.CollationFaultType.DUPLICATE);
|
|
109
157
|
}
|
|
110
158
|
}
|
|
@@ -142,6 +190,11 @@ class CollatorService {
|
|
|
142
190
|
}
|
|
143
191
|
}
|
|
144
192
|
static getDimensionsById(ancestors, dad) {
|
|
193
|
+
//ancestors is an ordered list of all ancestors, starting with the trigger (['t1', 'a1', 'a2'])
|
|
194
|
+
//dad is the dimensional address of the ancestors list (',0,5,3')
|
|
195
|
+
//loop through the ancestors list and create a map of the ancestor to the dimensional address.
|
|
196
|
+
//return { 't1': ',0', 'a1': ',0,5', 'a1': ',0,5,3', $ADJACENT: ',0,5,3,0' };
|
|
197
|
+
// `adjacent` is a special key that is used to track the dimensional address of adjacent activities
|
|
145
198
|
const map = { '$ADJACENT': `${dad},0` };
|
|
146
199
|
let dadStr = dad;
|
|
147
200
|
ancestors.reverse().forEach((ancestor) => {
|
|
@@ -150,15 +203,43 @@ class CollatorService {
|
|
|
150
203
|
});
|
|
151
204
|
return map;
|
|
152
205
|
}
|
|
206
|
+
/**
|
|
207
|
+
* All non-trigger activities are assigned a status seed by their parent
|
|
208
|
+
*/
|
|
153
209
|
static getSeed() {
|
|
154
210
|
return '999000000000000';
|
|
155
211
|
}
|
|
212
|
+
/**
|
|
213
|
+
* All trigger activities are assigned a status seed in a completed state
|
|
214
|
+
*/
|
|
156
215
|
static getTriggerSeed() {
|
|
157
216
|
return '888000001000001';
|
|
158
217
|
}
|
|
218
|
+
/**
|
|
219
|
+
* entry point for compiler-type activities. This is called by the compiler
|
|
220
|
+
* to bind the sorted activity IDs to the trigger activity. These are then used
|
|
221
|
+
* at runtime by the activities to track job/activity status.
|
|
222
|
+
* @param graphs
|
|
223
|
+
*/
|
|
159
224
|
static compile(graphs) {
|
|
160
225
|
CollatorService.bindAncestorArray(graphs);
|
|
161
226
|
}
|
|
227
|
+
/**
|
|
228
|
+
* binds the ancestor array to each activity.
|
|
229
|
+
* Used in conjunction with the dimensional
|
|
230
|
+
* address (dad). If dad is `,0,1,0,0` and the
|
|
231
|
+
* ancestor array is `['t1', 'a1', 'a2']` for
|
|
232
|
+
* activity 'a3', then the SAVED DAD
|
|
233
|
+
* will always have the trailing
|
|
234
|
+
* 0's removed. This ensures that the addressing
|
|
235
|
+
* remains consistent even if the graph changes.
|
|
236
|
+
* id DAD SAVED DAD
|
|
237
|
+
* * t1 => ,0 => [empty]
|
|
238
|
+
* * a1 => ,0,1 => ,0,1
|
|
239
|
+
* * a2 => ,0,1,0 => ,0,1
|
|
240
|
+
* * a3 => ,0,1,0,0 => ,0,1
|
|
241
|
+
*
|
|
242
|
+
*/
|
|
162
243
|
static bindAncestorArray(graphs) {
|
|
163
244
|
graphs.forEach((graph) => {
|
|
164
245
|
const ancestors = {};
|
|
@@ -174,12 +255,21 @@ class CollatorService {
|
|
|
174
255
|
dfs(transition.to, [...parentList, node]);
|
|
175
256
|
});
|
|
176
257
|
};
|
|
258
|
+
// Start the DFS traversal
|
|
177
259
|
dfs(startingNode, []);
|
|
178
260
|
});
|
|
179
261
|
}
|
|
262
|
+
/**
|
|
263
|
+
* All activities exist on a dimensional plane. Zero
|
|
264
|
+
* is the default. A value of
|
|
265
|
+
* `AxY,0,0,0,0,1,0,0` would reflect that
|
|
266
|
+
* an ancestor activity was dimensionalized beyond
|
|
267
|
+
* the default.
|
|
268
|
+
*/
|
|
180
269
|
static getDimensionalSeed(index = 0) {
|
|
181
270
|
return `,${index}`;
|
|
182
271
|
}
|
|
183
272
|
}
|
|
184
273
|
exports.CollatorService = CollatorService;
|
|
274
|
+
//max int digit count that supports `hincrby`
|
|
185
275
|
CollatorService.targetLength = 15;
|