@hotmeshio/hotmesh 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/cjs/package.json +3 -1
- package/build/cjs/services/activities/activity.d.ts +6 -0
- package/build/cjs/services/activities/activity.js +83 -7
- package/build/cjs/services/activities/await.d.ts +2 -2
- package/build/cjs/services/activities/await.js +5 -5
- package/build/cjs/services/activities/cycle.d.ts +19 -0
- package/build/cjs/services/activities/cycle.js +77 -0
- package/build/cjs/services/activities/index.d.ts +4 -2
- package/build/cjs/services/activities/index.js +4 -2
- package/build/cjs/services/activities/worker.d.ts +0 -8
- package/build/cjs/services/activities/worker.js +1 -87
- package/build/cjs/services/collator/index.d.ts +18 -1
- package/build/cjs/services/collator/index.js +41 -11
- package/build/cjs/services/durable/factory.js +17 -1
- package/build/cjs/services/durable/worker.d.ts +1 -0
- package/build/cjs/services/durable/worker.js +9 -14
- package/build/cjs/services/durable/workflow.js +5 -1
- package/build/cjs/services/mapper/index.js +3 -0
- package/build/cjs/services/signaler/stream.js +0 -1
- package/build/cjs/types/activity.d.ts +7 -2
- package/build/cjs/types/index.d.ts +1 -1
- package/build/esm/package.json +3 -1
- package/build/esm/services/activities/activity.d.ts +6 -0
- package/build/esm/services/activities/activity.js +83 -7
- package/build/esm/services/activities/await.d.ts +2 -2
- package/build/esm/services/activities/await.js +5 -5
- package/build/esm/services/activities/cycle.d.ts +19 -0
- package/build/esm/services/activities/cycle.js +74 -0
- package/build/esm/services/activities/index.d.ts +4 -2
- package/build/esm/services/activities/index.js +4 -2
- package/build/esm/services/activities/worker.d.ts +0 -8
- package/build/esm/services/activities/worker.js +1 -87
- package/build/esm/services/collator/index.d.ts +18 -1
- package/build/esm/services/collator/index.js +41 -11
- package/build/esm/services/durable/factory.js +17 -1
- package/build/esm/services/durable/worker.d.ts +1 -0
- package/build/esm/services/durable/worker.js +9 -14
- package/build/esm/services/durable/workflow.js +5 -1
- package/build/esm/services/mapper/index.js +3 -0
- package/build/esm/services/signaler/stream.js +0 -1
- package/build/esm/types/activity.d.ts +7 -2
- package/build/esm/types/index.d.ts +1 -1
- package/package.json +3 -1
- package/services/activities/activity.ts +90 -7
- package/services/activities/await.ts +5 -5
- package/services/activities/cycle.ts +96 -0
- package/services/activities/index.ts +4 -2
- package/services/activities/worker.ts +2 -93
- package/services/collator/index.ts +43 -11
- package/services/durable/factory.ts +17 -1
- package/services/durable/worker.ts +10 -13
- package/services/durable/workflow.ts +4 -1
- package/services/mapper/index.ts +3 -0
- package/services/signaler/stream.ts +0 -1
- package/types/activity.ts +8 -1
- package/types/index.ts +1 -0
package/build/cjs/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hotmeshio/hotmesh",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.8",
|
|
4
4
|
"description": "Durable Workflows",
|
|
5
5
|
"main": "build/cjs/index.js",
|
|
6
6
|
"module": "build/esm/index.js",
|
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
"test": "NODE_ENV=test jest --detectOpenHandles --forceExit --verbose",
|
|
30
30
|
"test:hmsh": "NODE_ENV=test jest ./tests/functional/index.test.ts --detectOpenHandles --verbose",
|
|
31
31
|
"test:compile": "NODE_ENV=test jest ./tests/functional/compile/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
32
|
+
"test:cycle": "NODE_ENV=test jest ./tests/functional/cycle/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
32
33
|
"test:connect": "NODE_ENV=test jest ./tests/unit/services/connector/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
33
34
|
"test:connect:redis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
|
|
34
35
|
"test:connect:ioredis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
|
|
@@ -48,6 +49,7 @@
|
|
|
48
49
|
"test:durable": "NODE_ENV=test jest ./tests/durable/*/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
49
50
|
"test:durable:hello": "NODE_ENV=test jest ./tests/durable/helloworld/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
50
51
|
"test:durable:goodbye": "NODE_ENV=test jest ./tests/durable/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
52
|
+
"test:durable:retry": "NODE_ENV=test jest ./tests/durable/retry/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
51
53
|
"test:durable:loopactivity": "NODE_ENV=test jest ./tests/durable/loopactivity/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
52
54
|
"test:durable:nested": "NODE_ENV=test jest ./tests/durable/nested/index.test.ts --detectOpenHandles --forceExit --verbose"
|
|
53
55
|
},
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { EngineService } from '../engine';
|
|
2
2
|
import { ILogger } from '../logger';
|
|
3
3
|
import { StoreService } from '../store';
|
|
4
|
+
import { TelemetryService } from '../telemetry';
|
|
4
5
|
import { ActivityData, ActivityLeg, ActivityMetadata, ActivityType } from '../../types/activity';
|
|
5
6
|
import { JobState, JobStatus } from '../../types/job';
|
|
6
7
|
import { MultiResponseFlags, RedisClient, RedisMulti } from '../../types/redis';
|
|
@@ -31,6 +32,11 @@ declare class Activity {
|
|
|
31
32
|
processWebHookEvent(): Promise<JobStatus | void>;
|
|
32
33
|
processTimeHookEvent(jobId: string): Promise<JobStatus | void>;
|
|
33
34
|
processHookEvent(jobId: string): Promise<JobStatus | void>;
|
|
35
|
+
processEvent(status?: StreamStatus, code?: StreamCode): Promise<void>;
|
|
36
|
+
processPending(telemetry: TelemetryService): Promise<MultiResponseFlags>;
|
|
37
|
+
processSuccess(telemetry: TelemetryService): Promise<MultiResponseFlags>;
|
|
38
|
+
processError(telemetry: TelemetryService): Promise<MultiResponseFlags>;
|
|
39
|
+
transitionAdjacent(multiResponse: MultiResponseFlags, telemetry: TelemetryService): Promise<void>;
|
|
34
40
|
resolveStatus(multiResponse: MultiResponseFlags): number;
|
|
35
41
|
mapJobData(): void;
|
|
36
42
|
mapInputData(): void;
|
|
@@ -87,7 +87,7 @@ class Activity {
|
|
|
87
87
|
setLeg(leg) {
|
|
88
88
|
this.leg = leg;
|
|
89
89
|
}
|
|
90
|
-
//********
|
|
90
|
+
//******** SIGNAL RE-ENTRY POINT ********//
|
|
91
91
|
doesHook() {
|
|
92
92
|
return !!(this.config.hook?.topic || this.config.sleep);
|
|
93
93
|
}
|
|
@@ -125,11 +125,6 @@ class Activity {
|
|
|
125
125
|
});
|
|
126
126
|
return await this.processHookEvent(jobId);
|
|
127
127
|
}
|
|
128
|
-
//todo: hooks are currently singletons. but they can support
|
|
129
|
-
// dimensional threads like `await` and `worker` do.
|
|
130
|
-
// Copy code from those activities to support cyclical
|
|
131
|
-
// timehook and eventhook inputs by adding a 'pending'
|
|
132
|
-
// flag to hooks that allows for repeated signals
|
|
133
128
|
async processHookEvent(jobId) {
|
|
134
129
|
this.logger.debug('activity-process-hook-event', { jobId });
|
|
135
130
|
let telemetry;
|
|
@@ -159,7 +154,6 @@ class Activity {
|
|
|
159
154
|
return jobStatus;
|
|
160
155
|
}
|
|
161
156
|
catch (error) {
|
|
162
|
-
console.error('this error?', error);
|
|
163
157
|
this.logger.error('engine-process-hook-event-error', error);
|
|
164
158
|
telemetry.setActivityError(error.message);
|
|
165
159
|
throw error;
|
|
@@ -168,6 +162,88 @@ class Activity {
|
|
|
168
162
|
telemetry.endActivitySpan();
|
|
169
163
|
}
|
|
170
164
|
}
|
|
165
|
+
//******** DUPLEX RE-ENTRY POINT ********//
|
|
166
|
+
async processEvent(status = stream_1.StreamStatus.SUCCESS, code = 200) {
|
|
167
|
+
this.setLeg(2);
|
|
168
|
+
const jid = this.context.metadata.jid;
|
|
169
|
+
const aid = this.metadata.aid;
|
|
170
|
+
this.status = status;
|
|
171
|
+
this.code = code;
|
|
172
|
+
this.logger.debug('activity-process-event', { topic: this.config.subtype, jid, aid, status, code });
|
|
173
|
+
let telemetry;
|
|
174
|
+
try {
|
|
175
|
+
await this.getState();
|
|
176
|
+
const aState = await collator_1.CollatorService.notarizeReentry(this);
|
|
177
|
+
this.adjacentIndex = collator_1.CollatorService.getDimensionalIndex(aState);
|
|
178
|
+
telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
179
|
+
let isComplete = collator_1.CollatorService.isActivityComplete(this.context.metadata.js);
|
|
180
|
+
if (isComplete) {
|
|
181
|
+
this.logger.warn('activity-process-event-duplicate', { jid, aid });
|
|
182
|
+
this.logger.debug('activity-process-event-duplicate-resolution', { resolution: 'Increase HotMesh config `reclaimDelay` timeout.' });
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
telemetry.startActivitySpan(this.leg);
|
|
186
|
+
let multiResponse;
|
|
187
|
+
if (status === stream_1.StreamStatus.PENDING) {
|
|
188
|
+
multiResponse = await this.processPending(telemetry);
|
|
189
|
+
}
|
|
190
|
+
else if (status === stream_1.StreamStatus.SUCCESS) {
|
|
191
|
+
multiResponse = await this.processSuccess(telemetry);
|
|
192
|
+
}
|
|
193
|
+
else {
|
|
194
|
+
multiResponse = await this.processError(telemetry);
|
|
195
|
+
}
|
|
196
|
+
this.transitionAdjacent(multiResponse, telemetry);
|
|
197
|
+
}
|
|
198
|
+
catch (error) {
|
|
199
|
+
this.logger.error('activity-process-event-error', error);
|
|
200
|
+
telemetry.setActivityError(error.message);
|
|
201
|
+
throw error;
|
|
202
|
+
}
|
|
203
|
+
finally {
|
|
204
|
+
telemetry.endActivitySpan();
|
|
205
|
+
this.logger.debug('activity-process-event-end', { jid, aid });
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
async processPending(telemetry) {
|
|
209
|
+
this.bindActivityData('output');
|
|
210
|
+
this.adjacencyList = await this.filterAdjacent();
|
|
211
|
+
this.mapJobData();
|
|
212
|
+
const multi = this.store.getMulti();
|
|
213
|
+
await this.setState(multi);
|
|
214
|
+
await collator_1.CollatorService.notarizeContinuation(this, multi);
|
|
215
|
+
await this.setStatus(this.adjacencyList.length, multi);
|
|
216
|
+
return await multi.exec();
|
|
217
|
+
}
|
|
218
|
+
async processSuccess(telemetry) {
|
|
219
|
+
this.bindActivityData('output');
|
|
220
|
+
this.adjacencyList = await this.filterAdjacent();
|
|
221
|
+
this.mapJobData();
|
|
222
|
+
const multi = this.store.getMulti();
|
|
223
|
+
await this.setState(multi);
|
|
224
|
+
await collator_1.CollatorService.notarizeCompletion(this, multi);
|
|
225
|
+
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
226
|
+
return await multi.exec();
|
|
227
|
+
}
|
|
228
|
+
async processError(telemetry) {
|
|
229
|
+
this.bindActivityError(this.data);
|
|
230
|
+
this.adjacencyList = await this.filterAdjacent();
|
|
231
|
+
const multi = this.store.getMulti();
|
|
232
|
+
await this.setState(multi);
|
|
233
|
+
await collator_1.CollatorService.notarizeCompletion(this, multi);
|
|
234
|
+
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
235
|
+
return await multi.exec();
|
|
236
|
+
}
|
|
237
|
+
async transitionAdjacent(multiResponse, telemetry) {
|
|
238
|
+
telemetry.mapActivityAttributes();
|
|
239
|
+
const jobStatus = this.resolveStatus(multiResponse);
|
|
240
|
+
const attrs = { 'app.job.jss': jobStatus };
|
|
241
|
+
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
242
|
+
if (messageIds.length) {
|
|
243
|
+
attrs['app.activity.mids'] = messageIds.join(',');
|
|
244
|
+
}
|
|
245
|
+
telemetry.setActivityAttributes(attrs);
|
|
246
|
+
}
|
|
171
247
|
resolveStatus(multiResponse) {
|
|
172
248
|
const activityStatus = multiResponse[multiResponse.length - 1];
|
|
173
249
|
if (Array.isArray(activityStatus)) {
|
|
@@ -10,7 +10,7 @@ declare class Await extends Activity {
|
|
|
10
10
|
process(): Promise<string>;
|
|
11
11
|
execActivity(): Promise<string>;
|
|
12
12
|
processEvent(status?: StreamStatus, code?: StreamCode): Promise<void>;
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
processSuccessResponse(adjacencyList: StreamData[]): Promise<MultiResponseFlags>;
|
|
14
|
+
processErrorResponse(adjacencyList: StreamData[]): Promise<MultiResponseFlags>;
|
|
15
15
|
}
|
|
16
16
|
export { Await };
|
|
@@ -22,7 +22,7 @@ class Await extends activity_1.Activity {
|
|
|
22
22
|
telemetry.startActivitySpan(this.leg);
|
|
23
23
|
this.mapInputData();
|
|
24
24
|
const multi = this.store.getMulti();
|
|
25
|
-
//await this.registerTimeout();
|
|
25
|
+
//todo: await this.registerTimeout();
|
|
26
26
|
await collator_1.CollatorService.authorizeReentry(this, multi);
|
|
27
27
|
await this.setState(multi);
|
|
28
28
|
await this.setStatus(0, multi);
|
|
@@ -95,12 +95,12 @@ class Await extends activity_1.Activity {
|
|
|
95
95
|
if (status === stream_1.StreamStatus.SUCCESS) {
|
|
96
96
|
this.bindActivityData('output');
|
|
97
97
|
this.adjacencyList = await this.filterAdjacent();
|
|
98
|
-
multiResponse = await this.
|
|
98
|
+
multiResponse = await this.processSuccessResponse(this.adjacencyList);
|
|
99
99
|
}
|
|
100
100
|
else {
|
|
101
101
|
this.bindActivityError(this.data);
|
|
102
102
|
this.adjacencyList = await this.filterAdjacent();
|
|
103
|
-
multiResponse = await this.
|
|
103
|
+
multiResponse = await this.processErrorResponse(this.adjacencyList);
|
|
104
104
|
}
|
|
105
105
|
telemetry.mapActivityAttributes();
|
|
106
106
|
const jobStatus = this.resolveStatus(multiResponse);
|
|
@@ -121,7 +121,7 @@ class Await extends activity_1.Activity {
|
|
|
121
121
|
this.logger.debug('await-resolve-await-end', { jid, aid, status, code });
|
|
122
122
|
}
|
|
123
123
|
}
|
|
124
|
-
async
|
|
124
|
+
async processSuccessResponse(adjacencyList) {
|
|
125
125
|
this.mapJobData();
|
|
126
126
|
const multi = this.store.getMulti();
|
|
127
127
|
await this.setState(multi);
|
|
@@ -129,7 +129,7 @@ class Await extends activity_1.Activity {
|
|
|
129
129
|
await this.setStatus(adjacencyList.length - 1, multi);
|
|
130
130
|
return await multi.exec();
|
|
131
131
|
}
|
|
132
|
-
async
|
|
132
|
+
async processErrorResponse(adjacencyList) {
|
|
133
133
|
//todo: if adjacencyList.length == 0, then map to the job output
|
|
134
134
|
// this method would be added to Base activity class
|
|
135
135
|
//this.mapJobData();
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { EngineService } from '../engine';
|
|
2
|
+
import { Activity, ActivityType } from './activity';
|
|
3
|
+
import { ActivityData, ActivityMetadata, CycleActivity } from '../../types/activity';
|
|
4
|
+
import { JobState } from '../../types/job';
|
|
5
|
+
import { RedisMulti } from '../../types/redis';
|
|
6
|
+
declare class Cycle extends Activity {
|
|
7
|
+
config: CycleActivity;
|
|
8
|
+
constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
|
|
9
|
+
process(): Promise<string>;
|
|
10
|
+
/**
|
|
11
|
+
* Trigger the target ancestor to execute in a cycle,
|
|
12
|
+
* without violating the constraints of the DAG. Immutable
|
|
13
|
+
* `individual activity state` will execute in a new dimensional
|
|
14
|
+
* thread while `shared job state` can change. This
|
|
15
|
+
* pattern allows for retries without violating the DAG.
|
|
16
|
+
*/
|
|
17
|
+
cycleAncestorActivity(multi: RedisMulti): Promise<string>;
|
|
18
|
+
}
|
|
19
|
+
export { Cycle };
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Cycle = void 0;
|
|
4
|
+
const errors_1 = require("../../modules/errors");
|
|
5
|
+
const collator_1 = require("../collator");
|
|
6
|
+
const activity_1 = require("./activity");
|
|
7
|
+
const telemetry_1 = require("../telemetry");
|
|
8
|
+
class Cycle extends activity_1.Activity {
|
|
9
|
+
constructor(config, data, metadata, hook, engine, context) {
|
|
10
|
+
super(config, data, metadata, hook, engine, context);
|
|
11
|
+
}
|
|
12
|
+
//******** LEG 1 ENTRY ********//
|
|
13
|
+
async process() {
|
|
14
|
+
this.logger.debug('cycle-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
15
|
+
let telemetry;
|
|
16
|
+
try {
|
|
17
|
+
//verify entry is allowed
|
|
18
|
+
this.setLeg(1);
|
|
19
|
+
await collator_1.CollatorService.notarizeEntry(this);
|
|
20
|
+
await this.getState();
|
|
21
|
+
telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
22
|
+
telemetry.startActivitySpan(this.leg);
|
|
23
|
+
this.mapInputData();
|
|
24
|
+
//set state/status
|
|
25
|
+
let multi = this.store.getMulti();
|
|
26
|
+
await this.setState(multi);
|
|
27
|
+
await this.setStatus(0, multi); //leg 1 never changes job status
|
|
28
|
+
const multiResponse = await multi.exec();
|
|
29
|
+
telemetry.mapActivityAttributes();
|
|
30
|
+
const jobStatus = this.resolveStatus(multiResponse);
|
|
31
|
+
//cycle the target ancestor
|
|
32
|
+
multi = this.store.getMulti();
|
|
33
|
+
const messageId = await this.cycleAncestorActivity(multi);
|
|
34
|
+
telemetry.setActivityAttributes({
|
|
35
|
+
'app.activity.mid': messageId,
|
|
36
|
+
'app.job.jss': jobStatus
|
|
37
|
+
});
|
|
38
|
+
//exit early (`Cycle` activities only execute Leg 1)
|
|
39
|
+
await collator_1.CollatorService.notarizeEarlyExit(this, multi);
|
|
40
|
+
await multi.exec();
|
|
41
|
+
return this.context.metadata.aid;
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
if (error instanceof errors_1.GetStateError) {
|
|
45
|
+
this.logger.error('cycle-get-state-error', error);
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
this.logger.error('cycle-process-error', error);
|
|
49
|
+
}
|
|
50
|
+
telemetry.setActivityError(error.message);
|
|
51
|
+
throw error;
|
|
52
|
+
}
|
|
53
|
+
finally {
|
|
54
|
+
telemetry.endActivitySpan();
|
|
55
|
+
this.logger.debug('cycle-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Trigger the target ancestor to execute in a cycle,
|
|
60
|
+
* without violating the constraints of the DAG. Immutable
|
|
61
|
+
* `individual activity state` will execute in a new dimensional
|
|
62
|
+
* thread while `shared job state` can change. This
|
|
63
|
+
* pattern allows for retries without violating the DAG.
|
|
64
|
+
*/
|
|
65
|
+
async cycleAncestorActivity(multi) {
|
|
66
|
+
const streamData = {
|
|
67
|
+
metadata: {
|
|
68
|
+
dad: collator_1.CollatorService.resolveReentryDimension(this),
|
|
69
|
+
jid: this.context.metadata.jid,
|
|
70
|
+
aid: this.config.ancestor,
|
|
71
|
+
},
|
|
72
|
+
data: {} //todo: verify immutability, before enabling: `this.context.data`
|
|
73
|
+
};
|
|
74
|
+
return (await this.engine.streamSignaler?.publishMessage(null, streamData, multi));
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
exports.Cycle = Cycle;
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { Activity } from './activity';
|
|
2
2
|
import { Await } from './await';
|
|
3
|
-
import {
|
|
4
|
-
import { Iterate } from './iterate';
|
|
3
|
+
import { Cycle } from './cycle';
|
|
5
4
|
import { Emit } from './emit';
|
|
5
|
+
import { Iterate } from './iterate';
|
|
6
6
|
import { Trigger } from './trigger';
|
|
7
|
+
import { Worker } from './worker';
|
|
7
8
|
declare const _default: {
|
|
8
9
|
activity: typeof Activity;
|
|
9
10
|
await: typeof Await;
|
|
11
|
+
cycle: typeof Cycle;
|
|
10
12
|
iterate: typeof Iterate;
|
|
11
13
|
emit: typeof Emit;
|
|
12
14
|
trigger: typeof Trigger;
|
|
@@ -2,13 +2,15 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const activity_1 = require("./activity");
|
|
4
4
|
const await_1 = require("./await");
|
|
5
|
-
const
|
|
6
|
-
const iterate_1 = require("./iterate");
|
|
5
|
+
const cycle_1 = require("./cycle");
|
|
7
6
|
const emit_1 = require("./emit");
|
|
7
|
+
const iterate_1 = require("./iterate");
|
|
8
8
|
const trigger_1 = require("./trigger");
|
|
9
|
+
const worker_1 = require("./worker");
|
|
9
10
|
exports.default = {
|
|
10
11
|
activity: activity_1.Activity,
|
|
11
12
|
await: await_1.Await,
|
|
13
|
+
cycle: cycle_1.Cycle,
|
|
12
14
|
iterate: iterate_1.Iterate,
|
|
13
15
|
emit: emit_1.Emit,
|
|
14
16
|
trigger: trigger_1.Trigger,
|
|
@@ -2,18 +2,10 @@ import { Activity } from './activity';
|
|
|
2
2
|
import { EngineService } from '../engine';
|
|
3
3
|
import { ActivityData, ActivityMetadata, ActivityType, WorkerActivity } from '../../types/activity';
|
|
4
4
|
import { JobState } from '../../types/job';
|
|
5
|
-
import { MultiResponseFlags } from '../../types/redis';
|
|
6
|
-
import { StreamCode, StreamStatus } from '../../types/stream';
|
|
7
|
-
import { TelemetryService } from '../telemetry';
|
|
8
5
|
declare class Worker extends Activity {
|
|
9
6
|
config: WorkerActivity;
|
|
10
7
|
constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
|
|
11
8
|
process(): Promise<string>;
|
|
12
9
|
execActivity(): Promise<string>;
|
|
13
|
-
processEvent(status?: StreamStatus, code?: StreamCode): Promise<void>;
|
|
14
|
-
processPending(telemetry: TelemetryService): Promise<void>;
|
|
15
|
-
processSuccess(telemetry: TelemetryService): Promise<void>;
|
|
16
|
-
processError(telemetry: TelemetryService): Promise<void>;
|
|
17
|
-
transitionAdjacent(multiResponse: MultiResponseFlags, telemetry: TelemetryService): Promise<void>;
|
|
18
10
|
}
|
|
19
11
|
export { Worker };
|
|
@@ -4,7 +4,6 @@ exports.Worker = void 0;
|
|
|
4
4
|
const errors_1 = require("../../modules/errors");
|
|
5
5
|
const activity_1 = require("./activity");
|
|
6
6
|
const collator_1 = require("../collator");
|
|
7
|
-
const stream_1 = require("../../types/stream");
|
|
8
7
|
const telemetry_1 = require("../telemetry");
|
|
9
8
|
class Worker extends activity_1.Activity {
|
|
10
9
|
constructor(config, data, metadata, hook, engine, context) {
|
|
@@ -22,7 +21,7 @@ class Worker extends activity_1.Activity {
|
|
|
22
21
|
telemetry.startActivitySpan(this.leg);
|
|
23
22
|
this.mapInputData();
|
|
24
23
|
const multi = this.store.getMulti();
|
|
25
|
-
//await this.registerTimeout();
|
|
24
|
+
//todo: await this.registerTimeout();
|
|
26
25
|
await collator_1.CollatorService.authorizeReentry(this, multi);
|
|
27
26
|
await this.setState(multi);
|
|
28
27
|
await this.setStatus(0, multi);
|
|
@@ -34,7 +33,6 @@ class Worker extends activity_1.Activity {
|
|
|
34
33
|
'app.activity.mid': messageId,
|
|
35
34
|
'app.job.jss': jobStatus
|
|
36
35
|
});
|
|
37
|
-
//TODO: UPDATE ACTIVITY STATE (LEG 1 EXIT)
|
|
38
36
|
return this.context.metadata.aid;
|
|
39
37
|
}
|
|
40
38
|
catch (error) {
|
|
@@ -42,7 +40,6 @@ class Worker extends activity_1.Activity {
|
|
|
42
40
|
this.logger.error('worker-get-state-error', error);
|
|
43
41
|
}
|
|
44
42
|
else {
|
|
45
|
-
console.error(error);
|
|
46
43
|
this.logger.error('worker-process-error', error);
|
|
47
44
|
}
|
|
48
45
|
telemetry.setActivityError(error.message);
|
|
@@ -72,88 +69,5 @@ class Worker extends activity_1.Activity {
|
|
|
72
69
|
}
|
|
73
70
|
return (await this.engine.streamSignaler?.publishMessage(this.config.subtype, streamData));
|
|
74
71
|
}
|
|
75
|
-
//******** SIGNAL RE-ENTRY POINT (DUPLEX LEG 2 of 2) ********//
|
|
76
|
-
async processEvent(status = stream_1.StreamStatus.SUCCESS, code = 200) {
|
|
77
|
-
this.setLeg(2);
|
|
78
|
-
const jid = this.context.metadata.jid;
|
|
79
|
-
const aid = this.metadata.aid;
|
|
80
|
-
this.status = status;
|
|
81
|
-
this.code = code;
|
|
82
|
-
this.logger.debug('worker-process-event', { topic: this.config.subtype, jid, aid, status, code });
|
|
83
|
-
let telemetry;
|
|
84
|
-
try {
|
|
85
|
-
await this.getState();
|
|
86
|
-
const aState = await collator_1.CollatorService.notarizeReentry(this);
|
|
87
|
-
this.adjacentIndex = collator_1.CollatorService.getDimensionalIndex(aState);
|
|
88
|
-
telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
89
|
-
let isComplete = collator_1.CollatorService.isActivityComplete(this.context.metadata.js);
|
|
90
|
-
if (isComplete) {
|
|
91
|
-
this.logger.warn('worker-process-event-duplicate', { jid, aid });
|
|
92
|
-
this.logger.debug('worker-process-event-duplicate-resolution', { resolution: 'Increase HotMesh config `reclaimDelay` timeout.' });
|
|
93
|
-
return;
|
|
94
|
-
}
|
|
95
|
-
telemetry.startActivitySpan(this.leg);
|
|
96
|
-
if (status === stream_1.StreamStatus.PENDING) {
|
|
97
|
-
await this.processPending(telemetry);
|
|
98
|
-
}
|
|
99
|
-
else if (status === stream_1.StreamStatus.SUCCESS) {
|
|
100
|
-
await this.processSuccess(telemetry);
|
|
101
|
-
}
|
|
102
|
-
else {
|
|
103
|
-
await this.processError(telemetry);
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
catch (error) {
|
|
107
|
-
this.logger.error('worker-process-event-error', error);
|
|
108
|
-
telemetry.setActivityError(error.message);
|
|
109
|
-
throw error;
|
|
110
|
-
}
|
|
111
|
-
finally {
|
|
112
|
-
telemetry.endActivitySpan();
|
|
113
|
-
this.logger.debug('worker-process-event-end', { jid, aid });
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
async processPending(telemetry) {
|
|
117
|
-
this.bindActivityData('output');
|
|
118
|
-
this.adjacencyList = await this.filterAdjacent();
|
|
119
|
-
this.mapJobData();
|
|
120
|
-
const multi = this.store.getMulti();
|
|
121
|
-
await this.setState(multi);
|
|
122
|
-
await collator_1.CollatorService.notarizeContinuation(this, multi);
|
|
123
|
-
await this.setStatus(this.adjacencyList.length, multi);
|
|
124
|
-
const multiResponse = await multi.exec();
|
|
125
|
-
this.transitionAdjacent(multiResponse, telemetry);
|
|
126
|
-
}
|
|
127
|
-
async processSuccess(telemetry) {
|
|
128
|
-
this.bindActivityData('output');
|
|
129
|
-
this.adjacencyList = await this.filterAdjacent();
|
|
130
|
-
this.mapJobData();
|
|
131
|
-
const multi = this.store.getMulti();
|
|
132
|
-
await this.setState(multi);
|
|
133
|
-
await collator_1.CollatorService.notarizeCompletion(this, multi);
|
|
134
|
-
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
135
|
-
const multiResponse = await multi.exec();
|
|
136
|
-
this.transitionAdjacent(multiResponse, telemetry);
|
|
137
|
-
}
|
|
138
|
-
async processError(telemetry) {
|
|
139
|
-
this.bindActivityError(this.data);
|
|
140
|
-
this.adjacencyList = await this.filterAdjacent();
|
|
141
|
-
const multi = this.store.getMulti();
|
|
142
|
-
await this.setState(multi);
|
|
143
|
-
await collator_1.CollatorService.notarizeCompletion(this, multi);
|
|
144
|
-
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
145
|
-
const multiResponse = await multi.exec();
|
|
146
|
-
this.transitionAdjacent(multiResponse, telemetry);
|
|
147
|
-
}
|
|
148
|
-
async transitionAdjacent(multiResponse, telemetry) {
|
|
149
|
-
telemetry.mapActivityAttributes();
|
|
150
|
-
const jobStatus = this.resolveStatus(multiResponse);
|
|
151
|
-
const attrs = { 'app.job.jss': jobStatus };
|
|
152
|
-
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
153
|
-
if (messageIds.length) {
|
|
154
|
-
attrs['app.activity.mids'] = messageIds.join(',');
|
|
155
|
-
}
|
|
156
|
-
telemetry.setActivityAttributes(attrs);
|
|
157
|
-
}
|
|
158
72
|
}
|
|
159
73
|
exports.Worker = Worker;
|
|
@@ -3,11 +3,28 @@ import { CollationStage } from '../../types/collator';
|
|
|
3
3
|
import { ActivityDuplex } from '../../types/activity';
|
|
4
4
|
import { HotMeshGraph } from '../../types/hotmesh';
|
|
5
5
|
import { Activity } from '../activities/activity';
|
|
6
|
+
import { Cycle } from '../activities/cycle';
|
|
6
7
|
declare class CollatorService {
|
|
7
8
|
static targetLength: number;
|
|
8
|
-
|
|
9
|
+
/**
|
|
10
|
+
* returns the dimensional address (dad) for the target; due
|
|
11
|
+
* to the nature of the notary system, the dad for leg 2 entry
|
|
12
|
+
* must target the `0` index while leg 2 exit must target the
|
|
13
|
+
* current index (0)
|
|
14
|
+
*/
|
|
15
|
+
static getDimensionalAddress(activity: Activity, isEntry?: boolean): Record<string, string>;
|
|
16
|
+
/**
|
|
17
|
+
* resolves the dimensional address for the
|
|
18
|
+
* ancestor in the graph to go back to. this address
|
|
19
|
+
* is determined by trimming the last digits from
|
|
20
|
+
* the `dad` (including the target).
|
|
21
|
+
* the target activity index is then set to `0`, so that
|
|
22
|
+
* the origin node can be queried for approval/entry.
|
|
23
|
+
*/
|
|
24
|
+
static resolveReentryDimension(activity: Cycle): string;
|
|
9
25
|
static notarizeEntry(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
10
26
|
static authorizeReentry(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
27
|
+
static notarizeEarlyExit(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
11
28
|
static notarizeEarlyCompletion(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
12
29
|
static notarizeReentry(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
13
30
|
static notarizeContinuation(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
@@ -4,15 +4,36 @@ exports.CollatorService = void 0;
|
|
|
4
4
|
const errors_1 = require("../../modules/errors");
|
|
5
5
|
const collator_1 = require("../../types/collator");
|
|
6
6
|
class CollatorService {
|
|
7
|
-
|
|
7
|
+
/**
|
|
8
|
+
* returns the dimensional address (dad) for the target; due
|
|
9
|
+
* to the nature of the notary system, the dad for leg 2 entry
|
|
10
|
+
* must target the `0` index while leg 2 exit must target the
|
|
11
|
+
* current index (0)
|
|
12
|
+
*/
|
|
13
|
+
static getDimensionalAddress(activity, isEntry = false) {
|
|
8
14
|
let dad = activity.context.metadata.dad || activity.metadata.dad;
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
// dad = `${dad.substring(0, dad.lastIndexOf(','))},0`;
|
|
13
|
-
// }
|
|
15
|
+
if (isEntry && dad && activity.leg === 2) {
|
|
16
|
+
dad = `${dad.substring(0, dad.lastIndexOf(','))},0`;
|
|
17
|
+
}
|
|
14
18
|
return CollatorService.getDimensionsById([...activity.config.ancestors, activity.metadata.aid], dad);
|
|
15
19
|
}
|
|
20
|
+
/**
|
|
21
|
+
* resolves the dimensional address for the
|
|
22
|
+
* ancestor in the graph to go back to. this address
|
|
23
|
+
* is determined by trimming the last digits from
|
|
24
|
+
* the `dad` (including the target).
|
|
25
|
+
* the target activity index is then set to `0`, so that
|
|
26
|
+
* the origin node can be queried for approval/entry.
|
|
27
|
+
*/
|
|
28
|
+
static resolveReentryDimension(activity) {
|
|
29
|
+
const targetActivityId = activity.config.ancestor;
|
|
30
|
+
const ancestors = activity.config.ancestors;
|
|
31
|
+
const ancestorIndex = ancestors.indexOf(targetActivityId);
|
|
32
|
+
const dimensions = activity.metadata.dad.split(','); //e.g., `,0,0,1,0`
|
|
33
|
+
dimensions.length = ancestorIndex + 1;
|
|
34
|
+
dimensions.push('0');
|
|
35
|
+
return dimensions.join(',');
|
|
36
|
+
}
|
|
16
37
|
static async notarizeEntry(activity, multi) {
|
|
17
38
|
//decrement by -100_000_000_000_000
|
|
18
39
|
const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -100000000000000, this.getDimensionalAddress(activity), multi);
|
|
@@ -27,14 +48,21 @@ class CollatorService {
|
|
|
27
48
|
//this.verifyInteger(amount, 1, 'exit');
|
|
28
49
|
return amount;
|
|
29
50
|
}
|
|
51
|
+
static async notarizeEarlyExit(activity, multi) {
|
|
52
|
+
//decrement the 2nd and 3rd digits to fully deactivate (`cycle` activities use this command to fully exit after leg 1) (should result in `888000000000000`)
|
|
53
|
+
return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -11000000000000, this.getDimensionalAddress(activity), multi);
|
|
54
|
+
}
|
|
55
|
+
;
|
|
30
56
|
static async notarizeEarlyCompletion(activity, multi) {
|
|
31
|
-
//initialize both `possible` (1m) and `actualized` (1) zero dimension, while decrementing the 2nd
|
|
32
|
-
|
|
57
|
+
//initialize both `possible` (1m) and `actualized` (1) zero dimension, while decrementing the 2nd
|
|
58
|
+
//3rd digit is optionally kept open if the activity might be used in a cycle
|
|
59
|
+
const decrement = activity.config.cycle ? 10000000000000 : 11000000000000;
|
|
60
|
+
return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1000001 - decrement, this.getDimensionalAddress(activity), multi);
|
|
33
61
|
}
|
|
34
62
|
;
|
|
35
63
|
static async notarizeReentry(activity, multi) {
|
|
36
64
|
//increment by 1_000_000 (indicates re-entry and is used to drive the 'dimensional address' for adjacent activities (minus 1))
|
|
37
|
-
const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1000000, this.getDimensionalAddress(activity), multi);
|
|
65
|
+
const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1000000, this.getDimensionalAddress(activity, true), multi);
|
|
38
66
|
this.verifyInteger(amount, 2, 'enter');
|
|
39
67
|
return amount;
|
|
40
68
|
}
|
|
@@ -45,8 +73,10 @@ class CollatorService {
|
|
|
45
73
|
}
|
|
46
74
|
;
|
|
47
75
|
static async notarizeCompletion(activity, multi) {
|
|
48
|
-
//
|
|
49
|
-
|
|
76
|
+
//1) ALWAYS actualize leg2 dimension (+1)
|
|
77
|
+
//2) IF the activity is used in a cycle, don't close leg 2!
|
|
78
|
+
const decrement = activity.config.cycle ? 0 : -1000000000000;
|
|
79
|
+
return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1 - decrement, this.getDimensionalAddress(activity), multi);
|
|
50
80
|
}
|
|
51
81
|
;
|
|
52
82
|
static getDigitAtIndex(num, targetDigitIndex) {
|
|
@@ -29,7 +29,12 @@ const getWorkflowYAML = (topic, version = '1') => {
|
|
|
29
29
|
type: trigger
|
|
30
30
|
stats:
|
|
31
31
|
id: '{$self.input.data.workflowId}'
|
|
32
|
+
|
|
32
33
|
a1:
|
|
34
|
+
type: activity
|
|
35
|
+
cycle: true
|
|
36
|
+
|
|
37
|
+
w1:
|
|
33
38
|
type: worker
|
|
34
39
|
topic: ${topic}
|
|
35
40
|
input:
|
|
@@ -52,9 +57,20 @@ const getWorkflowYAML = (topic, version = '1') => {
|
|
|
52
57
|
job:
|
|
53
58
|
maps:
|
|
54
59
|
response: '{$self.output.data.response}'
|
|
60
|
+
|
|
61
|
+
c1:
|
|
62
|
+
type: cycle
|
|
63
|
+
ancestor: a1
|
|
55
64
|
transitions:
|
|
56
65
|
t1:
|
|
57
|
-
- to: a1
|
|
66
|
+
- to: a1
|
|
67
|
+
a1:
|
|
68
|
+
- to: w1
|
|
69
|
+
w1:
|
|
70
|
+
- to: c1
|
|
71
|
+
conditions:
|
|
72
|
+
code: 500
|
|
73
|
+
`;
|
|
58
74
|
};
|
|
59
75
|
exports.getWorkflowYAML = getWorkflowYAML;
|
|
60
76
|
const getActivityYAML = (topic, version = '1') => {
|
|
@@ -5,6 +5,7 @@ export declare class WorkerService {
|
|
|
5
5
|
static connection: Connection;
|
|
6
6
|
static instances: Map<string, HotMesh | Promise<HotMesh>>;
|
|
7
7
|
workflowRunner: HotMesh;
|
|
8
|
+
activityRunner: HotMesh;
|
|
8
9
|
static getHotMesh: (worflowTopic: string) => Promise<HotMesh>;
|
|
9
10
|
static activateWorkflow(hotMesh: HotMesh, topic: string, factory: Function): Promise<void>;
|
|
10
11
|
/**
|