@hotmeshio/hotmesh 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/cjs/package.json +3 -1
- package/build/cjs/services/activities/activity.d.ts +6 -0
- package/build/cjs/services/activities/activity.js +83 -7
- package/build/cjs/services/activities/await.d.ts +2 -2
- package/build/cjs/services/activities/await.js +5 -5
- package/build/cjs/services/activities/cycle.d.ts +19 -0
- package/build/cjs/services/activities/cycle.js +77 -0
- package/build/cjs/services/activities/index.d.ts +4 -2
- package/build/cjs/services/activities/index.js +4 -2
- package/build/cjs/services/activities/worker.d.ts +0 -8
- package/build/cjs/services/activities/worker.js +1 -87
- package/build/cjs/services/collator/index.d.ts +18 -1
- package/build/cjs/services/collator/index.js +41 -11
- package/build/cjs/services/durable/factory.js +17 -1
- package/build/cjs/services/durable/worker.d.ts +1 -0
- package/build/cjs/services/durable/worker.js +9 -14
- package/build/cjs/services/durable/workflow.js +5 -1
- package/build/cjs/services/mapper/index.js +3 -0
- package/build/cjs/services/signaler/stream.js +0 -1
- package/build/cjs/types/activity.d.ts +7 -2
- package/build/cjs/types/index.d.ts +1 -1
- package/build/esm/package.json +3 -1
- package/build/esm/services/activities/activity.d.ts +6 -0
- package/build/esm/services/activities/activity.js +83 -7
- package/build/esm/services/activities/await.d.ts +2 -2
- package/build/esm/services/activities/await.js +5 -5
- package/build/esm/services/activities/cycle.d.ts +19 -0
- package/build/esm/services/activities/cycle.js +74 -0
- package/build/esm/services/activities/index.d.ts +4 -2
- package/build/esm/services/activities/index.js +4 -2
- package/build/esm/services/activities/worker.d.ts +0 -8
- package/build/esm/services/activities/worker.js +1 -87
- package/build/esm/services/collator/index.d.ts +18 -1
- package/build/esm/services/collator/index.js +41 -11
- package/build/esm/services/durable/factory.js +17 -1
- package/build/esm/services/durable/worker.d.ts +1 -0
- package/build/esm/services/durable/worker.js +9 -14
- package/build/esm/services/durable/workflow.js +5 -1
- package/build/esm/services/mapper/index.js +3 -0
- package/build/esm/services/signaler/stream.js +0 -1
- package/build/esm/types/activity.d.ts +7 -2
- package/build/esm/types/index.d.ts +1 -1
- package/package.json +3 -1
- package/services/activities/activity.ts +90 -7
- package/services/activities/await.ts +5 -5
- package/services/activities/cycle.ts +96 -0
- package/services/activities/index.ts +4 -2
- package/services/activities/worker.ts +2 -93
- package/services/collator/index.ts +43 -11
- package/services/durable/factory.ts +17 -1
- package/services/durable/worker.ts +10 -13
- package/services/durable/workflow.ts +4 -1
- package/services/mapper/index.ts +3 -0
- package/services/signaler/stream.ts +0 -1
- package/types/activity.ts +8 -1
- package/types/index.ts +1 -0
|
@@ -120,8 +120,8 @@ class WorkerService {
|
|
|
120
120
|
const workflowTopic = `${baseTopic}`;
|
|
121
121
|
//initialize supporting workflows
|
|
122
122
|
const worker = new WorkerService();
|
|
123
|
-
|
|
124
|
-
await WorkerService.activateWorkflow(activityRunner, activityTopic, factory_1.getActivityYAML);
|
|
123
|
+
worker.activityRunner = await worker.initActivityWorkflow(config, activityTopic);
|
|
124
|
+
await WorkerService.activateWorkflow(worker.activityRunner, activityTopic, factory_1.getActivityYAML);
|
|
125
125
|
worker.workflowRunner = await worker.initWorkerWorkflow(config, workflowTopic, workflowFunction);
|
|
126
126
|
await WorkerService.activateWorkflow(worker.workflowRunner, workflowTopic, factory_1.getWorkflowYAML);
|
|
127
127
|
return worker;
|
|
@@ -139,12 +139,7 @@ class WorkerService {
|
|
|
139
139
|
return [workflowFunction.name, workflowFunction];
|
|
140
140
|
}
|
|
141
141
|
async run() {
|
|
142
|
-
|
|
143
|
-
this.workflowRunner.engine.logger.info('WorkerService is running');
|
|
144
|
-
}
|
|
145
|
-
else {
|
|
146
|
-
console.log('WorkerService is running');
|
|
147
|
-
}
|
|
142
|
+
this.workflowRunner.engine.logger.info('WorkerService is running');
|
|
148
143
|
}
|
|
149
144
|
async initActivityWorkflow(config, activityTopic) {
|
|
150
145
|
const redisConfig = {
|
|
@@ -179,10 +174,11 @@ class WorkerService {
|
|
|
179
174
|
};
|
|
180
175
|
}
|
|
181
176
|
catch (err) {
|
|
182
|
-
|
|
183
|
-
//todo (make retry configurable)
|
|
177
|
+
this.activityRunner.engine.logger.error('durable-worker-activity-err', err);
|
|
184
178
|
return {
|
|
185
|
-
status: stream_1.StreamStatus.
|
|
179
|
+
status: stream_1.StreamStatus.ERROR,
|
|
180
|
+
code: 500,
|
|
181
|
+
message: err.message,
|
|
186
182
|
metadata: { ...data.metadata },
|
|
187
183
|
data: { error: err }
|
|
188
184
|
};
|
|
@@ -199,7 +195,7 @@ class WorkerService {
|
|
|
199
195
|
await hotMesh.activate(version);
|
|
200
196
|
}
|
|
201
197
|
catch (err) {
|
|
202
|
-
|
|
198
|
+
hotMesh.engine.logger.error('durable-worker-activity-deploy-activate-error', err);
|
|
203
199
|
throw err;
|
|
204
200
|
}
|
|
205
201
|
}
|
|
@@ -255,10 +251,9 @@ class WorkerService {
|
|
|
255
251
|
};
|
|
256
252
|
}
|
|
257
253
|
catch (err) {
|
|
258
|
-
//todo: (retryable error types)
|
|
259
254
|
return {
|
|
255
|
+
status: stream_1.StreamStatus.ERROR,
|
|
260
256
|
code: 500,
|
|
261
|
-
status: stream_1.StreamStatus.PENDING,
|
|
262
257
|
metadata: { ...data.metadata },
|
|
263
258
|
data: { error: err }
|
|
264
259
|
};
|
|
@@ -100,7 +100,11 @@ class WorkflowService {
|
|
|
100
100
|
try {
|
|
101
101
|
const hmshInstance = await worker_1.WorkerService.getHotMesh(activityTopic);
|
|
102
102
|
activityState = await hmshInstance.getState(activityTopic, activityJobId);
|
|
103
|
-
if (activityState.metadata.
|
|
103
|
+
if (activityState.metadata.err) {
|
|
104
|
+
await hmshInstance.scrub(activityJobId);
|
|
105
|
+
throw new Error(activityState.metadata.err);
|
|
106
|
+
}
|
|
107
|
+
else if (activityState.metadata.js === 0) {
|
|
104
108
|
//return immediately
|
|
105
109
|
return activityState.data?.response;
|
|
106
110
|
}
|
|
@@ -50,6 +50,9 @@ class MapperService {
|
|
|
50
50
|
return transitionRule;
|
|
51
51
|
}
|
|
52
52
|
if (code.toString() === (transitionRule.code || 200).toString()) {
|
|
53
|
+
if (!transitionRule.match) {
|
|
54
|
+
return true;
|
|
55
|
+
}
|
|
53
56
|
const orGate = transitionRule.gate === 'or';
|
|
54
57
|
let allAreTrue = true;
|
|
55
58
|
let someAreTrue = false;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { MetricTypes } from "./stats";
|
|
2
2
|
import { StreamRetryPolicy } from "./stream";
|
|
3
|
-
type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate';
|
|
3
|
+
type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate' | 'cycle';
|
|
4
4
|
type Consumes = Record<string, string[]>;
|
|
5
5
|
interface BaseActivity {
|
|
6
6
|
title?: string;
|
|
@@ -15,6 +15,7 @@ interface BaseActivity {
|
|
|
15
15
|
sleep?: number;
|
|
16
16
|
expire?: number;
|
|
17
17
|
retry?: StreamRetryPolicy;
|
|
18
|
+
cycle?: boolean;
|
|
18
19
|
collationInt?: number;
|
|
19
20
|
consumes?: Consumes;
|
|
20
21
|
PRODUCES?: string[];
|
|
@@ -55,6 +56,10 @@ interface WorkerActivity extends BaseActivity {
|
|
|
55
56
|
interface EmitActivity extends BaseActivity {
|
|
56
57
|
type: 'emit';
|
|
57
58
|
}
|
|
59
|
+
interface CycleActivity extends BaseActivity {
|
|
60
|
+
type: 'cycle';
|
|
61
|
+
ancestor: string;
|
|
62
|
+
}
|
|
58
63
|
interface IterateActivity extends BaseActivity {
|
|
59
64
|
type: 'iterate';
|
|
60
65
|
}
|
|
@@ -84,4 +89,4 @@ type ActivityDataType = {
|
|
|
84
89
|
hook?: Record<string, unknown>;
|
|
85
90
|
};
|
|
86
91
|
type ActivityLeg = 1 | 2;
|
|
87
|
-
export { ActivityContext, ActivityData, ActivityDataType, ActivityDuplex, ActivityLeg, ActivityMetadata, ActivityType, Consumes, TriggerActivityStats, AwaitActivity, BaseActivity, EmitActivity, IterateActivity, TriggerActivity, WorkerActivity };
|
|
92
|
+
export { ActivityContext, ActivityData, ActivityDataType, ActivityDuplex, ActivityLeg, ActivityMetadata, ActivityType, Consumes, TriggerActivityStats, AwaitActivity, CycleActivity, BaseActivity, EmitActivity, IterateActivity, TriggerActivity, WorkerActivity };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { ActivityType, ActivityDataType, ActivityContext, ActivityData, ActivityDuplex, ActivityLeg, ActivityMetadata, Consumes, AwaitActivity, BaseActivity, EmitActivity, WorkerActivity, IterateActivity, TriggerActivity, TriggerActivityStats } from './activity';
|
|
1
|
+
export { ActivityType, ActivityDataType, ActivityContext, ActivityData, ActivityDuplex, ActivityLeg, ActivityMetadata, Consumes, AwaitActivity, BaseActivity, CycleActivity, EmitActivity, WorkerActivity, IterateActivity, TriggerActivity, TriggerActivityStats } from './activity';
|
|
2
2
|
export { App, AppVID, AppTransitions, AppSubscriptions } from './app';
|
|
3
3
|
export { AsyncSignal } from './async';
|
|
4
4
|
export { CacheMode } from './cache';
|
package/build/esm/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hotmeshio/hotmesh",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.8",
|
|
4
4
|
"description": "Durable Workflows",
|
|
5
5
|
"main": "build/cjs/index.js",
|
|
6
6
|
"module": "build/esm/index.js",
|
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
"test": "NODE_ENV=test jest --detectOpenHandles --forceExit --verbose",
|
|
30
30
|
"test:hmsh": "NODE_ENV=test jest ./tests/functional/index.test.ts --detectOpenHandles --verbose",
|
|
31
31
|
"test:compile": "NODE_ENV=test jest ./tests/functional/compile/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
32
|
+
"test:cycle": "NODE_ENV=test jest ./tests/functional/cycle/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
32
33
|
"test:connect": "NODE_ENV=test jest ./tests/unit/services/connector/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
33
34
|
"test:connect:redis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
|
|
34
35
|
"test:connect:ioredis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
|
|
@@ -48,6 +49,7 @@
|
|
|
48
49
|
"test:durable": "NODE_ENV=test jest ./tests/durable/*/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
49
50
|
"test:durable:hello": "NODE_ENV=test jest ./tests/durable/helloworld/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
50
51
|
"test:durable:goodbye": "NODE_ENV=test jest ./tests/durable/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
52
|
+
"test:durable:retry": "NODE_ENV=test jest ./tests/durable/retry/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
51
53
|
"test:durable:loopactivity": "NODE_ENV=test jest ./tests/durable/loopactivity/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
52
54
|
"test:durable:nested": "NODE_ENV=test jest ./tests/durable/nested/index.test.ts --detectOpenHandles --forceExit --verbose"
|
|
53
55
|
},
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { EngineService } from '../engine';
|
|
2
2
|
import { ILogger } from '../logger';
|
|
3
3
|
import { StoreService } from '../store';
|
|
4
|
+
import { TelemetryService } from '../telemetry';
|
|
4
5
|
import { ActivityData, ActivityLeg, ActivityMetadata, ActivityType } from '../../types/activity';
|
|
5
6
|
import { JobState, JobStatus } from '../../types/job';
|
|
6
7
|
import { MultiResponseFlags, RedisClient, RedisMulti } from '../../types/redis';
|
|
@@ -31,6 +32,11 @@ declare class Activity {
|
|
|
31
32
|
processWebHookEvent(): Promise<JobStatus | void>;
|
|
32
33
|
processTimeHookEvent(jobId: string): Promise<JobStatus | void>;
|
|
33
34
|
processHookEvent(jobId: string): Promise<JobStatus | void>;
|
|
35
|
+
processEvent(status?: StreamStatus, code?: StreamCode): Promise<void>;
|
|
36
|
+
processPending(telemetry: TelemetryService): Promise<MultiResponseFlags>;
|
|
37
|
+
processSuccess(telemetry: TelemetryService): Promise<MultiResponseFlags>;
|
|
38
|
+
processError(telemetry: TelemetryService): Promise<MultiResponseFlags>;
|
|
39
|
+
transitionAdjacent(multiResponse: MultiResponseFlags, telemetry: TelemetryService): Promise<void>;
|
|
34
40
|
resolveStatus(multiResponse: MultiResponseFlags): number;
|
|
35
41
|
mapJobData(): void;
|
|
36
42
|
mapInputData(): void;
|
|
@@ -84,7 +84,7 @@ class Activity {
|
|
|
84
84
|
setLeg(leg) {
|
|
85
85
|
this.leg = leg;
|
|
86
86
|
}
|
|
87
|
-
//********
|
|
87
|
+
//******** SIGNAL RE-ENTRY POINT ********//
|
|
88
88
|
doesHook() {
|
|
89
89
|
return !!(this.config.hook?.topic || this.config.sleep);
|
|
90
90
|
}
|
|
@@ -122,11 +122,6 @@ class Activity {
|
|
|
122
122
|
});
|
|
123
123
|
return await this.processHookEvent(jobId);
|
|
124
124
|
}
|
|
125
|
-
//todo: hooks are currently singletons. but they can support
|
|
126
|
-
// dimensional threads like `await` and `worker` do.
|
|
127
|
-
// Copy code from those activities to support cyclical
|
|
128
|
-
// timehook and eventhook inputs by adding a 'pending'
|
|
129
|
-
// flag to hooks that allows for repeated signals
|
|
130
125
|
async processHookEvent(jobId) {
|
|
131
126
|
this.logger.debug('activity-process-hook-event', { jobId });
|
|
132
127
|
let telemetry;
|
|
@@ -156,7 +151,6 @@ class Activity {
|
|
|
156
151
|
return jobStatus;
|
|
157
152
|
}
|
|
158
153
|
catch (error) {
|
|
159
|
-
console.error('this error?', error);
|
|
160
154
|
this.logger.error('engine-process-hook-event-error', error);
|
|
161
155
|
telemetry.setActivityError(error.message);
|
|
162
156
|
throw error;
|
|
@@ -165,6 +159,88 @@ class Activity {
|
|
|
165
159
|
telemetry.endActivitySpan();
|
|
166
160
|
}
|
|
167
161
|
}
|
|
162
|
+
//******** DUPLEX RE-ENTRY POINT ********//
|
|
163
|
+
async processEvent(status = StreamStatus.SUCCESS, code = 200) {
|
|
164
|
+
this.setLeg(2);
|
|
165
|
+
const jid = this.context.metadata.jid;
|
|
166
|
+
const aid = this.metadata.aid;
|
|
167
|
+
this.status = status;
|
|
168
|
+
this.code = code;
|
|
169
|
+
this.logger.debug('activity-process-event', { topic: this.config.subtype, jid, aid, status, code });
|
|
170
|
+
let telemetry;
|
|
171
|
+
try {
|
|
172
|
+
await this.getState();
|
|
173
|
+
const aState = await CollatorService.notarizeReentry(this);
|
|
174
|
+
this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
|
|
175
|
+
telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
176
|
+
let isComplete = CollatorService.isActivityComplete(this.context.metadata.js);
|
|
177
|
+
if (isComplete) {
|
|
178
|
+
this.logger.warn('activity-process-event-duplicate', { jid, aid });
|
|
179
|
+
this.logger.debug('activity-process-event-duplicate-resolution', { resolution: 'Increase HotMesh config `reclaimDelay` timeout.' });
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
telemetry.startActivitySpan(this.leg);
|
|
183
|
+
let multiResponse;
|
|
184
|
+
if (status === StreamStatus.PENDING) {
|
|
185
|
+
multiResponse = await this.processPending(telemetry);
|
|
186
|
+
}
|
|
187
|
+
else if (status === StreamStatus.SUCCESS) {
|
|
188
|
+
multiResponse = await this.processSuccess(telemetry);
|
|
189
|
+
}
|
|
190
|
+
else {
|
|
191
|
+
multiResponse = await this.processError(telemetry);
|
|
192
|
+
}
|
|
193
|
+
this.transitionAdjacent(multiResponse, telemetry);
|
|
194
|
+
}
|
|
195
|
+
catch (error) {
|
|
196
|
+
this.logger.error('activity-process-event-error', error);
|
|
197
|
+
telemetry.setActivityError(error.message);
|
|
198
|
+
throw error;
|
|
199
|
+
}
|
|
200
|
+
finally {
|
|
201
|
+
telemetry.endActivitySpan();
|
|
202
|
+
this.logger.debug('activity-process-event-end', { jid, aid });
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
async processPending(telemetry) {
|
|
206
|
+
this.bindActivityData('output');
|
|
207
|
+
this.adjacencyList = await this.filterAdjacent();
|
|
208
|
+
this.mapJobData();
|
|
209
|
+
const multi = this.store.getMulti();
|
|
210
|
+
await this.setState(multi);
|
|
211
|
+
await CollatorService.notarizeContinuation(this, multi);
|
|
212
|
+
await this.setStatus(this.adjacencyList.length, multi);
|
|
213
|
+
return await multi.exec();
|
|
214
|
+
}
|
|
215
|
+
async processSuccess(telemetry) {
|
|
216
|
+
this.bindActivityData('output');
|
|
217
|
+
this.adjacencyList = await this.filterAdjacent();
|
|
218
|
+
this.mapJobData();
|
|
219
|
+
const multi = this.store.getMulti();
|
|
220
|
+
await this.setState(multi);
|
|
221
|
+
await CollatorService.notarizeCompletion(this, multi);
|
|
222
|
+
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
223
|
+
return await multi.exec();
|
|
224
|
+
}
|
|
225
|
+
async processError(telemetry) {
|
|
226
|
+
this.bindActivityError(this.data);
|
|
227
|
+
this.adjacencyList = await this.filterAdjacent();
|
|
228
|
+
const multi = this.store.getMulti();
|
|
229
|
+
await this.setState(multi);
|
|
230
|
+
await CollatorService.notarizeCompletion(this, multi);
|
|
231
|
+
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
232
|
+
return await multi.exec();
|
|
233
|
+
}
|
|
234
|
+
async transitionAdjacent(multiResponse, telemetry) {
|
|
235
|
+
telemetry.mapActivityAttributes();
|
|
236
|
+
const jobStatus = this.resolveStatus(multiResponse);
|
|
237
|
+
const attrs = { 'app.job.jss': jobStatus };
|
|
238
|
+
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
239
|
+
if (messageIds.length) {
|
|
240
|
+
attrs['app.activity.mids'] = messageIds.join(',');
|
|
241
|
+
}
|
|
242
|
+
telemetry.setActivityAttributes(attrs);
|
|
243
|
+
}
|
|
168
244
|
resolveStatus(multiResponse) {
|
|
169
245
|
const activityStatus = multiResponse[multiResponse.length - 1];
|
|
170
246
|
if (Array.isArray(activityStatus)) {
|
|
@@ -10,7 +10,7 @@ declare class Await extends Activity {
|
|
|
10
10
|
process(): Promise<string>;
|
|
11
11
|
execActivity(): Promise<string>;
|
|
12
12
|
processEvent(status?: StreamStatus, code?: StreamCode): Promise<void>;
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
processSuccessResponse(adjacencyList: StreamData[]): Promise<MultiResponseFlags>;
|
|
14
|
+
processErrorResponse(adjacencyList: StreamData[]): Promise<MultiResponseFlags>;
|
|
15
15
|
}
|
|
16
16
|
export { Await };
|
|
@@ -19,7 +19,7 @@ class Await extends Activity {
|
|
|
19
19
|
telemetry.startActivitySpan(this.leg);
|
|
20
20
|
this.mapInputData();
|
|
21
21
|
const multi = this.store.getMulti();
|
|
22
|
-
//await this.registerTimeout();
|
|
22
|
+
//todo: await this.registerTimeout();
|
|
23
23
|
await CollatorService.authorizeReentry(this, multi);
|
|
24
24
|
await this.setState(multi);
|
|
25
25
|
await this.setStatus(0, multi);
|
|
@@ -92,12 +92,12 @@ class Await extends Activity {
|
|
|
92
92
|
if (status === StreamStatus.SUCCESS) {
|
|
93
93
|
this.bindActivityData('output');
|
|
94
94
|
this.adjacencyList = await this.filterAdjacent();
|
|
95
|
-
multiResponse = await this.
|
|
95
|
+
multiResponse = await this.processSuccessResponse(this.adjacencyList);
|
|
96
96
|
}
|
|
97
97
|
else {
|
|
98
98
|
this.bindActivityError(this.data);
|
|
99
99
|
this.adjacencyList = await this.filterAdjacent();
|
|
100
|
-
multiResponse = await this.
|
|
100
|
+
multiResponse = await this.processErrorResponse(this.adjacencyList);
|
|
101
101
|
}
|
|
102
102
|
telemetry.mapActivityAttributes();
|
|
103
103
|
const jobStatus = this.resolveStatus(multiResponse);
|
|
@@ -118,7 +118,7 @@ class Await extends Activity {
|
|
|
118
118
|
this.logger.debug('await-resolve-await-end', { jid, aid, status, code });
|
|
119
119
|
}
|
|
120
120
|
}
|
|
121
|
-
async
|
|
121
|
+
async processSuccessResponse(adjacencyList) {
|
|
122
122
|
this.mapJobData();
|
|
123
123
|
const multi = this.store.getMulti();
|
|
124
124
|
await this.setState(multi);
|
|
@@ -126,7 +126,7 @@ class Await extends Activity {
|
|
|
126
126
|
await this.setStatus(adjacencyList.length - 1, multi);
|
|
127
127
|
return await multi.exec();
|
|
128
128
|
}
|
|
129
|
-
async
|
|
129
|
+
async processErrorResponse(adjacencyList) {
|
|
130
130
|
//todo: if adjacencyList.length == 0, then map to the job output
|
|
131
131
|
// this method would be added to Base activity class
|
|
132
132
|
//this.mapJobData();
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { EngineService } from '../engine';
|
|
2
|
+
import { Activity, ActivityType } from './activity';
|
|
3
|
+
import { ActivityData, ActivityMetadata, CycleActivity } from '../../types/activity';
|
|
4
|
+
import { JobState } from '../../types/job';
|
|
5
|
+
import { RedisMulti } from '../../types/redis';
|
|
6
|
+
declare class Cycle extends Activity {
|
|
7
|
+
config: CycleActivity;
|
|
8
|
+
constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
|
|
9
|
+
process(): Promise<string>;
|
|
10
|
+
/**
|
|
11
|
+
* Trigger the target ancestor to execute in a cycle,
|
|
12
|
+
* without violating the constraints of the DAG. Immutable
|
|
13
|
+
* `individual activity state` will execute in a new dimensional
|
|
14
|
+
* thread while `shared job state` can change. This
|
|
15
|
+
* pattern allows for retries without violating the DAG.
|
|
16
|
+
*/
|
|
17
|
+
cycleAncestorActivity(multi: RedisMulti): Promise<string>;
|
|
18
|
+
}
|
|
19
|
+
export { Cycle };
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { GetStateError } from '../../modules/errors';
|
|
2
|
+
import { CollatorService } from '../collator';
|
|
3
|
+
import { Activity } from './activity';
|
|
4
|
+
import { TelemetryService } from '../telemetry';
|
|
5
|
+
class Cycle extends Activity {
|
|
6
|
+
constructor(config, data, metadata, hook, engine, context) {
|
|
7
|
+
super(config, data, metadata, hook, engine, context);
|
|
8
|
+
}
|
|
9
|
+
//******** LEG 1 ENTRY ********//
|
|
10
|
+
async process() {
|
|
11
|
+
this.logger.debug('cycle-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
12
|
+
let telemetry;
|
|
13
|
+
try {
|
|
14
|
+
//verify entry is allowed
|
|
15
|
+
this.setLeg(1);
|
|
16
|
+
await CollatorService.notarizeEntry(this);
|
|
17
|
+
await this.getState();
|
|
18
|
+
telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
19
|
+
telemetry.startActivitySpan(this.leg);
|
|
20
|
+
this.mapInputData();
|
|
21
|
+
//set state/status
|
|
22
|
+
let multi = this.store.getMulti();
|
|
23
|
+
await this.setState(multi);
|
|
24
|
+
await this.setStatus(0, multi); //leg 1 never changes job status
|
|
25
|
+
const multiResponse = await multi.exec();
|
|
26
|
+
telemetry.mapActivityAttributes();
|
|
27
|
+
const jobStatus = this.resolveStatus(multiResponse);
|
|
28
|
+
//cycle the target ancestor
|
|
29
|
+
multi = this.store.getMulti();
|
|
30
|
+
const messageId = await this.cycleAncestorActivity(multi);
|
|
31
|
+
telemetry.setActivityAttributes({
|
|
32
|
+
'app.activity.mid': messageId,
|
|
33
|
+
'app.job.jss': jobStatus
|
|
34
|
+
});
|
|
35
|
+
//exit early (`Cycle` activities only execute Leg 1)
|
|
36
|
+
await CollatorService.notarizeEarlyExit(this, multi);
|
|
37
|
+
await multi.exec();
|
|
38
|
+
return this.context.metadata.aid;
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
if (error instanceof GetStateError) {
|
|
42
|
+
this.logger.error('cycle-get-state-error', error);
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
this.logger.error('cycle-process-error', error);
|
|
46
|
+
}
|
|
47
|
+
telemetry.setActivityError(error.message);
|
|
48
|
+
throw error;
|
|
49
|
+
}
|
|
50
|
+
finally {
|
|
51
|
+
telemetry.endActivitySpan();
|
|
52
|
+
this.logger.debug('cycle-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Trigger the target ancestor to execute in a cycle,
|
|
57
|
+
* without violating the constraints of the DAG. Immutable
|
|
58
|
+
* `individual activity state` will execute in a new dimensional
|
|
59
|
+
* thread while `shared job state` can change. This
|
|
60
|
+
* pattern allows for retries without violating the DAG.
|
|
61
|
+
*/
|
|
62
|
+
async cycleAncestorActivity(multi) {
|
|
63
|
+
const streamData = {
|
|
64
|
+
metadata: {
|
|
65
|
+
dad: CollatorService.resolveReentryDimension(this),
|
|
66
|
+
jid: this.context.metadata.jid,
|
|
67
|
+
aid: this.config.ancestor,
|
|
68
|
+
},
|
|
69
|
+
data: {} //todo: verify immutability, before enabling: `this.context.data`
|
|
70
|
+
};
|
|
71
|
+
return (await this.engine.streamSignaler?.publishMessage(null, streamData, multi));
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
export { Cycle };
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { Activity } from './activity';
|
|
2
2
|
import { Await } from './await';
|
|
3
|
-
import {
|
|
4
|
-
import { Iterate } from './iterate';
|
|
3
|
+
import { Cycle } from './cycle';
|
|
5
4
|
import { Emit } from './emit';
|
|
5
|
+
import { Iterate } from './iterate';
|
|
6
6
|
import { Trigger } from './trigger';
|
|
7
|
+
import { Worker } from './worker';
|
|
7
8
|
declare const _default: {
|
|
8
9
|
activity: typeof Activity;
|
|
9
10
|
await: typeof Await;
|
|
11
|
+
cycle: typeof Cycle;
|
|
10
12
|
iterate: typeof Iterate;
|
|
11
13
|
emit: typeof Emit;
|
|
12
14
|
trigger: typeof Trigger;
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { Activity } from './activity';
|
|
2
2
|
import { Await } from './await';
|
|
3
|
-
import {
|
|
4
|
-
import { Iterate } from './iterate';
|
|
3
|
+
import { Cycle } from './cycle';
|
|
5
4
|
import { Emit } from './emit';
|
|
5
|
+
import { Iterate } from './iterate';
|
|
6
6
|
import { Trigger } from './trigger';
|
|
7
|
+
import { Worker } from './worker';
|
|
7
8
|
export default {
|
|
8
9
|
activity: Activity,
|
|
9
10
|
await: Await,
|
|
11
|
+
cycle: Cycle,
|
|
10
12
|
iterate: Iterate,
|
|
11
13
|
emit: Emit,
|
|
12
14
|
trigger: Trigger,
|
|
@@ -2,18 +2,10 @@ import { Activity } from './activity';
|
|
|
2
2
|
import { EngineService } from '../engine';
|
|
3
3
|
import { ActivityData, ActivityMetadata, ActivityType, WorkerActivity } from '../../types/activity';
|
|
4
4
|
import { JobState } from '../../types/job';
|
|
5
|
-
import { MultiResponseFlags } from '../../types/redis';
|
|
6
|
-
import { StreamCode, StreamStatus } from '../../types/stream';
|
|
7
|
-
import { TelemetryService } from '../telemetry';
|
|
8
5
|
declare class Worker extends Activity {
|
|
9
6
|
config: WorkerActivity;
|
|
10
7
|
constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
|
|
11
8
|
process(): Promise<string>;
|
|
12
9
|
execActivity(): Promise<string>;
|
|
13
|
-
processEvent(status?: StreamStatus, code?: StreamCode): Promise<void>;
|
|
14
|
-
processPending(telemetry: TelemetryService): Promise<void>;
|
|
15
|
-
processSuccess(telemetry: TelemetryService): Promise<void>;
|
|
16
|
-
processError(telemetry: TelemetryService): Promise<void>;
|
|
17
|
-
transitionAdjacent(multiResponse: MultiResponseFlags, telemetry: TelemetryService): Promise<void>;
|
|
18
10
|
}
|
|
19
11
|
export { Worker };
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { GetStateError } from '../../modules/errors';
|
|
2
2
|
import { Activity } from './activity';
|
|
3
3
|
import { CollatorService } from '../collator';
|
|
4
|
-
import { StreamStatus } from '../../types/stream';
|
|
5
4
|
import { TelemetryService } from '../telemetry';
|
|
6
5
|
class Worker extends Activity {
|
|
7
6
|
constructor(config, data, metadata, hook, engine, context) {
|
|
@@ -19,7 +18,7 @@ class Worker extends Activity {
|
|
|
19
18
|
telemetry.startActivitySpan(this.leg);
|
|
20
19
|
this.mapInputData();
|
|
21
20
|
const multi = this.store.getMulti();
|
|
22
|
-
//await this.registerTimeout();
|
|
21
|
+
//todo: await this.registerTimeout();
|
|
23
22
|
await CollatorService.authorizeReentry(this, multi);
|
|
24
23
|
await this.setState(multi);
|
|
25
24
|
await this.setStatus(0, multi);
|
|
@@ -31,7 +30,6 @@ class Worker extends Activity {
|
|
|
31
30
|
'app.activity.mid': messageId,
|
|
32
31
|
'app.job.jss': jobStatus
|
|
33
32
|
});
|
|
34
|
-
//TODO: UPDATE ACTIVITY STATE (LEG 1 EXIT)
|
|
35
33
|
return this.context.metadata.aid;
|
|
36
34
|
}
|
|
37
35
|
catch (error) {
|
|
@@ -39,7 +37,6 @@ class Worker extends Activity {
|
|
|
39
37
|
this.logger.error('worker-get-state-error', error);
|
|
40
38
|
}
|
|
41
39
|
else {
|
|
42
|
-
console.error(error);
|
|
43
40
|
this.logger.error('worker-process-error', error);
|
|
44
41
|
}
|
|
45
42
|
telemetry.setActivityError(error.message);
|
|
@@ -69,88 +66,5 @@ class Worker extends Activity {
|
|
|
69
66
|
}
|
|
70
67
|
return (await this.engine.streamSignaler?.publishMessage(this.config.subtype, streamData));
|
|
71
68
|
}
|
|
72
|
-
//******** SIGNAL RE-ENTRY POINT (DUPLEX LEG 2 of 2) ********//
|
|
73
|
-
async processEvent(status = StreamStatus.SUCCESS, code = 200) {
|
|
74
|
-
this.setLeg(2);
|
|
75
|
-
const jid = this.context.metadata.jid;
|
|
76
|
-
const aid = this.metadata.aid;
|
|
77
|
-
this.status = status;
|
|
78
|
-
this.code = code;
|
|
79
|
-
this.logger.debug('worker-process-event', { topic: this.config.subtype, jid, aid, status, code });
|
|
80
|
-
let telemetry;
|
|
81
|
-
try {
|
|
82
|
-
await this.getState();
|
|
83
|
-
const aState = await CollatorService.notarizeReentry(this);
|
|
84
|
-
this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
|
|
85
|
-
telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
86
|
-
let isComplete = CollatorService.isActivityComplete(this.context.metadata.js);
|
|
87
|
-
if (isComplete) {
|
|
88
|
-
this.logger.warn('worker-process-event-duplicate', { jid, aid });
|
|
89
|
-
this.logger.debug('worker-process-event-duplicate-resolution', { resolution: 'Increase HotMesh config `reclaimDelay` timeout.' });
|
|
90
|
-
return;
|
|
91
|
-
}
|
|
92
|
-
telemetry.startActivitySpan(this.leg);
|
|
93
|
-
if (status === StreamStatus.PENDING) {
|
|
94
|
-
await this.processPending(telemetry);
|
|
95
|
-
}
|
|
96
|
-
else if (status === StreamStatus.SUCCESS) {
|
|
97
|
-
await this.processSuccess(telemetry);
|
|
98
|
-
}
|
|
99
|
-
else {
|
|
100
|
-
await this.processError(telemetry);
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
catch (error) {
|
|
104
|
-
this.logger.error('worker-process-event-error', error);
|
|
105
|
-
telemetry.setActivityError(error.message);
|
|
106
|
-
throw error;
|
|
107
|
-
}
|
|
108
|
-
finally {
|
|
109
|
-
telemetry.endActivitySpan();
|
|
110
|
-
this.logger.debug('worker-process-event-end', { jid, aid });
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
async processPending(telemetry) {
|
|
114
|
-
this.bindActivityData('output');
|
|
115
|
-
this.adjacencyList = await this.filterAdjacent();
|
|
116
|
-
this.mapJobData();
|
|
117
|
-
const multi = this.store.getMulti();
|
|
118
|
-
await this.setState(multi);
|
|
119
|
-
await CollatorService.notarizeContinuation(this, multi);
|
|
120
|
-
await this.setStatus(this.adjacencyList.length, multi);
|
|
121
|
-
const multiResponse = await multi.exec();
|
|
122
|
-
this.transitionAdjacent(multiResponse, telemetry);
|
|
123
|
-
}
|
|
124
|
-
async processSuccess(telemetry) {
|
|
125
|
-
this.bindActivityData('output');
|
|
126
|
-
this.adjacencyList = await this.filterAdjacent();
|
|
127
|
-
this.mapJobData();
|
|
128
|
-
const multi = this.store.getMulti();
|
|
129
|
-
await this.setState(multi);
|
|
130
|
-
await CollatorService.notarizeCompletion(this, multi);
|
|
131
|
-
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
132
|
-
const multiResponse = await multi.exec();
|
|
133
|
-
this.transitionAdjacent(multiResponse, telemetry);
|
|
134
|
-
}
|
|
135
|
-
async processError(telemetry) {
|
|
136
|
-
this.bindActivityError(this.data);
|
|
137
|
-
this.adjacencyList = await this.filterAdjacent();
|
|
138
|
-
const multi = this.store.getMulti();
|
|
139
|
-
await this.setState(multi);
|
|
140
|
-
await CollatorService.notarizeCompletion(this, multi);
|
|
141
|
-
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
142
|
-
const multiResponse = await multi.exec();
|
|
143
|
-
this.transitionAdjacent(multiResponse, telemetry);
|
|
144
|
-
}
|
|
145
|
-
async transitionAdjacent(multiResponse, telemetry) {
|
|
146
|
-
telemetry.mapActivityAttributes();
|
|
147
|
-
const jobStatus = this.resolveStatus(multiResponse);
|
|
148
|
-
const attrs = { 'app.job.jss': jobStatus };
|
|
149
|
-
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
150
|
-
if (messageIds.length) {
|
|
151
|
-
attrs['app.activity.mids'] = messageIds.join(',');
|
|
152
|
-
}
|
|
153
|
-
telemetry.setActivityAttributes(attrs);
|
|
154
|
-
}
|
|
155
69
|
}
|
|
156
70
|
export { Worker };
|
|
@@ -3,11 +3,28 @@ import { CollationStage } from '../../types/collator';
|
|
|
3
3
|
import { ActivityDuplex } from '../../types/activity';
|
|
4
4
|
import { HotMeshGraph } from '../../types/hotmesh';
|
|
5
5
|
import { Activity } from '../activities/activity';
|
|
6
|
+
import { Cycle } from '../activities/cycle';
|
|
6
7
|
declare class CollatorService {
|
|
7
8
|
static targetLength: number;
|
|
8
|
-
|
|
9
|
+
/**
|
|
10
|
+
* returns the dimensional address (dad) for the target; due
|
|
11
|
+
* to the nature of the notary system, the dad for leg 2 entry
|
|
12
|
+
* must target the `0` index while leg 2 exit must target the
|
|
13
|
+
* current index (0)
|
|
14
|
+
*/
|
|
15
|
+
static getDimensionalAddress(activity: Activity, isEntry?: boolean): Record<string, string>;
|
|
16
|
+
/**
|
|
17
|
+
* resolves the dimensional address for the
|
|
18
|
+
* ancestor in the graph to go back to. this address
|
|
19
|
+
* is determined by trimming the last digits from
|
|
20
|
+
* the `dad` (including the target).
|
|
21
|
+
* the target activity index is then set to `0`, so that
|
|
22
|
+
* the origin node can be queried for approval/entry.
|
|
23
|
+
*/
|
|
24
|
+
static resolveReentryDimension(activity: Cycle): string;
|
|
9
25
|
static notarizeEntry(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
10
26
|
static authorizeReentry(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
27
|
+
static notarizeEarlyExit(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
11
28
|
static notarizeEarlyCompletion(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
12
29
|
static notarizeReentry(activity: Activity, multi?: RedisMulti): Promise<number>;
|
|
13
30
|
static notarizeContinuation(activity: Activity, multi?: RedisMulti): Promise<number>;
|