@hotmeshio/hotmesh 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/cjs/package.json +3 -1
- package/build/cjs/services/activities/activity.d.ts +6 -0
- package/build/cjs/services/activities/activity.js +84 -9
- package/build/cjs/services/activities/await.d.ts +2 -2
- package/build/cjs/services/activities/await.js +5 -5
- package/build/cjs/services/activities/cycle.d.ts +19 -0
- package/build/cjs/services/activities/cycle.js +77 -0
- package/build/cjs/services/activities/index.d.ts +4 -2
- package/build/cjs/services/activities/index.js +4 -2
- package/build/cjs/services/activities/worker.d.ts +0 -6
- package/build/cjs/services/activities/worker.js +1 -93
- package/build/cjs/services/collator/index.d.ts +18 -1
- package/build/cjs/services/collator/index.js +41 -11
- package/build/cjs/services/durable/factory.js +17 -1
- package/build/cjs/services/durable/worker.d.ts +1 -0
- package/build/cjs/services/durable/worker.js +18 -19
- package/build/cjs/services/durable/workflow.js +5 -1
- package/build/cjs/services/mapper/index.js +3 -0
- package/build/cjs/services/signaler/stream.js +0 -1
- package/build/cjs/types/activity.d.ts +7 -2
- package/build/cjs/types/index.d.ts +1 -1
- package/build/esm/package.json +3 -1
- package/build/esm/services/activities/activity.d.ts +6 -0
- package/build/esm/services/activities/activity.js +84 -9
- package/build/esm/services/activities/await.d.ts +2 -2
- package/build/esm/services/activities/await.js +5 -5
- package/build/esm/services/activities/cycle.d.ts +19 -0
- package/build/esm/services/activities/cycle.js +74 -0
- package/build/esm/services/activities/index.d.ts +4 -2
- package/build/esm/services/activities/index.js +4 -2
- package/build/esm/services/activities/worker.d.ts +0 -6
- package/build/esm/services/activities/worker.js +1 -93
- package/build/esm/services/collator/index.d.ts +18 -1
- package/build/esm/services/collator/index.js +41 -11
- package/build/esm/services/durable/factory.js +17 -1
- package/build/esm/services/durable/worker.d.ts +1 -0
- package/build/esm/services/durable/worker.js +18 -19
- package/build/esm/services/durable/workflow.js +5 -1
- package/build/esm/services/mapper/index.js +3 -0
- package/build/esm/services/signaler/stream.js +0 -1
- package/build/esm/types/activity.d.ts +7 -2
- package/build/esm/types/index.d.ts +1 -1
- package/package.json +3 -1
- package/services/activities/activity.ts +91 -9
- package/services/activities/await.ts +5 -5
- package/services/activities/cycle.ts +96 -0
- package/services/activities/index.ts +4 -2
- package/services/activities/worker.ts +2 -97
- package/services/collator/index.ts +43 -11
- package/services/durable/factory.ts +17 -1
- package/services/durable/worker.ts +18 -19
- package/services/durable/workflow.ts +4 -1
- package/services/mapper/index.ts +3 -0
- package/services/signaler/stream.ts +0 -1
- package/types/activity.ts +8 -1
- package/types/index.ts +1 -0
|
@@ -99,16 +99,20 @@ class WorkerService {
|
|
|
99
99
|
* allowing proxyActivities to succeed.
|
|
100
100
|
*/
|
|
101
101
|
static registerActivities(activities) {
|
|
102
|
-
|
|
103
|
-
WorkerService.activityRegistry[
|
|
104
|
-
}
|
|
102
|
+
if (typeof activities === 'function') {
|
|
103
|
+
WorkerService.activityRegistry[activities.name] = activities;
|
|
104
|
+
}
|
|
105
|
+
else {
|
|
106
|
+
Object.keys(activities).forEach(key => {
|
|
107
|
+
WorkerService.activityRegistry[activities[key].name] = activities[key];
|
|
108
|
+
});
|
|
109
|
+
}
|
|
105
110
|
return WorkerService.activityRegistry;
|
|
106
111
|
}
|
|
107
112
|
static async create(config) {
|
|
113
|
+
//always call `registerActivities` before `import`
|
|
108
114
|
WorkerService.connection = config.connection;
|
|
109
|
-
//pre-cache user activity functions
|
|
110
115
|
WorkerService.registerActivities(config.activities);
|
|
111
|
-
//import the user's workflow file (triggers activity functions to be wrapped)
|
|
112
116
|
const workflow = await Promise.resolve(`${config.workflowsPath}`).then(s => __importStar(require(s)));
|
|
113
117
|
const [workflowFunctionName, workflowFunction] = WorkerService.resolveWorkflowTarget(workflow);
|
|
114
118
|
const baseTopic = `${config.taskQueue}-${workflowFunctionName}`;
|
|
@@ -116,8 +120,8 @@ class WorkerService {
|
|
|
116
120
|
const workflowTopic = `${baseTopic}`;
|
|
117
121
|
//initialize supporting workflows
|
|
118
122
|
const worker = new WorkerService();
|
|
119
|
-
|
|
120
|
-
await WorkerService.activateWorkflow(activityRunner, activityTopic, factory_1.getActivityYAML);
|
|
123
|
+
worker.activityRunner = await worker.initActivityWorkflow(config, activityTopic);
|
|
124
|
+
await WorkerService.activateWorkflow(worker.activityRunner, activityTopic, factory_1.getActivityYAML);
|
|
121
125
|
worker.workflowRunner = await worker.initWorkerWorkflow(config, workflowTopic, workflowFunction);
|
|
122
126
|
await WorkerService.activateWorkflow(worker.workflowRunner, workflowTopic, factory_1.getWorkflowYAML);
|
|
123
127
|
return worker;
|
|
@@ -135,12 +139,7 @@ class WorkerService {
|
|
|
135
139
|
return [workflowFunction.name, workflowFunction];
|
|
136
140
|
}
|
|
137
141
|
async run() {
|
|
138
|
-
|
|
139
|
-
this.workflowRunner.engine.logger.info('WorkerService is running');
|
|
140
|
-
}
|
|
141
|
-
else {
|
|
142
|
-
console.log('WorkerService is running');
|
|
143
|
-
}
|
|
142
|
+
this.workflowRunner.engine.logger.info('WorkerService is running');
|
|
144
143
|
}
|
|
145
144
|
async initActivityWorkflow(config, activityTopic) {
|
|
146
145
|
const redisConfig = {
|
|
@@ -175,10 +174,11 @@ class WorkerService {
|
|
|
175
174
|
};
|
|
176
175
|
}
|
|
177
176
|
catch (err) {
|
|
178
|
-
|
|
179
|
-
//todo (make retry configurable)
|
|
177
|
+
this.activityRunner.engine.logger.error('durable-worker-activity-err', err);
|
|
180
178
|
return {
|
|
181
|
-
status: stream_1.StreamStatus.
|
|
179
|
+
status: stream_1.StreamStatus.ERROR,
|
|
180
|
+
code: 500,
|
|
181
|
+
message: err.message,
|
|
182
182
|
metadata: { ...data.metadata },
|
|
183
183
|
data: { error: err }
|
|
184
184
|
};
|
|
@@ -195,7 +195,7 @@ class WorkerService {
|
|
|
195
195
|
await hotMesh.activate(version);
|
|
196
196
|
}
|
|
197
197
|
catch (err) {
|
|
198
|
-
|
|
198
|
+
hotMesh.engine.logger.error('durable-worker-activity-deploy-activate-error', err);
|
|
199
199
|
throw err;
|
|
200
200
|
}
|
|
201
201
|
}
|
|
@@ -251,10 +251,9 @@ class WorkerService {
|
|
|
251
251
|
};
|
|
252
252
|
}
|
|
253
253
|
catch (err) {
|
|
254
|
-
//todo: (retryable error types)
|
|
255
254
|
return {
|
|
255
|
+
status: stream_1.StreamStatus.ERROR,
|
|
256
256
|
code: 500,
|
|
257
|
-
status: stream_1.StreamStatus.PENDING,
|
|
258
257
|
metadata: { ...data.metadata },
|
|
259
258
|
data: { error: err }
|
|
260
259
|
};
|
|
@@ -100,7 +100,11 @@ class WorkflowService {
|
|
|
100
100
|
try {
|
|
101
101
|
const hmshInstance = await worker_1.WorkerService.getHotMesh(activityTopic);
|
|
102
102
|
activityState = await hmshInstance.getState(activityTopic, activityJobId);
|
|
103
|
-
if (activityState.metadata.
|
|
103
|
+
if (activityState.metadata.err) {
|
|
104
|
+
await hmshInstance.scrub(activityJobId);
|
|
105
|
+
throw new Error(activityState.metadata.err);
|
|
106
|
+
}
|
|
107
|
+
else if (activityState.metadata.js === 0) {
|
|
104
108
|
//return immediately
|
|
105
109
|
return activityState.data?.response;
|
|
106
110
|
}
|
|
@@ -50,6 +50,9 @@ class MapperService {
|
|
|
50
50
|
return transitionRule;
|
|
51
51
|
}
|
|
52
52
|
if (code.toString() === (transitionRule.code || 200).toString()) {
|
|
53
|
+
if (!transitionRule.match) {
|
|
54
|
+
return true;
|
|
55
|
+
}
|
|
53
56
|
const orGate = transitionRule.gate === 'or';
|
|
54
57
|
let allAreTrue = true;
|
|
55
58
|
let someAreTrue = false;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { MetricTypes } from "./stats";
|
|
2
2
|
import { StreamRetryPolicy } from "./stream";
|
|
3
|
-
type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate';
|
|
3
|
+
type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate' | 'cycle';
|
|
4
4
|
type Consumes = Record<string, string[]>;
|
|
5
5
|
interface BaseActivity {
|
|
6
6
|
title?: string;
|
|
@@ -15,6 +15,7 @@ interface BaseActivity {
|
|
|
15
15
|
sleep?: number;
|
|
16
16
|
expire?: number;
|
|
17
17
|
retry?: StreamRetryPolicy;
|
|
18
|
+
cycle?: boolean;
|
|
18
19
|
collationInt?: number;
|
|
19
20
|
consumes?: Consumes;
|
|
20
21
|
PRODUCES?: string[];
|
|
@@ -55,6 +56,10 @@ interface WorkerActivity extends BaseActivity {
|
|
|
55
56
|
interface EmitActivity extends BaseActivity {
|
|
56
57
|
type: 'emit';
|
|
57
58
|
}
|
|
59
|
+
interface CycleActivity extends BaseActivity {
|
|
60
|
+
type: 'cycle';
|
|
61
|
+
ancestor: string;
|
|
62
|
+
}
|
|
58
63
|
interface IterateActivity extends BaseActivity {
|
|
59
64
|
type: 'iterate';
|
|
60
65
|
}
|
|
@@ -84,4 +89,4 @@ type ActivityDataType = {
|
|
|
84
89
|
hook?: Record<string, unknown>;
|
|
85
90
|
};
|
|
86
91
|
type ActivityLeg = 1 | 2;
|
|
87
|
-
export { ActivityContext, ActivityData, ActivityDataType, ActivityDuplex, ActivityLeg, ActivityMetadata, ActivityType, Consumes, TriggerActivityStats, AwaitActivity, BaseActivity, EmitActivity, IterateActivity, TriggerActivity, WorkerActivity };
|
|
92
|
+
export { ActivityContext, ActivityData, ActivityDataType, ActivityDuplex, ActivityLeg, ActivityMetadata, ActivityType, Consumes, TriggerActivityStats, AwaitActivity, CycleActivity, BaseActivity, EmitActivity, IterateActivity, TriggerActivity, WorkerActivity };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { ActivityType, ActivityDataType, ActivityContext, ActivityData, ActivityDuplex, ActivityLeg, ActivityMetadata, Consumes, AwaitActivity, BaseActivity, EmitActivity, WorkerActivity, IterateActivity, TriggerActivity, TriggerActivityStats } from './activity';
|
|
1
|
+
export { ActivityType, ActivityDataType, ActivityContext, ActivityData, ActivityDuplex, ActivityLeg, ActivityMetadata, Consumes, AwaitActivity, BaseActivity, CycleActivity, EmitActivity, WorkerActivity, IterateActivity, TriggerActivity, TriggerActivityStats } from './activity';
|
|
2
2
|
export { App, AppVID, AppTransitions, AppSubscriptions } from './app';
|
|
3
3
|
export { AsyncSignal } from './async';
|
|
4
4
|
export { CacheMode } from './cache';
|
package/build/esm/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hotmeshio/hotmesh",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.8",
|
|
4
4
|
"description": "Durable Workflows",
|
|
5
5
|
"main": "build/cjs/index.js",
|
|
6
6
|
"module": "build/esm/index.js",
|
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
"test": "NODE_ENV=test jest --detectOpenHandles --forceExit --verbose",
|
|
30
30
|
"test:hmsh": "NODE_ENV=test jest ./tests/functional/index.test.ts --detectOpenHandles --verbose",
|
|
31
31
|
"test:compile": "NODE_ENV=test jest ./tests/functional/compile/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
32
|
+
"test:cycle": "NODE_ENV=test jest ./tests/functional/cycle/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
32
33
|
"test:connect": "NODE_ENV=test jest ./tests/unit/services/connector/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
33
34
|
"test:connect:redis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
|
|
34
35
|
"test:connect:ioredis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
|
|
@@ -48,6 +49,7 @@
|
|
|
48
49
|
"test:durable": "NODE_ENV=test jest ./tests/durable/*/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
49
50
|
"test:durable:hello": "NODE_ENV=test jest ./tests/durable/helloworld/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
50
51
|
"test:durable:goodbye": "NODE_ENV=test jest ./tests/durable/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
52
|
+
"test:durable:retry": "NODE_ENV=test jest ./tests/durable/retry/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
51
53
|
"test:durable:loopactivity": "NODE_ENV=test jest ./tests/durable/loopactivity/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
52
54
|
"test:durable:nested": "NODE_ENV=test jest ./tests/durable/nested/index.test.ts --detectOpenHandles --forceExit --verbose"
|
|
53
55
|
},
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { EngineService } from '../engine';
|
|
2
2
|
import { ILogger } from '../logger';
|
|
3
3
|
import { StoreService } from '../store';
|
|
4
|
+
import { TelemetryService } from '../telemetry';
|
|
4
5
|
import { ActivityData, ActivityLeg, ActivityMetadata, ActivityType } from '../../types/activity';
|
|
5
6
|
import { JobState, JobStatus } from '../../types/job';
|
|
6
7
|
import { MultiResponseFlags, RedisClient, RedisMulti } from '../../types/redis';
|
|
@@ -31,6 +32,11 @@ declare class Activity {
|
|
|
31
32
|
processWebHookEvent(): Promise<JobStatus | void>;
|
|
32
33
|
processTimeHookEvent(jobId: string): Promise<JobStatus | void>;
|
|
33
34
|
processHookEvent(jobId: string): Promise<JobStatus | void>;
|
|
35
|
+
processEvent(status?: StreamStatus, code?: StreamCode): Promise<void>;
|
|
36
|
+
processPending(telemetry: TelemetryService): Promise<MultiResponseFlags>;
|
|
37
|
+
processSuccess(telemetry: TelemetryService): Promise<MultiResponseFlags>;
|
|
38
|
+
processError(telemetry: TelemetryService): Promise<MultiResponseFlags>;
|
|
39
|
+
transitionAdjacent(multiResponse: MultiResponseFlags, telemetry: TelemetryService): Promise<void>;
|
|
34
40
|
resolveStatus(multiResponse: MultiResponseFlags): number;
|
|
35
41
|
mapJobData(): void;
|
|
36
42
|
mapInputData(): void;
|
|
@@ -84,7 +84,7 @@ class Activity {
|
|
|
84
84
|
setLeg(leg) {
|
|
85
85
|
this.leg = leg;
|
|
86
86
|
}
|
|
87
|
-
//********
|
|
87
|
+
//******** SIGNAL RE-ENTRY POINT ********//
|
|
88
88
|
doesHook() {
|
|
89
89
|
return !!(this.config.hook?.topic || this.config.sleep);
|
|
90
90
|
}
|
|
@@ -122,11 +122,6 @@ class Activity {
|
|
|
122
122
|
});
|
|
123
123
|
return await this.processHookEvent(jobId);
|
|
124
124
|
}
|
|
125
|
-
//todo: hooks are currently singletons. but they can support
|
|
126
|
-
// dimensional threads like `await` and `worker` do.
|
|
127
|
-
// Copy code from those activities to support cyclical
|
|
128
|
-
// timehook and eventhook inputs by adding a 'pending'
|
|
129
|
-
// flag to hooks that allows for repeated signals
|
|
130
125
|
async processHookEvent(jobId) {
|
|
131
126
|
this.logger.debug('activity-process-hook-event', { jobId });
|
|
132
127
|
let telemetry;
|
|
@@ -156,7 +151,6 @@ class Activity {
|
|
|
156
151
|
return jobStatus;
|
|
157
152
|
}
|
|
158
153
|
catch (error) {
|
|
159
|
-
console.error('this error?', error);
|
|
160
154
|
this.logger.error('engine-process-hook-event-error', error);
|
|
161
155
|
telemetry.setActivityError(error.message);
|
|
162
156
|
throw error;
|
|
@@ -165,6 +159,88 @@ class Activity {
|
|
|
165
159
|
telemetry.endActivitySpan();
|
|
166
160
|
}
|
|
167
161
|
}
|
|
162
|
+
//******** DUPLEX RE-ENTRY POINT ********//
|
|
163
|
+
async processEvent(status = StreamStatus.SUCCESS, code = 200) {
|
|
164
|
+
this.setLeg(2);
|
|
165
|
+
const jid = this.context.metadata.jid;
|
|
166
|
+
const aid = this.metadata.aid;
|
|
167
|
+
this.status = status;
|
|
168
|
+
this.code = code;
|
|
169
|
+
this.logger.debug('activity-process-event', { topic: this.config.subtype, jid, aid, status, code });
|
|
170
|
+
let telemetry;
|
|
171
|
+
try {
|
|
172
|
+
await this.getState();
|
|
173
|
+
const aState = await CollatorService.notarizeReentry(this);
|
|
174
|
+
this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
|
|
175
|
+
telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
176
|
+
let isComplete = CollatorService.isActivityComplete(this.context.metadata.js);
|
|
177
|
+
if (isComplete) {
|
|
178
|
+
this.logger.warn('activity-process-event-duplicate', { jid, aid });
|
|
179
|
+
this.logger.debug('activity-process-event-duplicate-resolution', { resolution: 'Increase HotMesh config `reclaimDelay` timeout.' });
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
telemetry.startActivitySpan(this.leg);
|
|
183
|
+
let multiResponse;
|
|
184
|
+
if (status === StreamStatus.PENDING) {
|
|
185
|
+
multiResponse = await this.processPending(telemetry);
|
|
186
|
+
}
|
|
187
|
+
else if (status === StreamStatus.SUCCESS) {
|
|
188
|
+
multiResponse = await this.processSuccess(telemetry);
|
|
189
|
+
}
|
|
190
|
+
else {
|
|
191
|
+
multiResponse = await this.processError(telemetry);
|
|
192
|
+
}
|
|
193
|
+
this.transitionAdjacent(multiResponse, telemetry);
|
|
194
|
+
}
|
|
195
|
+
catch (error) {
|
|
196
|
+
this.logger.error('activity-process-event-error', error);
|
|
197
|
+
telemetry.setActivityError(error.message);
|
|
198
|
+
throw error;
|
|
199
|
+
}
|
|
200
|
+
finally {
|
|
201
|
+
telemetry.endActivitySpan();
|
|
202
|
+
this.logger.debug('activity-process-event-end', { jid, aid });
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
async processPending(telemetry) {
|
|
206
|
+
this.bindActivityData('output');
|
|
207
|
+
this.adjacencyList = await this.filterAdjacent();
|
|
208
|
+
this.mapJobData();
|
|
209
|
+
const multi = this.store.getMulti();
|
|
210
|
+
await this.setState(multi);
|
|
211
|
+
await CollatorService.notarizeContinuation(this, multi);
|
|
212
|
+
await this.setStatus(this.adjacencyList.length, multi);
|
|
213
|
+
return await multi.exec();
|
|
214
|
+
}
|
|
215
|
+
async processSuccess(telemetry) {
|
|
216
|
+
this.bindActivityData('output');
|
|
217
|
+
this.adjacencyList = await this.filterAdjacent();
|
|
218
|
+
this.mapJobData();
|
|
219
|
+
const multi = this.store.getMulti();
|
|
220
|
+
await this.setState(multi);
|
|
221
|
+
await CollatorService.notarizeCompletion(this, multi);
|
|
222
|
+
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
223
|
+
return await multi.exec();
|
|
224
|
+
}
|
|
225
|
+
async processError(telemetry) {
|
|
226
|
+
this.bindActivityError(this.data);
|
|
227
|
+
this.adjacencyList = await this.filterAdjacent();
|
|
228
|
+
const multi = this.store.getMulti();
|
|
229
|
+
await this.setState(multi);
|
|
230
|
+
await CollatorService.notarizeCompletion(this, multi);
|
|
231
|
+
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
232
|
+
return await multi.exec();
|
|
233
|
+
}
|
|
234
|
+
async transitionAdjacent(multiResponse, telemetry) {
|
|
235
|
+
telemetry.mapActivityAttributes();
|
|
236
|
+
const jobStatus = this.resolveStatus(multiResponse);
|
|
237
|
+
const attrs = { 'app.job.jss': jobStatus };
|
|
238
|
+
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
239
|
+
if (messageIds.length) {
|
|
240
|
+
attrs['app.activity.mids'] = messageIds.join(',');
|
|
241
|
+
}
|
|
242
|
+
telemetry.setActivityAttributes(attrs);
|
|
243
|
+
}
|
|
168
244
|
resolveStatus(multiResponse) {
|
|
169
245
|
const activityStatus = multiResponse[multiResponse.length - 1];
|
|
170
246
|
if (Array.isArray(activityStatus)) {
|
|
@@ -318,9 +394,8 @@ class Activity {
|
|
|
318
394
|
let { dad, jid } = this.context.metadata;
|
|
319
395
|
jobId = jobId || jid;
|
|
320
396
|
const dIds = CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], dad);
|
|
321
|
-
//`state` is a flat hash
|
|
397
|
+
//`state` is a flat hash; context is a tree
|
|
322
398
|
const [state, status] = await this.store.getState(jobId, consumes, dIds);
|
|
323
|
-
//`context` is a tree
|
|
324
399
|
this.context = restoreHierarchy(state);
|
|
325
400
|
this.initDimensionalAddress(dad);
|
|
326
401
|
this.initSelf(this.context);
|
|
@@ -10,7 +10,7 @@ declare class Await extends Activity {
|
|
|
10
10
|
process(): Promise<string>;
|
|
11
11
|
execActivity(): Promise<string>;
|
|
12
12
|
processEvent(status?: StreamStatus, code?: StreamCode): Promise<void>;
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
processSuccessResponse(adjacencyList: StreamData[]): Promise<MultiResponseFlags>;
|
|
14
|
+
processErrorResponse(adjacencyList: StreamData[]): Promise<MultiResponseFlags>;
|
|
15
15
|
}
|
|
16
16
|
export { Await };
|
|
@@ -19,7 +19,7 @@ class Await extends Activity {
|
|
|
19
19
|
telemetry.startActivitySpan(this.leg);
|
|
20
20
|
this.mapInputData();
|
|
21
21
|
const multi = this.store.getMulti();
|
|
22
|
-
//await this.registerTimeout();
|
|
22
|
+
//todo: await this.registerTimeout();
|
|
23
23
|
await CollatorService.authorizeReentry(this, multi);
|
|
24
24
|
await this.setState(multi);
|
|
25
25
|
await this.setStatus(0, multi);
|
|
@@ -92,12 +92,12 @@ class Await extends Activity {
|
|
|
92
92
|
if (status === StreamStatus.SUCCESS) {
|
|
93
93
|
this.bindActivityData('output');
|
|
94
94
|
this.adjacencyList = await this.filterAdjacent();
|
|
95
|
-
multiResponse = await this.
|
|
95
|
+
multiResponse = await this.processSuccessResponse(this.adjacencyList);
|
|
96
96
|
}
|
|
97
97
|
else {
|
|
98
98
|
this.bindActivityError(this.data);
|
|
99
99
|
this.adjacencyList = await this.filterAdjacent();
|
|
100
|
-
multiResponse = await this.
|
|
100
|
+
multiResponse = await this.processErrorResponse(this.adjacencyList);
|
|
101
101
|
}
|
|
102
102
|
telemetry.mapActivityAttributes();
|
|
103
103
|
const jobStatus = this.resolveStatus(multiResponse);
|
|
@@ -118,7 +118,7 @@ class Await extends Activity {
|
|
|
118
118
|
this.logger.debug('await-resolve-await-end', { jid, aid, status, code });
|
|
119
119
|
}
|
|
120
120
|
}
|
|
121
|
-
async
|
|
121
|
+
async processSuccessResponse(adjacencyList) {
|
|
122
122
|
this.mapJobData();
|
|
123
123
|
const multi = this.store.getMulti();
|
|
124
124
|
await this.setState(multi);
|
|
@@ -126,7 +126,7 @@ class Await extends Activity {
|
|
|
126
126
|
await this.setStatus(adjacencyList.length - 1, multi);
|
|
127
127
|
return await multi.exec();
|
|
128
128
|
}
|
|
129
|
-
async
|
|
129
|
+
async processErrorResponse(adjacencyList) {
|
|
130
130
|
//todo: if adjacencyList.length == 0, then map to the job output
|
|
131
131
|
// this method would be added to Base activity class
|
|
132
132
|
//this.mapJobData();
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { EngineService } from '../engine';
|
|
2
|
+
import { Activity, ActivityType } from './activity';
|
|
3
|
+
import { ActivityData, ActivityMetadata, CycleActivity } from '../../types/activity';
|
|
4
|
+
import { JobState } from '../../types/job';
|
|
5
|
+
import { RedisMulti } from '../../types/redis';
|
|
6
|
+
declare class Cycle extends Activity {
|
|
7
|
+
config: CycleActivity;
|
|
8
|
+
constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
|
|
9
|
+
process(): Promise<string>;
|
|
10
|
+
/**
|
|
11
|
+
* Trigger the target ancestor to execute in a cycle,
|
|
12
|
+
* without violating the constraints of the DAG. Immutable
|
|
13
|
+
* `individual activity state` will execute in a new dimensional
|
|
14
|
+
* thread while `shared job state` can change. This
|
|
15
|
+
* pattern allows for retries without violating the DAG.
|
|
16
|
+
*/
|
|
17
|
+
cycleAncestorActivity(multi: RedisMulti): Promise<string>;
|
|
18
|
+
}
|
|
19
|
+
export { Cycle };
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { GetStateError } from '../../modules/errors';
|
|
2
|
+
import { CollatorService } from '../collator';
|
|
3
|
+
import { Activity } from './activity';
|
|
4
|
+
import { TelemetryService } from '../telemetry';
|
|
5
|
+
class Cycle extends Activity {
|
|
6
|
+
constructor(config, data, metadata, hook, engine, context) {
|
|
7
|
+
super(config, data, metadata, hook, engine, context);
|
|
8
|
+
}
|
|
9
|
+
//******** LEG 1 ENTRY ********//
|
|
10
|
+
async process() {
|
|
11
|
+
this.logger.debug('cycle-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
12
|
+
let telemetry;
|
|
13
|
+
try {
|
|
14
|
+
//verify entry is allowed
|
|
15
|
+
this.setLeg(1);
|
|
16
|
+
await CollatorService.notarizeEntry(this);
|
|
17
|
+
await this.getState();
|
|
18
|
+
telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
19
|
+
telemetry.startActivitySpan(this.leg);
|
|
20
|
+
this.mapInputData();
|
|
21
|
+
//set state/status
|
|
22
|
+
let multi = this.store.getMulti();
|
|
23
|
+
await this.setState(multi);
|
|
24
|
+
await this.setStatus(0, multi); //leg 1 never changes job status
|
|
25
|
+
const multiResponse = await multi.exec();
|
|
26
|
+
telemetry.mapActivityAttributes();
|
|
27
|
+
const jobStatus = this.resolveStatus(multiResponse);
|
|
28
|
+
//cycle the target ancestor
|
|
29
|
+
multi = this.store.getMulti();
|
|
30
|
+
const messageId = await this.cycleAncestorActivity(multi);
|
|
31
|
+
telemetry.setActivityAttributes({
|
|
32
|
+
'app.activity.mid': messageId,
|
|
33
|
+
'app.job.jss': jobStatus
|
|
34
|
+
});
|
|
35
|
+
//exit early (`Cycle` activities only execute Leg 1)
|
|
36
|
+
await CollatorService.notarizeEarlyExit(this, multi);
|
|
37
|
+
await multi.exec();
|
|
38
|
+
return this.context.metadata.aid;
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
if (error instanceof GetStateError) {
|
|
42
|
+
this.logger.error('cycle-get-state-error', error);
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
this.logger.error('cycle-process-error', error);
|
|
46
|
+
}
|
|
47
|
+
telemetry.setActivityError(error.message);
|
|
48
|
+
throw error;
|
|
49
|
+
}
|
|
50
|
+
finally {
|
|
51
|
+
telemetry.endActivitySpan();
|
|
52
|
+
this.logger.debug('cycle-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Trigger the target ancestor to execute in a cycle,
|
|
57
|
+
* without violating the constraints of the DAG. Immutable
|
|
58
|
+
* `individual activity state` will execute in a new dimensional
|
|
59
|
+
* thread while `shared job state` can change. This
|
|
60
|
+
* pattern allows for retries without violating the DAG.
|
|
61
|
+
*/
|
|
62
|
+
async cycleAncestorActivity(multi) {
|
|
63
|
+
const streamData = {
|
|
64
|
+
metadata: {
|
|
65
|
+
dad: CollatorService.resolveReentryDimension(this),
|
|
66
|
+
jid: this.context.metadata.jid,
|
|
67
|
+
aid: this.config.ancestor,
|
|
68
|
+
},
|
|
69
|
+
data: {} //todo: verify immutability, before enabling: `this.context.data`
|
|
70
|
+
};
|
|
71
|
+
return (await this.engine.streamSignaler?.publishMessage(null, streamData, multi));
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
export { Cycle };
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { Activity } from './activity';
|
|
2
2
|
import { Await } from './await';
|
|
3
|
-
import {
|
|
4
|
-
import { Iterate } from './iterate';
|
|
3
|
+
import { Cycle } from './cycle';
|
|
5
4
|
import { Emit } from './emit';
|
|
5
|
+
import { Iterate } from './iterate';
|
|
6
6
|
import { Trigger } from './trigger';
|
|
7
|
+
import { Worker } from './worker';
|
|
7
8
|
declare const _default: {
|
|
8
9
|
activity: typeof Activity;
|
|
9
10
|
await: typeof Await;
|
|
11
|
+
cycle: typeof Cycle;
|
|
10
12
|
iterate: typeof Iterate;
|
|
11
13
|
emit: typeof Emit;
|
|
12
14
|
trigger: typeof Trigger;
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { Activity } from './activity';
|
|
2
2
|
import { Await } from './await';
|
|
3
|
-
import {
|
|
4
|
-
import { Iterate } from './iterate';
|
|
3
|
+
import { Cycle } from './cycle';
|
|
5
4
|
import { Emit } from './emit';
|
|
5
|
+
import { Iterate } from './iterate';
|
|
6
6
|
import { Trigger } from './trigger';
|
|
7
|
+
import { Worker } from './worker';
|
|
7
8
|
export default {
|
|
8
9
|
activity: Activity,
|
|
9
10
|
await: Await,
|
|
11
|
+
cycle: Cycle,
|
|
10
12
|
iterate: Iterate,
|
|
11
13
|
emit: Emit,
|
|
12
14
|
trigger: Trigger,
|
|
@@ -2,16 +2,10 @@ import { Activity } from './activity';
|
|
|
2
2
|
import { EngineService } from '../engine';
|
|
3
3
|
import { ActivityData, ActivityMetadata, ActivityType, WorkerActivity } from '../../types/activity';
|
|
4
4
|
import { JobState } from '../../types/job';
|
|
5
|
-
import { StreamCode, StreamStatus } from '../../types/stream';
|
|
6
|
-
import { TelemetryService } from '../telemetry';
|
|
7
5
|
declare class Worker extends Activity {
|
|
8
6
|
config: WorkerActivity;
|
|
9
7
|
constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
|
|
10
8
|
process(): Promise<string>;
|
|
11
9
|
execActivity(): Promise<string>;
|
|
12
|
-
processEvent(status?: StreamStatus, code?: StreamCode): Promise<void>;
|
|
13
|
-
processPending(telemetry: TelemetryService): Promise<void>;
|
|
14
|
-
processSuccess(telemetry: TelemetryService): Promise<void>;
|
|
15
|
-
processError(telemetry: TelemetryService): Promise<void>;
|
|
16
10
|
}
|
|
17
11
|
export { Worker };
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { GetStateError } from '../../modules/errors';
|
|
2
2
|
import { Activity } from './activity';
|
|
3
3
|
import { CollatorService } from '../collator';
|
|
4
|
-
import { StreamStatus } from '../../types/stream';
|
|
5
4
|
import { TelemetryService } from '../telemetry';
|
|
6
5
|
class Worker extends Activity {
|
|
7
6
|
constructor(config, data, metadata, hook, engine, context) {
|
|
@@ -19,7 +18,7 @@ class Worker extends Activity {
|
|
|
19
18
|
telemetry.startActivitySpan(this.leg);
|
|
20
19
|
this.mapInputData();
|
|
21
20
|
const multi = this.store.getMulti();
|
|
22
|
-
//await this.registerTimeout();
|
|
21
|
+
//todo: await this.registerTimeout();
|
|
23
22
|
await CollatorService.authorizeReentry(this, multi);
|
|
24
23
|
await this.setState(multi);
|
|
25
24
|
await this.setStatus(0, multi);
|
|
@@ -31,7 +30,6 @@ class Worker extends Activity {
|
|
|
31
30
|
'app.activity.mid': messageId,
|
|
32
31
|
'app.job.jss': jobStatus
|
|
33
32
|
});
|
|
34
|
-
//TODO: UPDATE ACTIVITY STATE (LEG 1 EXIT)
|
|
35
33
|
return this.context.metadata.aid;
|
|
36
34
|
}
|
|
37
35
|
catch (error) {
|
|
@@ -39,7 +37,6 @@ class Worker extends Activity {
|
|
|
39
37
|
this.logger.error('worker-get-state-error', error);
|
|
40
38
|
}
|
|
41
39
|
else {
|
|
42
|
-
console.error(error);
|
|
43
40
|
this.logger.error('worker-process-error', error);
|
|
44
41
|
}
|
|
45
42
|
telemetry.setActivityError(error.message);
|
|
@@ -69,94 +66,5 @@ class Worker extends Activity {
|
|
|
69
66
|
}
|
|
70
67
|
return (await this.engine.streamSignaler?.publishMessage(this.config.subtype, streamData));
|
|
71
68
|
}
|
|
72
|
-
//******** SIGNAL RE-ENTRY POINT (DUPLEX LEG 2 of 2) ********//
|
|
73
|
-
async processEvent(status = StreamStatus.SUCCESS, code = 200) {
|
|
74
|
-
this.setLeg(2);
|
|
75
|
-
const jid = this.context.metadata.jid;
|
|
76
|
-
const aid = this.metadata.aid;
|
|
77
|
-
this.status = status;
|
|
78
|
-
this.code = code;
|
|
79
|
-
this.logger.debug('worker-process-event', { topic: this.config.subtype, jid, aid, status, code });
|
|
80
|
-
let telemetry;
|
|
81
|
-
try {
|
|
82
|
-
await this.getState();
|
|
83
|
-
const aState = await CollatorService.notarizeReentry(this);
|
|
84
|
-
this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
|
|
85
|
-
telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
86
|
-
let isComplete = CollatorService.isActivityComplete(this.context.metadata.js);
|
|
87
|
-
if (isComplete) {
|
|
88
|
-
this.logger.warn('worker-process-event-duplicate', { jid, aid });
|
|
89
|
-
this.logger.debug('worker-process-event-duplicate-resolution', { resolution: 'Increase HotMesh config `reclaimDelay` timeout.' });
|
|
90
|
-
return;
|
|
91
|
-
}
|
|
92
|
-
telemetry.startActivitySpan(this.leg);
|
|
93
|
-
if (status === StreamStatus.PENDING) {
|
|
94
|
-
await this.processPending(telemetry);
|
|
95
|
-
}
|
|
96
|
-
else if (status === StreamStatus.SUCCESS) {
|
|
97
|
-
await this.processSuccess(telemetry);
|
|
98
|
-
}
|
|
99
|
-
else {
|
|
100
|
-
await this.processError(telemetry);
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
catch (error) {
|
|
104
|
-
this.logger.error('worker-process-event-error', error);
|
|
105
|
-
telemetry.setActivityError(error.message);
|
|
106
|
-
throw error;
|
|
107
|
-
}
|
|
108
|
-
finally {
|
|
109
|
-
telemetry.endActivitySpan();
|
|
110
|
-
this.logger.debug('worker-process-event-end', { jid, aid });
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
async processPending(telemetry) {
|
|
114
|
-
this.bindActivityData('output');
|
|
115
|
-
this.adjacencyList = await this.filterAdjacent();
|
|
116
|
-
this.mapJobData();
|
|
117
|
-
const multi = this.store.getMulti();
|
|
118
|
-
await this.setState(multi);
|
|
119
|
-
await CollatorService.notarizeContinuation(this, multi);
|
|
120
|
-
await this.setStatus(0, multi);
|
|
121
|
-
const multiResponse = await multi.exec();
|
|
122
|
-
telemetry.mapActivityAttributes();
|
|
123
|
-
const jobStatus = this.resolveStatus(multiResponse);
|
|
124
|
-
telemetry.setActivityAttributes({ 'app.job.jss': jobStatus });
|
|
125
|
-
}
|
|
126
|
-
async processSuccess(telemetry) {
|
|
127
|
-
this.bindActivityData('output');
|
|
128
|
-
this.adjacencyList = await this.filterAdjacent();
|
|
129
|
-
this.mapJobData();
|
|
130
|
-
const multi = this.store.getMulti();
|
|
131
|
-
await this.setState(multi);
|
|
132
|
-
await CollatorService.notarizeCompletion(this, multi);
|
|
133
|
-
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
134
|
-
const multiResponse = await multi.exec();
|
|
135
|
-
telemetry.mapActivityAttributes();
|
|
136
|
-
const jobStatus = this.resolveStatus(multiResponse);
|
|
137
|
-
const attrs = { 'app.job.jss': jobStatus };
|
|
138
|
-
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
139
|
-
if (messageIds.length) {
|
|
140
|
-
attrs['app.activity.mids'] = messageIds.join(',');
|
|
141
|
-
}
|
|
142
|
-
telemetry.setActivityAttributes(attrs);
|
|
143
|
-
}
|
|
144
|
-
async processError(telemetry) {
|
|
145
|
-
this.bindActivityError(this.data);
|
|
146
|
-
this.adjacencyList = await this.filterAdjacent();
|
|
147
|
-
const multi = this.store.getMulti();
|
|
148
|
-
await this.setState(multi);
|
|
149
|
-
await CollatorService.notarizeCompletion(this, multi);
|
|
150
|
-
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
151
|
-
const multiResponse = await multi.exec();
|
|
152
|
-
telemetry.mapActivityAttributes();
|
|
153
|
-
const jobStatus = this.resolveStatus(multiResponse);
|
|
154
|
-
const attrs = { 'app.job.jss': jobStatus };
|
|
155
|
-
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
156
|
-
if (messageIds.length) {
|
|
157
|
-
attrs['app.activity.mids'] = messageIds.join(',');
|
|
158
|
-
}
|
|
159
|
-
telemetry.setActivityAttributes(attrs);
|
|
160
|
-
}
|
|
161
69
|
}
|
|
162
70
|
export { Worker };
|