@hotmeshio/hotmesh 0.0.17 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/modules/utils.d.ts +3 -0
- package/build/modules/utils.js +10 -1
- package/build/package.json +1 -1
- package/build/services/activities/activity.d.ts +4 -12
- package/build/services/activities/activity.js +14 -156
- package/build/services/activities/hook.d.ts +20 -0
- package/build/services/activities/hook.js +124 -0
- package/build/services/activities/index.d.ts +2 -0
- package/build/services/activities/index.js +2 -0
- package/build/services/collator/index.js +0 -1
- package/build/services/compiler/deployer.d.ts +2 -0
- package/build/services/compiler/deployer.js +29 -2
- package/build/services/durable/client.d.ts +8 -1
- package/build/services/durable/client.js +46 -0
- package/build/services/durable/factory.js +11 -10
- package/build/services/durable/search.d.ts +15 -0
- package/build/services/durable/search.js +45 -0
- package/build/services/durable/workflow.d.ts +1 -0
- package/build/services/durable/workflow.js +34 -0
- package/build/services/engine/index.d.ts +7 -2
- package/build/services/engine/index.js +2 -1
- package/build/services/store/clients/ioredis.d.ts +1 -0
- package/build/services/store/clients/ioredis.js +12 -0
- package/build/services/store/clients/redis.d.ts +1 -0
- package/build/services/store/clients/redis.js +3 -0
- package/build/services/store/index.d.ts +1 -0
- package/build/services/telemetry/index.js +2 -1
- package/build/types/activity.d.ts +6 -3
- package/build/types/durable.d.ts +10 -1
- package/build/types/hook.d.ts +1 -0
- package/build/types/index.d.ts +2 -2
- package/modules/utils.ts +11 -0
- package/package.json +1 -1
- package/services/activities/activity.ts +15 -167
- package/services/activities/hook.ts +149 -0
- package/services/activities/index.ts +2 -0
- package/services/collator/index.ts +0 -1
- package/services/compiler/deployer.ts +32 -2
- package/services/durable/client.ts +50 -2
- package/services/durable/factory.ts +11 -10
- package/services/durable/search.ts +54 -0
- package/services/durable/workflow.ts +32 -1
- package/services/engine/index.ts +8 -4
- package/services/store/clients/ioredis.ts +13 -0
- package/services/store/clients/redis.ts +4 -0
- package/services/store/index.ts +1 -0
- package/services/telemetry/index.ts +2 -1
- package/types/activity.ts +7 -2
- package/types/durable.ts +8 -0
- package/types/hook.ts +1 -0
- package/types/index.ts +2 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CollationError
|
|
1
|
+
import { CollationError } from '../../modules/errors';
|
|
2
2
|
import {
|
|
3
3
|
formatISODate,
|
|
4
4
|
getValueByPath,
|
|
@@ -9,7 +9,6 @@ import { ILogger } from '../logger';
|
|
|
9
9
|
import { MapperService } from '../mapper';
|
|
10
10
|
import { Pipe } from '../pipe';
|
|
11
11
|
import { MDATA_SYMBOLS } from '../serializer';
|
|
12
|
-
import { StoreSignaler } from '../signaler/store';
|
|
13
12
|
import { StoreService } from '../store';
|
|
14
13
|
import { TelemetryService } from '../telemetry';
|
|
15
14
|
import {
|
|
@@ -30,7 +29,6 @@ import {
|
|
|
30
29
|
StreamDataType,
|
|
31
30
|
StreamStatus } from '../../types/stream';
|
|
32
31
|
import { TransitionRule } from '../../types/transition';
|
|
33
|
-
import { HookRule } from '../../types/hook';
|
|
34
32
|
|
|
35
33
|
/**
|
|
36
34
|
* The base class for all activities
|
|
@@ -48,7 +46,7 @@ class Activity {
|
|
|
48
46
|
code: StreamCode = 200;
|
|
49
47
|
leg: ActivityLeg;
|
|
50
48
|
adjacencyList: StreamData[];
|
|
51
|
-
adjacentIndex = 0;
|
|
49
|
+
adjacentIndex = 0;
|
|
52
50
|
|
|
53
51
|
constructor(
|
|
54
52
|
config: ActivityType,
|
|
@@ -67,171 +65,21 @@ class Activity {
|
|
|
67
65
|
this.store = engine.store;
|
|
68
66
|
}
|
|
69
67
|
|
|
70
|
-
//******** INITIAL ENTRY POINT (A) ********//
|
|
71
|
-
async process(): Promise<string> {
|
|
72
|
-
this.logger.debug('activity-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
73
|
-
let telemetry: TelemetryService;
|
|
74
|
-
try {
|
|
75
|
-
this.setLeg(1);
|
|
76
|
-
await CollatorService.notarizeEntry(this);
|
|
77
|
-
|
|
78
|
-
await this.getState();
|
|
79
|
-
telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
80
|
-
telemetry.startActivitySpan(this.leg);
|
|
81
|
-
let multiResponse: MultiResponseFlags;
|
|
82
|
-
|
|
83
|
-
const multi = this.store.getMulti();
|
|
84
|
-
if (this.doesHook()) {
|
|
85
|
-
//sleep and wait to awaken upon a signal
|
|
86
|
-
await this.registerHook(multi);
|
|
87
|
-
this.mapOutputData();
|
|
88
|
-
this.mapJobData();
|
|
89
|
-
await this.setState(multi);
|
|
90
|
-
await CollatorService.authorizeReentry(this, multi);
|
|
91
|
-
|
|
92
|
-
await this.setStatus(0, multi);
|
|
93
|
-
await multi.exec();
|
|
94
|
-
telemetry.mapActivityAttributes();
|
|
95
|
-
} else {
|
|
96
|
-
//end the activity and transition to its children
|
|
97
|
-
this.adjacencyList = await this.filterAdjacent();
|
|
98
|
-
this.mapOutputData();
|
|
99
|
-
this.mapJobData();
|
|
100
|
-
await this.setState(multi);
|
|
101
|
-
await CollatorService.notarizeEarlyCompletion(this, multi);
|
|
102
|
-
|
|
103
|
-
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
104
|
-
multiResponse = await multi.exec() as MultiResponseFlags;
|
|
105
|
-
telemetry.mapActivityAttributes();
|
|
106
|
-
const jobStatus = this.resolveStatus(multiResponse);
|
|
107
|
-
const attrs: StringScalarType = { 'app.job.jss': jobStatus };
|
|
108
|
-
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
109
|
-
if (messageIds.length) {
|
|
110
|
-
attrs['app.activity.mids'] = messageIds.join(',')
|
|
111
|
-
}
|
|
112
|
-
telemetry.setActivityAttributes(attrs);
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
return this.context.metadata.aid;
|
|
116
|
-
} catch (error) {
|
|
117
|
-
if (error instanceof GetStateError) {
|
|
118
|
-
this.logger.error('activity-get-state-error', { error });
|
|
119
|
-
} else {
|
|
120
|
-
this.logger.error('activity-process-error', { error });
|
|
121
|
-
}
|
|
122
|
-
telemetry.setActivityError(error.message);
|
|
123
|
-
throw error;
|
|
124
|
-
} finally {
|
|
125
|
-
telemetry.endActivitySpan();
|
|
126
|
-
this.logger.debug('activity-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
|
|
130
68
|
setLeg(leg: ActivityLeg): void {
|
|
131
69
|
this.leg = leg;
|
|
132
70
|
}
|
|
133
71
|
|
|
134
|
-
//******** SIGNAL RE-ENTRY POINT ********//
|
|
135
|
-
doesHook(): boolean {
|
|
136
|
-
return !!(this.config.hook?.topic || this.config.sleep);
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
async getHookRule(topic: string): Promise<HookRule | undefined> {
|
|
140
|
-
const rules = await this.store.getHookRules();
|
|
141
|
-
return rules?.[topic]?.[0] as HookRule;
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
async registerHook(multi?: RedisMulti): Promise<string | void> {
|
|
145
|
-
if (this.config.hook?.topic) {
|
|
146
|
-
const signaler = new StoreSignaler(this.store, this.logger);
|
|
147
|
-
return await signaler.registerWebHook(this.config.hook.topic, this.context, multi);
|
|
148
|
-
} else if (this.config.sleep) {
|
|
149
|
-
const durationInSeconds = Pipe.resolve(this.config.sleep, this.context);
|
|
150
|
-
const jobId = this.context.metadata.jid;
|
|
151
|
-
const activityId = this.metadata.aid;
|
|
152
|
-
const dId = this.metadata.dad;
|
|
153
|
-
await this.engine.task.registerTimeHook(jobId, `${activityId}${dId||''}`, 'sleep', durationInSeconds);
|
|
154
|
-
return jobId;
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
async processWebHookEvent(): Promise<JobStatus | void> {
|
|
159
|
-
this.logger.debug('engine-process-web-hook-event', {
|
|
160
|
-
topic: this.config.hook.topic,
|
|
161
|
-
aid: this.metadata.aid
|
|
162
|
-
});
|
|
163
|
-
const signaler = new StoreSignaler(this.store, this.logger);
|
|
164
|
-
const data = { ...this.data };
|
|
165
|
-
const jobId = await signaler.processWebHookSignal(this.config.hook.topic, data);
|
|
166
|
-
if (jobId) {
|
|
167
|
-
await this.processHookEvent(jobId);
|
|
168
|
-
await signaler.deleteWebHookSignal(this.config.hook.topic, data);
|
|
169
|
-
} //else => already resolved
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
async processTimeHookEvent(jobId: string): Promise<JobStatus | void> {
|
|
173
|
-
this.logger.debug('engine-process-time-hook-event', {
|
|
174
|
-
jid: jobId,
|
|
175
|
-
aid: this.metadata.aid
|
|
176
|
-
});
|
|
177
|
-
return await this.processHookEvent(jobId);
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
async processHookEvent(jobId: string): Promise<JobStatus | void> {
|
|
181
|
-
this.logger.debug('activity-process-hook-event', { jobId });
|
|
182
|
-
let telemetry: TelemetryService;
|
|
183
|
-
try {
|
|
184
|
-
this.setLeg(2);
|
|
185
|
-
await this.getState(jobId);
|
|
186
|
-
telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
187
|
-
telemetry.startActivitySpan(this.leg);
|
|
188
|
-
const aState = await CollatorService.notarizeReentry(this);
|
|
189
|
-
this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
|
|
190
|
-
|
|
191
|
-
this.bindActivityData('hook');
|
|
192
|
-
this.mapJobData();
|
|
193
|
-
this.adjacencyList = await this.filterAdjacent();
|
|
194
|
-
|
|
195
|
-
const multi = this.engine.store.getMulti();
|
|
196
|
-
await this.setState(multi);
|
|
197
|
-
await CollatorService.notarizeCompletion(this, multi);
|
|
198
|
-
|
|
199
|
-
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
200
|
-
const multiResponse = await multi.exec() as MultiResponseFlags;
|
|
201
|
-
|
|
202
|
-
telemetry.mapActivityAttributes();
|
|
203
|
-
const jobStatus = this.resolveStatus(multiResponse);
|
|
204
|
-
const attrs: StringScalarType = { 'app.job.jss': jobStatus };
|
|
205
|
-
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
206
|
-
if (messageIds.length) {
|
|
207
|
-
attrs['app.activity.mids'] = messageIds.join(',')
|
|
208
|
-
}
|
|
209
|
-
telemetry.setActivityAttributes(attrs);
|
|
210
|
-
return jobStatus as number;
|
|
211
|
-
} catch (error) {
|
|
212
|
-
if (error instanceof CollationError && error.fault === 'inactive') {
|
|
213
|
-
this.logger.info('process-hook-event-inactive-error', { error });
|
|
214
|
-
return;
|
|
215
|
-
}
|
|
216
|
-
this.logger.error('engine-process-hook-event-error', { error });
|
|
217
|
-
telemetry.setActivityError(error.message);
|
|
218
|
-
throw error;
|
|
219
|
-
} finally {
|
|
220
|
-
telemetry.endActivitySpan();
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
|
|
224
72
|
//******** DUPLEX RE-ENTRY POINT ********//
|
|
225
|
-
async processEvent(status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200): Promise<void> {
|
|
73
|
+
async processEvent(status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200, type: 'hook' | 'output' = 'output', jobId?: string): Promise<void> {
|
|
226
74
|
this.setLeg(2);
|
|
227
|
-
const jid = this.context.metadata.jid;
|
|
75
|
+
const jid = this.context.metadata.jid || jobId;
|
|
228
76
|
const aid = this.metadata.aid;
|
|
229
77
|
this.status = status;
|
|
230
78
|
this.code = code;
|
|
231
79
|
this.logger.debug('activity-process-event', { topic: this.config.subtype, jid, aid, status, code });
|
|
232
80
|
let telemetry: TelemetryService;
|
|
233
81
|
try {
|
|
234
|
-
await this.getState();
|
|
82
|
+
await this.getState(jobId);
|
|
235
83
|
const aState = await CollatorService.notarizeReentry(this);
|
|
236
84
|
this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
|
|
237
85
|
|
|
@@ -247,18 +95,19 @@ class Activity {
|
|
|
247
95
|
telemetry.startActivitySpan(this.leg);
|
|
248
96
|
let multiResponse: MultiResponseFlags;
|
|
249
97
|
if (status === StreamStatus.PENDING) {
|
|
250
|
-
multiResponse = await this.processPending(telemetry);
|
|
98
|
+
multiResponse = await this.processPending(telemetry, type);
|
|
251
99
|
} else if (status === StreamStatus.SUCCESS) {
|
|
252
|
-
multiResponse = await this.processSuccess(telemetry);
|
|
100
|
+
multiResponse = await this.processSuccess(telemetry, type);
|
|
253
101
|
} else {
|
|
254
|
-
multiResponse = await this.processError(telemetry);
|
|
102
|
+
multiResponse = await this.processError(telemetry, type);
|
|
255
103
|
}
|
|
256
104
|
this.transitionAdjacent(multiResponse, telemetry);
|
|
257
105
|
} catch (error) {
|
|
258
|
-
if (error instanceof CollationError
|
|
106
|
+
if (error instanceof CollationError) {
|
|
259
107
|
this.logger.info('process-event-inactive-error', { error });
|
|
260
108
|
return;
|
|
261
109
|
}
|
|
110
|
+
console.error(error);
|
|
262
111
|
this.logger.error('activity-process-event-error', { error });
|
|
263
112
|
telemetry && telemetry.setActivityError(error.message);
|
|
264
113
|
throw error;
|
|
@@ -268,8 +117,8 @@ class Activity {
|
|
|
268
117
|
}
|
|
269
118
|
}
|
|
270
119
|
|
|
271
|
-
async processPending(telemetry: TelemetryService): Promise<MultiResponseFlags> {
|
|
272
|
-
this.bindActivityData(
|
|
120
|
+
async processPending(telemetry: TelemetryService, type: 'hook' | 'output'): Promise<MultiResponseFlags> {
|
|
121
|
+
this.bindActivityData(type);
|
|
273
122
|
this.adjacencyList = await this.filterAdjacent();
|
|
274
123
|
this.mapJobData();
|
|
275
124
|
const multi = this.store.getMulti();
|
|
@@ -280,8 +129,8 @@ class Activity {
|
|
|
280
129
|
return await multi.exec() as MultiResponseFlags;
|
|
281
130
|
}
|
|
282
131
|
|
|
283
|
-
async processSuccess(telemetry: TelemetryService): Promise<MultiResponseFlags> {
|
|
284
|
-
this.bindActivityData(
|
|
132
|
+
async processSuccess(telemetry: TelemetryService, type: 'hook' | 'output'): Promise<MultiResponseFlags> {
|
|
133
|
+
this.bindActivityData(type);
|
|
285
134
|
this.adjacencyList = await this.filterAdjacent();
|
|
286
135
|
this.mapJobData();
|
|
287
136
|
const multi = this.store.getMulti();
|
|
@@ -292,7 +141,7 @@ class Activity {
|
|
|
292
141
|
return await multi.exec() as MultiResponseFlags;
|
|
293
142
|
}
|
|
294
143
|
|
|
295
|
-
async processError(telemetry: TelemetryService): Promise<MultiResponseFlags> {
|
|
144
|
+
async processError(telemetry: TelemetryService, type: string): Promise<MultiResponseFlags> {
|
|
296
145
|
this.bindActivityError(this.data);
|
|
297
146
|
this.adjacencyList = await this.filterAdjacent();
|
|
298
147
|
const multi = this.store.getMulti();
|
|
@@ -584,7 +433,6 @@ class Activity {
|
|
|
584
433
|
|
|
585
434
|
async transition(adjacencyList: StreamData[], jobStatus: JobStatus): Promise<string[]> {
|
|
586
435
|
let mIds: string[] = [];
|
|
587
|
-
//emit can be a mapping (allows emissions to be driven by the job state)
|
|
588
436
|
let emit: boolean = false;
|
|
589
437
|
if (this.config.emit) {
|
|
590
438
|
emit = Pipe.resolve(this.config.emit, this.context);
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { CollationError, GetStateError } from '../../modules/errors';
|
|
2
|
+
import { CollatorService } from '../collator';
|
|
3
|
+
import { EngineService } from '../engine';
|
|
4
|
+
import { Pipe } from '../pipe';
|
|
5
|
+
import { StoreSignaler } from '../signaler/store';
|
|
6
|
+
import { TelemetryService } from '../telemetry';
|
|
7
|
+
import {
|
|
8
|
+
ActivityData,
|
|
9
|
+
ActivityMetadata,
|
|
10
|
+
ActivityType,
|
|
11
|
+
HookActivity} from '../../types/activity';
|
|
12
|
+
import { JobState, JobStatus } from '../../types/job';
|
|
13
|
+
import {
|
|
14
|
+
MultiResponseFlags,
|
|
15
|
+
RedisMulti } from '../../types/redis';
|
|
16
|
+
import { StringScalarType } from '../../types/serializer';
|
|
17
|
+
import { HookRule } from '../../types/hook';
|
|
18
|
+
import { Activity } from './activity';
|
|
19
|
+
import { StreamStatus } from '../../types';
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Listens for `webhook`, `timehook`, and `cycle` (repeat) signals
|
|
23
|
+
*/
|
|
24
|
+
class Hook extends Activity {
|
|
25
|
+
config: HookActivity;
|
|
26
|
+
|
|
27
|
+
constructor(
|
|
28
|
+
config: ActivityType,
|
|
29
|
+
data: ActivityData,
|
|
30
|
+
metadata: ActivityMetadata,
|
|
31
|
+
hook: ActivityData | null,
|
|
32
|
+
engine: EngineService,
|
|
33
|
+
context?: JobState) {
|
|
34
|
+
super(config, data, metadata, hook, engine, context);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
//******** INITIAL ENTRY POINT (A) ********//
|
|
38
|
+
async process(): Promise<string> {
|
|
39
|
+
this.logger.debug('hook-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
40
|
+
let telemetry: TelemetryService;
|
|
41
|
+
try {
|
|
42
|
+
this.setLeg(1);
|
|
43
|
+
await CollatorService.notarizeEntry(this);
|
|
44
|
+
|
|
45
|
+
await this.getState();
|
|
46
|
+
telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
47
|
+
telemetry.startActivitySpan(this.leg);
|
|
48
|
+
let multiResponse: MultiResponseFlags;
|
|
49
|
+
|
|
50
|
+
const multi = this.store.getMulti();
|
|
51
|
+
if (this.doesHook()) {
|
|
52
|
+
//sleep and wait to awaken upon a signal
|
|
53
|
+
await this.registerHook(multi);
|
|
54
|
+
this.mapOutputData();
|
|
55
|
+
this.mapJobData();
|
|
56
|
+
await this.setState(multi);
|
|
57
|
+
await CollatorService.authorizeReentry(this, multi);
|
|
58
|
+
|
|
59
|
+
await this.setStatus(0, multi);
|
|
60
|
+
await multi.exec();
|
|
61
|
+
telemetry.mapActivityAttributes();
|
|
62
|
+
} else {
|
|
63
|
+
//end the activity and transition to its children
|
|
64
|
+
this.adjacencyList = await this.filterAdjacent();
|
|
65
|
+
this.mapOutputData();
|
|
66
|
+
this.mapJobData();
|
|
67
|
+
await this.setState(multi);
|
|
68
|
+
await CollatorService.notarizeEarlyCompletion(this, multi);
|
|
69
|
+
|
|
70
|
+
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
71
|
+
multiResponse = await multi.exec() as MultiResponseFlags;
|
|
72
|
+
telemetry.mapActivityAttributes();
|
|
73
|
+
const jobStatus = this.resolveStatus(multiResponse);
|
|
74
|
+
const attrs: StringScalarType = { 'app.job.jss': jobStatus };
|
|
75
|
+
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
76
|
+
if (messageIds.length) {
|
|
77
|
+
attrs['app.activity.mids'] = messageIds.join(',')
|
|
78
|
+
}
|
|
79
|
+
telemetry.setActivityAttributes(attrs);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
return this.context.metadata.aid;
|
|
83
|
+
} catch (error) {
|
|
84
|
+
if (error instanceof GetStateError) {
|
|
85
|
+
this.logger.error('hook-get-state-error', { error });
|
|
86
|
+
} else {
|
|
87
|
+
this.logger.error('hook-process-error', { error });
|
|
88
|
+
}
|
|
89
|
+
telemetry.setActivityError(error.message);
|
|
90
|
+
throw error;
|
|
91
|
+
} finally {
|
|
92
|
+
telemetry.endActivitySpan();
|
|
93
|
+
this.logger.debug('hook-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
//******** SIGNAL RE-ENTRY POINT ********//
|
|
98
|
+
doesHook(): boolean {
|
|
99
|
+
//does this activity use a time-hook or web-hook
|
|
100
|
+
return !!(this.config.hook?.topic || this.config.sleep);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async getHookRule(topic: string): Promise<HookRule | undefined> {
|
|
104
|
+
const rules = await this.store.getHookRules();
|
|
105
|
+
return rules?.[topic]?.[0] as HookRule;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
async registerHook(multi?: RedisMulti): Promise<string | void> {
|
|
109
|
+
if (this.config.hook?.topic) {
|
|
110
|
+
const signaler = new StoreSignaler(this.store, this.logger);
|
|
111
|
+
return await signaler.registerWebHook(this.config.hook.topic, this.context, multi);
|
|
112
|
+
} else if (this.config.sleep) {
|
|
113
|
+
const durationInSeconds = Pipe.resolve(this.config.sleep, this.context);
|
|
114
|
+
const jobId = this.context.metadata.jid;
|
|
115
|
+
const activityId = this.metadata.aid;
|
|
116
|
+
const dId = this.metadata.dad;
|
|
117
|
+
await this.engine.task.registerTimeHook(jobId, `${activityId}${dId||''}`, 'sleep', durationInSeconds);
|
|
118
|
+
return jobId;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async processWebHookEvent(): Promise<JobStatus | void> {
|
|
123
|
+
this.logger.debug('hook-process-web-hook-event', {
|
|
124
|
+
topic: this.config.hook.topic,
|
|
125
|
+
aid: this.metadata.aid
|
|
126
|
+
});
|
|
127
|
+
const signaler = new StoreSignaler(this.store, this.logger);
|
|
128
|
+
const data = { ...this.data };
|
|
129
|
+
const jobId = await signaler.processWebHookSignal(this.config.hook.topic, data);
|
|
130
|
+
if (jobId) {
|
|
131
|
+
//if a webhook signal is sent that includes 'keep_alive' the hook will remain open
|
|
132
|
+
const code = data.keep_alive ? 202 : 200;
|
|
133
|
+
await this.processEvent(StreamStatus.SUCCESS, code, 'hook', jobId);
|
|
134
|
+
if (code === 200) {
|
|
135
|
+
await signaler.deleteWebHookSignal(this.config.hook.topic, data);
|
|
136
|
+
}
|
|
137
|
+
} //else => already resolved
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
async processTimeHookEvent(jobId: string): Promise<JobStatus | void> {
|
|
141
|
+
this.logger.debug('hook-process-time-hook-event', {
|
|
142
|
+
jid: jobId,
|
|
143
|
+
aid: this.metadata.aid
|
|
144
|
+
});
|
|
145
|
+
await this.processEvent(StreamStatus.SUCCESS, 200, 'hook');
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export { Hook };
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Activity } from './activity';
|
|
2
2
|
import { Await } from './await';
|
|
3
3
|
import { Cycle } from './cycle';
|
|
4
|
+
import { Hook } from './hook';
|
|
4
5
|
import { Iterate } from './iterate';
|
|
5
6
|
import { Signal } from './signal';
|
|
6
7
|
import { Trigger } from './trigger';
|
|
@@ -10,6 +11,7 @@ export default {
|
|
|
10
11
|
activity: Activity,
|
|
11
12
|
await: Await,
|
|
12
13
|
cycle: Cycle,
|
|
14
|
+
hook: Hook,
|
|
13
15
|
iterate: Iterate,
|
|
14
16
|
signal: Signal,
|
|
15
17
|
trigger: Trigger,
|
|
@@ -54,7 +54,6 @@ class CollatorService {
|
|
|
54
54
|
//set second digit to 8, allowing for re-entry
|
|
55
55
|
//decrement by -10_000_000_000_000
|
|
56
56
|
const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -10_000_000_000_000, this.getDimensionalAddress(activity), multi);
|
|
57
|
-
//this.verifyInteger(amount, 1, 'exit');
|
|
58
57
|
return amount;
|
|
59
58
|
}
|
|
60
59
|
|
|
@@ -25,10 +25,12 @@ class Deployer {
|
|
|
25
25
|
async deploy(store: StoreService<RedisClient, RedisMulti>) {
|
|
26
26
|
this.store = store;
|
|
27
27
|
CollatorService.compile(this.manifest.app.graphs);
|
|
28
|
+
this.convertActivitiesToHooks();
|
|
28
29
|
this.convertTopicsToTypes();
|
|
29
30
|
this.copyJobSchemas();
|
|
30
31
|
this.bindBackRefs();
|
|
31
32
|
this.bindParents();
|
|
33
|
+
this.bindCycleTarget();
|
|
32
34
|
this.resolveMappingDependencies();
|
|
33
35
|
this.resolveJobMapsPaths();
|
|
34
36
|
await this.generateSymKeys();
|
|
@@ -148,8 +150,22 @@ class Deployer {
|
|
|
148
150
|
}
|
|
149
151
|
}
|
|
150
152
|
|
|
151
|
-
//
|
|
152
|
-
//
|
|
153
|
+
//the cycle/goto activity includes and ancestor target;
|
|
154
|
+
//update with the cycle flag, so it can be rerun
|
|
155
|
+
bindCycleTarget() {
|
|
156
|
+
for (const graph of this.manifest!.app.graphs) {
|
|
157
|
+
const activities = graph.activities;
|
|
158
|
+
for (const activityKey in activities) {
|
|
159
|
+
const activity = activities[activityKey];
|
|
160
|
+
if (activity.type === 'cycle') {
|
|
161
|
+
activities[activity.ancestor].cycle = true;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
//it's more intuitive for SDK users to use 'topic',
|
|
168
|
+
//but the compiler is desiged to be generic and uses the attribute, 'subtypes'
|
|
153
169
|
convertTopicsToTypes() {
|
|
154
170
|
for (const graph of this.manifest!.app.graphs) {
|
|
155
171
|
const activities = graph.activities;
|
|
@@ -162,6 +178,20 @@ class Deployer {
|
|
|
162
178
|
}
|
|
163
179
|
}
|
|
164
180
|
|
|
181
|
+
//legacy; remove at beta (assume no legacy refs to 'activity' at that point)
|
|
182
|
+
convertActivitiesToHooks() {
|
|
183
|
+
for (const graph of this.manifest!.app.graphs) {
|
|
184
|
+
const activities = graph.activities;
|
|
185
|
+
for (const activityKey in activities) {
|
|
186
|
+
const activity = activities[activityKey];
|
|
187
|
+
if (['activity'].includes(activity.type)) {
|
|
188
|
+
activity.type = 'hook';
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
|
|
165
195
|
async bindParents() {
|
|
166
196
|
const graphs = this.manifest.app.graphs;
|
|
167
197
|
for (const graph of graphs) {
|
|
@@ -5,9 +5,10 @@ import { HotMeshService as HotMesh } from '../hotmesh';
|
|
|
5
5
|
import {
|
|
6
6
|
ClientConfig,
|
|
7
7
|
Connection,
|
|
8
|
-
WorkflowOptions
|
|
8
|
+
WorkflowOptions,
|
|
9
|
+
WorkflowSearchOptions} from '../../types/durable';
|
|
9
10
|
import { JobState } from '../../types/job';
|
|
10
|
-
import { KeyType } from '../../modules/key';
|
|
11
|
+
import { KeyService, KeyType } from '../../modules/key';
|
|
11
12
|
|
|
12
13
|
/*
|
|
13
14
|
Here is an example of how the methods in this file are used:
|
|
@@ -90,6 +91,41 @@ export class ClientService {
|
|
|
90
91
|
return hotMeshClient;
|
|
91
92
|
}
|
|
92
93
|
|
|
94
|
+
/**
|
|
95
|
+
* For those deployments with a redis stack backend (with the FT module),
|
|
96
|
+
* this method will configure the search index for the workflow.
|
|
97
|
+
*/
|
|
98
|
+
configureSearchIndex = async (hotMeshClient: HotMesh, search?: WorkflowSearchOptions): Promise<void> => {
|
|
99
|
+
if (search) {
|
|
100
|
+
const store = hotMeshClient.engine.store;
|
|
101
|
+
const schema: string[] = [];
|
|
102
|
+
for (const [key, value] of Object.entries(search.schema)) {
|
|
103
|
+
//prefix with a comma (avoids collisions with hotmesh reserved words)
|
|
104
|
+
schema.push(`_${key}`);
|
|
105
|
+
schema.push(value.type);
|
|
106
|
+
if (value.sortable) {
|
|
107
|
+
schema.push('SORTABLE');
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
try {
|
|
111
|
+
const keyParams = {
|
|
112
|
+
appId: hotMeshClient.appId,
|
|
113
|
+
jobId: ''
|
|
114
|
+
}
|
|
115
|
+
const hotMeshPrefix = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
|
|
116
|
+
const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
|
|
117
|
+
await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length, ...prefixes, 'SCHEMA', ...schema);
|
|
118
|
+
} catch (err) {
|
|
119
|
+
hotMeshClient.engine.logger.info('durable-client-search-err', { err });
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
search = async (hotMeshClient: HotMesh, index: string, query: string[]): Promise<string[]> => {
|
|
125
|
+
const store = hotMeshClient.engine.store;
|
|
126
|
+
return await store.exec('FT.SEARCH', index, ...query) as string[];
|
|
127
|
+
}
|
|
128
|
+
|
|
93
129
|
workflow = {
|
|
94
130
|
start: async (options: WorkflowOptions): Promise<WorkflowHandleService> => {
|
|
95
131
|
const taskQueueName = options.taskQueue;
|
|
@@ -99,6 +135,7 @@ export class ClientService {
|
|
|
99
135
|
//topic is concat of taskQueue and workflowName
|
|
100
136
|
const workflowTopic = `${taskQueueName}-${workflowName}`;
|
|
101
137
|
const hotMeshClient = await this.getHotMeshClient(workflowTopic);
|
|
138
|
+
this.configureSearchIndex(hotMeshClient, options.search)
|
|
102
139
|
const payload = {
|
|
103
140
|
arguments: [...options.args],
|
|
104
141
|
parentWorkflowId: options.parentWorkflowId,
|
|
@@ -122,6 +159,17 @@ export class ClientService {
|
|
|
122
159
|
const workflowTopic = `${taskQueue}-${workflowName}`;
|
|
123
160
|
const hotMeshClient = await this.getHotMeshClient(workflowTopic);
|
|
124
161
|
return new WorkflowHandleService(hotMeshClient, workflowTopic, workflowId);
|
|
162
|
+
},
|
|
163
|
+
|
|
164
|
+
search: async (taskQueue: string, workflowName: string, index: string, ...query: string[]): Promise<string[]> => {
|
|
165
|
+
const workflowTopic = `${taskQueue}-${workflowName}`;
|
|
166
|
+
const hotMeshClient = await this.getHotMeshClient(workflowTopic);
|
|
167
|
+
try {
|
|
168
|
+
return await this.search(hotMeshClient, index, query);
|
|
169
|
+
} catch (err) {
|
|
170
|
+
hotMeshClient.engine.logger.error('durable-client-search-err', { err });
|
|
171
|
+
throw err;
|
|
172
|
+
}
|
|
125
173
|
}
|
|
126
174
|
}
|
|
127
175
|
|
|
@@ -58,7 +58,7 @@ const getWorkflowYAML = (app: string, version: string) => {
|
|
|
58
58
|
done: false
|
|
59
59
|
|
|
60
60
|
a1:
|
|
61
|
-
type:
|
|
61
|
+
type: hook
|
|
62
62
|
cycle: true
|
|
63
63
|
output:
|
|
64
64
|
schema:
|
|
@@ -131,7 +131,7 @@ const getWorkflowYAML = (app: string, version: string) => {
|
|
|
131
131
|
done: '{$self.output.data.done}'
|
|
132
132
|
|
|
133
133
|
a2:
|
|
134
|
-
type:
|
|
134
|
+
type: hook
|
|
135
135
|
title: Wait for cleanup signal
|
|
136
136
|
hook:
|
|
137
137
|
type: object
|
|
@@ -247,7 +247,7 @@ const getWorkflowYAML = (app: string, version: string) => {
|
|
|
247
247
|
|
|
248
248
|
a599:
|
|
249
249
|
title: Sleep exponentially longer before retrying
|
|
250
|
-
type:
|
|
250
|
+
type: hook
|
|
251
251
|
sleep: '{a1.output.data.duration}'
|
|
252
252
|
|
|
253
253
|
c599:
|
|
@@ -539,7 +539,7 @@ const getWorkflowYAML = (app: string, version: string) => {
|
|
|
539
539
|
done: true
|
|
540
540
|
|
|
541
541
|
s1a:
|
|
542
|
-
type:
|
|
542
|
+
type: hook
|
|
543
543
|
title: Wait for cleanup signal
|
|
544
544
|
hook:
|
|
545
545
|
type: object
|
|
@@ -559,6 +559,7 @@ const getWorkflowYAML = (app: string, version: string) => {
|
|
|
559
559
|
hooks:
|
|
560
560
|
${app}.activity.awaken:
|
|
561
561
|
- to: s1a
|
|
562
|
+
keep_alive: true
|
|
562
563
|
conditions:
|
|
563
564
|
match:
|
|
564
565
|
- expected: '{t1a.output.data.workflowId}'
|
|
@@ -605,13 +606,13 @@ const getWorkflowYAML = (app: string, version: string) => {
|
|
|
605
606
|
target: '{$self.input.data.parentWorkflowId}'
|
|
606
607
|
|
|
607
608
|
a1s:
|
|
608
|
-
type:
|
|
609
|
+
type: hook
|
|
609
610
|
title: Sleep for a duration
|
|
610
611
|
sleep: '{t1s.output.data.duration}'
|
|
611
612
|
emit: true
|
|
612
613
|
|
|
613
614
|
a2s:
|
|
614
|
-
type:
|
|
615
|
+
type: hook
|
|
615
616
|
title: Wait for cleanup signal
|
|
616
617
|
hook:
|
|
617
618
|
type: object
|
|
@@ -683,7 +684,7 @@ const getWorkflowYAML = (app: string, version: string) => {
|
|
|
683
684
|
|
|
684
685
|
a1wc:
|
|
685
686
|
title: Split signal data
|
|
686
|
-
type:
|
|
687
|
+
type: hook
|
|
687
688
|
cycle: true
|
|
688
689
|
output:
|
|
689
690
|
schema:
|
|
@@ -721,7 +722,7 @@ const getWorkflowYAML = (app: string, version: string) => {
|
|
|
721
722
|
- ['{t1wc.output.data.signals}', 1]
|
|
722
723
|
- ['{@array.slice}']
|
|
723
724
|
a2wc:
|
|
724
|
-
type:
|
|
725
|
+
type: hook
|
|
725
726
|
output:
|
|
726
727
|
schema:
|
|
727
728
|
type: object
|
|
@@ -848,7 +849,7 @@ const getWorkflowYAML = (app: string, version: string) => {
|
|
|
848
849
|
target: '{$self.input.data.parentWorkflowId}'
|
|
849
850
|
|
|
850
851
|
a1ww:
|
|
851
|
-
type:
|
|
852
|
+
type: hook
|
|
852
853
|
title: Wait for custom signal
|
|
853
854
|
emit: true
|
|
854
855
|
hook:
|
|
@@ -863,7 +864,7 @@ const getWorkflowYAML = (app: string, version: string) => {
|
|
|
863
864
|
signalId: '{t1ww.output.data.signalId}'
|
|
864
865
|
|
|
865
866
|
a2ww:
|
|
866
|
-
type:
|
|
867
|
+
type: hook
|
|
867
868
|
title: Wait for cleanup signal
|
|
868
869
|
hook:
|
|
869
870
|
type: object
|