@hotmeshio/hotmesh 0.0.17 → 0.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/modules/utils.d.ts +3 -0
- package/build/modules/utils.js +17 -1
- package/build/package.json +1 -1
- package/build/services/activities/activity.d.ts +4 -12
- package/build/services/activities/activity.js +14 -156
- package/build/services/activities/hook.d.ts +20 -0
- package/build/services/activities/hook.js +124 -0
- package/build/services/activities/index.d.ts +2 -0
- package/build/services/activities/index.js +2 -0
- package/build/services/collator/index.js +0 -1
- package/build/services/compiler/deployer.d.ts +2 -0
- package/build/services/compiler/deployer.js +29 -2
- package/build/services/durable/client.d.ts +8 -1
- package/build/services/durable/client.js +54 -40
- package/build/services/durable/factory.js +11 -10
- package/build/services/durable/search.d.ts +15 -0
- package/build/services/durable/search.js +45 -0
- package/build/services/durable/worker.d.ts +6 -1
- package/build/services/durable/worker.js +34 -30
- package/build/services/durable/workflow.d.ts +2 -0
- package/build/services/durable/workflow.js +11 -28
- package/build/services/engine/index.d.ts +7 -2
- package/build/services/engine/index.js +2 -1
- package/build/services/store/clients/ioredis.d.ts +1 -0
- package/build/services/store/clients/ioredis.js +12 -0
- package/build/services/store/clients/redis.d.ts +1 -0
- package/build/services/store/clients/redis.js +3 -0
- package/build/services/store/index.d.ts +1 -0
- package/build/services/telemetry/index.js +2 -1
- package/build/types/activity.d.ts +6 -3
- package/build/types/durable.d.ts +12 -1
- package/build/types/hook.d.ts +1 -0
- package/build/types/index.d.ts +2 -2
- package/modules/utils.ts +17 -0
- package/package.json +1 -1
- package/services/activities/activity.ts +15 -167
- package/services/activities/hook.ts +149 -0
- package/services/activities/index.ts +2 -0
- package/services/collator/index.ts +0 -1
- package/services/compiler/deployer.ts +32 -2
- package/services/durable/client.ts +58 -43
- package/services/durable/factory.ts +11 -10
- package/services/durable/search.ts +54 -0
- package/services/durable/worker.ts +36 -32
- package/services/durable/workflow.ts +14 -30
- package/services/engine/index.ts +8 -4
- package/services/store/clients/ioredis.ts +13 -0
- package/services/store/clients/redis.ts +4 -0
- package/services/store/index.ts +1 -0
- package/services/telemetry/index.ts +2 -1
- package/types/activity.ts +7 -2
- package/types/durable.ts +10 -0
- package/types/hook.ts +1 -0
- package/types/index.ts +2 -0
package/modules/utils.ts
CHANGED
|
@@ -9,6 +9,12 @@ export async function sleepFor(ms: number) {
|
|
|
9
9
|
}
|
|
10
10
|
|
|
11
11
|
export function identifyRedisType(redisInstance: any): 'redis' | 'ioredis' | null {
|
|
12
|
+
const prototype = Object.getPrototypeOf(redisInstance);
|
|
13
|
+
if ('defineCommand' in prototype || Object.keys(prototype).includes('multi')) {
|
|
14
|
+
return 'ioredis';
|
|
15
|
+
} else if (Object.keys(prototype).includes('Multi')) {
|
|
16
|
+
return 'redis';
|
|
17
|
+
}
|
|
12
18
|
if (redisInstance.constructor) {
|
|
13
19
|
if (redisInstance.constructor.name === 'Redis' || redisInstance.constructor.name === 'EventEmitter') {
|
|
14
20
|
if ('hset' in redisInstance) {
|
|
@@ -23,6 +29,17 @@ export function identifyRedisType(redisInstance: any): 'redis' | 'ioredis' | nul
|
|
|
23
29
|
return null;
|
|
24
30
|
}
|
|
25
31
|
|
|
32
|
+
//todo: the polyfill methods will all be deleted in the `beta` release.
|
|
33
|
+
export const polyfill = {
|
|
34
|
+
resolveActivityType(activityType: string): string {
|
|
35
|
+
if (activityType === 'activity') {
|
|
36
|
+
return 'hook';
|
|
37
|
+
}
|
|
38
|
+
return activityType;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
|
|
26
43
|
export function identifyRedisTypeFromClass(redisClass: any): 'redis' | 'ioredis' | null {
|
|
27
44
|
if (redisClass && redisClass.name === 'Redis' || redisClass.name === 'EventEmitter') {
|
|
28
45
|
return 'ioredis';
|
package/package.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CollationError
|
|
1
|
+
import { CollationError } from '../../modules/errors';
|
|
2
2
|
import {
|
|
3
3
|
formatISODate,
|
|
4
4
|
getValueByPath,
|
|
@@ -9,7 +9,6 @@ import { ILogger } from '../logger';
|
|
|
9
9
|
import { MapperService } from '../mapper';
|
|
10
10
|
import { Pipe } from '../pipe';
|
|
11
11
|
import { MDATA_SYMBOLS } from '../serializer';
|
|
12
|
-
import { StoreSignaler } from '../signaler/store';
|
|
13
12
|
import { StoreService } from '../store';
|
|
14
13
|
import { TelemetryService } from '../telemetry';
|
|
15
14
|
import {
|
|
@@ -30,7 +29,6 @@ import {
|
|
|
30
29
|
StreamDataType,
|
|
31
30
|
StreamStatus } from '../../types/stream';
|
|
32
31
|
import { TransitionRule } from '../../types/transition';
|
|
33
|
-
import { HookRule } from '../../types/hook';
|
|
34
32
|
|
|
35
33
|
/**
|
|
36
34
|
* The base class for all activities
|
|
@@ -48,7 +46,7 @@ class Activity {
|
|
|
48
46
|
code: StreamCode = 200;
|
|
49
47
|
leg: ActivityLeg;
|
|
50
48
|
adjacencyList: StreamData[];
|
|
51
|
-
adjacentIndex = 0;
|
|
49
|
+
adjacentIndex = 0;
|
|
52
50
|
|
|
53
51
|
constructor(
|
|
54
52
|
config: ActivityType,
|
|
@@ -67,171 +65,21 @@ class Activity {
|
|
|
67
65
|
this.store = engine.store;
|
|
68
66
|
}
|
|
69
67
|
|
|
70
|
-
//******** INITIAL ENTRY POINT (A) ********//
|
|
71
|
-
async process(): Promise<string> {
|
|
72
|
-
this.logger.debug('activity-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
73
|
-
let telemetry: TelemetryService;
|
|
74
|
-
try {
|
|
75
|
-
this.setLeg(1);
|
|
76
|
-
await CollatorService.notarizeEntry(this);
|
|
77
|
-
|
|
78
|
-
await this.getState();
|
|
79
|
-
telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
80
|
-
telemetry.startActivitySpan(this.leg);
|
|
81
|
-
let multiResponse: MultiResponseFlags;
|
|
82
|
-
|
|
83
|
-
const multi = this.store.getMulti();
|
|
84
|
-
if (this.doesHook()) {
|
|
85
|
-
//sleep and wait to awaken upon a signal
|
|
86
|
-
await this.registerHook(multi);
|
|
87
|
-
this.mapOutputData();
|
|
88
|
-
this.mapJobData();
|
|
89
|
-
await this.setState(multi);
|
|
90
|
-
await CollatorService.authorizeReentry(this, multi);
|
|
91
|
-
|
|
92
|
-
await this.setStatus(0, multi);
|
|
93
|
-
await multi.exec();
|
|
94
|
-
telemetry.mapActivityAttributes();
|
|
95
|
-
} else {
|
|
96
|
-
//end the activity and transition to its children
|
|
97
|
-
this.adjacencyList = await this.filterAdjacent();
|
|
98
|
-
this.mapOutputData();
|
|
99
|
-
this.mapJobData();
|
|
100
|
-
await this.setState(multi);
|
|
101
|
-
await CollatorService.notarizeEarlyCompletion(this, multi);
|
|
102
|
-
|
|
103
|
-
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
104
|
-
multiResponse = await multi.exec() as MultiResponseFlags;
|
|
105
|
-
telemetry.mapActivityAttributes();
|
|
106
|
-
const jobStatus = this.resolveStatus(multiResponse);
|
|
107
|
-
const attrs: StringScalarType = { 'app.job.jss': jobStatus };
|
|
108
|
-
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
109
|
-
if (messageIds.length) {
|
|
110
|
-
attrs['app.activity.mids'] = messageIds.join(',')
|
|
111
|
-
}
|
|
112
|
-
telemetry.setActivityAttributes(attrs);
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
return this.context.metadata.aid;
|
|
116
|
-
} catch (error) {
|
|
117
|
-
if (error instanceof GetStateError) {
|
|
118
|
-
this.logger.error('activity-get-state-error', { error });
|
|
119
|
-
} else {
|
|
120
|
-
this.logger.error('activity-process-error', { error });
|
|
121
|
-
}
|
|
122
|
-
telemetry.setActivityError(error.message);
|
|
123
|
-
throw error;
|
|
124
|
-
} finally {
|
|
125
|
-
telemetry.endActivitySpan();
|
|
126
|
-
this.logger.debug('activity-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
|
|
130
68
|
setLeg(leg: ActivityLeg): void {
|
|
131
69
|
this.leg = leg;
|
|
132
70
|
}
|
|
133
71
|
|
|
134
|
-
//******** SIGNAL RE-ENTRY POINT ********//
|
|
135
|
-
doesHook(): boolean {
|
|
136
|
-
return !!(this.config.hook?.topic || this.config.sleep);
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
async getHookRule(topic: string): Promise<HookRule | undefined> {
|
|
140
|
-
const rules = await this.store.getHookRules();
|
|
141
|
-
return rules?.[topic]?.[0] as HookRule;
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
async registerHook(multi?: RedisMulti): Promise<string | void> {
|
|
145
|
-
if (this.config.hook?.topic) {
|
|
146
|
-
const signaler = new StoreSignaler(this.store, this.logger);
|
|
147
|
-
return await signaler.registerWebHook(this.config.hook.topic, this.context, multi);
|
|
148
|
-
} else if (this.config.sleep) {
|
|
149
|
-
const durationInSeconds = Pipe.resolve(this.config.sleep, this.context);
|
|
150
|
-
const jobId = this.context.metadata.jid;
|
|
151
|
-
const activityId = this.metadata.aid;
|
|
152
|
-
const dId = this.metadata.dad;
|
|
153
|
-
await this.engine.task.registerTimeHook(jobId, `${activityId}${dId||''}`, 'sleep', durationInSeconds);
|
|
154
|
-
return jobId;
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
async processWebHookEvent(): Promise<JobStatus | void> {
|
|
159
|
-
this.logger.debug('engine-process-web-hook-event', {
|
|
160
|
-
topic: this.config.hook.topic,
|
|
161
|
-
aid: this.metadata.aid
|
|
162
|
-
});
|
|
163
|
-
const signaler = new StoreSignaler(this.store, this.logger);
|
|
164
|
-
const data = { ...this.data };
|
|
165
|
-
const jobId = await signaler.processWebHookSignal(this.config.hook.topic, data);
|
|
166
|
-
if (jobId) {
|
|
167
|
-
await this.processHookEvent(jobId);
|
|
168
|
-
await signaler.deleteWebHookSignal(this.config.hook.topic, data);
|
|
169
|
-
} //else => already resolved
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
async processTimeHookEvent(jobId: string): Promise<JobStatus | void> {
|
|
173
|
-
this.logger.debug('engine-process-time-hook-event', {
|
|
174
|
-
jid: jobId,
|
|
175
|
-
aid: this.metadata.aid
|
|
176
|
-
});
|
|
177
|
-
return await this.processHookEvent(jobId);
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
async processHookEvent(jobId: string): Promise<JobStatus | void> {
|
|
181
|
-
this.logger.debug('activity-process-hook-event', { jobId });
|
|
182
|
-
let telemetry: TelemetryService;
|
|
183
|
-
try {
|
|
184
|
-
this.setLeg(2);
|
|
185
|
-
await this.getState(jobId);
|
|
186
|
-
telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
187
|
-
telemetry.startActivitySpan(this.leg);
|
|
188
|
-
const aState = await CollatorService.notarizeReentry(this);
|
|
189
|
-
this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
|
|
190
|
-
|
|
191
|
-
this.bindActivityData('hook');
|
|
192
|
-
this.mapJobData();
|
|
193
|
-
this.adjacencyList = await this.filterAdjacent();
|
|
194
|
-
|
|
195
|
-
const multi = this.engine.store.getMulti();
|
|
196
|
-
await this.setState(multi);
|
|
197
|
-
await CollatorService.notarizeCompletion(this, multi);
|
|
198
|
-
|
|
199
|
-
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
200
|
-
const multiResponse = await multi.exec() as MultiResponseFlags;
|
|
201
|
-
|
|
202
|
-
telemetry.mapActivityAttributes();
|
|
203
|
-
const jobStatus = this.resolveStatus(multiResponse);
|
|
204
|
-
const attrs: StringScalarType = { 'app.job.jss': jobStatus };
|
|
205
|
-
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
206
|
-
if (messageIds.length) {
|
|
207
|
-
attrs['app.activity.mids'] = messageIds.join(',')
|
|
208
|
-
}
|
|
209
|
-
telemetry.setActivityAttributes(attrs);
|
|
210
|
-
return jobStatus as number;
|
|
211
|
-
} catch (error) {
|
|
212
|
-
if (error instanceof CollationError && error.fault === 'inactive') {
|
|
213
|
-
this.logger.info('process-hook-event-inactive-error', { error });
|
|
214
|
-
return;
|
|
215
|
-
}
|
|
216
|
-
this.logger.error('engine-process-hook-event-error', { error });
|
|
217
|
-
telemetry.setActivityError(error.message);
|
|
218
|
-
throw error;
|
|
219
|
-
} finally {
|
|
220
|
-
telemetry.endActivitySpan();
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
|
|
224
72
|
//******** DUPLEX RE-ENTRY POINT ********//
|
|
225
|
-
async processEvent(status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200): Promise<void> {
|
|
73
|
+
async processEvent(status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200, type: 'hook' | 'output' = 'output', jobId?: string): Promise<void> {
|
|
226
74
|
this.setLeg(2);
|
|
227
|
-
const jid = this.context.metadata.jid;
|
|
75
|
+
const jid = this.context.metadata.jid || jobId;
|
|
228
76
|
const aid = this.metadata.aid;
|
|
229
77
|
this.status = status;
|
|
230
78
|
this.code = code;
|
|
231
79
|
this.logger.debug('activity-process-event', { topic: this.config.subtype, jid, aid, status, code });
|
|
232
80
|
let telemetry: TelemetryService;
|
|
233
81
|
try {
|
|
234
|
-
await this.getState();
|
|
82
|
+
await this.getState(jobId);
|
|
235
83
|
const aState = await CollatorService.notarizeReentry(this);
|
|
236
84
|
this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
|
|
237
85
|
|
|
@@ -247,18 +95,19 @@ class Activity {
|
|
|
247
95
|
telemetry.startActivitySpan(this.leg);
|
|
248
96
|
let multiResponse: MultiResponseFlags;
|
|
249
97
|
if (status === StreamStatus.PENDING) {
|
|
250
|
-
multiResponse = await this.processPending(telemetry);
|
|
98
|
+
multiResponse = await this.processPending(telemetry, type);
|
|
251
99
|
} else if (status === StreamStatus.SUCCESS) {
|
|
252
|
-
multiResponse = await this.processSuccess(telemetry);
|
|
100
|
+
multiResponse = await this.processSuccess(telemetry, type);
|
|
253
101
|
} else {
|
|
254
|
-
multiResponse = await this.processError(telemetry);
|
|
102
|
+
multiResponse = await this.processError(telemetry, type);
|
|
255
103
|
}
|
|
256
104
|
this.transitionAdjacent(multiResponse, telemetry);
|
|
257
105
|
} catch (error) {
|
|
258
|
-
if (error instanceof CollationError
|
|
106
|
+
if (error instanceof CollationError) {
|
|
259
107
|
this.logger.info('process-event-inactive-error', { error });
|
|
260
108
|
return;
|
|
261
109
|
}
|
|
110
|
+
console.error(error);
|
|
262
111
|
this.logger.error('activity-process-event-error', { error });
|
|
263
112
|
telemetry && telemetry.setActivityError(error.message);
|
|
264
113
|
throw error;
|
|
@@ -268,8 +117,8 @@ class Activity {
|
|
|
268
117
|
}
|
|
269
118
|
}
|
|
270
119
|
|
|
271
|
-
async processPending(telemetry: TelemetryService): Promise<MultiResponseFlags> {
|
|
272
|
-
this.bindActivityData(
|
|
120
|
+
async processPending(telemetry: TelemetryService, type: 'hook' | 'output'): Promise<MultiResponseFlags> {
|
|
121
|
+
this.bindActivityData(type);
|
|
273
122
|
this.adjacencyList = await this.filterAdjacent();
|
|
274
123
|
this.mapJobData();
|
|
275
124
|
const multi = this.store.getMulti();
|
|
@@ -280,8 +129,8 @@ class Activity {
|
|
|
280
129
|
return await multi.exec() as MultiResponseFlags;
|
|
281
130
|
}
|
|
282
131
|
|
|
283
|
-
async processSuccess(telemetry: TelemetryService): Promise<MultiResponseFlags> {
|
|
284
|
-
this.bindActivityData(
|
|
132
|
+
async processSuccess(telemetry: TelemetryService, type: 'hook' | 'output'): Promise<MultiResponseFlags> {
|
|
133
|
+
this.bindActivityData(type);
|
|
285
134
|
this.adjacencyList = await this.filterAdjacent();
|
|
286
135
|
this.mapJobData();
|
|
287
136
|
const multi = this.store.getMulti();
|
|
@@ -292,7 +141,7 @@ class Activity {
|
|
|
292
141
|
return await multi.exec() as MultiResponseFlags;
|
|
293
142
|
}
|
|
294
143
|
|
|
295
|
-
async processError(telemetry: TelemetryService): Promise<MultiResponseFlags> {
|
|
144
|
+
async processError(telemetry: TelemetryService, type: string): Promise<MultiResponseFlags> {
|
|
296
145
|
this.bindActivityError(this.data);
|
|
297
146
|
this.adjacencyList = await this.filterAdjacent();
|
|
298
147
|
const multi = this.store.getMulti();
|
|
@@ -584,7 +433,6 @@ class Activity {
|
|
|
584
433
|
|
|
585
434
|
async transition(adjacencyList: StreamData[], jobStatus: JobStatus): Promise<string[]> {
|
|
586
435
|
let mIds: string[] = [];
|
|
587
|
-
//emit can be a mapping (allows emissions to be driven by the job state)
|
|
588
436
|
let emit: boolean = false;
|
|
589
437
|
if (this.config.emit) {
|
|
590
438
|
emit = Pipe.resolve(this.config.emit, this.context);
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { GetStateError } from '../../modules/errors';
|
|
2
|
+
import { Activity } from './activity';
|
|
3
|
+
import { CollatorService } from '../collator';
|
|
4
|
+
import { EngineService } from '../engine';
|
|
5
|
+
import { Pipe } from '../pipe';
|
|
6
|
+
import { StoreSignaler } from '../signaler/store';
|
|
7
|
+
import { TelemetryService } from '../telemetry';
|
|
8
|
+
import {
|
|
9
|
+
ActivityData,
|
|
10
|
+
ActivityMetadata,
|
|
11
|
+
ActivityType,
|
|
12
|
+
HookActivity } from '../../types/activity';
|
|
13
|
+
import { HookRule } from '../../types/hook';
|
|
14
|
+
import { JobState, JobStatus } from '../../types/job';
|
|
15
|
+
import {
|
|
16
|
+
MultiResponseFlags,
|
|
17
|
+
RedisMulti } from '../../types/redis';
|
|
18
|
+
import { StringScalarType } from '../../types/serializer';
|
|
19
|
+
import { StreamStatus } from '../../types/stream';
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Listens for `webhook`, `timehook`, and `cycle` (repeat) signals
|
|
23
|
+
*/
|
|
24
|
+
class Hook extends Activity {
|
|
25
|
+
config: HookActivity;
|
|
26
|
+
|
|
27
|
+
constructor(
|
|
28
|
+
config: ActivityType,
|
|
29
|
+
data: ActivityData,
|
|
30
|
+
metadata: ActivityMetadata,
|
|
31
|
+
hook: ActivityData | null,
|
|
32
|
+
engine: EngineService,
|
|
33
|
+
context?: JobState) {
|
|
34
|
+
super(config, data, metadata, hook, engine, context);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
//******** INITIAL ENTRY POINT (A) ********//
|
|
38
|
+
async process(): Promise<string> {
|
|
39
|
+
this.logger.debug('hook-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
40
|
+
let telemetry: TelemetryService;
|
|
41
|
+
try {
|
|
42
|
+
this.setLeg(1);
|
|
43
|
+
await CollatorService.notarizeEntry(this);
|
|
44
|
+
|
|
45
|
+
await this.getState();
|
|
46
|
+
telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
47
|
+
telemetry.startActivitySpan(this.leg);
|
|
48
|
+
let multiResponse: MultiResponseFlags;
|
|
49
|
+
|
|
50
|
+
const multi = this.store.getMulti();
|
|
51
|
+
if (this.doesHook()) {
|
|
52
|
+
//sleep and wait to awaken upon a signal
|
|
53
|
+
await this.registerHook(multi);
|
|
54
|
+
this.mapOutputData();
|
|
55
|
+
this.mapJobData();
|
|
56
|
+
await this.setState(multi);
|
|
57
|
+
await CollatorService.authorizeReentry(this, multi);
|
|
58
|
+
|
|
59
|
+
await this.setStatus(0, multi);
|
|
60
|
+
await multi.exec();
|
|
61
|
+
telemetry.mapActivityAttributes();
|
|
62
|
+
} else {
|
|
63
|
+
//end the activity and transition to its children
|
|
64
|
+
this.adjacencyList = await this.filterAdjacent();
|
|
65
|
+
this.mapOutputData();
|
|
66
|
+
this.mapJobData();
|
|
67
|
+
await this.setState(multi);
|
|
68
|
+
await CollatorService.notarizeEarlyCompletion(this, multi);
|
|
69
|
+
|
|
70
|
+
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
71
|
+
multiResponse = await multi.exec() as MultiResponseFlags;
|
|
72
|
+
telemetry.mapActivityAttributes();
|
|
73
|
+
const jobStatus = this.resolveStatus(multiResponse);
|
|
74
|
+
const attrs: StringScalarType = { 'app.job.jss': jobStatus };
|
|
75
|
+
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
76
|
+
if (messageIds.length) {
|
|
77
|
+
attrs['app.activity.mids'] = messageIds.join(',')
|
|
78
|
+
}
|
|
79
|
+
telemetry.setActivityAttributes(attrs);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
return this.context.metadata.aid;
|
|
83
|
+
} catch (error) {
|
|
84
|
+
if (error instanceof GetStateError) {
|
|
85
|
+
this.logger.error('hook-get-state-error', { error });
|
|
86
|
+
} else {
|
|
87
|
+
this.logger.error('hook-process-error', { error });
|
|
88
|
+
}
|
|
89
|
+
telemetry.setActivityError(error.message);
|
|
90
|
+
throw error;
|
|
91
|
+
} finally {
|
|
92
|
+
telemetry.endActivitySpan();
|
|
93
|
+
this.logger.debug('hook-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
//******** SIGNAL RE-ENTRY POINT ********//
|
|
98
|
+
doesHook(): boolean {
|
|
99
|
+
//does this activity use a time-hook or web-hook
|
|
100
|
+
return !!(this.config.hook?.topic || this.config.sleep);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async getHookRule(topic: string): Promise<HookRule | undefined> {
|
|
104
|
+
const rules = await this.store.getHookRules();
|
|
105
|
+
return rules?.[topic]?.[0] as HookRule;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
async registerHook(multi?: RedisMulti): Promise<string | void> {
|
|
109
|
+
if (this.config.hook?.topic) {
|
|
110
|
+
const signaler = new StoreSignaler(this.store, this.logger);
|
|
111
|
+
return await signaler.registerWebHook(this.config.hook.topic, this.context, multi);
|
|
112
|
+
} else if (this.config.sleep) {
|
|
113
|
+
const durationInSeconds = Pipe.resolve(this.config.sleep, this.context);
|
|
114
|
+
const jobId = this.context.metadata.jid;
|
|
115
|
+
const activityId = this.metadata.aid;
|
|
116
|
+
const dId = this.metadata.dad;
|
|
117
|
+
await this.engine.task.registerTimeHook(jobId, `${activityId}${dId||''}`, 'sleep', durationInSeconds);
|
|
118
|
+
return jobId;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async processWebHookEvent(): Promise<JobStatus | void> {
|
|
123
|
+
this.logger.debug('hook-process-web-hook-event', {
|
|
124
|
+
topic: this.config.hook.topic,
|
|
125
|
+
aid: this.metadata.aid
|
|
126
|
+
});
|
|
127
|
+
const signaler = new StoreSignaler(this.store, this.logger);
|
|
128
|
+
const data = { ...this.data };
|
|
129
|
+
const jobId = await signaler.processWebHookSignal(this.config.hook.topic, data);
|
|
130
|
+
if (jobId) {
|
|
131
|
+
//if a webhook signal is sent that includes 'keep_alive' the hook will remain open
|
|
132
|
+
const code = data.keep_alive ? 202 : 200;
|
|
133
|
+
await this.processEvent(StreamStatus.SUCCESS, code, 'hook', jobId);
|
|
134
|
+
if (code === 200) {
|
|
135
|
+
await signaler.deleteWebHookSignal(this.config.hook.topic, data);
|
|
136
|
+
}
|
|
137
|
+
} //else => already resolved
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
async processTimeHookEvent(jobId: string): Promise<JobStatus | void> {
|
|
141
|
+
this.logger.debug('hook-process-time-hook-event', {
|
|
142
|
+
jid: jobId,
|
|
143
|
+
aid: this.metadata.aid
|
|
144
|
+
});
|
|
145
|
+
await this.processEvent(StreamStatus.SUCCESS, 200, 'hook');
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export { Hook };
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Activity } from './activity';
|
|
2
2
|
import { Await } from './await';
|
|
3
3
|
import { Cycle } from './cycle';
|
|
4
|
+
import { Hook } from './hook';
|
|
4
5
|
import { Iterate } from './iterate';
|
|
5
6
|
import { Signal } from './signal';
|
|
6
7
|
import { Trigger } from './trigger';
|
|
@@ -10,6 +11,7 @@ export default {
|
|
|
10
11
|
activity: Activity,
|
|
11
12
|
await: Await,
|
|
12
13
|
cycle: Cycle,
|
|
14
|
+
hook: Hook,
|
|
13
15
|
iterate: Iterate,
|
|
14
16
|
signal: Signal,
|
|
15
17
|
trigger: Trigger,
|
|
@@ -54,7 +54,6 @@ class CollatorService {
|
|
|
54
54
|
//set second digit to 8, allowing for re-entry
|
|
55
55
|
//decrement by -10_000_000_000_000
|
|
56
56
|
const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -10_000_000_000_000, this.getDimensionalAddress(activity), multi);
|
|
57
|
-
//this.verifyInteger(amount, 1, 'exit');
|
|
58
57
|
return amount;
|
|
59
58
|
}
|
|
60
59
|
|
|
@@ -25,10 +25,12 @@ class Deployer {
|
|
|
25
25
|
async deploy(store: StoreService<RedisClient, RedisMulti>) {
|
|
26
26
|
this.store = store;
|
|
27
27
|
CollatorService.compile(this.manifest.app.graphs);
|
|
28
|
+
this.convertActivitiesToHooks();
|
|
28
29
|
this.convertTopicsToTypes();
|
|
29
30
|
this.copyJobSchemas();
|
|
30
31
|
this.bindBackRefs();
|
|
31
32
|
this.bindParents();
|
|
33
|
+
this.bindCycleTarget();
|
|
32
34
|
this.resolveMappingDependencies();
|
|
33
35
|
this.resolveJobMapsPaths();
|
|
34
36
|
await this.generateSymKeys();
|
|
@@ -148,8 +150,22 @@ class Deployer {
|
|
|
148
150
|
}
|
|
149
151
|
}
|
|
150
152
|
|
|
151
|
-
//
|
|
152
|
-
//
|
|
153
|
+
//the cycle/goto activity includes and ancestor target;
|
|
154
|
+
//update with the cycle flag, so it can be rerun
|
|
155
|
+
bindCycleTarget() {
|
|
156
|
+
for (const graph of this.manifest!.app.graphs) {
|
|
157
|
+
const activities = graph.activities;
|
|
158
|
+
for (const activityKey in activities) {
|
|
159
|
+
const activity = activities[activityKey];
|
|
160
|
+
if (activity.type === 'cycle') {
|
|
161
|
+
activities[activity.ancestor].cycle = true;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
//it's more intuitive for SDK users to use 'topic',
|
|
168
|
+
//but the compiler is desiged to be generic and uses the attribute, 'subtypes'
|
|
153
169
|
convertTopicsToTypes() {
|
|
154
170
|
for (const graph of this.manifest!.app.graphs) {
|
|
155
171
|
const activities = graph.activities;
|
|
@@ -162,6 +178,20 @@ class Deployer {
|
|
|
162
178
|
}
|
|
163
179
|
}
|
|
164
180
|
|
|
181
|
+
//legacy; remove at beta (assume no legacy refs to 'activity' at that point)
|
|
182
|
+
convertActivitiesToHooks() {
|
|
183
|
+
for (const graph of this.manifest!.app.graphs) {
|
|
184
|
+
const activities = graph.activities;
|
|
185
|
+
for (const activityKey in activities) {
|
|
186
|
+
const activity = activities[activityKey];
|
|
187
|
+
if (['activity'].includes(activity.type)) {
|
|
188
|
+
activity.type = 'hook';
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
|
|
165
195
|
async bindParents() {
|
|
166
196
|
const graphs = this.manifest.app.graphs;
|
|
167
197
|
for (const graph of graphs) {
|
|
@@ -5,50 +5,11 @@ import { HotMeshService as HotMesh } from '../hotmesh';
|
|
|
5
5
|
import {
|
|
6
6
|
ClientConfig,
|
|
7
7
|
Connection,
|
|
8
|
-
WorkflowOptions
|
|
8
|
+
WorkflowOptions,
|
|
9
|
+
WorkflowSearchOptions} from '../../types/durable';
|
|
9
10
|
import { JobState } from '../../types/job';
|
|
10
|
-
import { KeyType } from '../../modules/key';
|
|
11
|
-
|
|
12
|
-
/*
|
|
13
|
-
Here is an example of how the methods in this file are used:
|
|
14
|
-
|
|
15
|
-
./client.ts
|
|
16
|
-
|
|
17
|
-
import { Durable } from '@hotmeshio/hotmesh';
|
|
18
|
-
import Redis from 'ioredis';
|
|
19
|
-
import { example } from './workflows';
|
|
20
|
-
import { nanoid } from 'nanoid';
|
|
21
|
-
|
|
22
|
-
async function run() {
|
|
23
|
-
const connection = await Durable.Connection.connect({
|
|
24
|
-
class: Redis,
|
|
25
|
-
options: {
|
|
26
|
-
host: 'localhost',
|
|
27
|
-
port: 6379,
|
|
28
|
-
},
|
|
29
|
-
});
|
|
30
|
-
|
|
31
|
-
const client = new Durable.Client({
|
|
32
|
-
connection,
|
|
33
|
-
});
|
|
34
|
-
|
|
35
|
-
const handle = await client.workflow.start({
|
|
36
|
-
args: ['HotMesh'],
|
|
37
|
-
taskQueue: 'hello-world',
|
|
38
|
-
workflowName: 'example',
|
|
39
|
-
workflowId: 'workflow-' + nanoid(),
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
console.log(`Started workflow ${handle.workflowId}`);
|
|
43
|
-
console.log(await handle.result());
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
run().catch((err) => {
|
|
47
|
-
console.error(err);
|
|
48
|
-
process.exit(1);
|
|
49
|
-
});
|
|
50
|
-
|
|
51
|
-
*/
|
|
11
|
+
import { KeyService, KeyType } from '../../modules/key';
|
|
12
|
+
import { Search } from './search';
|
|
52
13
|
|
|
53
14
|
export class ClientService {
|
|
54
15
|
|
|
@@ -90,6 +51,41 @@ export class ClientService {
|
|
|
90
51
|
return hotMeshClient;
|
|
91
52
|
}
|
|
92
53
|
|
|
54
|
+
/**
|
|
55
|
+
* For those deployments with a redis stack backend (with the FT module),
|
|
56
|
+
* this method will configure the search index for the workflow.
|
|
57
|
+
*/
|
|
58
|
+
configureSearchIndex = async (hotMeshClient: HotMesh, search?: WorkflowSearchOptions): Promise<void> => {
|
|
59
|
+
if (search?.schema) {
|
|
60
|
+
const store = hotMeshClient.engine.store;
|
|
61
|
+
const schema: string[] = [];
|
|
62
|
+
for (const [key, value] of Object.entries(search.schema)) {
|
|
63
|
+
//prefix with a comma (avoids collisions with hotmesh reserved words)
|
|
64
|
+
schema.push(`_${key}`);
|
|
65
|
+
schema.push(value.type);
|
|
66
|
+
if (value.sortable) {
|
|
67
|
+
schema.push('SORTABLE');
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
try {
|
|
71
|
+
const keyParams = {
|
|
72
|
+
appId: hotMeshClient.appId,
|
|
73
|
+
jobId: ''
|
|
74
|
+
}
|
|
75
|
+
const hotMeshPrefix = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
|
|
76
|
+
const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
|
|
77
|
+
await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length, ...prefixes, 'SCHEMA', ...schema);
|
|
78
|
+
} catch (err) {
|
|
79
|
+
hotMeshClient.engine.logger.info('durable-client-search-err', { err });
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
search = async (hotMeshClient: HotMesh, index: string, query: string[]): Promise<string[]> => {
|
|
85
|
+
const store = hotMeshClient.engine.store;
|
|
86
|
+
return await store.exec('FT.SEARCH', index, ...query) as string[];
|
|
87
|
+
}
|
|
88
|
+
|
|
93
89
|
workflow = {
|
|
94
90
|
start: async (options: WorkflowOptions): Promise<WorkflowHandleService> => {
|
|
95
91
|
const taskQueueName = options.taskQueue;
|
|
@@ -99,6 +95,7 @@ export class ClientService {
|
|
|
99
95
|
//topic is concat of taskQueue and workflowName
|
|
100
96
|
const workflowTopic = `${taskQueueName}-${workflowName}`;
|
|
101
97
|
const hotMeshClient = await this.getHotMeshClient(workflowTopic);
|
|
98
|
+
this.configureSearchIndex(hotMeshClient, options.search)
|
|
102
99
|
const payload = {
|
|
103
100
|
arguments: [...options.args],
|
|
104
101
|
parentWorkflowId: options.parentWorkflowId,
|
|
@@ -111,6 +108,13 @@ export class ClientService {
|
|
|
111
108
|
SUBSCRIBES_TOPIC,
|
|
112
109
|
payload,
|
|
113
110
|
context as JobState);
|
|
111
|
+
if (jobId && options.search?.data) {
|
|
112
|
+
//job successfully kicked off; there is default job data to persist
|
|
113
|
+
const search = new Search(jobId, hotMeshClient);
|
|
114
|
+
for (const [key, value] of Object.entries(options.search.data)) {
|
|
115
|
+
search.set(key, value);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
114
118
|
return new WorkflowHandleService(hotMeshClient, workflowTopic, jobId);
|
|
115
119
|
},
|
|
116
120
|
|
|
@@ -122,6 +126,17 @@ export class ClientService {
|
|
|
122
126
|
const workflowTopic = `${taskQueue}-${workflowName}`;
|
|
123
127
|
const hotMeshClient = await this.getHotMeshClient(workflowTopic);
|
|
124
128
|
return new WorkflowHandleService(hotMeshClient, workflowTopic, workflowId);
|
|
129
|
+
},
|
|
130
|
+
|
|
131
|
+
search: async (taskQueue: string, workflowName: string, index: string, ...query: string[]): Promise<string[]> => {
|
|
132
|
+
const workflowTopic = `${taskQueue}-${workflowName}`;
|
|
133
|
+
const hotMeshClient = await this.getHotMeshClient(workflowTopic);
|
|
134
|
+
try {
|
|
135
|
+
return await this.search(hotMeshClient, index, query);
|
|
136
|
+
} catch (err) {
|
|
137
|
+
hotMeshClient.engine.logger.error('durable-client-search-err', { err });
|
|
138
|
+
throw err;
|
|
139
|
+
}
|
|
125
140
|
}
|
|
126
141
|
}
|
|
127
142
|
|