@hotmeshio/hotmesh 0.0.23 → 0.0.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -67
- package/build/index.d.ts +2 -1
- package/build/index.js +3 -1
- package/build/package.json +2 -2
- package/build/services/durable/factory.js +6 -6
- package/build/services/durable/handle.js +2 -4
- package/build/services/durable/index.d.ts +2 -2
- package/build/services/durable/index.js +2 -2
- package/build/services/durable/meshos.d.ts +108 -0
- package/build/services/durable/meshos.js +289 -0
- package/build/services/durable/search.js +0 -1
- package/build/services/durable/worker.d.ts +1 -1
- package/build/services/durable/worker.js +8 -4
- package/build/services/durable/workflow.d.ts +4 -0
- package/build/services/durable/workflow.js +21 -9
- package/build/services/signaler/stream.js +1 -2
- package/build/services/store/clients/ioredis.js +2 -2
- package/build/services/store/clients/redis.js +1 -1
- package/build/types/durable.d.ts +19 -5
- package/build/types/index.d.ts +1 -1
- package/index.ts +2 -1
- package/package.json +2 -2
- package/services/durable/factory.ts +6 -6
- package/services/durable/handle.ts +2 -4
- package/services/durable/index.ts +2 -2
- package/services/durable/meshos.ts +344 -0
- package/services/durable/search.ts +0 -1
- package/services/durable/worker.ts +8 -5
- package/services/durable/workflow.ts +23 -10
- package/services/signaler/stream.ts +1 -2
- package/services/store/clients/ioredis.ts +2 -3
- package/services/store/clients/redis.ts +1 -1
- package/types/durable.ts +26 -6
- package/types/index.ts +6 -2
- package/build/services/durable/meshdb.d.ts +0 -113
- package/build/services/durable/meshdb.js +0 -211
- package/services/durable/meshdb.ts +0 -254
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ClientService } from './client';
|
|
2
2
|
import { ConnectionService } from './connection';
|
|
3
|
-
import {
|
|
3
|
+
import { MeshOSService } from './meshos';
|
|
4
4
|
import { WorkerService } from './worker';
|
|
5
5
|
import { WorkflowService } from './workflow';
|
|
6
6
|
import { ContextType } from '../../types/durable';
|
|
@@ -8,7 +8,7 @@ import { ContextType } from '../../types/durable';
|
|
|
8
8
|
export const Durable = {
|
|
9
9
|
Client: ClientService,
|
|
10
10
|
Connection: ConnectionService,
|
|
11
|
-
|
|
11
|
+
MeshOS: MeshOSService,
|
|
12
12
|
Worker: WorkerService,
|
|
13
13
|
workflow: WorkflowService,
|
|
14
14
|
};
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
import { nanoid } from 'nanoid';
|
|
2
|
+
|
|
3
|
+
import { ClientService as Client } from './client';
|
|
4
|
+
import { WorkflowHandleService } from './handle';
|
|
5
|
+
import { Search } from './search';
|
|
6
|
+
import { WorkerService as Worker } from './worker';
|
|
7
|
+
import { FindOptions, MeshOSActivityOptions, MeshOSOptions, WorkflowSearchOptions } from '../../types/durable';
|
|
8
|
+
import { RedisOptions, RedisClass } from '../../types/redis';
|
|
9
|
+
import { StringAnyType } from '../../types';
|
|
10
|
+
import { Durable } from '.';
|
|
11
|
+
import { asyncLocalStorage } from './asyncLocalStorage';
|
|
12
|
+
import { WorkflowService } from './workflow';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* The base class for running MeshOS workflows.
|
|
16
|
+
* Extend and register subclass methods by name to
|
|
17
|
+
* execute as durable workflows, backed by Redis.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
export class MeshOSService {
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* The top-level Redis isolation. All workflow data is
|
|
24
|
+
* isolated within this namespace. Values should be
|
|
25
|
+
* lower-case with no spaces (e.g, 'staging', 'prod', 'test',
|
|
26
|
+
* 'routing-stagig', 'reporting-prod', etc.).
|
|
27
|
+
* 1) only url-safe values are allowed;
|
|
28
|
+
* 2) the 'a' symbol is reserved by HotMesh for indexing apps
|
|
29
|
+
*/
|
|
30
|
+
namespace = 'durable';
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Data is routed to workers that specify this task queue.
|
|
34
|
+
* Setting the task queue when the worker is created will
|
|
35
|
+
* ensure that the worker only receives messages destined
|
|
36
|
+
* for the queue. Callers can specify the taskQue to when
|
|
37
|
+
* starting a job to call those workers.
|
|
38
|
+
*/
|
|
39
|
+
taskQueue = 'default';
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* These methods run as durable workflows
|
|
43
|
+
*/
|
|
44
|
+
workflowFunctions: Array<MeshOSOptions | string> = [];
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* These methods run as hooks (hook into a running workflow)
|
|
48
|
+
*/
|
|
49
|
+
hookFunctions: Array<MeshOSOptions | string> = [];
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* These methods run as proxied activities (and are safely memoized)
|
|
53
|
+
*/
|
|
54
|
+
proxyFunctions: Array<MeshOSActivityOptions | string> = [];
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* The workflow GUID. Workflows will be persisted to
|
|
58
|
+
* Redis using the pattern hmsh:<namespace>:j:<id>.
|
|
59
|
+
*/
|
|
60
|
+
id: string;
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* The Redis connection options. NOTE: Redis and IORedis
|
|
64
|
+
* use different formats for their connection config.
|
|
65
|
+
*/
|
|
66
|
+
redisOptions: RedisOptions = {
|
|
67
|
+
host: 'localhost',
|
|
68
|
+
port: 6379,
|
|
69
|
+
password: '',
|
|
70
|
+
db: 0,
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* The Redis connection class.
|
|
75
|
+
*
|
|
76
|
+
* @example
|
|
77
|
+
* import Redis from 'ioredis';
|
|
78
|
+
* import * as Redis from 'redis';
|
|
79
|
+
*/
|
|
80
|
+
redisClass: RedisClass;
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Optional model declaration (custom workflow state)
|
|
84
|
+
*/
|
|
85
|
+
model: StringAnyType;
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Optional configuration for Redis FT search
|
|
89
|
+
*/
|
|
90
|
+
search: WorkflowSearchOptions;
|
|
91
|
+
|
|
92
|
+
static MeshOS = WorkflowService
|
|
93
|
+
|
|
94
|
+
static async getHotMeshClient (redisClass: RedisClass, redisOptions: RedisOptions, namespace: string, taskQueue: string) {
|
|
95
|
+
const client = new Client({
|
|
96
|
+
connection: {
|
|
97
|
+
class: redisClass,
|
|
98
|
+
options: redisOptions,
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
return await client.getHotMeshClient(taskQueue, namespace);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* mints a workflow ID, using the search prefix.
|
|
106
|
+
* NOTE: The prefix is necesary when indexing
|
|
107
|
+
* HASHes when FT search is enabled.
|
|
108
|
+
* @returns {string}
|
|
109
|
+
*/
|
|
110
|
+
static mintGuid(): string {
|
|
111
|
+
const my = new this();
|
|
112
|
+
return `${my.search?.prefix?.[0]}${nanoid()}}`;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Creates an FT search index
|
|
117
|
+
*/
|
|
118
|
+
static async createIndex() {
|
|
119
|
+
const my = new this();
|
|
120
|
+
const hmClient = await MeshOSService.getHotMeshClient(my.redisClass, my.redisOptions, my.namespace, my.taskQueue);
|
|
121
|
+
Search.configureSearchIndex(hmClient, my.search)
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Initialize the worker(s) for the entity. This is a static
|
|
126
|
+
* method that allows for optional task Queue targeting.
|
|
127
|
+
* NOTE: Allow List may be optionally used to only wrap
|
|
128
|
+
* specific methods in this class.
|
|
129
|
+
* @param {string} taskQueue
|
|
130
|
+
* @param {string[]} allowList
|
|
131
|
+
*/
|
|
132
|
+
static async startWorkers(taskQueue?: string, allowList: Array<MeshOSOptions | string> = []) {
|
|
133
|
+
const my = new this();
|
|
134
|
+
|
|
135
|
+
//helper functions
|
|
136
|
+
const resolveFunctionNames = (arr: any[]) => arr.map(item => typeof item === 'string' ? item : item.name);
|
|
137
|
+
const belongsTo = (name: string, target: Array<MeshOSOptions | MeshOSActivityOptions | string>): boolean => {
|
|
138
|
+
const isWorkflow = target.find((item: MeshOSOptions | string) => {
|
|
139
|
+
return typeof item === 'string' ? item === name : item.name === name;
|
|
140
|
+
});
|
|
141
|
+
return isWorkflow !== undefined;
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
// proxy registered activities
|
|
145
|
+
const proxyFunctionNames = resolveFunctionNames([...my.proxyFunctions]);
|
|
146
|
+
if (proxyFunctionNames.length) {
|
|
147
|
+
const proxyActivities = proxyFunctionNames.reduce((acc, funcName) => {
|
|
148
|
+
let originalMethod = my[funcName];
|
|
149
|
+
if (typeof originalMethod === 'function') {
|
|
150
|
+
acc[funcName] = async (...args: any[]) => {
|
|
151
|
+
return await originalMethod.apply(my, args);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return acc;
|
|
155
|
+
}, {});
|
|
156
|
+
const proxiedActivities = Durable.workflow.proxyActivities({
|
|
157
|
+
activities: proxyActivities
|
|
158
|
+
});
|
|
159
|
+
//WATCH!: unsure if this will pollute the scope; don't think
|
|
160
|
+
// so as activity functions are terminal in the chain.
|
|
161
|
+
Object.assign(my, proxiedActivities);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const functionsToIterate = allowList.length ? resolveFunctionNames(allowList) : resolveFunctionNames([...my.workflowFunctions, ...my.hookFunctions]);
|
|
165
|
+
|
|
166
|
+
// Iterating through the functions sequentially
|
|
167
|
+
for (const funcName of functionsToIterate) {
|
|
168
|
+
const originalMethod = my[funcName];
|
|
169
|
+
if (typeof originalMethod === 'function') {
|
|
170
|
+
|
|
171
|
+
//wrap the function to return
|
|
172
|
+
const wrappedFunction = {
|
|
173
|
+
[funcName]: async (...args: any[]) => {
|
|
174
|
+
const store = asyncLocalStorage.getStore();
|
|
175
|
+
const workflowId = store.get('workflowId');
|
|
176
|
+
|
|
177
|
+
//use a Proxy to wrap hook methods
|
|
178
|
+
const context = new Proxy(my, {
|
|
179
|
+
get: (target, prop, receiver) => {
|
|
180
|
+
if (prop === 'id') {
|
|
181
|
+
return workflowId;
|
|
182
|
+
} else if (typeof target[prop] === 'function') {
|
|
183
|
+
return (...args: any[]) => {
|
|
184
|
+
return new Promise(async (resolve, reject) => {
|
|
185
|
+
if (belongsTo(prop as string, my.hookFunctions)) {
|
|
186
|
+
return WorkflowService.hook({
|
|
187
|
+
namespace: my.namespace,
|
|
188
|
+
taskQueue: my.taskQueue,
|
|
189
|
+
workflowName: prop as string,
|
|
190
|
+
workflowId,
|
|
191
|
+
args,
|
|
192
|
+
}).then(resolve).catch(reject);
|
|
193
|
+
}
|
|
194
|
+
//otherwise, call the method as a standard instance method.
|
|
195
|
+
target[prop].apply(this, args).then(resolve).catch(reject);
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return Reflect.get(target, prop, receiver);
|
|
200
|
+
},
|
|
201
|
+
});
|
|
202
|
+
return await originalMethod.apply(context, args);
|
|
203
|
+
}
|
|
204
|
+
};
|
|
205
|
+
|
|
206
|
+
//start the worker
|
|
207
|
+
await Worker.create({
|
|
208
|
+
namespace: my.namespace,
|
|
209
|
+
connection: {
|
|
210
|
+
class: my.redisClass,
|
|
211
|
+
options: my.redisOptions,
|
|
212
|
+
},
|
|
213
|
+
taskQueue: taskQueue ?? my.taskQueue,
|
|
214
|
+
workflow: wrappedFunction,
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* executes the redis FT search query
|
|
222
|
+
* @example '@_quantity:[89 89]'
|
|
223
|
+
* @param {any[]} args
|
|
224
|
+
* @returns {string}
|
|
225
|
+
*/
|
|
226
|
+
static async find(options: FindOptions, ...args: string[]): Promise<string[] | [number]> {
|
|
227
|
+
const my = new this();
|
|
228
|
+
const client = new Client({ connection: {
|
|
229
|
+
class: my.redisClass,
|
|
230
|
+
options: my.redisOptions
|
|
231
|
+
}});
|
|
232
|
+
//workflow name is the function name driving the workflow
|
|
233
|
+
let workflowName: string;
|
|
234
|
+
if (options?.workflowName) {
|
|
235
|
+
workflowName = options?.workflowName
|
|
236
|
+
} else if(my.workflowFunctions?.length) {
|
|
237
|
+
let target = my.workflowFunctions[0];
|
|
238
|
+
if (typeof target === 'string') {
|
|
239
|
+
workflowName = target;
|
|
240
|
+
} else {
|
|
241
|
+
workflowName = target.name;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
return await client.workflow.search(
|
|
245
|
+
options?.taskQueue ?? my.taskQueue,
|
|
246
|
+
workflowName,
|
|
247
|
+
my.namespace,
|
|
248
|
+
my.search.index,
|
|
249
|
+
...args,
|
|
250
|
+
); //[count, [id, fields[], id, fields[], id, fields[], ...]]
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* returns the workflow handle. The handle can then be
|
|
255
|
+
* used to query for status, state, custom state, etc.
|
|
256
|
+
* @param {string} id
|
|
257
|
+
* @returns {Promise<WorkflowHandleService>}
|
|
258
|
+
*/
|
|
259
|
+
static async get(id: string): Promise<WorkflowHandleService> {
|
|
260
|
+
const my = new this();
|
|
261
|
+
const client = new Client({ connection: {
|
|
262
|
+
class: my.redisClass,
|
|
263
|
+
options: my.redisOptions
|
|
264
|
+
}});
|
|
265
|
+
let workflowName: string;
|
|
266
|
+
let target = my.workflowFunctions[0];
|
|
267
|
+
if (typeof target === 'string') {
|
|
268
|
+
workflowName = target;
|
|
269
|
+
} else {
|
|
270
|
+
workflowName = target.name;
|
|
271
|
+
}
|
|
272
|
+
return await client.workflow.getHandle(
|
|
273
|
+
my.taskQueue,
|
|
274
|
+
workflowName,
|
|
275
|
+
id,
|
|
276
|
+
my.namespace,
|
|
277
|
+
);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Optionally include a target taskQueue to exec the
|
|
282
|
+
* workflow's call on a specific worker queue.
|
|
283
|
+
*/
|
|
284
|
+
constructor(id?: string, options?: Record<string, any>) {
|
|
285
|
+
this.id = id;
|
|
286
|
+
if (options?.taskQueue) {
|
|
287
|
+
this.taskQueue = options.taskQueue;
|
|
288
|
+
} else if (!id && !options?.taskQueue) {
|
|
289
|
+
return this;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
function belongsTo(name: string, target: Array<MeshOSOptions | MeshOSActivityOptions | string>): boolean {
|
|
293
|
+
const isWorkflow = target.find((item: MeshOSOptions | string) => {
|
|
294
|
+
return typeof item === 'string' ? item === name : item.name === name;
|
|
295
|
+
});
|
|
296
|
+
return isWorkflow !== undefined;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
return new Proxy(this, {
|
|
300
|
+
get: (target, prop, receiver) => {
|
|
301
|
+
if (typeof target[prop] === 'function') {
|
|
302
|
+
return (...args: any[]) => {
|
|
303
|
+
return new Promise(async (resolve, reject) => {
|
|
304
|
+
const client = new Client({ connection: {
|
|
305
|
+
class: this.redisClass,
|
|
306
|
+
options: this.redisOptions
|
|
307
|
+
}});
|
|
308
|
+
if (belongsTo(prop as string, this.workflowFunctions)) {
|
|
309
|
+
//start a new workflow
|
|
310
|
+
const handle = await client.workflow.start({
|
|
311
|
+
namespace: this.namespace,
|
|
312
|
+
args,
|
|
313
|
+
taskQueue: this.taskQueue,
|
|
314
|
+
workflowName: prop as string,
|
|
315
|
+
workflowId: this.id,
|
|
316
|
+
});
|
|
317
|
+
if (options?.await) {
|
|
318
|
+
//wait for the workflow to complete
|
|
319
|
+
const result = await handle.result();
|
|
320
|
+
return resolve(result);
|
|
321
|
+
} else {
|
|
322
|
+
//return the workflow handle
|
|
323
|
+
return resolve(handle);
|
|
324
|
+
}
|
|
325
|
+
} else if (belongsTo(prop as string, this.hookFunctions)) {
|
|
326
|
+
//hook into a running workflow
|
|
327
|
+
return client.workflow.hook({
|
|
328
|
+
namespace: this.namespace,
|
|
329
|
+
taskQueue: this.taskQueue,
|
|
330
|
+
workflowName: prop as string,
|
|
331
|
+
workflowId: this.id,
|
|
332
|
+
args,
|
|
333
|
+
}).then(resolve).catch(reject);
|
|
334
|
+
}
|
|
335
|
+
//otherwise, call the method as a standard instance method.
|
|
336
|
+
target[prop].apply(this, args).then(resolve).catch(reject);
|
|
337
|
+
});
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
return Reflect.get(target, prop, receiver);
|
|
341
|
+
},
|
|
342
|
+
});
|
|
343
|
+
}
|
|
344
|
+
}
|
|
@@ -44,7 +44,6 @@ export class Search {
|
|
|
44
44
|
const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
|
|
45
45
|
await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length.toString(), ...prefixes, 'SCHEMA', ...schema);
|
|
46
46
|
} catch (err) {
|
|
47
|
-
console.error(err);
|
|
48
47
|
hotMeshClient.engine.logger.info('durable-client-search-err', { err });
|
|
49
48
|
}
|
|
50
49
|
}
|
|
@@ -72,13 +72,15 @@ export class WorkerService {
|
|
|
72
72
|
Object.keys(activities).forEach(key => {
|
|
73
73
|
if (activities[key].name && typeof WorkerService.activityRegistry[activities[key].name] !== 'function') {
|
|
74
74
|
WorkerService.activityRegistry[activities[key].name] = (activities as any)[key] as Function;
|
|
75
|
+
} else if (typeof (activities as any)[key] === 'function') {
|
|
76
|
+
WorkerService.activityRegistry[key] = (activities as any)[key] as Function;
|
|
75
77
|
}
|
|
76
78
|
});
|
|
77
79
|
}
|
|
78
80
|
return WorkerService.activityRegistry;
|
|
79
81
|
}
|
|
80
82
|
|
|
81
|
-
static async create(config: WorkerConfig) {
|
|
83
|
+
static async create(config: WorkerConfig): Promise<WorkerService> {
|
|
82
84
|
WorkerService.connection = config.connection;
|
|
83
85
|
const workflow = config.workflow;
|
|
84
86
|
const [workflowFunctionName, workflowFunction] = WorkerService.resolveWorkflowTarget(workflow);
|
|
@@ -95,16 +97,17 @@ export class WorkerService {
|
|
|
95
97
|
return worker;
|
|
96
98
|
}
|
|
97
99
|
|
|
98
|
-
static resolveWorkflowTarget(workflow: object | Function): [string, Function] {
|
|
100
|
+
static resolveWorkflowTarget(workflow: object | Function, name?: string): [string, Function] {
|
|
99
101
|
let workflowFunction: Function;
|
|
100
102
|
if (typeof workflow === 'function') {
|
|
101
103
|
workflowFunction = workflow;
|
|
104
|
+
return [workflowFunction.name ?? name, workflowFunction];
|
|
102
105
|
} else {
|
|
103
106
|
const workflowFunctionNames = Object.keys(workflow);
|
|
104
|
-
|
|
105
|
-
|
|
107
|
+
const lastFunctionName = workflowFunctionNames[workflowFunctionNames.length - 1];
|
|
108
|
+
workflowFunction = workflow[lastFunctionName];
|
|
109
|
+
return WorkerService.resolveWorkflowTarget(workflowFunction, lastFunctionName);
|
|
106
110
|
}
|
|
107
|
-
return [workflowFunction.name, workflowFunction];
|
|
108
111
|
}
|
|
109
112
|
|
|
110
113
|
async run() {
|
|
@@ -37,7 +37,7 @@ export class WorkflowService {
|
|
|
37
37
|
//this is risky but MUST be allowed. Users MAY set the workflowId,
|
|
38
38
|
//but if there is a naming collision, the data from the target entity will be used
|
|
39
39
|
//as there is know way of knowing if the item was generated via a prior run of the workflow
|
|
40
|
-
const childJobId = options.workflowId ??
|
|
40
|
+
const childJobId = options.workflowId ?? `-${workflowId}-$${options.workflowName}${workflowDimension}-${execIndex}`;
|
|
41
41
|
const parentWorkflowId = `${workflowId}-f`;
|
|
42
42
|
|
|
43
43
|
const client = new Client({
|
|
@@ -80,7 +80,7 @@ export class WorkflowService {
|
|
|
80
80
|
const workflowSpan = store.get('workflowSpan');
|
|
81
81
|
const COUNTER = store.get('counter');
|
|
82
82
|
const execIndex = COUNTER.counter = COUNTER.counter + 1;
|
|
83
|
-
const childJobId = options.workflowId ??
|
|
83
|
+
const childJobId = options.workflowId ?? `-${workflowId}-$${options.workflowName}${workflowDimension}-${execIndex}`;
|
|
84
84
|
const parentWorkflowId = `${workflowId}-f`;
|
|
85
85
|
const workflowTopic = `${options.taskQueue}-${options.workflowName}`;
|
|
86
86
|
|
|
@@ -111,7 +111,7 @@ export class WorkflowService {
|
|
|
111
111
|
*/
|
|
112
112
|
static proxyActivities<ACT>(options?: ActivityConfig): ProxyType<ACT> {
|
|
113
113
|
if (options.activities) {
|
|
114
|
-
WorkerService.registerActivities(options.activities)
|
|
114
|
+
WorkerService.registerActivities(options.activities);
|
|
115
115
|
}
|
|
116
116
|
|
|
117
117
|
const proxy: any = {};
|
|
@@ -142,6 +142,16 @@ export class WorkflowService {
|
|
|
142
142
|
return new Search(workflowId, hotMeshClient, searchSessionId);
|
|
143
143
|
}
|
|
144
144
|
|
|
145
|
+
/**
|
|
146
|
+
* return a handle to the hotmesh client currently running the workflow
|
|
147
|
+
*/
|
|
148
|
+
static async getHotMesh(): Promise<HotMesh> {
|
|
149
|
+
const store = asyncLocalStorage.getStore();
|
|
150
|
+
const workflowTopic = store.get('workflowTopic');
|
|
151
|
+
const namespace = store.get('namespace');
|
|
152
|
+
return await WorkerService.getHotMesh(workflowTopic, { namespace });
|
|
153
|
+
}
|
|
154
|
+
|
|
145
155
|
/**
|
|
146
156
|
* those methods that may only be called once must be protected by flagging
|
|
147
157
|
* their execution with a unique key (the key is stored in the HASH alongside
|
|
@@ -169,8 +179,11 @@ export class WorkflowService {
|
|
|
169
179
|
*/
|
|
170
180
|
static async signal(signalId: string, data: Record<any, any>): Promise<string> {
|
|
171
181
|
const store = asyncLocalStorage.getStore();
|
|
182
|
+
const workflowTopic = store.get('workflowTopic');
|
|
172
183
|
const namespace = store.get('namespace');
|
|
173
|
-
const hotMeshClient = await WorkerService.getHotMesh(
|
|
184
|
+
const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
|
|
185
|
+
//todo: this particular one is better patterned as a get/set,
|
|
186
|
+
//since the receipt is a meaningful string (the stream id)
|
|
174
187
|
if (await WorkflowService.isSideEffectAllowed(hotMeshClient, 'signal')) {
|
|
175
188
|
return await hotMeshClient.hook(`${namespace}.wfs.signal`, { id: signalId, data });
|
|
176
189
|
}
|
|
@@ -183,8 +196,9 @@ export class WorkflowService {
|
|
|
183
196
|
*/
|
|
184
197
|
static async hook(options: HookOptions): Promise<string> {
|
|
185
198
|
const store = asyncLocalStorage.getStore();
|
|
199
|
+
const workflowTopic = store.get('workflowTopic');
|
|
186
200
|
const namespace = store.get('namespace');
|
|
187
|
-
const hotMeshClient = await WorkerService.getHotMesh(
|
|
201
|
+
const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
|
|
188
202
|
if (await WorkflowService.isSideEffectAllowed(hotMeshClient, 'hook')) {
|
|
189
203
|
const store = asyncLocalStorage.getStore();
|
|
190
204
|
const workflowId = options.workflowId ?? store.get('workflowId');
|
|
@@ -212,7 +226,7 @@ export class WorkflowService {
|
|
|
212
226
|
const workflowTopic = store.get('workflowTopic');
|
|
213
227
|
const workflowDimension = store.get('workflowDimension') ?? '';
|
|
214
228
|
const namespace = store.get('namespace');
|
|
215
|
-
const sleepJobId =
|
|
229
|
+
const sleepJobId = `-${workflowId}-$sleep${workflowDimension}-${execIndex}`;
|
|
216
230
|
|
|
217
231
|
try {
|
|
218
232
|
const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
|
|
@@ -220,9 +234,8 @@ export class WorkflowService {
|
|
|
220
234
|
//if no error is thrown, we've already slept, return the delay
|
|
221
235
|
return seconds;
|
|
222
236
|
} catch (e) {
|
|
223
|
-
//if an error, the sleep job was not found...rethrow error; sleep job
|
|
224
|
-
// will be automatically created according to the DAG rules (they
|
|
225
237
|
// spawn a new sleep job if error code 595 is thrown by the worker)
|
|
238
|
+
// NOTE: If this message shows up in your stack trace, you forgot to await `Durable.workflow.sleep()` in your workflow code.
|
|
226
239
|
throw new DurableSleepError(workflowId, seconds, execIndex, workflowDimension);
|
|
227
240
|
}
|
|
228
241
|
}
|
|
@@ -242,7 +255,7 @@ export class WorkflowService {
|
|
|
242
255
|
const signalResults: any[] = [];
|
|
243
256
|
for (const signal of signals) {
|
|
244
257
|
const execIndex = COUNTER.counter = COUNTER.counter + 1;
|
|
245
|
-
const wfsJobId =
|
|
258
|
+
const wfsJobId = `-${workflowId}-$wfs${workflowDimension}-${execIndex}`;
|
|
246
259
|
try {
|
|
247
260
|
if (allAreComplete) {
|
|
248
261
|
const state = await hotMeshClient.getState(`${hotMeshClient.appId}.wfs.execute`, wfsJobId);
|
|
@@ -289,7 +302,7 @@ export class WorkflowService {
|
|
|
289
302
|
const spn = store.get('workflowSpan');
|
|
290
303
|
const namespace = store.get('namespace');
|
|
291
304
|
const activityTopic = `${workflowTopic}-activity`;
|
|
292
|
-
const activityJobId =
|
|
305
|
+
const activityJobId = `-${workflowId}-$${activityName}${workflowDimension}-${execIndex}`;
|
|
293
306
|
|
|
294
307
|
let activityState: JobOutput
|
|
295
308
|
try {
|
|
@@ -61,7 +61,7 @@ class StreamSignaler {
|
|
|
61
61
|
try {
|
|
62
62
|
await this.store.xgroup('CREATE', stream, group, '$', 'MKSTREAM');
|
|
63
63
|
} catch (err) {
|
|
64
|
-
this.logger.
|
|
64
|
+
this.logger.debug('consumer-group-exists', { stream, group });
|
|
65
65
|
}
|
|
66
66
|
}
|
|
67
67
|
|
|
@@ -150,7 +150,6 @@ class StreamSignaler {
|
|
|
150
150
|
try {
|
|
151
151
|
output = await callback(input);
|
|
152
152
|
} catch (error) {
|
|
153
|
-
console.error(error);
|
|
154
153
|
this.logger.error(`stream-call-function-error`, { error });
|
|
155
154
|
output = this.structureUnhandledError(input, error);
|
|
156
155
|
}
|
|
@@ -5,7 +5,6 @@ import { Cache } from '../cache';
|
|
|
5
5
|
import { StoreService } from '../index';
|
|
6
6
|
import { RedisClientType, RedisMultiType } from '../../../types/ioredisclient';
|
|
7
7
|
import { ReclaimedMessageType } from '../../../types/stream';
|
|
8
|
-
import { type } from 'os';
|
|
9
8
|
|
|
10
9
|
class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
|
|
11
10
|
redisClient: RedisClientType;
|
|
@@ -60,14 +59,14 @@ class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType>
|
|
|
60
59
|
try {
|
|
61
60
|
return (await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK';
|
|
62
61
|
} catch (err) {
|
|
63
|
-
this.logger.
|
|
62
|
+
this.logger.debug(`Consumer group not created with MKSTREAM for key: ${key} and group: ${groupName}`);
|
|
64
63
|
throw err;
|
|
65
64
|
}
|
|
66
65
|
} else {
|
|
67
66
|
try {
|
|
68
67
|
return (await this.redisClient.xgroup(command, key, groupName, id)) === 'OK';
|
|
69
68
|
} catch (err) {
|
|
70
|
-
this.logger.
|
|
69
|
+
this.logger.debug(`Consumer group not created for key: ${key} and group: ${groupName}`);
|
|
71
70
|
throw err;
|
|
72
71
|
}
|
|
73
72
|
}
|
|
@@ -86,7 +86,7 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
|
|
|
86
86
|
return (await this.redisClient.sendCommand(['XGROUP', 'CREATE', key, groupName, id, ...args])) === 1;
|
|
87
87
|
} catch (error) {
|
|
88
88
|
const streamType = mkStream === 'MKSTREAM' ? 'with MKSTREAM' : 'without MKSTREAM';
|
|
89
|
-
this.logger.
|
|
89
|
+
this.logger.debug(`x-group-error ${streamType} for key: ${key} and group: ${groupName}`, { error });
|
|
90
90
|
throw error;
|
|
91
91
|
}
|
|
92
92
|
}
|
package/types/durable.ts
CHANGED
|
@@ -10,7 +10,7 @@ type WorkflowConfig = {
|
|
|
10
10
|
type WorkflowSearchOptions = {
|
|
11
11
|
index?: string; //FT index name (myapp:myindex)
|
|
12
12
|
prefix?: string[]; //FT prefixes (['myapp:myindex:prefix1', 'myapp:myindex:prefix2'])
|
|
13
|
-
schema?: Record<string, {type: 'TEXT' | 'NUMERIC' | 'TAG', sortable
|
|
13
|
+
schema?: Record<string, {type: 'TEXT' | 'NUMERIC' | 'TAG', sortable?: boolean}>;
|
|
14
14
|
data?: Record<string, string>;
|
|
15
15
|
}
|
|
16
16
|
|
|
@@ -57,14 +57,14 @@ type WorkflowDataType = {
|
|
|
57
57
|
workflowTopic: string;
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
-
type
|
|
60
|
+
type MeshOSClassConfig = {
|
|
61
61
|
namespace: string;
|
|
62
62
|
taskQueue: string;
|
|
63
63
|
redisOptions: RedisOptions;
|
|
64
64
|
redisClass: RedisClass;
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
-
type
|
|
67
|
+
type MeshOSConfig = {
|
|
68
68
|
taskQueue?: string;
|
|
69
69
|
index?: {
|
|
70
70
|
index: string;
|
|
@@ -91,11 +91,28 @@ type WorkerConfig = {
|
|
|
91
91
|
connection: Connection;
|
|
92
92
|
namespace?: string; //`appid` in the YAML (e.g, 'default')
|
|
93
93
|
taskQueue: string; //`subscribes` in the YAML (e.g, 'hello-world')
|
|
94
|
-
workflow: Function
|
|
94
|
+
workflow: Function | Record<string | symbol, Function>; //target function to run
|
|
95
95
|
options?: WorkerOptions;
|
|
96
96
|
search?: WorkflowSearchOptions;
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
+
type FindOptions = {
|
|
100
|
+
workflowName?: string; //also the function name
|
|
101
|
+
taskQueue?: string;
|
|
102
|
+
namespace?: string;
|
|
103
|
+
index?: string; //the FT search index name
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
type MeshOSOptions = {
|
|
107
|
+
name: string;
|
|
108
|
+
options: WorkerOptions;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
type MeshOSActivityOptions = {
|
|
112
|
+
name: string;
|
|
113
|
+
options: ActivityConfig;
|
|
114
|
+
}
|
|
115
|
+
|
|
99
116
|
type WorkerOptions = {
|
|
100
117
|
logLevel?: string; //debug, info, warn, error
|
|
101
118
|
maxSystemRetries?: number; //1-3 (10ms, 100ms, 1_000ms)
|
|
@@ -133,9 +150,12 @@ export {
|
|
|
133
150
|
ProxyType,
|
|
134
151
|
Registry,
|
|
135
152
|
SignalOptions,
|
|
153
|
+
FindOptions,
|
|
136
154
|
HookOptions,
|
|
137
|
-
|
|
138
|
-
|
|
155
|
+
MeshOSActivityOptions,
|
|
156
|
+
MeshOSClassConfig,
|
|
157
|
+
MeshOSConfig,
|
|
158
|
+
MeshOSOptions,
|
|
139
159
|
WorkerConfig,
|
|
140
160
|
WorkflowConfig,
|
|
141
161
|
WorkerOptions,
|
package/types/index.ts
CHANGED
|
@@ -36,9 +36,13 @@ export {
|
|
|
36
36
|
Connection,
|
|
37
37
|
ProxyType,
|
|
38
38
|
Registry,
|
|
39
|
+
SignalOptions,
|
|
40
|
+
FindOptions,
|
|
39
41
|
HookOptions,
|
|
40
|
-
|
|
41
|
-
|
|
42
|
+
MeshOSActivityOptions,
|
|
43
|
+
MeshOSClassConfig,
|
|
44
|
+
MeshOSConfig,
|
|
45
|
+
MeshOSOptions,
|
|
42
46
|
WorkflowConfig,
|
|
43
47
|
WorkerConfig,
|
|
44
48
|
WorkerOptions,
|