@hotmeshio/hotmesh 0.0.23 → 0.0.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -67
- package/build/index.d.ts +2 -1
- package/build/index.js +3 -1
- package/build/package.json +2 -2
- package/build/services/durable/factory.js +6 -6
- package/build/services/durable/handle.js +2 -4
- package/build/services/durable/index.d.ts +2 -2
- package/build/services/durable/index.js +2 -2
- package/build/services/durable/meshos.d.ts +108 -0
- package/build/services/durable/meshos.js +289 -0
- package/build/services/durable/search.js +0 -1
- package/build/services/durable/worker.d.ts +1 -1
- package/build/services/durable/worker.js +8 -4
- package/build/services/durable/workflow.d.ts +4 -0
- package/build/services/durable/workflow.js +21 -9
- package/build/services/signaler/stream.js +1 -2
- package/build/services/store/clients/ioredis.js +2 -2
- package/build/services/store/clients/redis.js +1 -1
- package/build/types/durable.d.ts +19 -5
- package/build/types/index.d.ts +1 -1
- package/index.ts +2 -1
- package/package.json +2 -2
- package/services/durable/factory.ts +6 -6
- package/services/durable/handle.ts +2 -4
- package/services/durable/index.ts +2 -2
- package/services/durable/meshos.ts +344 -0
- package/services/durable/search.ts +0 -1
- package/services/durable/worker.ts +8 -5
- package/services/durable/workflow.ts +23 -10
- package/services/signaler/stream.ts +1 -2
- package/services/store/clients/ioredis.ts +2 -3
- package/services/store/clients/redis.ts +1 -1
- package/types/durable.ts +26 -6
- package/types/index.ts +6 -2
- package/build/services/durable/meshdb.d.ts +0 -113
- package/build/services/durable/meshdb.js +0 -211
- package/services/durable/meshdb.ts +0 -254
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MeshOSService = void 0;
|
|
4
|
+
const nanoid_1 = require("nanoid");
|
|
5
|
+
const client_1 = require("./client");
|
|
6
|
+
const search_1 = require("./search");
|
|
7
|
+
const worker_1 = require("./worker");
|
|
8
|
+
const _1 = require(".");
|
|
9
|
+
const asyncLocalStorage_1 = require("./asyncLocalStorage");
|
|
10
|
+
const workflow_1 = require("./workflow");
|
|
11
|
+
/**
|
|
12
|
+
* The base class for running MeshOS workflows.
|
|
13
|
+
* Extend and register subclass methods by name to
|
|
14
|
+
* execute as durable workflows, backed by Redis.
|
|
15
|
+
*/
|
|
16
|
+
class MeshOSService {
|
|
17
|
+
static async getHotMeshClient(redisClass, redisOptions, namespace, taskQueue) {
|
|
18
|
+
const client = new client_1.ClientService({
|
|
19
|
+
connection: {
|
|
20
|
+
class: redisClass,
|
|
21
|
+
options: redisOptions,
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
return await client.getHotMeshClient(taskQueue, namespace);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* mints a workflow ID, using the search prefix.
|
|
28
|
+
* NOTE: The prefix is necesary when indexing
|
|
29
|
+
* HASHes when FT search is enabled.
|
|
30
|
+
* @returns {string}
|
|
31
|
+
*/
|
|
32
|
+
static mintGuid() {
|
|
33
|
+
const my = new this();
|
|
34
|
+
return `${my.search?.prefix?.[0]}${(0, nanoid_1.nanoid)()}}`;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Creates an FT search index
|
|
38
|
+
*/
|
|
39
|
+
static async createIndex() {
|
|
40
|
+
const my = new this();
|
|
41
|
+
const hmClient = await MeshOSService.getHotMeshClient(my.redisClass, my.redisOptions, my.namespace, my.taskQueue);
|
|
42
|
+
search_1.Search.configureSearchIndex(hmClient, my.search);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Initialize the worker(s) for the entity. This is a static
|
|
46
|
+
* method that allows for optional task Queue targeting.
|
|
47
|
+
* NOTE: Allow List may be optionally used to only wrap
|
|
48
|
+
* specific methods in this class.
|
|
49
|
+
* @param {string} taskQueue
|
|
50
|
+
* @param {string[]} allowList
|
|
51
|
+
*/
|
|
52
|
+
static async startWorkers(taskQueue, allowList = []) {
|
|
53
|
+
const my = new this();
|
|
54
|
+
//helper functions
|
|
55
|
+
const resolveFunctionNames = (arr) => arr.map(item => typeof item === 'string' ? item : item.name);
|
|
56
|
+
const belongsTo = (name, target) => {
|
|
57
|
+
const isWorkflow = target.find((item) => {
|
|
58
|
+
return typeof item === 'string' ? item === name : item.name === name;
|
|
59
|
+
});
|
|
60
|
+
return isWorkflow !== undefined;
|
|
61
|
+
};
|
|
62
|
+
// proxy registered activities
|
|
63
|
+
const proxyFunctionNames = resolveFunctionNames([...my.proxyFunctions]);
|
|
64
|
+
if (proxyFunctionNames.length) {
|
|
65
|
+
const proxyActivities = proxyFunctionNames.reduce((acc, funcName) => {
|
|
66
|
+
let originalMethod = my[funcName];
|
|
67
|
+
if (typeof originalMethod === 'function') {
|
|
68
|
+
acc[funcName] = async (...args) => {
|
|
69
|
+
return await originalMethod.apply(my, args);
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
return acc;
|
|
73
|
+
}, {});
|
|
74
|
+
const proxiedActivities = _1.Durable.workflow.proxyActivities({
|
|
75
|
+
activities: proxyActivities
|
|
76
|
+
});
|
|
77
|
+
//WATCH!: unsure if this will pollute the scope; don't think
|
|
78
|
+
// so as activity functions are terminal in the chain.
|
|
79
|
+
Object.assign(my, proxiedActivities);
|
|
80
|
+
}
|
|
81
|
+
const functionsToIterate = allowList.length ? resolveFunctionNames(allowList) : resolveFunctionNames([...my.workflowFunctions, ...my.hookFunctions]);
|
|
82
|
+
// Iterating through the functions sequentially
|
|
83
|
+
for (const funcName of functionsToIterate) {
|
|
84
|
+
const originalMethod = my[funcName];
|
|
85
|
+
if (typeof originalMethod === 'function') {
|
|
86
|
+
//wrap the function to return
|
|
87
|
+
const wrappedFunction = {
|
|
88
|
+
[funcName]: async (...args) => {
|
|
89
|
+
const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
|
|
90
|
+
const workflowId = store.get('workflowId');
|
|
91
|
+
//use a Proxy to wrap hook methods
|
|
92
|
+
const context = new Proxy(my, {
|
|
93
|
+
get: (target, prop, receiver) => {
|
|
94
|
+
if (prop === 'id') {
|
|
95
|
+
return workflowId;
|
|
96
|
+
}
|
|
97
|
+
else if (typeof target[prop] === 'function') {
|
|
98
|
+
return (...args) => {
|
|
99
|
+
return new Promise(async (resolve, reject) => {
|
|
100
|
+
if (belongsTo(prop, my.hookFunctions)) {
|
|
101
|
+
return workflow_1.WorkflowService.hook({
|
|
102
|
+
namespace: my.namespace,
|
|
103
|
+
taskQueue: my.taskQueue,
|
|
104
|
+
workflowName: prop,
|
|
105
|
+
workflowId,
|
|
106
|
+
args,
|
|
107
|
+
}).then(resolve).catch(reject);
|
|
108
|
+
}
|
|
109
|
+
//otherwise, call the method as a standard instance method.
|
|
110
|
+
target[prop].apply(this, args).then(resolve).catch(reject);
|
|
111
|
+
});
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
return Reflect.get(target, prop, receiver);
|
|
115
|
+
},
|
|
116
|
+
});
|
|
117
|
+
return await originalMethod.apply(context, args);
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
//start the worker
|
|
121
|
+
await worker_1.WorkerService.create({
|
|
122
|
+
namespace: my.namespace,
|
|
123
|
+
connection: {
|
|
124
|
+
class: my.redisClass,
|
|
125
|
+
options: my.redisOptions,
|
|
126
|
+
},
|
|
127
|
+
taskQueue: taskQueue ?? my.taskQueue,
|
|
128
|
+
workflow: wrappedFunction,
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* executes the redis FT search query
|
|
135
|
+
* @example '@_quantity:[89 89]'
|
|
136
|
+
* @param {any[]} args
|
|
137
|
+
* @returns {string}
|
|
138
|
+
*/
|
|
139
|
+
static async find(options, ...args) {
|
|
140
|
+
const my = new this();
|
|
141
|
+
const client = new client_1.ClientService({ connection: {
|
|
142
|
+
class: my.redisClass,
|
|
143
|
+
options: my.redisOptions
|
|
144
|
+
} });
|
|
145
|
+
//workflow name is the function name driving the workflow
|
|
146
|
+
let workflowName;
|
|
147
|
+
if (options?.workflowName) {
|
|
148
|
+
workflowName = options?.workflowName;
|
|
149
|
+
}
|
|
150
|
+
else if (my.workflowFunctions?.length) {
|
|
151
|
+
let target = my.workflowFunctions[0];
|
|
152
|
+
if (typeof target === 'string') {
|
|
153
|
+
workflowName = target;
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
workflowName = target.name;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
return await client.workflow.search(options?.taskQueue ?? my.taskQueue, workflowName, my.namespace, my.search.index, ...args); //[count, [id, fields[], id, fields[], id, fields[], ...]]
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* returns the workflow handle. The handle can then be
|
|
163
|
+
* used to query for status, state, custom state, etc.
|
|
164
|
+
* @param {string} id
|
|
165
|
+
* @returns {Promise<WorkflowHandleService>}
|
|
166
|
+
*/
|
|
167
|
+
static async get(id) {
|
|
168
|
+
const my = new this();
|
|
169
|
+
const client = new client_1.ClientService({ connection: {
|
|
170
|
+
class: my.redisClass,
|
|
171
|
+
options: my.redisOptions
|
|
172
|
+
} });
|
|
173
|
+
let workflowName;
|
|
174
|
+
let target = my.workflowFunctions[0];
|
|
175
|
+
if (typeof target === 'string') {
|
|
176
|
+
workflowName = target;
|
|
177
|
+
}
|
|
178
|
+
else {
|
|
179
|
+
workflowName = target.name;
|
|
180
|
+
}
|
|
181
|
+
return await client.workflow.getHandle(my.taskQueue, workflowName, id, my.namespace);
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Optionally include a target taskQueue to exec the
|
|
185
|
+
* workflow's call on a specific worker queue.
|
|
186
|
+
*/
|
|
187
|
+
constructor(id, options) {
|
|
188
|
+
/**
|
|
189
|
+
* The top-level Redis isolation. All workflow data is
|
|
190
|
+
* isolated within this namespace. Values should be
|
|
191
|
+
* lower-case with no spaces (e.g, 'staging', 'prod', 'test',
|
|
192
|
+
* 'routing-stagig', 'reporting-prod', etc.).
|
|
193
|
+
* 1) only url-safe values are allowed;
|
|
194
|
+
* 2) the 'a' symbol is reserved by HotMesh for indexing apps
|
|
195
|
+
*/
|
|
196
|
+
this.namespace = 'durable';
|
|
197
|
+
/**
|
|
198
|
+
* Data is routed to workers that specify this task queue.
|
|
199
|
+
* Setting the task queue when the worker is created will
|
|
200
|
+
* ensure that the worker only receives messages destined
|
|
201
|
+
* for the queue. Callers can specify the taskQue to when
|
|
202
|
+
* starting a job to call those workers.
|
|
203
|
+
*/
|
|
204
|
+
this.taskQueue = 'default';
|
|
205
|
+
/**
|
|
206
|
+
* These methods run as durable workflows
|
|
207
|
+
*/
|
|
208
|
+
this.workflowFunctions = [];
|
|
209
|
+
/**
|
|
210
|
+
* These methods run as hooks (hook into a running workflow)
|
|
211
|
+
*/
|
|
212
|
+
this.hookFunctions = [];
|
|
213
|
+
/**
|
|
214
|
+
* These methods run as proxied activities (and are safely memoized)
|
|
215
|
+
*/
|
|
216
|
+
this.proxyFunctions = [];
|
|
217
|
+
/**
|
|
218
|
+
* The Redis connection options. NOTE: Redis and IORedis
|
|
219
|
+
* use different formats for their connection config.
|
|
220
|
+
*/
|
|
221
|
+
this.redisOptions = {
|
|
222
|
+
host: 'localhost',
|
|
223
|
+
port: 6379,
|
|
224
|
+
password: '',
|
|
225
|
+
db: 0,
|
|
226
|
+
};
|
|
227
|
+
this.id = id;
|
|
228
|
+
if (options?.taskQueue) {
|
|
229
|
+
this.taskQueue = options.taskQueue;
|
|
230
|
+
}
|
|
231
|
+
else if (!id && !options?.taskQueue) {
|
|
232
|
+
return this;
|
|
233
|
+
}
|
|
234
|
+
function belongsTo(name, target) {
|
|
235
|
+
const isWorkflow = target.find((item) => {
|
|
236
|
+
return typeof item === 'string' ? item === name : item.name === name;
|
|
237
|
+
});
|
|
238
|
+
return isWorkflow !== undefined;
|
|
239
|
+
}
|
|
240
|
+
return new Proxy(this, {
|
|
241
|
+
get: (target, prop, receiver) => {
|
|
242
|
+
if (typeof target[prop] === 'function') {
|
|
243
|
+
return (...args) => {
|
|
244
|
+
return new Promise(async (resolve, reject) => {
|
|
245
|
+
const client = new client_1.ClientService({ connection: {
|
|
246
|
+
class: this.redisClass,
|
|
247
|
+
options: this.redisOptions
|
|
248
|
+
} });
|
|
249
|
+
if (belongsTo(prop, this.workflowFunctions)) {
|
|
250
|
+
//start a new workflow
|
|
251
|
+
const handle = await client.workflow.start({
|
|
252
|
+
namespace: this.namespace,
|
|
253
|
+
args,
|
|
254
|
+
taskQueue: this.taskQueue,
|
|
255
|
+
workflowName: prop,
|
|
256
|
+
workflowId: this.id,
|
|
257
|
+
});
|
|
258
|
+
if (options?.await) {
|
|
259
|
+
//wait for the workflow to complete
|
|
260
|
+
const result = await handle.result();
|
|
261
|
+
return resolve(result);
|
|
262
|
+
}
|
|
263
|
+
else {
|
|
264
|
+
//return the workflow handle
|
|
265
|
+
return resolve(handle);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
else if (belongsTo(prop, this.hookFunctions)) {
|
|
269
|
+
//hook into a running workflow
|
|
270
|
+
return client.workflow.hook({
|
|
271
|
+
namespace: this.namespace,
|
|
272
|
+
taskQueue: this.taskQueue,
|
|
273
|
+
workflowName: prop,
|
|
274
|
+
workflowId: this.id,
|
|
275
|
+
args,
|
|
276
|
+
}).then(resolve).catch(reject);
|
|
277
|
+
}
|
|
278
|
+
//otherwise, call the method as a standard instance method.
|
|
279
|
+
target[prop].apply(this, args).then(resolve).catch(reject);
|
|
280
|
+
});
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
return Reflect.get(target, prop, receiver);
|
|
284
|
+
},
|
|
285
|
+
});
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
MeshOSService.MeshOS = workflow_1.WorkflowService;
|
|
289
|
+
exports.MeshOSService = MeshOSService;
|
|
@@ -36,7 +36,6 @@ class Search {
|
|
|
36
36
|
await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length.toString(), ...prefixes, 'SCHEMA', ...schema);
|
|
37
37
|
}
|
|
38
38
|
catch (err) {
|
|
39
|
-
console.error(err);
|
|
40
39
|
hotMeshClient.engine.logger.info('durable-client-search-err', { err });
|
|
41
40
|
}
|
|
42
41
|
}
|
|
@@ -10,7 +10,7 @@ export declare class WorkerService {
|
|
|
10
10
|
static activateWorkflow(hotMesh: HotMesh): Promise<void>;
|
|
11
11
|
static registerActivities<ACT>(activities: ACT): Registry;
|
|
12
12
|
static create(config: WorkerConfig): Promise<WorkerService>;
|
|
13
|
-
static resolveWorkflowTarget(workflow: object | Function): [string, Function];
|
|
13
|
+
static resolveWorkflowTarget(workflow: object | Function, name?: string): [string, Function];
|
|
14
14
|
run(): Promise<void>;
|
|
15
15
|
initActivityWorker(config: WorkerConfig, activityTopic: string): Promise<HotMesh>;
|
|
16
16
|
wrapActivityFunctions(): Function;
|
|
@@ -41,6 +41,9 @@ class WorkerService {
|
|
|
41
41
|
if (activities[key].name && typeof WorkerService.activityRegistry[activities[key].name] !== 'function') {
|
|
42
42
|
WorkerService.activityRegistry[activities[key].name] = activities[key];
|
|
43
43
|
}
|
|
44
|
+
else if (typeof activities[key] === 'function') {
|
|
45
|
+
WorkerService.activityRegistry[key] = activities[key];
|
|
46
|
+
}
|
|
44
47
|
});
|
|
45
48
|
}
|
|
46
49
|
return WorkerService.activityRegistry;
|
|
@@ -60,17 +63,18 @@ class WorkerService {
|
|
|
60
63
|
await WorkerService.activateWorkflow(worker.workflowRunner);
|
|
61
64
|
return worker;
|
|
62
65
|
}
|
|
63
|
-
static resolveWorkflowTarget(workflow) {
|
|
66
|
+
static resolveWorkflowTarget(workflow, name) {
|
|
64
67
|
let workflowFunction;
|
|
65
68
|
if (typeof workflow === 'function') {
|
|
66
69
|
workflowFunction = workflow;
|
|
70
|
+
return [workflowFunction.name ?? name, workflowFunction];
|
|
67
71
|
}
|
|
68
72
|
else {
|
|
69
73
|
const workflowFunctionNames = Object.keys(workflow);
|
|
70
|
-
|
|
71
|
-
|
|
74
|
+
const lastFunctionName = workflowFunctionNames[workflowFunctionNames.length - 1];
|
|
75
|
+
workflowFunction = workflow[lastFunctionName];
|
|
76
|
+
return WorkerService.resolveWorkflowTarget(workflowFunction, lastFunctionName);
|
|
72
77
|
}
|
|
73
|
-
return [workflowFunction.name, workflowFunction];
|
|
74
78
|
}
|
|
75
79
|
async run() {
|
|
76
80
|
this.workflowRunner.engine.logger.info('WorkerService is running');
|
|
@@ -18,6 +18,10 @@ export declare class WorkflowService {
|
|
|
18
18
|
* return a search session for use when reading/writing to the workflow HASH
|
|
19
19
|
*/
|
|
20
20
|
static search(): Promise<Search>;
|
|
21
|
+
/**
|
|
22
|
+
* return a handle to the hotmesh client currently running the workflow
|
|
23
|
+
*/
|
|
24
|
+
static getHotMesh(): Promise<HotMesh>;
|
|
21
25
|
/**
|
|
22
26
|
* those methods that may only be called once must be protected by flagging
|
|
23
27
|
* their execution with a unique key (the key is stored in the HASH alongside
|
|
@@ -30,7 +30,7 @@ class WorkflowService {
|
|
|
30
30
|
//this is risky but MUST be allowed. Users MAY set the workflowId,
|
|
31
31
|
//but if there is a naming collision, the data from the target entity will be used
|
|
32
32
|
//as there is know way of knowing if the item was generated via a prior run of the workflow
|
|
33
|
-
const childJobId = options.workflowId ??
|
|
33
|
+
const childJobId = options.workflowId ?? `-${workflowId}-$${options.workflowName}${workflowDimension}-${execIndex}`;
|
|
34
34
|
const parentWorkflowId = `${workflowId}-f`;
|
|
35
35
|
const client = new client_1.ClientService({
|
|
36
36
|
connection: await connection_1.ConnectionService.connect(worker_1.WorkerService.connection),
|
|
@@ -65,7 +65,7 @@ class WorkflowService {
|
|
|
65
65
|
const workflowSpan = store.get('workflowSpan');
|
|
66
66
|
const COUNTER = store.get('counter');
|
|
67
67
|
const execIndex = COUNTER.counter = COUNTER.counter + 1;
|
|
68
|
-
const childJobId = options.workflowId ??
|
|
68
|
+
const childJobId = options.workflowId ?? `-${workflowId}-$${options.workflowName}${workflowDimension}-${execIndex}`;
|
|
69
69
|
const parentWorkflowId = `${workflowId}-f`;
|
|
70
70
|
const workflowTopic = `${options.taskQueue}-${options.workflowName}`;
|
|
71
71
|
try {
|
|
@@ -122,6 +122,15 @@ class WorkflowService {
|
|
|
122
122
|
const searchSessionId = `-search${workflowDimension}-${execIndex}`;
|
|
123
123
|
return new search_1.Search(workflowId, hotMeshClient, searchSessionId);
|
|
124
124
|
}
|
|
125
|
+
/**
|
|
126
|
+
* return a handle to the hotmesh client currently running the workflow
|
|
127
|
+
*/
|
|
128
|
+
static async getHotMesh() {
|
|
129
|
+
const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
|
|
130
|
+
const workflowTopic = store.get('workflowTopic');
|
|
131
|
+
const namespace = store.get('namespace');
|
|
132
|
+
return await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
|
|
133
|
+
}
|
|
125
134
|
/**
|
|
126
135
|
* those methods that may only be called once must be protected by flagging
|
|
127
136
|
* their execution with a unique key (the key is stored in the HASH alongside
|
|
@@ -148,8 +157,11 @@ class WorkflowService {
|
|
|
148
157
|
*/
|
|
149
158
|
static async signal(signalId, data) {
|
|
150
159
|
const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
|
|
160
|
+
const workflowTopic = store.get('workflowTopic');
|
|
151
161
|
const namespace = store.get('namespace');
|
|
152
|
-
const hotMeshClient = await worker_1.WorkerService.getHotMesh(
|
|
162
|
+
const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
|
|
163
|
+
//todo: this particular one is better patterned as a get/set,
|
|
164
|
+
//since the receipt is a meaningful string (the stream id)
|
|
153
165
|
if (await WorkflowService.isSideEffectAllowed(hotMeshClient, 'signal')) {
|
|
154
166
|
return await hotMeshClient.hook(`${namespace}.wfs.signal`, { id: signalId, data });
|
|
155
167
|
}
|
|
@@ -161,8 +173,9 @@ class WorkflowService {
|
|
|
161
173
|
*/
|
|
162
174
|
static async hook(options) {
|
|
163
175
|
const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
|
|
176
|
+
const workflowTopic = store.get('workflowTopic');
|
|
164
177
|
const namespace = store.get('namespace');
|
|
165
|
-
const hotMeshClient = await worker_1.WorkerService.getHotMesh(
|
|
178
|
+
const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
|
|
166
179
|
if (await WorkflowService.isSideEffectAllowed(hotMeshClient, 'hook')) {
|
|
167
180
|
const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
|
|
168
181
|
const workflowId = options.workflowId ?? store.get('workflowId');
|
|
@@ -188,7 +201,7 @@ class WorkflowService {
|
|
|
188
201
|
const workflowTopic = store.get('workflowTopic');
|
|
189
202
|
const workflowDimension = store.get('workflowDimension') ?? '';
|
|
190
203
|
const namespace = store.get('namespace');
|
|
191
|
-
const sleepJobId =
|
|
204
|
+
const sleepJobId = `-${workflowId}-$sleep${workflowDimension}-${execIndex}`;
|
|
192
205
|
try {
|
|
193
206
|
const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
|
|
194
207
|
await hotMeshClient.getState(`${hotMeshClient.appId}.sleep.execute`, sleepJobId);
|
|
@@ -196,9 +209,8 @@ class WorkflowService {
|
|
|
196
209
|
return seconds;
|
|
197
210
|
}
|
|
198
211
|
catch (e) {
|
|
199
|
-
//if an error, the sleep job was not found...rethrow error; sleep job
|
|
200
|
-
// will be automatically created according to the DAG rules (they
|
|
201
212
|
// spawn a new sleep job if error code 595 is thrown by the worker)
|
|
213
|
+
// NOTE: If this message shows up in your stack trace, you forgot to await `Durable.workflow.sleep()` in your workflow code.
|
|
202
214
|
throw new errors_1.DurableSleepError(workflowId, seconds, execIndex, workflowDimension);
|
|
203
215
|
}
|
|
204
216
|
}
|
|
@@ -216,7 +228,7 @@ class WorkflowService {
|
|
|
216
228
|
const signalResults = [];
|
|
217
229
|
for (const signal of signals) {
|
|
218
230
|
const execIndex = COUNTER.counter = COUNTER.counter + 1;
|
|
219
|
-
const wfsJobId =
|
|
231
|
+
const wfsJobId = `-${workflowId}-$wfs${workflowDimension}-${execIndex}`;
|
|
220
232
|
try {
|
|
221
233
|
if (allAreComplete) {
|
|
222
234
|
const state = await hotMeshClient.getState(`${hotMeshClient.appId}.wfs.execute`, wfsJobId);
|
|
@@ -267,7 +279,7 @@ class WorkflowService {
|
|
|
267
279
|
const spn = store.get('workflowSpan');
|
|
268
280
|
const namespace = store.get('namespace');
|
|
269
281
|
const activityTopic = `${workflowTopic}-activity`;
|
|
270
|
-
const activityJobId =
|
|
282
|
+
const activityJobId = `-${workflowId}-$${activityName}${workflowDimension}-${execIndex}`;
|
|
271
283
|
let activityState;
|
|
272
284
|
try {
|
|
273
285
|
const hotMeshClient = await worker_1.WorkerService.getHotMesh(activityTopic, { namespace });
|
|
@@ -37,7 +37,7 @@ class StreamSignaler {
|
|
|
37
37
|
await this.store.xgroup('CREATE', stream, group, '$', 'MKSTREAM');
|
|
38
38
|
}
|
|
39
39
|
catch (err) {
|
|
40
|
-
this.logger.
|
|
40
|
+
this.logger.debug('consumer-group-exists', { stream, group });
|
|
41
41
|
}
|
|
42
42
|
}
|
|
43
43
|
async publishMessage(topic, streamData, multi) {
|
|
@@ -121,7 +121,6 @@ class StreamSignaler {
|
|
|
121
121
|
output = await callback(input);
|
|
122
122
|
}
|
|
123
123
|
catch (error) {
|
|
124
|
-
console.error(error);
|
|
125
124
|
this.logger.error(`stream-call-function-error`, { error });
|
|
126
125
|
output = this.structureUnhandledError(input, error);
|
|
127
126
|
}
|
|
@@ -45,7 +45,7 @@ class IORedisStoreService extends index_1.StoreService {
|
|
|
45
45
|
return (await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK';
|
|
46
46
|
}
|
|
47
47
|
catch (err) {
|
|
48
|
-
this.logger.
|
|
48
|
+
this.logger.debug(`Consumer group not created with MKSTREAM for key: ${key} and group: ${groupName}`);
|
|
49
49
|
throw err;
|
|
50
50
|
}
|
|
51
51
|
}
|
|
@@ -54,7 +54,7 @@ class IORedisStoreService extends index_1.StoreService {
|
|
|
54
54
|
return (await this.redisClient.xgroup(command, key, groupName, id)) === 'OK';
|
|
55
55
|
}
|
|
56
56
|
catch (err) {
|
|
57
|
-
this.logger.
|
|
57
|
+
this.logger.debug(`Consumer group not created for key: ${key} and group: ${groupName}`);
|
|
58
58
|
throw err;
|
|
59
59
|
}
|
|
60
60
|
}
|
|
@@ -68,7 +68,7 @@ class RedisStoreService extends index_1.StoreService {
|
|
|
68
68
|
}
|
|
69
69
|
catch (error) {
|
|
70
70
|
const streamType = mkStream === 'MKSTREAM' ? 'with MKSTREAM' : 'without MKSTREAM';
|
|
71
|
-
this.logger.
|
|
71
|
+
this.logger.debug(`x-group-error ${streamType} for key: ${key} and group: ${groupName}`, { error });
|
|
72
72
|
throw error;
|
|
73
73
|
}
|
|
74
74
|
}
|
package/build/types/durable.d.ts
CHANGED
|
@@ -10,7 +10,7 @@ type WorkflowSearchOptions = {
|
|
|
10
10
|
prefix?: string[];
|
|
11
11
|
schema?: Record<string, {
|
|
12
12
|
type: 'TEXT' | 'NUMERIC' | 'TAG';
|
|
13
|
-
sortable
|
|
13
|
+
sortable?: boolean;
|
|
14
14
|
}>;
|
|
15
15
|
data?: Record<string, string>;
|
|
16
16
|
};
|
|
@@ -52,13 +52,13 @@ type WorkflowDataType = {
|
|
|
52
52
|
workflowId: string;
|
|
53
53
|
workflowTopic: string;
|
|
54
54
|
};
|
|
55
|
-
type
|
|
55
|
+
type MeshOSClassConfig = {
|
|
56
56
|
namespace: string;
|
|
57
57
|
taskQueue: string;
|
|
58
58
|
redisOptions: RedisOptions;
|
|
59
59
|
redisClass: RedisClass;
|
|
60
60
|
};
|
|
61
|
-
type
|
|
61
|
+
type MeshOSConfig = {
|
|
62
62
|
taskQueue?: string;
|
|
63
63
|
index?: {
|
|
64
64
|
index: string;
|
|
@@ -84,10 +84,24 @@ type WorkerConfig = {
|
|
|
84
84
|
connection: Connection;
|
|
85
85
|
namespace?: string;
|
|
86
86
|
taskQueue: string;
|
|
87
|
-
workflow: Function
|
|
87
|
+
workflow: Function | Record<string | symbol, Function>;
|
|
88
88
|
options?: WorkerOptions;
|
|
89
89
|
search?: WorkflowSearchOptions;
|
|
90
90
|
};
|
|
91
|
+
type FindOptions = {
|
|
92
|
+
workflowName?: string;
|
|
93
|
+
taskQueue?: string;
|
|
94
|
+
namespace?: string;
|
|
95
|
+
index?: string;
|
|
96
|
+
};
|
|
97
|
+
type MeshOSOptions = {
|
|
98
|
+
name: string;
|
|
99
|
+
options: WorkerOptions;
|
|
100
|
+
};
|
|
101
|
+
type MeshOSActivityOptions = {
|
|
102
|
+
name: string;
|
|
103
|
+
options: ActivityConfig;
|
|
104
|
+
};
|
|
91
105
|
type WorkerOptions = {
|
|
92
106
|
logLevel?: string;
|
|
93
107
|
maxSystemRetries?: number;
|
|
@@ -111,4 +125,4 @@ type ActivityConfig = {
|
|
|
111
125
|
maximumInterval: string;
|
|
112
126
|
};
|
|
113
127
|
};
|
|
114
|
-
export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, ProxyType, Registry, SignalOptions, HookOptions,
|
|
128
|
+
export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, ProxyType, Registry, SignalOptions, FindOptions, HookOptions, MeshOSActivityOptions, MeshOSClassConfig, MeshOSConfig, MeshOSOptions, WorkerConfig, WorkflowConfig, WorkerOptions, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, };
|
package/build/types/index.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ export { App, AppVID, AppTransitions, AppSubscriptions } from './app';
|
|
|
3
3
|
export { AsyncSignal } from './async';
|
|
4
4
|
export { CacheMode } from './cache';
|
|
5
5
|
export { CollationFaultType, CollationStage } from './collator';
|
|
6
|
-
export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, ProxyType, Registry, HookOptions,
|
|
6
|
+
export { ActivityConfig, ActivityWorkflowDataType, ClientConfig, ContextType, ConnectionConfig, Connection, ProxyType, Registry, SignalOptions, FindOptions, HookOptions, MeshOSActivityOptions, MeshOSClassConfig, MeshOSConfig, MeshOSOptions, WorkflowConfig, WorkerConfig, WorkerOptions, WorkflowSearchOptions, WorkflowDataType, WorkflowOptions, } from './durable';
|
|
7
7
|
export { HookCondition, HookConditions, HookGate, HookInterface, HookRule, HookRules, HookSignal } from './hook';
|
|
8
8
|
export { RedisClientType as IORedisClientType, RedisMultiType as IORedisMultiType } from './ioredisclient';
|
|
9
9
|
export { ILogger } from './logger';
|
package/index.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Durable } from './services/durable';
|
|
2
|
+
import { MeshOSService as MeshOS } from './services/durable/meshos';
|
|
2
3
|
import { HotMeshService as HotMesh } from './services/hotmesh';
|
|
3
4
|
import { HotMeshConfig } from './types/hotmesh';
|
|
4
5
|
|
|
5
|
-
export { Durable, HotMesh, HotMeshConfig };
|
|
6
|
+
export { Durable, HotMesh, HotMeshConfig, MeshOS };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hotmeshio/hotmesh",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.24",
|
|
4
4
|
"description": "Unbreakable Workflows",
|
|
5
5
|
"main": "build/index.js",
|
|
6
6
|
"types": "build/index.d.ts",
|
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
"test:sub:redis": "NODE_ENV=test jest ./tests/functional/sub/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
|
|
44
44
|
"test:sub:ioredis": "NODE_ENV=test jest ./tests/functional/sub/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
|
|
45
45
|
"test:durable": "NODE_ENV=test jest ./tests/durable/*/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
46
|
-
"test:durable:
|
|
46
|
+
"test:durable:meshos": "NODE_ENV=test jest ./tests/durable/meshos/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
47
47
|
"test:durable:hello": "NODE_ENV=test jest ./tests/durable/helloworld/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
48
48
|
"test:durable:goodbye": "NODE_ENV=test jest ./tests/durable/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
49
49
|
"test:durable:hook": "NODE_ENV=test jest ./tests/durable/hook/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
@@ -264,11 +264,11 @@ const getWorkflowYAML = (app: string, version: string) => {
|
|
|
264
264
|
- ['{@string.concat}']
|
|
265
265
|
cycleWorkflowId:
|
|
266
266
|
'@pipe':
|
|
267
|
-
- ['{$job.metadata.jid}', '-$wfc', '{sig.output.metadata.dad}', '-', '{sigw1.output.data.index}']
|
|
267
|
+
- ['-', '{$job.metadata.jid}', '-$wfc', '{sig.output.metadata.dad}', '-', '{sigw1.output.data.index}']
|
|
268
268
|
- ['{@string.concat}']
|
|
269
269
|
baseWorkflowId:
|
|
270
270
|
'@pipe':
|
|
271
|
-
- ['{$job.metadata.jid}', '-$wfs', '{sig.output.metadata.dad}', '-']
|
|
271
|
+
- ['-', '{$job.metadata.jid}', '-$wfs', '{sig.output.metadata.dad}', '-']
|
|
272
272
|
- ['{@string.concat}']
|
|
273
273
|
output:
|
|
274
274
|
schema:
|
|
@@ -312,7 +312,7 @@ const getWorkflowYAML = (app: string, version: string) => {
|
|
|
312
312
|
- ['{@string.concat}']
|
|
313
313
|
workflowId:
|
|
314
314
|
'@pipe':
|
|
315
|
-
- ['{$job.metadata.jid}', '-$sleep', '{sig.output.metadata.dad}', '-', '{sigw1.output.data.index}']
|
|
315
|
+
- ['-', '{$job.metadata.jid}', '-$sleep', '{sig.output.metadata.dad}', '-', '{sigw1.output.data.index}']
|
|
316
316
|
- ['{@string.concat}']
|
|
317
317
|
output:
|
|
318
318
|
schema:
|
|
@@ -382,11 +382,11 @@ const getWorkflowYAML = (app: string, version: string) => {
|
|
|
382
382
|
- ['{@string.concat}']
|
|
383
383
|
cycleWorkflowId:
|
|
384
384
|
'@pipe':
|
|
385
|
-
- ['{$job.metadata.jid}', '-$wfc-', '{w1.output.data.index}']
|
|
385
|
+
- ['-', '{$job.metadata.jid}', '-$wfc-', '{w1.output.data.index}']
|
|
386
386
|
- ['{@string.concat}']
|
|
387
387
|
baseWorkflowId:
|
|
388
388
|
'@pipe':
|
|
389
|
-
- ['{$job.metadata.jid}', '-$wfs-']
|
|
389
|
+
- ['-', '{$job.metadata.jid}', '-$wfs-']
|
|
390
390
|
- ['{@string.concat}']
|
|
391
391
|
output:
|
|
392
392
|
schema:
|
|
@@ -430,7 +430,7 @@ const getWorkflowYAML = (app: string, version: string) => {
|
|
|
430
430
|
- ['{@string.concat}']
|
|
431
431
|
workflowId:
|
|
432
432
|
'@pipe':
|
|
433
|
-
- ['{$job.metadata.jid}', '-$sleep-', '{w1.output.data.index}']
|
|
433
|
+
- ['-', '{$job.metadata.jid}', '-$sleep-', '{w1.output.data.index}']
|
|
434
434
|
- ['{@string.concat}']
|
|
435
435
|
output:
|
|
436
436
|
schema:
|
|
@@ -39,10 +39,8 @@ export class WorkflowHandleService {
|
|
|
39
39
|
throw new Error(JSON.parse(state.metadata.err));
|
|
40
40
|
}
|
|
41
41
|
if (state?.data?.done) {
|
|
42
|
-
//child flows are never
|
|
43
|
-
//that
|
|
44
|
-
//the 'done' flag on the child flow's payload (not the 'js' metadata field
|
|
45
|
-
//which is typically used); the `loadState` parameter triggers this
|
|
42
|
+
//child flows are never 'done'; they use a hook
|
|
43
|
+
//that only closes upon parent flow completion.
|
|
46
44
|
return state.data.response;
|
|
47
45
|
}
|
|
48
46
|
}
|