@hotmeshio/hotmesh 0.0.22 → 0.0.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -67
- package/build/index.d.ts +2 -1
- package/build/index.js +3 -1
- package/build/package.json +2 -1
- package/build/services/durable/connection.js +0 -39
- package/build/services/durable/factory.d.ts +6 -4
- package/build/services/durable/factory.js +12 -10
- package/build/services/durable/handle.d.ts +3 -0
- package/build/services/durable/handle.js +15 -5
- package/build/services/durable/index.d.ts +2 -45
- package/build/services/durable/index.js +2 -45
- package/build/services/durable/meshos.d.ts +108 -0
- package/build/services/durable/meshos.js +289 -0
- package/build/services/durable/search.d.ts +9 -0
- package/build/services/durable/search.js +34 -2
- package/build/services/durable/worker.d.ts +2 -10
- package/build/services/durable/worker.js +10 -39
- package/build/services/durable/workflow.d.ts +12 -1
- package/build/services/durable/workflow.js +32 -16
- package/build/services/engine/index.d.ts +2 -0
- package/build/services/engine/index.js +3 -0
- package/build/services/hotmesh/index.d.ts +3 -1
- package/build/services/hotmesh/index.js +5 -2
- package/build/services/signaler/stream.js +1 -2
- package/build/services/store/clients/ioredis.js +2 -2
- package/build/services/store/clients/redis.js +1 -1
- package/build/services/store/index.d.ts +5 -0
- package/build/services/store/index.js +14 -0
- package/build/types/durable.d.ts +34 -4
- package/build/types/index.d.ts +1 -1
- package/index.ts +2 -1
- package/package.json +2 -1
- package/services/durable/connection.ts +0 -40
- package/services/durable/factory.ts +12 -10
- package/services/durable/handle.ts +18 -5
- package/services/durable/index.ts +2 -46
- package/services/durable/meshos.ts +344 -0
- package/services/durable/search.ts +35 -2
- package/services/durable/worker.ts +11 -42
- package/services/durable/workflow.ts +34 -17
- package/services/engine/index.ts +4 -1
- package/services/hotmesh/index.ts +6 -2
- package/services/signaler/stream.ts +1 -2
- package/services/store/clients/ioredis.ts +2 -3
- package/services/store/clients/redis.ts +1 -1
- package/services/store/index.ts +15 -0
- package/types/durable.ts +40 -4
- package/types/index.ts +6 -1
- package/build/services/durable/native.d.ts +0 -4
- package/build/services/durable/native.js +0 -46
- package/services/durable/native.ts +0 -45
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MeshOSService = void 0;
|
|
4
|
+
const nanoid_1 = require("nanoid");
|
|
5
|
+
const client_1 = require("./client");
|
|
6
|
+
const search_1 = require("./search");
|
|
7
|
+
const worker_1 = require("./worker");
|
|
8
|
+
const _1 = require(".");
|
|
9
|
+
const asyncLocalStorage_1 = require("./asyncLocalStorage");
|
|
10
|
+
const workflow_1 = require("./workflow");
|
|
11
|
+
/**
|
|
12
|
+
* The base class for running MeshOS workflows.
|
|
13
|
+
* Extend and register subclass methods by name to
|
|
14
|
+
* execute as durable workflows, backed by Redis.
|
|
15
|
+
*/
|
|
16
|
+
class MeshOSService {
|
|
17
|
+
static async getHotMeshClient(redisClass, redisOptions, namespace, taskQueue) {
|
|
18
|
+
const client = new client_1.ClientService({
|
|
19
|
+
connection: {
|
|
20
|
+
class: redisClass,
|
|
21
|
+
options: redisOptions,
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
return await client.getHotMeshClient(taskQueue, namespace);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* mints a workflow ID, using the search prefix.
|
|
28
|
+
* NOTE: The prefix is necesary when indexing
|
|
29
|
+
* HASHes when FT search is enabled.
|
|
30
|
+
* @returns {string}
|
|
31
|
+
*/
|
|
32
|
+
static mintGuid() {
|
|
33
|
+
const my = new this();
|
|
34
|
+
return `${my.search?.prefix?.[0]}${(0, nanoid_1.nanoid)()}}`;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Creates an FT search index
|
|
38
|
+
*/
|
|
39
|
+
static async createIndex() {
|
|
40
|
+
const my = new this();
|
|
41
|
+
const hmClient = await MeshOSService.getHotMeshClient(my.redisClass, my.redisOptions, my.namespace, my.taskQueue);
|
|
42
|
+
search_1.Search.configureSearchIndex(hmClient, my.search);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Initialize the worker(s) for the entity. This is a static
|
|
46
|
+
* method that allows for optional task Queue targeting.
|
|
47
|
+
* NOTE: Allow List may be optionally used to only wrap
|
|
48
|
+
* specific methods in this class.
|
|
49
|
+
* @param {string} taskQueue
|
|
50
|
+
* @param {string[]} allowList
|
|
51
|
+
*/
|
|
52
|
+
static async startWorkers(taskQueue, allowList = []) {
|
|
53
|
+
const my = new this();
|
|
54
|
+
//helper functions
|
|
55
|
+
const resolveFunctionNames = (arr) => arr.map(item => typeof item === 'string' ? item : item.name);
|
|
56
|
+
const belongsTo = (name, target) => {
|
|
57
|
+
const isWorkflow = target.find((item) => {
|
|
58
|
+
return typeof item === 'string' ? item === name : item.name === name;
|
|
59
|
+
});
|
|
60
|
+
return isWorkflow !== undefined;
|
|
61
|
+
};
|
|
62
|
+
// proxy registered activities
|
|
63
|
+
const proxyFunctionNames = resolveFunctionNames([...my.proxyFunctions]);
|
|
64
|
+
if (proxyFunctionNames.length) {
|
|
65
|
+
const proxyActivities = proxyFunctionNames.reduce((acc, funcName) => {
|
|
66
|
+
let originalMethod = my[funcName];
|
|
67
|
+
if (typeof originalMethod === 'function') {
|
|
68
|
+
acc[funcName] = async (...args) => {
|
|
69
|
+
return await originalMethod.apply(my, args);
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
return acc;
|
|
73
|
+
}, {});
|
|
74
|
+
const proxiedActivities = _1.Durable.workflow.proxyActivities({
|
|
75
|
+
activities: proxyActivities
|
|
76
|
+
});
|
|
77
|
+
//WATCH!: unsure if this will pollute the scope; don't think
|
|
78
|
+
// so as activity functions are terminal in the chain.
|
|
79
|
+
Object.assign(my, proxiedActivities);
|
|
80
|
+
}
|
|
81
|
+
const functionsToIterate = allowList.length ? resolveFunctionNames(allowList) : resolveFunctionNames([...my.workflowFunctions, ...my.hookFunctions]);
|
|
82
|
+
// Iterating through the functions sequentially
|
|
83
|
+
for (const funcName of functionsToIterate) {
|
|
84
|
+
const originalMethod = my[funcName];
|
|
85
|
+
if (typeof originalMethod === 'function') {
|
|
86
|
+
//wrap the function to return
|
|
87
|
+
const wrappedFunction = {
|
|
88
|
+
[funcName]: async (...args) => {
|
|
89
|
+
const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
|
|
90
|
+
const workflowId = store.get('workflowId');
|
|
91
|
+
//use a Proxy to wrap hook methods
|
|
92
|
+
const context = new Proxy(my, {
|
|
93
|
+
get: (target, prop, receiver) => {
|
|
94
|
+
if (prop === 'id') {
|
|
95
|
+
return workflowId;
|
|
96
|
+
}
|
|
97
|
+
else if (typeof target[prop] === 'function') {
|
|
98
|
+
return (...args) => {
|
|
99
|
+
return new Promise(async (resolve, reject) => {
|
|
100
|
+
if (belongsTo(prop, my.hookFunctions)) {
|
|
101
|
+
return workflow_1.WorkflowService.hook({
|
|
102
|
+
namespace: my.namespace,
|
|
103
|
+
taskQueue: my.taskQueue,
|
|
104
|
+
workflowName: prop,
|
|
105
|
+
workflowId,
|
|
106
|
+
args,
|
|
107
|
+
}).then(resolve).catch(reject);
|
|
108
|
+
}
|
|
109
|
+
//otherwise, call the method as a standard instance method.
|
|
110
|
+
target[prop].apply(this, args).then(resolve).catch(reject);
|
|
111
|
+
});
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
return Reflect.get(target, prop, receiver);
|
|
115
|
+
},
|
|
116
|
+
});
|
|
117
|
+
return await originalMethod.apply(context, args);
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
//start the worker
|
|
121
|
+
await worker_1.WorkerService.create({
|
|
122
|
+
namespace: my.namespace,
|
|
123
|
+
connection: {
|
|
124
|
+
class: my.redisClass,
|
|
125
|
+
options: my.redisOptions,
|
|
126
|
+
},
|
|
127
|
+
taskQueue: taskQueue ?? my.taskQueue,
|
|
128
|
+
workflow: wrappedFunction,
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* executes the redis FT search query
|
|
135
|
+
* @example '@_quantity:[89 89]'
|
|
136
|
+
* @param {any[]} args
|
|
137
|
+
* @returns {string}
|
|
138
|
+
*/
|
|
139
|
+
static async find(options, ...args) {
|
|
140
|
+
const my = new this();
|
|
141
|
+
const client = new client_1.ClientService({ connection: {
|
|
142
|
+
class: my.redisClass,
|
|
143
|
+
options: my.redisOptions
|
|
144
|
+
} });
|
|
145
|
+
//workflow name is the function name driving the workflow
|
|
146
|
+
let workflowName;
|
|
147
|
+
if (options?.workflowName) {
|
|
148
|
+
workflowName = options?.workflowName;
|
|
149
|
+
}
|
|
150
|
+
else if (my.workflowFunctions?.length) {
|
|
151
|
+
let target = my.workflowFunctions[0];
|
|
152
|
+
if (typeof target === 'string') {
|
|
153
|
+
workflowName = target;
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
workflowName = target.name;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
return await client.workflow.search(options?.taskQueue ?? my.taskQueue, workflowName, my.namespace, my.search.index, ...args); //[count, [id, fields[], id, fields[], id, fields[], ...]]
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* returns the workflow handle. The handle can then be
|
|
163
|
+
* used to query for status, state, custom state, etc.
|
|
164
|
+
* @param {string} id
|
|
165
|
+
* @returns {Promise<WorkflowHandleService>}
|
|
166
|
+
*/
|
|
167
|
+
static async get(id) {
|
|
168
|
+
const my = new this();
|
|
169
|
+
const client = new client_1.ClientService({ connection: {
|
|
170
|
+
class: my.redisClass,
|
|
171
|
+
options: my.redisOptions
|
|
172
|
+
} });
|
|
173
|
+
let workflowName;
|
|
174
|
+
let target = my.workflowFunctions[0];
|
|
175
|
+
if (typeof target === 'string') {
|
|
176
|
+
workflowName = target;
|
|
177
|
+
}
|
|
178
|
+
else {
|
|
179
|
+
workflowName = target.name;
|
|
180
|
+
}
|
|
181
|
+
return await client.workflow.getHandle(my.taskQueue, workflowName, id, my.namespace);
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Optionally include a target taskQueue to exec the
|
|
185
|
+
* workflow's call on a specific worker queue.
|
|
186
|
+
*/
|
|
187
|
+
constructor(id, options) {
|
|
188
|
+
/**
|
|
189
|
+
* The top-level Redis isolation. All workflow data is
|
|
190
|
+
* isolated within this namespace. Values should be
|
|
191
|
+
* lower-case with no spaces (e.g, 'staging', 'prod', 'test',
|
|
192
|
+
* 'routing-stagig', 'reporting-prod', etc.).
|
|
193
|
+
* 1) only url-safe values are allowed;
|
|
194
|
+
* 2) the 'a' symbol is reserved by HotMesh for indexing apps
|
|
195
|
+
*/
|
|
196
|
+
this.namespace = 'durable';
|
|
197
|
+
/**
|
|
198
|
+
* Data is routed to workers that specify this task queue.
|
|
199
|
+
* Setting the task queue when the worker is created will
|
|
200
|
+
* ensure that the worker only receives messages destined
|
|
201
|
+
* for the queue. Callers can specify the taskQue to when
|
|
202
|
+
* starting a job to call those workers.
|
|
203
|
+
*/
|
|
204
|
+
this.taskQueue = 'default';
|
|
205
|
+
/**
|
|
206
|
+
* These methods run as durable workflows
|
|
207
|
+
*/
|
|
208
|
+
this.workflowFunctions = [];
|
|
209
|
+
/**
|
|
210
|
+
* These methods run as hooks (hook into a running workflow)
|
|
211
|
+
*/
|
|
212
|
+
this.hookFunctions = [];
|
|
213
|
+
/**
|
|
214
|
+
* These methods run as proxied activities (and are safely memoized)
|
|
215
|
+
*/
|
|
216
|
+
this.proxyFunctions = [];
|
|
217
|
+
/**
|
|
218
|
+
* The Redis connection options. NOTE: Redis and IORedis
|
|
219
|
+
* use different formats for their connection config.
|
|
220
|
+
*/
|
|
221
|
+
this.redisOptions = {
|
|
222
|
+
host: 'localhost',
|
|
223
|
+
port: 6379,
|
|
224
|
+
password: '',
|
|
225
|
+
db: 0,
|
|
226
|
+
};
|
|
227
|
+
this.id = id;
|
|
228
|
+
if (options?.taskQueue) {
|
|
229
|
+
this.taskQueue = options.taskQueue;
|
|
230
|
+
}
|
|
231
|
+
else if (!id && !options?.taskQueue) {
|
|
232
|
+
return this;
|
|
233
|
+
}
|
|
234
|
+
function belongsTo(name, target) {
|
|
235
|
+
const isWorkflow = target.find((item) => {
|
|
236
|
+
return typeof item === 'string' ? item === name : item.name === name;
|
|
237
|
+
});
|
|
238
|
+
return isWorkflow !== undefined;
|
|
239
|
+
}
|
|
240
|
+
return new Proxy(this, {
|
|
241
|
+
get: (target, prop, receiver) => {
|
|
242
|
+
if (typeof target[prop] === 'function') {
|
|
243
|
+
return (...args) => {
|
|
244
|
+
return new Promise(async (resolve, reject) => {
|
|
245
|
+
const client = new client_1.ClientService({ connection: {
|
|
246
|
+
class: this.redisClass,
|
|
247
|
+
options: this.redisOptions
|
|
248
|
+
} });
|
|
249
|
+
if (belongsTo(prop, this.workflowFunctions)) {
|
|
250
|
+
//start a new workflow
|
|
251
|
+
const handle = await client.workflow.start({
|
|
252
|
+
namespace: this.namespace,
|
|
253
|
+
args,
|
|
254
|
+
taskQueue: this.taskQueue,
|
|
255
|
+
workflowName: prop,
|
|
256
|
+
workflowId: this.id,
|
|
257
|
+
});
|
|
258
|
+
if (options?.await) {
|
|
259
|
+
//wait for the workflow to complete
|
|
260
|
+
const result = await handle.result();
|
|
261
|
+
return resolve(result);
|
|
262
|
+
}
|
|
263
|
+
else {
|
|
264
|
+
//return the workflow handle
|
|
265
|
+
return resolve(handle);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
else if (belongsTo(prop, this.hookFunctions)) {
|
|
269
|
+
//hook into a running workflow
|
|
270
|
+
return client.workflow.hook({
|
|
271
|
+
namespace: this.namespace,
|
|
272
|
+
taskQueue: this.taskQueue,
|
|
273
|
+
workflowName: prop,
|
|
274
|
+
workflowId: this.id,
|
|
275
|
+
args,
|
|
276
|
+
}).then(resolve).catch(reject);
|
|
277
|
+
}
|
|
278
|
+
//otherwise, call the method as a standard instance method.
|
|
279
|
+
target[prop].apply(this, args).then(resolve).catch(reject);
|
|
280
|
+
});
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
return Reflect.get(target, prop, receiver);
|
|
284
|
+
},
|
|
285
|
+
});
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
MeshOSService.MeshOS = workflow_1.WorkflowService;
|
|
289
|
+
exports.MeshOSService = MeshOSService;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { HotMeshService as HotMesh } from '../hotmesh';
|
|
2
2
|
import { RedisClient, RedisMulti } from '../../types/redis';
|
|
3
3
|
import { StoreService } from '../store';
|
|
4
|
+
import { WorkflowSearchOptions } from '../../types/durable';
|
|
4
5
|
export declare class Search {
|
|
5
6
|
jobId: string;
|
|
6
7
|
searchSessionId: string;
|
|
@@ -8,6 +9,14 @@ export declare class Search {
|
|
|
8
9
|
hotMeshClient: HotMesh;
|
|
9
10
|
store: StoreService<RedisClient, RedisMulti> | null;
|
|
10
11
|
safeKey(key: string): string;
|
|
12
|
+
/**
|
|
13
|
+
* For those deployments with a redis stack backend (with the FT module),
|
|
14
|
+
* this method will configure the search index for the workflow. For all
|
|
15
|
+
* others, this method will exit/fail gracefully and not index
|
|
16
|
+
* the fields in the HASH. However, all values are still available
|
|
17
|
+
* in the HASH.
|
|
18
|
+
*/
|
|
19
|
+
static configureSearchIndex(hotMeshClient: HotMesh, search?: WorkflowSearchOptions): Promise<void>;
|
|
11
20
|
constructor(workflowId: string, hotMeshClient: HotMesh, searchSessionId: string);
|
|
12
21
|
/**
|
|
13
22
|
* increments the index to return a unique search session guid when
|
|
@@ -4,10 +4,42 @@ exports.Search = void 0;
|
|
|
4
4
|
const key_1 = require("../../modules/key");
|
|
5
5
|
class Search {
|
|
6
6
|
safeKey(key) {
|
|
7
|
-
//note: protect the execution namespace with a prefix
|
|
8
|
-
//so its design never conflicts with the hotmesh keyspace
|
|
7
|
+
//note: protect the execution namespace with a prefix
|
|
9
8
|
return `_${key}`;
|
|
10
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* For those deployments with a redis stack backend (with the FT module),
|
|
12
|
+
* this method will configure the search index for the workflow. For all
|
|
13
|
+
* others, this method will exit/fail gracefully and not index
|
|
14
|
+
* the fields in the HASH. However, all values are still available
|
|
15
|
+
* in the HASH.
|
|
16
|
+
*/
|
|
17
|
+
static async configureSearchIndex(hotMeshClient, search) {
|
|
18
|
+
if (search?.schema) {
|
|
19
|
+
const store = hotMeshClient.engine.store;
|
|
20
|
+
const schema = [];
|
|
21
|
+
for (const [key, value] of Object.entries(search.schema)) {
|
|
22
|
+
//prefix with a comma (avoids collisions with hotmesh reserved words)
|
|
23
|
+
schema.push(`_${key}`);
|
|
24
|
+
schema.push(value.type);
|
|
25
|
+
if (value.sortable) {
|
|
26
|
+
schema.push('SORTABLE');
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
try {
|
|
30
|
+
const keyParams = {
|
|
31
|
+
appId: hotMeshClient.appId,
|
|
32
|
+
jobId: ''
|
|
33
|
+
};
|
|
34
|
+
const hotMeshPrefix = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
|
|
35
|
+
const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
|
|
36
|
+
await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length.toString(), ...prefixes, 'SCHEMA', ...schema);
|
|
37
|
+
}
|
|
38
|
+
catch (err) {
|
|
39
|
+
hotMeshClient.engine.logger.info('durable-client-search-err', { err });
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
11
43
|
constructor(workflowId, hotMeshClient, searchSessionId) {
|
|
12
44
|
this.searchSessionIndex = 0;
|
|
13
45
|
const keyParams = {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { HotMeshService as HotMesh } from '../hotmesh';
|
|
2
|
-
import { Connection, Registry, WorkerConfig, WorkerOptions
|
|
2
|
+
import { Connection, Registry, WorkerConfig, WorkerOptions } from '../../types/durable';
|
|
3
3
|
export declare class WorkerService {
|
|
4
4
|
static activityRegistry: Registry;
|
|
5
5
|
static connection: Connection;
|
|
@@ -9,16 +9,8 @@ export declare class WorkerService {
|
|
|
9
9
|
static getHotMesh: (workflowTopic: string, config?: Partial<WorkerConfig>, options?: WorkerOptions) => Promise<HotMesh>;
|
|
10
10
|
static activateWorkflow(hotMesh: HotMesh): Promise<void>;
|
|
11
11
|
static registerActivities<ACT>(activities: ACT): Registry;
|
|
12
|
-
/**
|
|
13
|
-
* For those deployments with a redis stack backend (with the FT module),
|
|
14
|
-
* this method will configure the search index for the workflow. For all
|
|
15
|
-
* others, this method will fail gracefully. In all cases, the values
|
|
16
|
-
* will be stored in the workflow's central HASH data structure, allowing
|
|
17
|
-
* for manual traversal and inspection as well.
|
|
18
|
-
*/
|
|
19
|
-
static configureSearchIndex(hotMeshClient: HotMesh, search?: WorkflowSearchOptions): Promise<void>;
|
|
20
12
|
static create(config: WorkerConfig): Promise<WorkerService>;
|
|
21
|
-
static resolveWorkflowTarget(workflow: object | Function): [string, Function];
|
|
13
|
+
static resolveWorkflowTarget(workflow: object | Function, name?: string): [string, Function];
|
|
22
14
|
run(): Promise<void>;
|
|
23
15
|
initActivityWorker(config: WorkerConfig, activityTopic: string): Promise<HotMesh>;
|
|
24
16
|
wrapActivityFunctions(): Function;
|
|
@@ -3,10 +3,10 @@ var _a;
|
|
|
3
3
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
4
|
exports.WorkerService = void 0;
|
|
5
5
|
const errors_1 = require("../../modules/errors");
|
|
6
|
-
const key_1 = require("../../modules/key");
|
|
7
6
|
const asyncLocalStorage_1 = require("./asyncLocalStorage");
|
|
8
7
|
const factory_1 = require("./factory");
|
|
9
8
|
const hotmesh_1 = require("../hotmesh");
|
|
9
|
+
const search_1 = require("./search");
|
|
10
10
|
const stream_1 = require("../../types/stream");
|
|
11
11
|
class WorkerService {
|
|
12
12
|
static async activateWorkflow(hotMesh) {
|
|
@@ -41,43 +41,13 @@ class WorkerService {
|
|
|
41
41
|
if (activities[key].name && typeof WorkerService.activityRegistry[activities[key].name] !== 'function') {
|
|
42
42
|
WorkerService.activityRegistry[activities[key].name] = activities[key];
|
|
43
43
|
}
|
|
44
|
+
else if (typeof activities[key] === 'function') {
|
|
45
|
+
WorkerService.activityRegistry[key] = activities[key];
|
|
46
|
+
}
|
|
44
47
|
});
|
|
45
48
|
}
|
|
46
49
|
return WorkerService.activityRegistry;
|
|
47
50
|
}
|
|
48
|
-
/**
|
|
49
|
-
* For those deployments with a redis stack backend (with the FT module),
|
|
50
|
-
* this method will configure the search index for the workflow. For all
|
|
51
|
-
* others, this method will fail gracefully. In all cases, the values
|
|
52
|
-
* will be stored in the workflow's central HASH data structure, allowing
|
|
53
|
-
* for manual traversal and inspection as well.
|
|
54
|
-
*/
|
|
55
|
-
static async configureSearchIndex(hotMeshClient, search) {
|
|
56
|
-
if (search?.schema) {
|
|
57
|
-
const store = hotMeshClient.engine.store;
|
|
58
|
-
const schema = [];
|
|
59
|
-
for (const [key, value] of Object.entries(search.schema)) {
|
|
60
|
-
//prefix with a comma (avoids collisions with hotmesh reserved words)
|
|
61
|
-
schema.push(`_${key}`);
|
|
62
|
-
schema.push(value.type);
|
|
63
|
-
if (value.sortable) {
|
|
64
|
-
schema.push('SORTABLE');
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
try {
|
|
68
|
-
const keyParams = {
|
|
69
|
-
appId: hotMeshClient.appId,
|
|
70
|
-
jobId: ''
|
|
71
|
-
};
|
|
72
|
-
const hotMeshPrefix = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
|
|
73
|
-
const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
|
|
74
|
-
await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length, ...prefixes, 'SCHEMA', ...schema);
|
|
75
|
-
}
|
|
76
|
-
catch (err) {
|
|
77
|
-
hotMeshClient.engine.logger.info('durable-client-search-err', { err });
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
51
|
static async create(config) {
|
|
82
52
|
WorkerService.connection = config.connection;
|
|
83
53
|
const workflow = config.workflow;
|
|
@@ -89,21 +59,22 @@ class WorkerService {
|
|
|
89
59
|
const worker = new WorkerService();
|
|
90
60
|
worker.activityRunner = await worker.initActivityWorker(config, activityTopic);
|
|
91
61
|
worker.workflowRunner = await worker.initWorkflowWorker(config, workflowTopic, workflowFunction);
|
|
92
|
-
|
|
62
|
+
search_1.Search.configureSearchIndex(worker.workflowRunner, config.search);
|
|
93
63
|
await WorkerService.activateWorkflow(worker.workflowRunner);
|
|
94
64
|
return worker;
|
|
95
65
|
}
|
|
96
|
-
static resolveWorkflowTarget(workflow) {
|
|
66
|
+
static resolveWorkflowTarget(workflow, name) {
|
|
97
67
|
let workflowFunction;
|
|
98
68
|
if (typeof workflow === 'function') {
|
|
99
69
|
workflowFunction = workflow;
|
|
70
|
+
return [workflowFunction.name ?? name, workflowFunction];
|
|
100
71
|
}
|
|
101
72
|
else {
|
|
102
73
|
const workflowFunctionNames = Object.keys(workflow);
|
|
103
|
-
|
|
104
|
-
|
|
74
|
+
const lastFunctionName = workflowFunctionNames[workflowFunctionNames.length - 1];
|
|
75
|
+
workflowFunction = workflow[lastFunctionName];
|
|
76
|
+
return WorkerService.resolveWorkflowTarget(workflowFunction, lastFunctionName);
|
|
105
77
|
}
|
|
106
|
-
return [workflowFunction.name, workflowFunction];
|
|
107
78
|
}
|
|
108
79
|
async run() {
|
|
109
80
|
this.workflowRunner.engine.logger.info('WorkerService is running');
|
|
@@ -10,11 +10,22 @@ export declare class WorkflowService {
|
|
|
10
10
|
* spawn a child workflow. return the childJobId.
|
|
11
11
|
*/
|
|
12
12
|
static startChild<T>(options: WorkflowOptions): Promise<string>;
|
|
13
|
+
/**
|
|
14
|
+
* wrap all activities in a proxy that will durably run them
|
|
15
|
+
*/
|
|
13
16
|
static proxyActivities<ACT>(options?: ActivityConfig): ProxyType<ACT>;
|
|
17
|
+
/**
|
|
18
|
+
* return a search session for use when reading/writing to the workflow HASH
|
|
19
|
+
*/
|
|
14
20
|
static search(): Promise<Search>;
|
|
21
|
+
/**
|
|
22
|
+
* return a handle to the hotmesh client currently running the workflow
|
|
23
|
+
*/
|
|
24
|
+
static getHotMesh(): Promise<HotMesh>;
|
|
15
25
|
/**
|
|
16
26
|
* those methods that may only be called once must be protected by flagging
|
|
17
|
-
* their execution with a unique key (the key is stored in the
|
|
27
|
+
* their execution with a unique key (the key is stored in the HASH alongside
|
|
28
|
+
* process state and job state)
|
|
18
29
|
*/
|
|
19
30
|
static isSideEffectAllowed(hotMeshClient: HotMesh, prefix: string): Promise<boolean>;
|
|
20
31
|
/**
|
|
@@ -30,7 +30,7 @@ class WorkflowService {
|
|
|
30
30
|
//this is risky but MUST be allowed. Users MAY set the workflowId,
|
|
31
31
|
//but if there is a naming collision, the data from the target entity will be used
|
|
32
32
|
//as there is know way of knowing if the item was generated via a prior run of the workflow
|
|
33
|
-
const childJobId = options.workflowId ??
|
|
33
|
+
const childJobId = options.workflowId ?? `-${workflowId}-$${options.workflowName}${workflowDimension}-${execIndex}`;
|
|
34
34
|
const parentWorkflowId = `${workflowId}-f`;
|
|
35
35
|
const client = new client_1.ClientService({
|
|
36
36
|
connection: await connection_1.ConnectionService.connect(worker_1.WorkerService.connection),
|
|
@@ -65,7 +65,7 @@ class WorkflowService {
|
|
|
65
65
|
const workflowSpan = store.get('workflowSpan');
|
|
66
66
|
const COUNTER = store.get('counter');
|
|
67
67
|
const execIndex = COUNTER.counter = COUNTER.counter + 1;
|
|
68
|
-
const childJobId = options.workflowId ??
|
|
68
|
+
const childJobId = options.workflowId ?? `-${workflowId}-$${options.workflowName}${workflowDimension}-${execIndex}`;
|
|
69
69
|
const parentWorkflowId = `${workflowId}-f`;
|
|
70
70
|
const workflowTopic = `${options.taskQueue}-${options.workflowName}`;
|
|
71
71
|
try {
|
|
@@ -89,6 +89,9 @@ class WorkflowService {
|
|
|
89
89
|
return childJobId;
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
|
+
/**
|
|
93
|
+
* wrap all activities in a proxy that will durably run them
|
|
94
|
+
*/
|
|
92
95
|
static proxyActivities(options) {
|
|
93
96
|
if (options.activities) {
|
|
94
97
|
worker_1.WorkerService.registerActivities(options.activities);
|
|
@@ -103,6 +106,9 @@ class WorkflowService {
|
|
|
103
106
|
}
|
|
104
107
|
return proxy;
|
|
105
108
|
}
|
|
109
|
+
/**
|
|
110
|
+
* return a search session for use when reading/writing to the workflow HASH
|
|
111
|
+
*/
|
|
106
112
|
static async search() {
|
|
107
113
|
const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
|
|
108
114
|
const workflowId = store.get('workflowId');
|
|
@@ -111,14 +117,24 @@ class WorkflowService {
|
|
|
111
117
|
const namespace = store.get('namespace');
|
|
112
118
|
const COUNTER = store.get('counter');
|
|
113
119
|
const execIndex = COUNTER.counter = COUNTER.counter + 1;
|
|
114
|
-
//this ID is used as a item key with a hash (dash prefix ensures no collision)
|
|
115
120
|
const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
|
|
121
|
+
//this ID is used as a item key with a hash (dash prefix ensures no collision)
|
|
116
122
|
const searchSessionId = `-search${workflowDimension}-${execIndex}`;
|
|
117
123
|
return new search_1.Search(workflowId, hotMeshClient, searchSessionId);
|
|
118
124
|
}
|
|
125
|
+
/**
|
|
126
|
+
* return a handle to the hotmesh client currently running the workflow
|
|
127
|
+
*/
|
|
128
|
+
static async getHotMesh() {
|
|
129
|
+
const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
|
|
130
|
+
const workflowTopic = store.get('workflowTopic');
|
|
131
|
+
const namespace = store.get('namespace');
|
|
132
|
+
return await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
|
|
133
|
+
}
|
|
119
134
|
/**
|
|
120
135
|
* those methods that may only be called once must be protected by flagging
|
|
121
|
-
* their execution with a unique key (the key is stored in the
|
|
136
|
+
* their execution with a unique key (the key is stored in the HASH alongside
|
|
137
|
+
* process state and job state)
|
|
122
138
|
*/
|
|
123
139
|
static async isSideEffectAllowed(hotMeshClient, prefix) {
|
|
124
140
|
const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
|
|
@@ -126,15 +142,12 @@ class WorkflowService {
|
|
|
126
142
|
const workflowDimension = store.get('workflowDimension') ?? '';
|
|
127
143
|
const COUNTER = store.get('counter');
|
|
128
144
|
const execIndex = COUNTER.counter = COUNTER.counter + 1;
|
|
129
|
-
//this ID is used as a item key with a hash (dash prefix ensures no collision)
|
|
130
145
|
const sessionId = `-${prefix}${workflowDimension}-${execIndex}-`;
|
|
131
|
-
//this ID is used as a item key with a hash (dash prefix ensures no collision)
|
|
132
146
|
const keyParams = {
|
|
133
147
|
appId: hotMeshClient.appId,
|
|
134
|
-
jobId:
|
|
148
|
+
jobId: workflowId
|
|
135
149
|
};
|
|
136
|
-
const
|
|
137
|
-
const workflowGuid = `${hotMeshPrefix}${workflowId}`;
|
|
150
|
+
const workflowGuid = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
|
|
138
151
|
const guidValue = Number(await hotMeshClient.engine.store.exec('HINCRBYFLOAT', workflowGuid, sessionId, '1'));
|
|
139
152
|
return guidValue === 1;
|
|
140
153
|
}
|
|
@@ -144,8 +157,11 @@ class WorkflowService {
|
|
|
144
157
|
*/
|
|
145
158
|
static async signal(signalId, data) {
|
|
146
159
|
const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
|
|
160
|
+
const workflowTopic = store.get('workflowTopic');
|
|
147
161
|
const namespace = store.get('namespace');
|
|
148
|
-
const hotMeshClient = await worker_1.WorkerService.getHotMesh(
|
|
162
|
+
const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
|
|
163
|
+
//todo: this particular one is better patterned as a get/set,
|
|
164
|
+
//since the receipt is a meaningful string (the stream id)
|
|
149
165
|
if (await WorkflowService.isSideEffectAllowed(hotMeshClient, 'signal')) {
|
|
150
166
|
return await hotMeshClient.hook(`${namespace}.wfs.signal`, { id: signalId, data });
|
|
151
167
|
}
|
|
@@ -157,8 +173,9 @@ class WorkflowService {
|
|
|
157
173
|
*/
|
|
158
174
|
static async hook(options) {
|
|
159
175
|
const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
|
|
176
|
+
const workflowTopic = store.get('workflowTopic');
|
|
160
177
|
const namespace = store.get('namespace');
|
|
161
|
-
const hotMeshClient = await worker_1.WorkerService.getHotMesh(
|
|
178
|
+
const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
|
|
162
179
|
if (await WorkflowService.isSideEffectAllowed(hotMeshClient, 'hook')) {
|
|
163
180
|
const store = asyncLocalStorage_1.asyncLocalStorage.getStore();
|
|
164
181
|
const workflowId = options.workflowId ?? store.get('workflowId');
|
|
@@ -184,7 +201,7 @@ class WorkflowService {
|
|
|
184
201
|
const workflowTopic = store.get('workflowTopic');
|
|
185
202
|
const workflowDimension = store.get('workflowDimension') ?? '';
|
|
186
203
|
const namespace = store.get('namespace');
|
|
187
|
-
const sleepJobId =
|
|
204
|
+
const sleepJobId = `-${workflowId}-$sleep${workflowDimension}-${execIndex}`;
|
|
188
205
|
try {
|
|
189
206
|
const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
|
|
190
207
|
await hotMeshClient.getState(`${hotMeshClient.appId}.sleep.execute`, sleepJobId);
|
|
@@ -192,9 +209,8 @@ class WorkflowService {
|
|
|
192
209
|
return seconds;
|
|
193
210
|
}
|
|
194
211
|
catch (e) {
|
|
195
|
-
//if an error, the sleep job was not found...rethrow error; sleep job
|
|
196
|
-
// will be automatically created according to the DAG rules (they
|
|
197
212
|
// spawn a new sleep job if error code 595 is thrown by the worker)
|
|
213
|
+
// NOTE: If this message shows up in your stack trace, you forgot to await `Durable.workflow.sleep()` in your workflow code.
|
|
198
214
|
throw new errors_1.DurableSleepError(workflowId, seconds, execIndex, workflowDimension);
|
|
199
215
|
}
|
|
200
216
|
}
|
|
@@ -212,7 +228,7 @@ class WorkflowService {
|
|
|
212
228
|
const signalResults = [];
|
|
213
229
|
for (const signal of signals) {
|
|
214
230
|
const execIndex = COUNTER.counter = COUNTER.counter + 1;
|
|
215
|
-
const wfsJobId =
|
|
231
|
+
const wfsJobId = `-${workflowId}-$wfs${workflowDimension}-${execIndex}`;
|
|
216
232
|
try {
|
|
217
233
|
if (allAreComplete) {
|
|
218
234
|
const state = await hotMeshClient.getState(`${hotMeshClient.appId}.wfs.execute`, wfsJobId);
|
|
@@ -263,7 +279,7 @@ class WorkflowService {
|
|
|
263
279
|
const spn = store.get('workflowSpan');
|
|
264
280
|
const namespace = store.get('namespace');
|
|
265
281
|
const activityTopic = `${workflowTopic}-activity`;
|
|
266
|
-
const activityJobId =
|
|
282
|
+
const activityJobId = `-${workflowId}-$${activityName}${workflowDimension}-${execIndex}`;
|
|
267
283
|
let activityState;
|
|
268
284
|
try {
|
|
269
285
|
const hotMeshClient = await worker_1.WorkerService.getHotMesh(activityTopic, { namespace });
|
|
@@ -18,6 +18,7 @@ import { JobState, JobData, JobMetadata, JobOutput, JobStatus } from '../../type
|
|
|
18
18
|
import { HotMeshApps, HotMeshConfig, HotMeshManifest, HotMeshSettings } from '../../types/hotmesh';
|
|
19
19
|
import { JobMessageCallback } from '../../types/quorum';
|
|
20
20
|
import { RedisClient, RedisMulti } from '../../types/redis';
|
|
21
|
+
import { StringAnyType } from '../../types/serializer';
|
|
21
22
|
import { GetStatsOptions, IdsResponse, JobStatsInput, StatsResponse } from '../../types/stats';
|
|
22
23
|
import { StreamCode, StreamData, StreamDataResponse, StreamError, StreamStatus } from '../../types/stream';
|
|
23
24
|
declare class EngineService {
|
|
@@ -82,6 +83,7 @@ declare class EngineService {
|
|
|
82
83
|
runJobCompletionTasks(context: JobState, emit?: boolean): Promise<void>;
|
|
83
84
|
getStatus(jobId: string): Promise<JobStatus>;
|
|
84
85
|
getState(topic: string, jobId: string): Promise<JobOutput>;
|
|
86
|
+
getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
|
|
85
87
|
compress(terms: string[]): Promise<boolean>;
|
|
86
88
|
}
|
|
87
89
|
export { EngineService };
|
|
@@ -523,6 +523,9 @@ class EngineService {
|
|
|
523
523
|
}
|
|
524
524
|
return stateTree;
|
|
525
525
|
}
|
|
526
|
+
async getQueryState(jobId, fields) {
|
|
527
|
+
return await this.store.getQueryState(jobId, fields);
|
|
528
|
+
}
|
|
526
529
|
async compress(terms) {
|
|
527
530
|
const existingSymbols = await this.store.getSymbolValues();
|
|
528
531
|
const startIndex = Object.keys(existingSymbols).length;
|