@hotmeshio/hotmesh 0.3.5 → 0.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.d.ts +6 -4
- package/build/index.js +13 -5
- package/build/modules/key.js +1 -62
- package/build/modules/utils.d.ts +1 -0
- package/build/modules/utils.js +1 -242
- package/build/package.json +14 -9
- package/build/services/activities/activity.js +1 -495
- package/build/services/activities/await.js +1 -109
- package/build/services/activities/cycle.js +1 -96
- package/build/services/activities/hook.js +1 -154
- package/build/services/activities/index.js +1 -20
- package/build/services/activities/interrupt.js +1 -149
- package/build/services/activities/signal.js +1 -118
- package/build/services/activities/trigger.d.ts +0 -1
- package/build/services/activities/trigger.js +1 -269
- package/build/services/activities/worker.js +1 -101
- package/build/services/collator/index.js +1 -197
- package/build/services/compiler/deployer.d.ts +3 -1
- package/build/services/compiler/deployer.js +1 -455
- package/build/services/compiler/index.d.ts +3 -1
- package/build/services/compiler/index.js +1 -91
- package/build/services/compiler/validator.js +1 -122
- package/build/services/engine/index.d.ts +5 -2
- package/build/services/engine/index.js +1 -583
- package/build/services/exporter/index.js +1 -93
- package/build/services/mapper/index.js +1 -67
- package/build/services/meshdata/index.d.ts +0 -1
- package/build/services/meshdata/index.js +16 -24
- package/build/services/meshflow/client.js +4 -8
- package/build/services/meshflow/exporter.js +1 -186
- package/build/services/meshflow/search.d.ts +4 -5
- package/build/services/meshflow/search.js +48 -36
- package/build/services/meshflow/worker.js +1 -1
- package/build/services/meshflow/workflow.d.ts +1 -1
- package/build/services/meshflow/workflow.js +5 -30
- package/build/services/meshos/index.d.ts +81 -0
- package/build/services/meshos/index.js +339 -0
- package/build/services/pipe/functions/array.js +1 -74
- package/build/services/pipe/functions/bitwise.js +1 -24
- package/build/services/pipe/functions/conditional.js +1 -36
- package/build/services/pipe/functions/cron.js +1 -32
- package/build/services/pipe/functions/date.js +1 -164
- package/build/services/pipe/functions/index.js +1 -30
- package/build/services/pipe/functions/json.js +1 -12
- package/build/services/pipe/functions/logical.js +1 -12
- package/build/services/pipe/functions/math.js +1 -182
- package/build/services/pipe/functions/number.js +1 -60
- package/build/services/pipe/functions/object.js +1 -81
- package/build/services/pipe/functions/string.js +1 -69
- package/build/services/pipe/functions/symbol.js +1 -33
- package/build/services/pipe/functions/unary.js +1 -18
- package/build/services/pipe/index.js +1 -221
- package/build/services/quorum/index.d.ts +1 -1
- package/build/services/quorum/index.js +1 -233
- package/build/services/reporter/index.js +1 -331
- package/build/services/router/index.js +1 -420
- package/build/services/search/factory.d.ts +7 -0
- package/build/services/search/factory.js +20 -0
- package/build/services/search/index.d.ts +21 -0
- package/build/services/search/index.js +10 -0
- package/build/services/search/providers/redis/ioredis.d.ts +18 -0
- package/build/services/search/providers/redis/ioredis.js +1 -0
- package/build/services/search/providers/redis/redis.d.ts +18 -0
- package/build/services/search/providers/redis/redis.js +1 -0
- package/build/services/serializer/index.js +1 -265
- package/build/services/store/factory.d.ts +8 -0
- package/build/services/store/factory.js +20 -0
- package/build/services/store/index.d.ts +71 -98
- package/build/services/store/index.js +2 -941
- package/build/services/store/providers/postgres/postgres.d.ts +0 -0
- package/build/services/store/providers/postgres/postgres.js +0 -0
- package/build/services/store/providers/postgres/types/hash.d.ts +0 -0
- package/build/services/store/providers/postgres/types/hash.js +0 -0
- package/build/services/store/providers/postgres/types/list.d.ts +0 -0
- package/build/services/store/providers/postgres/types/list.js +0 -0
- package/build/services/store/providers/postgres/types/string.d.ts +0 -0
- package/build/services/store/providers/postgres/types/string.js +0 -0
- package/build/services/store/providers/postgres/types/zset.d.ts +0 -0
- package/build/services/store/providers/postgres/types/zset.js +0 -0
- package/build/services/store/providers/redis/_base.d.ts +98 -0
- package/build/services/store/providers/redis/_base.js +1 -0
- package/build/services/store/providers/redis/ioredis.d.ts +12 -0
- package/build/services/store/providers/redis/ioredis.js +1 -0
- package/build/services/store/providers/redis/redis.d.ts +13 -0
- package/build/services/store/providers/redis/redis.js +1 -0
- package/build/services/store/providers/store-initializable.d.ts +5 -0
- package/build/services/store/providers/store-initializable.js +1 -0
- package/build/services/stream/factory.d.ts +8 -0
- package/build/services/stream/factory.js +20 -0
- package/build/services/stream/index.d.ts +13 -14
- package/build/services/stream/index.js +3 -2
- package/build/services/stream/providers/postgres/_deploy.d.ts +4 -0
- package/build/services/stream/providers/postgres/_deploy.js +1 -0
- package/build/services/stream/providers/redis/ioredis.d.ts +21 -0
- package/build/services/stream/providers/redis/ioredis.js +1 -0
- package/build/services/stream/providers/redis/redis.d.ts +21 -0
- package/build/services/stream/providers/redis/redis.js +1 -0
- package/build/services/stream/providers/stream-initializable.d.ts +5 -0
- package/build/services/stream/providers/stream-initializable.js +1 -0
- package/build/services/sub/factory.d.ts +7 -0
- package/build/services/sub/factory.js +20 -0
- package/build/services/sub/index.d.ts +9 -7
- package/build/services/sub/index.js +3 -2
- package/build/services/sub/{clients → providers/redis}/ioredis.d.ts +7 -10
- package/build/services/sub/providers/redis/ioredis.js +1 -0
- package/build/services/sub/{clients → providers/redis}/redis.d.ts +7 -10
- package/build/services/sub/providers/redis/redis.js +1 -0
- package/build/services/task/index.js +1 -171
- package/build/services/telemetry/index.js +1 -225
- package/build/services/worker/index.d.ts +2 -2
- package/build/services/worker/index.js +1 -179
- package/build/types/index.d.ts +1 -0
- package/build/types/manifest.d.ts +49 -0
- package/build/types/manifest.js +2 -0
- package/build/types/redis.d.ts +5 -5
- package/index.ts +19 -4
- package/package.json +14 -9
- package/typedoc.json +1 -0
- package/types/index.ts +15 -0
- package/types/manifest.ts +68 -0
- package/types/meshflow.ts +1 -1
- package/types/redis.ts +5 -5
- package/build/services/store/clients/ioredis.d.ts +0 -30
- package/build/services/store/clients/ioredis.js +0 -220
- package/build/services/store/clients/redis.d.ts +0 -32
- package/build/services/store/clients/redis.js +0 -319
- package/build/services/stream/clients/ioredis.d.ts +0 -24
- package/build/services/stream/clients/ioredis.js +0 -121
- package/build/services/stream/clients/redis.d.ts +0 -24
- package/build/services/stream/clients/redis.js +0 -161
- package/build/services/sub/clients/ioredis.js +0 -72
- package/build/services/sub/clients/redis.js +0 -63
|
@@ -14,7 +14,7 @@ class Search {
|
|
|
14
14
|
this.jobId = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
|
|
15
15
|
this.searchSessionId = searchSessionId;
|
|
16
16
|
this.hotMeshClient = hotMeshClient;
|
|
17
|
-
this.
|
|
17
|
+
this.search = hotMeshClient.engine.search;
|
|
18
18
|
}
|
|
19
19
|
safeKey(key) {
|
|
20
20
|
if (key.startsWith('"')) {
|
|
@@ -24,7 +24,7 @@ class Search {
|
|
|
24
24
|
}
|
|
25
25
|
static async configureSearchIndex(hotMeshClient, search) {
|
|
26
26
|
if (search?.schema) {
|
|
27
|
-
const
|
|
27
|
+
const searchService = hotMeshClient.engine.search;
|
|
28
28
|
const schema = [];
|
|
29
29
|
for (const [key, value] of Object.entries(search.schema)) {
|
|
30
30
|
if (value.indexed !== false) {
|
|
@@ -50,7 +50,7 @@ class Search {
|
|
|
50
50
|
};
|
|
51
51
|
const hotMeshPrefix = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
|
|
52
52
|
const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
|
|
53
|
-
await
|
|
53
|
+
await searchService.createSearchIndex(`${search.index}`, prefixes, schema);
|
|
54
54
|
}
|
|
55
55
|
catch (error) {
|
|
56
56
|
hotMeshClient.engine.logger.info('meshflow-client-search-err', {
|
|
@@ -61,9 +61,8 @@ class Search {
|
|
|
61
61
|
}
|
|
62
62
|
static async listSearchIndexes(hotMeshClient) {
|
|
63
63
|
try {
|
|
64
|
-
const
|
|
65
|
-
|
|
66
|
-
return searchIndexes;
|
|
64
|
+
const searchService = hotMeshClient.engine.search;
|
|
65
|
+
return await searchService.listSearchIndexes();
|
|
67
66
|
}
|
|
68
67
|
catch (error) {
|
|
69
68
|
hotMeshClient.engine.logger.info('meshflow-client-search-list-err', {
|
|
@@ -79,32 +78,42 @@ class Search {
|
|
|
79
78
|
const ssGuid = this.getSearchSessionGuid();
|
|
80
79
|
const store = storage_1.asyncLocalStorage.getStore();
|
|
81
80
|
const replay = store?.get('replay') ?? {};
|
|
82
|
-
const safeArgs = [];
|
|
83
|
-
for (let i = 0; i < args.length; i += 2) {
|
|
84
|
-
const keyName = args[i];
|
|
85
|
-
delete this.cachedFields[keyName];
|
|
86
|
-
const key = this.safeKey(keyName);
|
|
87
|
-
const value = args[i + 1].toString();
|
|
88
|
-
safeArgs.push(key, value);
|
|
89
|
-
}
|
|
90
81
|
if (ssGuid in replay) {
|
|
91
82
|
return Number(replay[ssGuid]);
|
|
92
83
|
}
|
|
93
|
-
const
|
|
94
|
-
|
|
95
|
-
|
|
84
|
+
const fields = {};
|
|
85
|
+
if (typeof args[0] === 'object') {
|
|
86
|
+
for (const [key, value] of Object.entries(args[0])) {
|
|
87
|
+
delete this.cachedFields[key];
|
|
88
|
+
fields[this.safeKey(key)] = value.toString();
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
for (let i = 0; i < args.length; i += 2) {
|
|
93
|
+
const keyName = args[i];
|
|
94
|
+
delete this.cachedFields[keyName];
|
|
95
|
+
const key = this.safeKey(keyName);
|
|
96
|
+
const value = args[i + 1].toString();
|
|
97
|
+
fields[key] = value;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
const fieldCount = await this.search.setFields(this.jobId, fields);
|
|
101
|
+
await this.search.setFields(this.jobId, { [ssGuid]: fieldCount.toString() });
|
|
102
|
+
return fieldCount;
|
|
96
103
|
}
|
|
97
|
-
async get(
|
|
104
|
+
async get(id) {
|
|
98
105
|
try {
|
|
99
|
-
if (
|
|
100
|
-
return this.cachedFields[
|
|
106
|
+
if (id in this.cachedFields) {
|
|
107
|
+
return this.cachedFields[id];
|
|
101
108
|
}
|
|
102
|
-
const value =
|
|
103
|
-
this.cachedFields[
|
|
109
|
+
const value = await this.search.getField(this.jobId, this.safeKey(id));
|
|
110
|
+
this.cachedFields[id] = value;
|
|
104
111
|
return value;
|
|
105
112
|
}
|
|
106
113
|
catch (error) {
|
|
107
|
-
this.hotMeshClient.logger.error('meshflow-search-get-error', {
|
|
114
|
+
this.hotMeshClient.logger.error('meshflow-search-get-error', {
|
|
115
|
+
...error,
|
|
116
|
+
});
|
|
108
117
|
return '';
|
|
109
118
|
}
|
|
110
119
|
}
|
|
@@ -125,7 +134,7 @@ class Search {
|
|
|
125
134
|
if (isCached) {
|
|
126
135
|
return values;
|
|
127
136
|
}
|
|
128
|
-
const returnValues =
|
|
137
|
+
const returnValues = await this.search.getFields(this.jobId, safeArgs);
|
|
129
138
|
returnValues.forEach((value, index) => {
|
|
130
139
|
if (value !== null) {
|
|
131
140
|
this.cachedFields[args[index]] = value;
|
|
@@ -153,11 +162,9 @@ class Search {
|
|
|
153
162
|
if (ssGuid in replay) {
|
|
154
163
|
return Number(replay[ssGuid]);
|
|
155
164
|
}
|
|
156
|
-
const response = await this.
|
|
157
|
-
const formattedResponse = isNaN(response)
|
|
158
|
-
|
|
159
|
-
: Number(response);
|
|
160
|
-
this.store.exec('HSET', this.jobId, ssGuid, formattedResponse.toString());
|
|
165
|
+
const response = await this.search.deleteFields(this.jobId, safeArgs);
|
|
166
|
+
const formattedResponse = isNaN(response) ? 0 : Number(response);
|
|
167
|
+
await this.search.setFields(this.jobId, { [ssGuid]: formattedResponse.toString() });
|
|
161
168
|
return formattedResponse;
|
|
162
169
|
}
|
|
163
170
|
async incr(key, val) {
|
|
@@ -168,9 +175,9 @@ class Search {
|
|
|
168
175
|
if (ssGuid in replay) {
|
|
169
176
|
return Number(replay[ssGuid]);
|
|
170
177
|
}
|
|
171
|
-
const num =
|
|
172
|
-
this.
|
|
173
|
-
return
|
|
178
|
+
const num = await this.search.incrementFieldByFloat(this.jobId, this.safeKey(key), val);
|
|
179
|
+
await this.search.setFields(this.jobId, { [ssGuid]: num.toString() });
|
|
180
|
+
return num;
|
|
174
181
|
}
|
|
175
182
|
async mult(key, val) {
|
|
176
183
|
delete this.cachedFields[key];
|
|
@@ -180,12 +187,17 @@ class Search {
|
|
|
180
187
|
if (ssGuid in replay) {
|
|
181
188
|
return Math.exp(Number(replay[ssGuid]));
|
|
182
189
|
}
|
|
183
|
-
const ssGuidValue =
|
|
190
|
+
const ssGuidValue = await this.search.incrementFieldByFloat(this.jobId, ssGuid, 1);
|
|
184
191
|
if (ssGuidValue === 1) {
|
|
185
192
|
const log = Math.log(val);
|
|
186
|
-
const logTotal =
|
|
187
|
-
this.
|
|
188
|
-
return Math.exp(
|
|
193
|
+
const logTotal = await this.search.incrementFieldByFloat(this.jobId, this.safeKey(key), log);
|
|
194
|
+
await this.search.setFields(this.jobId, { [ssGuid]: logTotal.toString() });
|
|
195
|
+
return Math.exp(logTotal);
|
|
196
|
+
}
|
|
197
|
+
else {
|
|
198
|
+
const logTotalStr = await this.search.getField(this.jobId, ssGuid);
|
|
199
|
+
const logTotal = Number(logTotalStr);
|
|
200
|
+
return Math.exp(logTotal);
|
|
189
201
|
}
|
|
190
202
|
}
|
|
191
203
|
}
|
|
@@ -237,7 +237,7 @@ class WorkerService {
|
|
|
237
237
|
isProcessing = true;
|
|
238
238
|
const workflowInput = data.data;
|
|
239
239
|
const execIndex = counter.counter - interruptionRegistry.length + 1;
|
|
240
|
-
const { workflowId, workflowTopic, workflowDimension, originJobId, expire } = workflowInput;
|
|
240
|
+
const { workflowId, workflowTopic, workflowDimension, originJobId, expire, } = workflowInput;
|
|
241
241
|
const collatorFlowId = `${(0, utils_1.guid)()}$C`;
|
|
242
242
|
return {
|
|
243
243
|
status: stream_1.StreamStatus.SUCCESS,
|
|
@@ -10,6 +10,7 @@ export declare class WorkflowService {
|
|
|
10
10
|
static getContext(): WorkflowContext;
|
|
11
11
|
static getHotMesh(): Promise<HotMesh>;
|
|
12
12
|
static execChild<T>(options: WorkflowOptions): Promise<T>;
|
|
13
|
+
static executeChild: typeof WorkflowService.execChild;
|
|
13
14
|
static getChildInterruptPayload(context: WorkflowContext, options: WorkflowOptions, execIndex: number): MeshFlowChildErrorType;
|
|
14
15
|
static startChild(options: WorkflowOptions): Promise<string>;
|
|
15
16
|
static proxyActivities<ACT>(options?: ActivityConfig): ProxyType<ACT>;
|
|
@@ -19,7 +20,6 @@ export declare class WorkflowService {
|
|
|
19
20
|
static random(): number;
|
|
20
21
|
static signal(signalId: string, data: Record<any, any>): Promise<string>;
|
|
21
22
|
static hook(options: HookOptions): Promise<string>;
|
|
22
|
-
static once<T>(fn: (...args: any[]) => Promise<T>, ...args: any[]): Promise<T>;
|
|
23
23
|
static interrupt(jobId: string, options?: JobInterruptOptions): Promise<string | void>;
|
|
24
24
|
static all<T>(...promises: Promise<T>[]): Promise<T[]>;
|
|
25
25
|
static sleepFor(duration: string): Promise<number>;
|
|
@@ -38,7 +38,8 @@ class WorkflowService {
|
|
|
38
38
|
jobId: workflowId,
|
|
39
39
|
};
|
|
40
40
|
const workflowGuid = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
|
|
41
|
-
const
|
|
41
|
+
const searchClient = hotMeshClient.engine.search;
|
|
42
|
+
const guidValue = await searchClient.incrementFieldByFloat(workflowGuid, sessionId, 1);
|
|
42
43
|
return guidValue === 1;
|
|
43
44
|
}
|
|
44
45
|
static getContext() {
|
|
@@ -116,7 +117,7 @@ class WorkflowService {
|
|
|
116
117
|
}
|
|
117
118
|
return result.$error;
|
|
118
119
|
}
|
|
119
|
-
else {
|
|
120
|
+
else if (!result?.$error) {
|
|
120
121
|
return result.data;
|
|
121
122
|
}
|
|
122
123
|
}
|
|
@@ -135,7 +136,7 @@ class WorkflowService {
|
|
|
135
136
|
childJobId = options.workflowId;
|
|
136
137
|
}
|
|
137
138
|
else if (options.entity) {
|
|
138
|
-
childJobId = `${options.entity}-${
|
|
139
|
+
childJobId = `${options.entity}-${(0, utils_1.guid)()}-${workflowDimension}-${execIndex}`;
|
|
139
140
|
}
|
|
140
141
|
else {
|
|
141
142
|
childJobId = `-${options.workflowName}-${(0, utils_1.guid)()}-${workflowDimension}-${execIndex}`;
|
|
@@ -299,33 +300,6 @@ class WorkflowService {
|
|
|
299
300
|
return await hotMeshClient.hook(`${namespace}.flow.signal`, payload, stream_1.StreamStatus.PENDING, 202);
|
|
300
301
|
}
|
|
301
302
|
}
|
|
302
|
-
static async once(fn, ...args) {
|
|
303
|
-
const { COUNTER, connection, namespace, workflowId, workflowTopic, workflowDimension, replay, } = WorkflowService.getContext();
|
|
304
|
-
const execIndex = COUNTER.counter = COUNTER.counter + 1;
|
|
305
|
-
const sessionId = `-once${workflowDimension}-${execIndex}-`;
|
|
306
|
-
if (sessionId in replay) {
|
|
307
|
-
return serializer_1.SerializerService.fromString(replay[sessionId]).data;
|
|
308
|
-
}
|
|
309
|
-
const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, {
|
|
310
|
-
connection,
|
|
311
|
-
namespace,
|
|
312
|
-
});
|
|
313
|
-
const keyParams = {
|
|
314
|
-
appId: hotMeshClient.appId,
|
|
315
|
-
jobId: workflowId,
|
|
316
|
-
};
|
|
317
|
-
const workflowGuid = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
|
|
318
|
-
const t1 = new Date();
|
|
319
|
-
const response = await fn(...args);
|
|
320
|
-
const t2 = new Date();
|
|
321
|
-
const payload = {
|
|
322
|
-
data: response,
|
|
323
|
-
ac: (0, utils_1.formatISODate)(t1),
|
|
324
|
-
au: (0, utils_1.formatISODate)(t2),
|
|
325
|
-
};
|
|
326
|
-
await hotMeshClient.engine.store.exec('HSET', workflowGuid, sessionId, serializer_1.SerializerService.toString(payload));
|
|
327
|
-
return response;
|
|
328
|
-
}
|
|
329
303
|
static async interrupt(jobId, options = {}) {
|
|
330
304
|
const { workflowTopic, connection, namespace } = WorkflowService.getContext();
|
|
331
305
|
const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, {
|
|
@@ -385,4 +359,5 @@ class WorkflowService {
|
|
|
385
359
|
throw new errors_1.MeshFlowWaitForError(interruptionMessage);
|
|
386
360
|
}
|
|
387
361
|
}
|
|
362
|
+
WorkflowService.executeChild = WorkflowService.execChild;
|
|
388
363
|
exports.WorkflowService = WorkflowService;
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { MeshData } from '../meshdata/index';
|
|
2
|
+
import * as Types from '../../types';
|
|
3
|
+
declare abstract class MeshOS {
|
|
4
|
+
meshData: MeshData;
|
|
5
|
+
connected: boolean;
|
|
6
|
+
namespace: string;
|
|
7
|
+
namespaceType: string;
|
|
8
|
+
static databases: Record<string, Types.DB>;
|
|
9
|
+
static namespaces: Types.Namespaces;
|
|
10
|
+
static entities: Record<string, Types.Entity>;
|
|
11
|
+
static schemas: Record<string, Types.WorkflowSearchSchema>;
|
|
12
|
+
static profiles: Types.Profiles;
|
|
13
|
+
static classes: Record<string, typeof MeshOS>;
|
|
14
|
+
static logger: Types.ILogger;
|
|
15
|
+
constructor(namespace: string, namespaceType: string, config: Types.DBConfig);
|
|
16
|
+
protected abstract getEntity(): string;
|
|
17
|
+
abstract getSearchOptions(): Types.WorkflowSearchOptions;
|
|
18
|
+
protected abstract getTaskQueue(): string;
|
|
19
|
+
private initializeMeshData;
|
|
20
|
+
protected defaultTargetFn(): Promise<string>;
|
|
21
|
+
getNamespace(): string;
|
|
22
|
+
getRedisUrl: (config: Types.DBConfig) => {
|
|
23
|
+
url: string;
|
|
24
|
+
};
|
|
25
|
+
connect(): Promise<void>;
|
|
26
|
+
index(): Promise<void>;
|
|
27
|
+
static shutdown(): Promise<void>;
|
|
28
|
+
getIndexName(): string;
|
|
29
|
+
create(body: Record<string, any>): Promise<Types.StringStringType>;
|
|
30
|
+
retrieve(id: string, sparse?: boolean): Promise<Types.StringStringType>;
|
|
31
|
+
update(id: string, body: Record<string, any>): Promise<Types.StringStringType>;
|
|
32
|
+
delete(id: string): Promise<boolean>;
|
|
33
|
+
find(query?: {
|
|
34
|
+
field: string;
|
|
35
|
+
is: '=' | '[]' | '>=' | '<=';
|
|
36
|
+
value: string;
|
|
37
|
+
}[], start?: number, size?: number): Promise<{
|
|
38
|
+
count: number;
|
|
39
|
+
query: string;
|
|
40
|
+
data: Types.StringStringType[];
|
|
41
|
+
}>;
|
|
42
|
+
count(query: {
|
|
43
|
+
field: string;
|
|
44
|
+
is: '=' | '[]' | '>=' | '<=';
|
|
45
|
+
value: string;
|
|
46
|
+
}[]): Promise<number>;
|
|
47
|
+
aggregate(filter?: {
|
|
48
|
+
field: string;
|
|
49
|
+
is: '=' | '[]' | '>=' | '<=';
|
|
50
|
+
value: string;
|
|
51
|
+
}[], apply?: {
|
|
52
|
+
expression: string;
|
|
53
|
+
as: string;
|
|
54
|
+
}[], rows?: string[], columns?: string[], reduce?: {
|
|
55
|
+
operation: string;
|
|
56
|
+
as: string;
|
|
57
|
+
property?: string;
|
|
58
|
+
}[], sort?: {
|
|
59
|
+
field: string;
|
|
60
|
+
order: 'ASC' | 'DESC';
|
|
61
|
+
}[], start?: number, size?: number): Promise<{
|
|
62
|
+
count: number;
|
|
63
|
+
query: string;
|
|
64
|
+
data: Types.StringStringType[];
|
|
65
|
+
}>;
|
|
66
|
+
private buildAggregateCommand;
|
|
67
|
+
private buildFilterCommand;
|
|
68
|
+
init(search?: boolean): Promise<void>;
|
|
69
|
+
static registerDatabase(id: string, config: Types.DB): void;
|
|
70
|
+
static registerNamespace(id: string, config: Types.Namespace): void;
|
|
71
|
+
static registerEntity(id: string, config: Types.Entity): void;
|
|
72
|
+
static registerSchema(id: string, schema: Types.WorkflowSearchSchema): void;
|
|
73
|
+
static registerProfile(id: string, config: Types.Profile): void;
|
|
74
|
+
static registerClass(id: string, entityClass: typeof MeshOS): void;
|
|
75
|
+
static init(p?: Types.Profiles): Promise<void>;
|
|
76
|
+
static findEntity(database: string, namespace: string, entity: string): Types.EntityInstanceTypes | undefined;
|
|
77
|
+
static findSchemas(database: string, ns: string): Record<string, Types.WorkflowSearchSchema>;
|
|
78
|
+
static toJSON(p?: Types.Profiles): any;
|
|
79
|
+
workflow: {};
|
|
80
|
+
}
|
|
81
|
+
export { MeshOS };
|
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.MeshOS = void 0;
|
|
27
|
+
const Redis = __importStar(require("redis"));
|
|
28
|
+
const index_1 = require("../meshdata/index");
|
|
29
|
+
const utils_1 = require("../../modules/utils");
|
|
30
|
+
const logger_1 = require("../logger");
|
|
31
|
+
class MeshOS {
|
|
32
|
+
constructor(namespace, namespaceType, config) {
|
|
33
|
+
this.connected = false;
|
|
34
|
+
this.getRedisUrl = (config) => {
|
|
35
|
+
return {
|
|
36
|
+
url: `redis${config.REDIS_USE_TLS ? 's' : ''}://${config.REDIS_USERNAME ?? ''}:${config.REDIS_PASSWORD}@${config.REDIS_HOST}:${config.REDIS_PORT}`,
|
|
37
|
+
};
|
|
38
|
+
};
|
|
39
|
+
this.workflow = {};
|
|
40
|
+
this.namespace = namespace;
|
|
41
|
+
this.namespaceType = namespaceType;
|
|
42
|
+
this.meshData = this.initializeMeshData(config);
|
|
43
|
+
}
|
|
44
|
+
initializeMeshData(dbConfig) {
|
|
45
|
+
return new index_1.MeshData(Redis, this.getRedisUrl(dbConfig), this.getSearchOptions());
|
|
46
|
+
}
|
|
47
|
+
async defaultTargetFn() {
|
|
48
|
+
return 'OK';
|
|
49
|
+
}
|
|
50
|
+
getNamespace() {
|
|
51
|
+
return this.namespace;
|
|
52
|
+
}
|
|
53
|
+
async connect() {
|
|
54
|
+
this.connected = await this.meshData.connect({
|
|
55
|
+
entity: this.getEntity(),
|
|
56
|
+
target: this.defaultTargetFn,
|
|
57
|
+
options: {
|
|
58
|
+
namespace: this.getNamespace(),
|
|
59
|
+
taskQueue: this.getTaskQueue(),
|
|
60
|
+
},
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
async index() {
|
|
64
|
+
await this.meshData.createSearchIndex(this.getEntity(), { namespace: this.getNamespace() }, this.getSearchOptions());
|
|
65
|
+
}
|
|
66
|
+
static async shutdown() {
|
|
67
|
+
await index_1.MeshData.shutdown();
|
|
68
|
+
}
|
|
69
|
+
getIndexName() {
|
|
70
|
+
return this.getSearchOptions().index;
|
|
71
|
+
}
|
|
72
|
+
async create(body) {
|
|
73
|
+
const id = body.id || (0, utils_1.guid)();
|
|
74
|
+
await this.meshData.set(this.getEntity(), id, {
|
|
75
|
+
search: { data: body },
|
|
76
|
+
namespace: this.getNamespace(),
|
|
77
|
+
});
|
|
78
|
+
return this.retrieve(id);
|
|
79
|
+
}
|
|
80
|
+
async retrieve(id, sparse = false) {
|
|
81
|
+
const opts = this.getSearchOptions();
|
|
82
|
+
const fields = sparse ? ['id'] : Object.keys(opts?.schema || {});
|
|
83
|
+
const result = await this.meshData.get(this.getEntity(), id, {
|
|
84
|
+
fields,
|
|
85
|
+
namespace: this.getNamespace(),
|
|
86
|
+
});
|
|
87
|
+
if (!result?.id)
|
|
88
|
+
throw new Error(`${this.getEntity()} not found`);
|
|
89
|
+
return result;
|
|
90
|
+
}
|
|
91
|
+
async update(id, body) {
|
|
92
|
+
await this.retrieve(id);
|
|
93
|
+
await this.meshData.set(this.getEntity(), id, {
|
|
94
|
+
search: { data: body },
|
|
95
|
+
namespace: this.getNamespace(),
|
|
96
|
+
});
|
|
97
|
+
return this.retrieve(id);
|
|
98
|
+
}
|
|
99
|
+
async delete(id) {
|
|
100
|
+
await this.retrieve(id);
|
|
101
|
+
await this.meshData.flush(this.getEntity(), id, this.getNamespace());
|
|
102
|
+
return true;
|
|
103
|
+
}
|
|
104
|
+
async find(query = [], start = 0, size = 100) {
|
|
105
|
+
const opts = this.getSearchOptions();
|
|
106
|
+
return this.meshData.findWhere(this.getEntity(), {
|
|
107
|
+
query,
|
|
108
|
+
return: Object.keys(opts?.schema || {}),
|
|
109
|
+
limit: { start, size },
|
|
110
|
+
options: { namespace: this.getNamespace() },
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
async count(query) {
|
|
114
|
+
return this.meshData.findWhere(this.getEntity(), {
|
|
115
|
+
query,
|
|
116
|
+
count: true,
|
|
117
|
+
options: { namespace: this.getNamespace() },
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
async aggregate(filter = [], apply = [], rows = [], columns = [], reduce = [], sort = [], start = 0, size = 100) {
|
|
121
|
+
const command = this.buildAggregateCommand(filter, apply, rows, columns, reduce, sort);
|
|
122
|
+
try {
|
|
123
|
+
const results = await this.meshData.find(this.getEntity(), {
|
|
124
|
+
index: this.getIndexName(),
|
|
125
|
+
namespace: this.getNamespace(),
|
|
126
|
+
taskQueue: this.getTaskQueue(),
|
|
127
|
+
search: this.getSearchOptions(),
|
|
128
|
+
}, ...command);
|
|
129
|
+
return {
|
|
130
|
+
count: results[0],
|
|
131
|
+
query: command.join(' '),
|
|
132
|
+
data: (0, utils_1.arrayToHash)(results),
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
catch (e) {
|
|
136
|
+
throw e;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
buildAggregateCommand(filter, apply, rows, columns, reduce, sort) {
|
|
140
|
+
const command = ['FT.AGGREGATE', this.getIndexName() || 'default'];
|
|
141
|
+
const opts = this.getSearchOptions();
|
|
142
|
+
command.push(this.buildFilterCommand(filter));
|
|
143
|
+
apply.forEach((a) => command.push('APPLY', a.expression, 'AS', a.as));
|
|
144
|
+
const groupBy = rows.concat(columns);
|
|
145
|
+
if (groupBy.length > 0) {
|
|
146
|
+
command.push('GROUPBY', `${groupBy.length}`, ...groupBy.map((g) => opts?.schema?.[g] ? `@_${g}` : `@${g}`));
|
|
147
|
+
}
|
|
148
|
+
reduce.forEach((r) => {
|
|
149
|
+
const op = r.operation.toUpperCase();
|
|
150
|
+
if (op === 'COUNT') {
|
|
151
|
+
command.push('REDUCE', op, '0', 'AS', r.as ?? 'count');
|
|
152
|
+
}
|
|
153
|
+
else if ([
|
|
154
|
+
'COUNT_DISTINCT',
|
|
155
|
+
'COUNT_DISTINCTISH',
|
|
156
|
+
'SUM',
|
|
157
|
+
'AVG',
|
|
158
|
+
'MIN',
|
|
159
|
+
'MAX',
|
|
160
|
+
'STDDEV',
|
|
161
|
+
'TOLIST',
|
|
162
|
+
].includes(op)) {
|
|
163
|
+
const property = r.property
|
|
164
|
+
? opts?.schema?.[r.property]
|
|
165
|
+
? `@_${r.property}`
|
|
166
|
+
: `@${r.property}`
|
|
167
|
+
: '';
|
|
168
|
+
command.push('REDUCE', op, '1', property, 'AS', r.as ?? `${r.operation}_${r.property}`);
|
|
169
|
+
}
|
|
170
|
+
});
|
|
171
|
+
if (sort.length > 0) {
|
|
172
|
+
command.push('SORTBY', `${2 * sort.length}`, ...sort.flatMap((s) => [
|
|
173
|
+
opts?.schema?.[s.field] ? `@_${s.field}` : `@${s.field}`,
|
|
174
|
+
s.order.toUpperCase() || 'DESC',
|
|
175
|
+
]));
|
|
176
|
+
}
|
|
177
|
+
return command;
|
|
178
|
+
}
|
|
179
|
+
buildFilterCommand(filter) {
|
|
180
|
+
if (filter.length === 0)
|
|
181
|
+
return '*';
|
|
182
|
+
const opts = this.getSearchOptions();
|
|
183
|
+
return filter
|
|
184
|
+
.map((q) => {
|
|
185
|
+
const type = opts?.schema?.[q.field]?.type ?? 'TEXT';
|
|
186
|
+
switch (type) {
|
|
187
|
+
case 'TAG':
|
|
188
|
+
return `@_${q.field}:{${q.value}}`;
|
|
189
|
+
case 'TEXT':
|
|
190
|
+
return `@_${q.field}:${q.value}`;
|
|
191
|
+
case 'NUMERIC':
|
|
192
|
+
return `@_${q.field}:[${q.value}]`;
|
|
193
|
+
}
|
|
194
|
+
})
|
|
195
|
+
.join(' ');
|
|
196
|
+
}
|
|
197
|
+
async init(search = true) {
|
|
198
|
+
await this.connect();
|
|
199
|
+
if (search) {
|
|
200
|
+
await this.index();
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
static registerDatabase(id, config) {
|
|
204
|
+
MeshOS.databases[id] = config;
|
|
205
|
+
}
|
|
206
|
+
static registerNamespace(id, config) {
|
|
207
|
+
MeshOS.namespaces[id] = config;
|
|
208
|
+
}
|
|
209
|
+
static registerEntity(id, config) {
|
|
210
|
+
MeshOS.entities[id] = config;
|
|
211
|
+
}
|
|
212
|
+
static registerSchema(id, schema) {
|
|
213
|
+
MeshOS.schemas[id] = schema;
|
|
214
|
+
}
|
|
215
|
+
static registerProfile(id, config) {
|
|
216
|
+
MeshOS.profiles[id] = config;
|
|
217
|
+
}
|
|
218
|
+
static registerClass(id, entityClass) {
|
|
219
|
+
MeshOS.classes[id] = entityClass;
|
|
220
|
+
}
|
|
221
|
+
static async init(p = MeshOS.profiles) {
|
|
222
|
+
for (const key in p) {
|
|
223
|
+
const profile = p[key];
|
|
224
|
+
if (profile.db.config.REDIS_HOST) {
|
|
225
|
+
this.logger.info(`meshos-initializing`, {
|
|
226
|
+
db: profile.db.name,
|
|
227
|
+
key,
|
|
228
|
+
});
|
|
229
|
+
profile.instances = {};
|
|
230
|
+
for (const ns in profile.namespaces) {
|
|
231
|
+
const namespace = profile.namespaces[ns];
|
|
232
|
+
this.logger.info(`meshos-initializing-namespace`, {
|
|
233
|
+
namespace: ns,
|
|
234
|
+
label: namespace.label,
|
|
235
|
+
});
|
|
236
|
+
let pinstances = profile.instances[ns];
|
|
237
|
+
if (!pinstances) {
|
|
238
|
+
pinstances = {};
|
|
239
|
+
profile.instances[ns] = pinstances;
|
|
240
|
+
}
|
|
241
|
+
for (const entity of namespace.entities) {
|
|
242
|
+
this.logger.info(`meshos-initializing-entity`, {
|
|
243
|
+
entity: entity.name,
|
|
244
|
+
label: entity.label,
|
|
245
|
+
});
|
|
246
|
+
const instance = pinstances[entity.name] = new entity.class(ns, namespace.type, profile.db.config);
|
|
247
|
+
await instance.init(profile.db.search);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
static findEntity(database, namespace, entity) {
|
|
254
|
+
if (!database ||
|
|
255
|
+
!MeshOS.profiles[database] ||
|
|
256
|
+
!MeshOS.profiles[database]?.db?.config?.REDIS_HOST) {
|
|
257
|
+
const activeProfiles = Object.keys(MeshOS.profiles).filter((key) => MeshOS.profiles[key]?.db?.config?.REDIS_HOST);
|
|
258
|
+
throw new Error(`The database query parameter [${database}] was not found. Use one of: ${activeProfiles.join(', ')}`);
|
|
259
|
+
}
|
|
260
|
+
if (!namespace || !MeshOS.profiles[database]?.instances?.[namespace]) {
|
|
261
|
+
const activeNamespaces = Object.keys(MeshOS.profiles[database]?.instances ?? {});
|
|
262
|
+
throw new Error(`The namespace query parameter [${namespace}] was not found. Use one of: ${activeNamespaces.join(', ')}`);
|
|
263
|
+
}
|
|
264
|
+
const entities = MeshOS.profiles[database]?.instances?.[namespace] ?? {};
|
|
265
|
+
if (!entity || entity?.startsWith('-') || entity === '*') {
|
|
266
|
+
entity = Object.keys(entities)[0];
|
|
267
|
+
}
|
|
268
|
+
else if (entity?.endsWith('*')) {
|
|
269
|
+
entity = entity.slice(0, -1);
|
|
270
|
+
}
|
|
271
|
+
const target = MeshOS.profiles[database]?.instances?.[namespace]?.[entity];
|
|
272
|
+
if (!target) {
|
|
273
|
+
this.logger.error(`meshos-entity-not-found`, {
|
|
274
|
+
database,
|
|
275
|
+
namespace,
|
|
276
|
+
entity,
|
|
277
|
+
});
|
|
278
|
+
entity = Object.keys(entities)[0];
|
|
279
|
+
return MeshOS.profiles[database]?.instances?.[namespace]?.[entity];
|
|
280
|
+
}
|
|
281
|
+
return target;
|
|
282
|
+
}
|
|
283
|
+
static findSchemas(database, ns) {
|
|
284
|
+
if (!database ||
|
|
285
|
+
!MeshOS.profiles[database] ||
|
|
286
|
+
!MeshOS.profiles[database]?.db?.config?.REDIS_HOST) {
|
|
287
|
+
const activeProfiles = Object.keys(MeshOS.profiles).filter((key) => MeshOS.profiles[key]?.db?.config?.REDIS_HOST);
|
|
288
|
+
throw new Error(`The database query parameter [${database}] was not found. Use one of: ${activeProfiles.join(', ')}`);
|
|
289
|
+
}
|
|
290
|
+
const profile = MeshOS.profiles[database];
|
|
291
|
+
const namespacedInstance = profile.instances[ns];
|
|
292
|
+
const schemas = {};
|
|
293
|
+
for (const entityName in namespacedInstance) {
|
|
294
|
+
const entityInstance = namespacedInstance[entityName];
|
|
295
|
+
const opts = entityInstance.getSearchOptions();
|
|
296
|
+
schemas[opts.index ?? entityName] = opts.schema;
|
|
297
|
+
}
|
|
298
|
+
return schemas;
|
|
299
|
+
}
|
|
300
|
+
static toJSON(p = MeshOS.profiles) {
|
|
301
|
+
const result = {};
|
|
302
|
+
for (const key in p) {
|
|
303
|
+
const profile = p[key];
|
|
304
|
+
if (!profile.db.config.REDIS_HOST) {
|
|
305
|
+
continue;
|
|
306
|
+
}
|
|
307
|
+
else {
|
|
308
|
+
result[key] = {
|
|
309
|
+
db: { ...profile.db, config: undefined },
|
|
310
|
+
namespaces: {},
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
for (const ns in profile.namespaces) {
|
|
314
|
+
const namespace = profile.namespaces[ns];
|
|
315
|
+
result[key].namespaces[ns] = {
|
|
316
|
+
name: namespace.name,
|
|
317
|
+
label: namespace.label,
|
|
318
|
+
entities: [],
|
|
319
|
+
};
|
|
320
|
+
for (const entity of namespace.entities) {
|
|
321
|
+
result[key].namespaces[ns].entities.push({
|
|
322
|
+
name: entity.name,
|
|
323
|
+
label: entity.label,
|
|
324
|
+
schema: entity.schema,
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
return result;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
exports.MeshOS = MeshOS;
|
|
333
|
+
MeshOS.databases = {};
|
|
334
|
+
MeshOS.namespaces = {};
|
|
335
|
+
MeshOS.entities = {};
|
|
336
|
+
MeshOS.schemas = {};
|
|
337
|
+
MeshOS.profiles = {};
|
|
338
|
+
MeshOS.classes = {};
|
|
339
|
+
MeshOS.logger = new logger_1.LoggerService('hotmesh', 'meshos');
|