@hotmeshio/hotmesh 0.0.59 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/modules/enums.d.ts +2 -1
- package/build/modules/enums.js +13 -3
- package/build/modules/errors.d.ts +3 -3
- package/build/modules/errors.js +8 -8
- package/build/modules/key.d.ts +1 -1
- package/build/modules/key.js +3 -3
- package/build/modules/utils.d.ts +6 -6
- package/build/modules/utils.js +22 -14
- package/build/package.json +41 -38
- package/build/services/activities/activity.js +37 -20
- package/build/services/activities/await.d.ts +1 -1
- package/build/services/activities/await.js +15 -7
- package/build/services/activities/cycle.d.ts +1 -1
- package/build/services/activities/cycle.js +16 -8
- package/build/services/activities/hook.d.ts +1 -1
- package/build/services/activities/hook.js +8 -4
- package/build/services/activities/interrupt.d.ts +1 -1
- package/build/services/activities/interrupt.js +14 -6
- package/build/services/activities/signal.d.ts +1 -1
- package/build/services/activities/signal.js +12 -4
- package/build/services/activities/trigger.d.ts +1 -1
- package/build/services/activities/trigger.js +19 -12
- package/build/services/activities/worker.d.ts +1 -1
- package/build/services/activities/worker.js +15 -7
- package/build/services/collator/index.js +12 -12
- package/build/services/compiler/deployer.js +17 -12
- package/build/services/compiler/index.js +4 -4
- package/build/services/compiler/validator.d.ts +3 -3
- package/build/services/compiler/validator.js +12 -3
- package/build/services/durable/client.d.ts +1 -1
- package/build/services/durable/client.js +18 -12
- package/build/services/durable/connection.d.ts +1 -1
- package/build/services/durable/exporter.d.ts +1 -1
- package/build/services/durable/exporter.js +3 -4
- package/build/services/durable/handle.d.ts +1 -1
- package/build/services/durable/handle.js +4 -1
- package/build/services/durable/index.d.ts +1 -1
- package/build/services/durable/index.js +2 -2
- package/build/services/durable/schemas/factory.d.ts +1 -1
- package/build/services/durable/search.js +19 -11
- package/build/services/durable/worker.js +50 -30
- package/build/services/durable/workflow.d.ts +5 -5
- package/build/services/durable/workflow.js +34 -18
- package/build/services/engine/index.js +33 -26
- package/build/services/exporter/index.d.ts +1 -1
- package/build/services/exporter/index.js +3 -3
- package/build/services/hotmesh/index.js +1 -1
- package/build/services/logger/index.js +1 -1
- package/build/services/mapper/index.js +3 -1
- package/build/services/pipe/functions/date.js +1 -1
- package/build/services/pipe/index.js +37 -10
- package/build/services/quorum/index.js +14 -11
- package/build/services/reporter/index.js +15 -12
- package/build/services/router/index.d.ts +2 -2
- package/build/services/router/index.js +73 -23
- package/build/services/serializer/index.js +48 -26
- package/build/services/store/cache.d.ts +5 -5
- package/build/services/store/cache.js +2 -2
- package/build/services/store/clients/ioredis.js +3 -3
- package/build/services/store/clients/redis.js +24 -4
- package/build/services/store/index.d.ts +9 -3
- package/build/services/store/index.js +122 -60
- package/build/services/stream/clients/ioredis.js +4 -4
- package/build/services/stream/clients/redis.js +31 -4
- package/build/services/task/index.js +8 -11
- package/build/services/telemetry/index.js +21 -14
- package/build/services/worker/index.d.ts +6 -6
- package/build/services/worker/index.js +12 -7
- package/build/types/activity.d.ts +3 -3
- package/build/types/exporter.d.ts +2 -2
- package/build/types/exporter.js +0 -6
- package/build/types/hook.d.ts +1 -1
- package/build/types/hotmesh.js +0 -1
- package/build/types/index.d.ts +12 -12
- package/build/types/job.d.ts +1 -1
- package/build/types/logger.js +0 -1
- package/build/types/quorum.d.ts +1 -1
- package/build/types/stats.d.ts +1 -1
- package/build/types/stream.d.ts +1 -2
- package/build/types/telemetry.d.ts +1 -1
- package/build/types/transition.d.ts +1 -1
- package/package.json +41 -38
- package/types/activity.ts +56 -39
- package/types/async.ts +2 -3
- package/types/collator.ts +5 -5
- package/types/durable.ts +161 -161
- package/types/error.ts +37 -37
- package/types/exporter.ts +14 -9
- package/types/hook.ts +11 -4
- package/types/hotmesh.ts +26 -25
- package/types/index.ts +53 -53
- package/types/job.ts +33 -33
- package/types/logger.ts +1 -1
- package/types/map.ts +1 -1
- package/types/pipe.ts +10 -8
- package/types/quorum.ts +20 -13
- package/types/redis.ts +70 -15
- package/types/serializer.ts +8 -6
- package/types/stats.ts +22 -6
- package/types/stream.ts +9 -9
- package/types/task.ts +7 -1
- package/types/telemetry.ts +2 -1
- package/types/transition.ts +8 -8
|
@@ -43,7 +43,7 @@ class IORedisStoreService extends index_1.StoreService {
|
|
|
43
43
|
async xgroup(command, key, groupName, id, mkStream) {
|
|
44
44
|
if (mkStream === 'MKSTREAM') {
|
|
45
45
|
try {
|
|
46
|
-
return (await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK';
|
|
46
|
+
return ((await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK');
|
|
47
47
|
}
|
|
48
48
|
catch (err) {
|
|
49
49
|
this.logger.debug(`Consumer group not created with MKSTREAM for key: ${key} and group: ${groupName}`);
|
|
@@ -52,7 +52,7 @@ class IORedisStoreService extends index_1.StoreService {
|
|
|
52
52
|
}
|
|
53
53
|
else {
|
|
54
54
|
try {
|
|
55
|
-
return (await this.redisClient.xgroup(command, key, groupName, id)) === 'OK';
|
|
55
|
+
return ((await this.redisClient.xgroup(command, key, groupName, id)) === 'OK');
|
|
56
56
|
}
|
|
57
57
|
catch (err) {
|
|
58
58
|
this.logger.debug(`Consumer group not created for key: ${key} and group: ${groupName}`);
|
|
@@ -80,7 +80,7 @@ class IORedisStoreService extends index_1.StoreService {
|
|
|
80
80
|
}
|
|
81
81
|
async xclaim(key, group, consumer, minIdleTime, id, ...args) {
|
|
82
82
|
try {
|
|
83
|
-
return await this.redisClient.xclaim(key, group, consumer, minIdleTime, id, ...args);
|
|
83
|
+
return (await this.redisClient.xclaim(key, group, consumer, minIdleTime, id, ...args));
|
|
84
84
|
}
|
|
85
85
|
catch (error) {
|
|
86
86
|
this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { ...error });
|
|
@@ -48,7 +48,10 @@ class RedisStoreService extends index_1.StoreService {
|
|
|
48
48
|
return this.isSuccessful(status);
|
|
49
49
|
}
|
|
50
50
|
async zAdd(key, score, value, redisMulti) {
|
|
51
|
-
return await (redisMulti || this.redisClient)[this.commands.zadd](key, {
|
|
51
|
+
return await (redisMulti || this.redisClient)[this.commands.zadd](key, {
|
|
52
|
+
score: score,
|
|
53
|
+
value: value.toString(),
|
|
54
|
+
});
|
|
52
55
|
}
|
|
53
56
|
async zRangeByScoreWithScores(key, score, value) {
|
|
54
57
|
const result = await this.redisClient[this.commands.zrangebyscore_withscores](key, score, value);
|
|
@@ -67,7 +70,14 @@ class RedisStoreService extends index_1.StoreService {
|
|
|
67
70
|
async xgroup(command, key, groupName, id, mkStream) {
|
|
68
71
|
const args = mkStream === 'MKSTREAM' ? ['MKSTREAM'] : [];
|
|
69
72
|
try {
|
|
70
|
-
return (await this.redisClient.sendCommand([
|
|
73
|
+
return ((await this.redisClient.sendCommand([
|
|
74
|
+
'XGROUP',
|
|
75
|
+
'CREATE',
|
|
76
|
+
key,
|
|
77
|
+
groupName,
|
|
78
|
+
id,
|
|
79
|
+
...args,
|
|
80
|
+
])) === 1);
|
|
71
81
|
}
|
|
72
82
|
catch (error) {
|
|
73
83
|
const streamType = mkStream === 'MKSTREAM' ? 'with MKSTREAM' : 'without MKSTREAM';
|
|
@@ -81,7 +91,9 @@ class RedisStoreService extends index_1.StoreService {
|
|
|
81
91
|
multi = args.pop();
|
|
82
92
|
}
|
|
83
93
|
try {
|
|
84
|
-
return await (multi || this.redisClient).XADD(key, id, {
|
|
94
|
+
return await (multi || this.redisClient).XADD(key, id, {
|
|
95
|
+
[args[0]]: args[1],
|
|
96
|
+
});
|
|
85
97
|
}
|
|
86
98
|
catch (error) {
|
|
87
99
|
this.logger.error(`Error publishing 'xadd'; key: ${key}`, { ...error });
|
|
@@ -108,7 +120,15 @@ class RedisStoreService extends index_1.StoreService {
|
|
|
108
120
|
}
|
|
109
121
|
async xclaim(key, group, consumer, minIdleTime, id, ...args) {
|
|
110
122
|
try {
|
|
111
|
-
return await this.redisClient.sendCommand([
|
|
123
|
+
return (await this.redisClient.sendCommand([
|
|
124
|
+
'XCLAIM',
|
|
125
|
+
key,
|
|
126
|
+
group,
|
|
127
|
+
consumer,
|
|
128
|
+
minIdleTime.toString(),
|
|
129
|
+
id,
|
|
130
|
+
...args,
|
|
131
|
+
]));
|
|
112
132
|
}
|
|
113
133
|
catch (error) {
|
|
114
134
|
this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { ...error });
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { KeyStoreParams, KeyType } from '../../modules/key';
|
|
2
2
|
import { ILogger } from '../logger';
|
|
3
3
|
import { SerializerService as Serializer } from '../serializer';
|
|
4
|
-
import { Cache } from './cache';
|
|
5
4
|
import { ActivityType, Consumes } from '../../types/activity';
|
|
6
5
|
import { AppVID } from '../../types/app';
|
|
7
6
|
import { HookRule, HookSignal } from '../../types/hook';
|
|
@@ -12,6 +11,7 @@ import { Transitions } from '../../types/transition';
|
|
|
12
11
|
import { ReclaimedMessageType } from '../../types/stream';
|
|
13
12
|
import { JobCompletionOptions, JobInterruptOptions } from '../../types/job';
|
|
14
13
|
import { WorkListTaskType } from '../../types/task';
|
|
14
|
+
import { Cache } from './cache';
|
|
15
15
|
interface AbstractRedisClient {
|
|
16
16
|
exec(): any;
|
|
17
17
|
}
|
|
@@ -50,7 +50,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
50
50
|
releaseScoutRole(scoutType: 'time' | 'signal' | 'activate'): Promise<boolean>;
|
|
51
51
|
getSettings(bCreate?: boolean): Promise<HotMeshSettings>;
|
|
52
52
|
setSettings(manifest: HotMeshSettings): Promise<any>;
|
|
53
|
-
reserveSymbolRange(target: string, size: number, type: 'JOB' | 'ACTIVITY'): Promise<[number, number, Symbols]>;
|
|
53
|
+
reserveSymbolRange(target: string, size: number, type: 'JOB' | 'ACTIVITY', tryCount?: number): Promise<[number, number, Symbols]>;
|
|
54
54
|
getAllSymbols(): Promise<Symbols>;
|
|
55
55
|
getSymbols(activityId: string): Promise<Symbols>;
|
|
56
56
|
addSymbols(activityId: string, symbols: Symbols): Promise<boolean>;
|
|
@@ -134,7 +134,13 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
134
134
|
* organized into 'n'-second blocks (LISTS))
|
|
135
135
|
*/
|
|
136
136
|
registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, deletionTime: number, dad: string, multi?: U): Promise<void>;
|
|
137
|
-
getNextTask(listKey?: string): Promise<[
|
|
137
|
+
getNextTask(listKey?: string): Promise<[
|
|
138
|
+
listKey: string,
|
|
139
|
+
jobId: string,
|
|
140
|
+
gId: string,
|
|
141
|
+
activityId: string,
|
|
142
|
+
type: WorkListTaskType
|
|
143
|
+
] | boolean>;
|
|
138
144
|
/**
|
|
139
145
|
* when processing time jobs, the target LIST ID returned
|
|
140
146
|
* from the ZSET query can be prefixed to denote what to
|
|
@@ -26,10 +26,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
26
26
|
exports.StoreService = void 0;
|
|
27
27
|
const key_1 = require("../../modules/key");
|
|
28
28
|
const serializer_1 = require("../serializer");
|
|
29
|
-
const cache_1 = require("./cache");
|
|
30
29
|
const utils_1 = require("../../modules/utils");
|
|
31
30
|
const enums_1 = require("../../modules/enums");
|
|
32
31
|
const errors_1 = require("../../modules/errors");
|
|
32
|
+
const cache_1 = require("./cache");
|
|
33
33
|
class StoreService {
|
|
34
34
|
constructor(redisClient) {
|
|
35
35
|
this.commands = {
|
|
@@ -108,12 +108,18 @@ class StoreService {
|
|
|
108
108
|
* time and signal task queues.
|
|
109
109
|
*/
|
|
110
110
|
async reserveScoutRole(scoutType, delay = enums_1.HMSH_SCOUT_INTERVAL_SECONDS) {
|
|
111
|
-
const key = this.mintKey(key_1.KeyType.WORK_ITEMS, {
|
|
111
|
+
const key = this.mintKey(key_1.KeyType.WORK_ITEMS, {
|
|
112
|
+
appId: this.appId,
|
|
113
|
+
scoutType,
|
|
114
|
+
});
|
|
112
115
|
const success = await this.exec('SET', key, `${scoutType}:${(0, utils_1.formatISODate)(new Date())}`, 'NX', 'EX', `${delay - 1}`);
|
|
113
116
|
return this.isSuccessful(success);
|
|
114
117
|
}
|
|
115
118
|
async releaseScoutRole(scoutType) {
|
|
116
|
-
const key = this.mintKey(key_1.KeyType.WORK_ITEMS, {
|
|
119
|
+
const key = this.mintKey(key_1.KeyType.WORK_ITEMS, {
|
|
120
|
+
appId: this.appId,
|
|
121
|
+
scoutType,
|
|
122
|
+
});
|
|
117
123
|
const success = await this.exec('DEL', key);
|
|
118
124
|
return this.isSuccessful(success);
|
|
119
125
|
}
|
|
@@ -139,9 +145,12 @@ class StoreService {
|
|
|
139
145
|
const key = this.mintKey(key_1.KeyType.HOTMESH, params);
|
|
140
146
|
return await this.redisClient[this.commands.hset](key, manifest);
|
|
141
147
|
}
|
|
142
|
-
async reserveSymbolRange(target, size, type) {
|
|
148
|
+
async reserveSymbolRange(target, size, type, tryCount = 1) {
|
|
143
149
|
const rangeKey = this.mintKey(key_1.KeyType.SYMKEYS, { appId: this.appId });
|
|
144
|
-
const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, {
|
|
150
|
+
const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, {
|
|
151
|
+
activityId: target,
|
|
152
|
+
appId: this.appId,
|
|
153
|
+
});
|
|
145
154
|
//reserve the slot in a `pending` state (range will be established in the next step)
|
|
146
155
|
const response = await this.redisClient[this.commands.hsetnx](rangeKey, target, '?:?');
|
|
147
156
|
if (response) {
|
|
@@ -158,12 +167,24 @@ class StoreService {
|
|
|
158
167
|
//if the key already existed, get the lower limit and add the number of symbols
|
|
159
168
|
const range = await this.redisClient[this.commands.hget](rangeKey, target);
|
|
160
169
|
const [lowerLimitString] = range.split(':');
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
170
|
+
if (lowerLimitString === '?') {
|
|
171
|
+
console.log('symbol range collision!!!', tryCount);
|
|
172
|
+
await (0, utils_1.sleepFor)(tryCount * 1000);
|
|
173
|
+
if (tryCount < 5) {
|
|
174
|
+
return this.reserveSymbolRange(target, size, type, tryCount + 1);
|
|
175
|
+
}
|
|
176
|
+
else {
|
|
177
|
+
throw new Error('Symbol range reservation failed due to deployment contention');
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
else {
|
|
181
|
+
const lowerLimit = parseInt(lowerLimitString, 10);
|
|
182
|
+
const symbols = await this.redisClient[this.commands.hgetall](symbolKey);
|
|
183
|
+
const symbolCount = Object.keys(symbols).length;
|
|
184
|
+
const actualLowerLimit = lowerLimit + serializer_1.MDATA_SYMBOLS.SLOTS + symbolCount;
|
|
185
|
+
const upperLimit = Number(lowerLimit + size - 1);
|
|
186
|
+
return [actualLowerLimit, upperLimit, symbols];
|
|
187
|
+
}
|
|
167
188
|
}
|
|
168
189
|
}
|
|
169
190
|
async getAllSymbols() {
|
|
@@ -174,10 +195,13 @@ class StoreService {
|
|
|
174
195
|
delete rangeKeys[':cursor'];
|
|
175
196
|
const multi = this.getMulti();
|
|
176
197
|
for (const rangeKey of rangeKeys) {
|
|
177
|
-
const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, {
|
|
198
|
+
const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, {
|
|
199
|
+
activityId: rangeKey,
|
|
200
|
+
appId: this.appId,
|
|
201
|
+
});
|
|
178
202
|
multi[this.commands.hgetall](symbolKey);
|
|
179
203
|
}
|
|
180
|
-
const results = await multi.exec();
|
|
204
|
+
const results = (await multi.exec());
|
|
181
205
|
const symbolSets = {};
|
|
182
206
|
results.forEach((result, index) => {
|
|
183
207
|
if (result) {
|
|
@@ -189,7 +213,9 @@ class StoreService {
|
|
|
189
213
|
vals = result;
|
|
190
214
|
}
|
|
191
215
|
for (const [key, value] of Object.entries(vals)) {
|
|
192
|
-
symbolSets[value] = key.startsWith(rangeKeys[index])
|
|
216
|
+
symbolSets[value] = key.startsWith(rangeKeys[index])
|
|
217
|
+
? key
|
|
218
|
+
: `${rangeKeys[index]}/${key}`;
|
|
193
219
|
}
|
|
194
220
|
}
|
|
195
221
|
});
|
|
@@ -320,7 +346,7 @@ class StoreService {
|
|
|
320
346
|
id,
|
|
321
347
|
version: version.toString(),
|
|
322
348
|
[versionId]: `activated:${(0, utils_1.formatISODate)(new Date())}`,
|
|
323
|
-
active: true
|
|
349
|
+
active: true,
|
|
324
350
|
};
|
|
325
351
|
Object.entries(payload).forEach(([key, value]) => {
|
|
326
352
|
payload[key] = value.toString();
|
|
@@ -352,13 +378,7 @@ class StoreService {
|
|
|
352
378
|
jobId: originJobId,
|
|
353
379
|
};
|
|
354
380
|
const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
|
|
355
|
-
const expireTask = [
|
|
356
|
-
depType,
|
|
357
|
-
topic,
|
|
358
|
-
gId,
|
|
359
|
-
pd,
|
|
360
|
-
jobId,
|
|
361
|
-
].join(key_1.VALSEP);
|
|
381
|
+
const expireTask = [depType, topic, gId, pd, jobId].join(key_1.VALSEP);
|
|
362
382
|
privateMulti[this.commands.rpush](depKey, expireTask);
|
|
363
383
|
if (!multi) {
|
|
364
384
|
return await privateMulti.exec();
|
|
@@ -373,20 +393,19 @@ class StoreService {
|
|
|
373
393
|
const dependencyParams = { appId: this.appId, jobId };
|
|
374
394
|
const dependencyKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
|
|
375
395
|
//persiste dependency tasks as multi-segment composite keys
|
|
376
|
-
const delistTask = [
|
|
377
|
-
'delist',
|
|
378
|
-
'signal',
|
|
379
|
-
jobId,
|
|
380
|
-
dad,
|
|
381
|
-
signalKey
|
|
382
|
-
].join(key_1.VALSEP);
|
|
396
|
+
const delistTask = ['delist', 'signal', jobId, dad, signalKey].join(key_1.VALSEP);
|
|
383
397
|
privateMulti[this.commands.rpush](dependencyKey, delistTask);
|
|
384
398
|
if (!multi) {
|
|
385
399
|
return await privateMulti.exec();
|
|
386
400
|
}
|
|
387
401
|
}
|
|
388
402
|
async setStats(jobKey, jobId, dateTime, stats, appVersion, multi) {
|
|
389
|
-
const params = {
|
|
403
|
+
const params = {
|
|
404
|
+
appId: appVersion.id,
|
|
405
|
+
jobId,
|
|
406
|
+
jobKey,
|
|
407
|
+
dateTime,
|
|
408
|
+
};
|
|
390
409
|
const privateMulti = multi || this.getMulti();
|
|
391
410
|
if (stats.general.length) {
|
|
392
411
|
const generalStatsKey = this.mintKey(key_1.KeyType.JOB_STATS_GENERAL, params);
|
|
@@ -469,7 +488,10 @@ class StoreService {
|
|
|
469
488
|
}
|
|
470
489
|
async setState({ ...state }, status, jobId, symbolNames, dIds, multi) {
|
|
471
490
|
delete state['metadata/js'];
|
|
472
|
-
const hashKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
491
|
+
const hashKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
492
|
+
appId: this.appId,
|
|
493
|
+
jobId,
|
|
494
|
+
});
|
|
473
495
|
const symKeys = await this.getSymbolKeys(symbolNames);
|
|
474
496
|
const symVals = await this.getSymbolValues();
|
|
475
497
|
this.serializer.resetSymbols(symKeys, symVals, dIds);
|
|
@@ -489,7 +511,7 @@ class StoreService {
|
|
|
489
511
|
*/
|
|
490
512
|
async getQueryState(jobId, fields) {
|
|
491
513
|
const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
|
|
492
|
-
const _fields = fields.map(field => `_${field}`);
|
|
514
|
+
const _fields = fields.map((field) => `_${field}`);
|
|
493
515
|
const jobDataArray = await this.redisClient[this.commands.hmget](key, _fields);
|
|
494
516
|
const jobData = {};
|
|
495
517
|
fields.forEach((field, index) => {
|
|
@@ -530,7 +552,10 @@ class StoreService {
|
|
|
530
552
|
}
|
|
531
553
|
}
|
|
532
554
|
async getRaw(jobId) {
|
|
533
|
-
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
555
|
+
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
556
|
+
appId: this.appId,
|
|
557
|
+
jobId,
|
|
558
|
+
});
|
|
534
559
|
const job = await this.redisClient[this.commands.hgetall](jobKey);
|
|
535
560
|
if (!job) {
|
|
536
561
|
throw new errors_1.GetStateError(jobId);
|
|
@@ -542,7 +567,10 @@ class StoreService {
|
|
|
542
567
|
* in order to track their progress during processing.
|
|
543
568
|
*/
|
|
544
569
|
async collate(jobId, activityId, amount, dIds, multi) {
|
|
545
|
-
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
570
|
+
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
571
|
+
appId: this.appId,
|
|
572
|
+
jobId,
|
|
573
|
+
});
|
|
546
574
|
const collationKey = `${activityId}/output/metadata/as`; //activity state
|
|
547
575
|
const symbolNames = [activityId];
|
|
548
576
|
const symKeys = await this.getSymbolKeys(symbolNames);
|
|
@@ -560,7 +588,10 @@ class StoreService {
|
|
|
560
588
|
* purposeful re-entry.
|
|
561
589
|
*/
|
|
562
590
|
async collateSynthetic(jobId, guid, amount, multi) {
|
|
563
|
-
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
591
|
+
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
592
|
+
appId: this.appId,
|
|
593
|
+
jobId,
|
|
594
|
+
});
|
|
564
595
|
return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, guid, amount);
|
|
565
596
|
}
|
|
566
597
|
async setStateNX(jobId, appId) {
|
|
@@ -584,7 +615,10 @@ class StoreService {
|
|
|
584
615
|
return schemas;
|
|
585
616
|
}
|
|
586
617
|
else {
|
|
587
|
-
const params = {
|
|
618
|
+
const params = {
|
|
619
|
+
appId: appVersion.id,
|
|
620
|
+
appVersion: appVersion.version,
|
|
621
|
+
};
|
|
588
622
|
const key = this.mintKey(key_1.KeyType.SCHEMAS, params);
|
|
589
623
|
schemas = {};
|
|
590
624
|
const hash = await this.redisClient[this.commands.hgetall](key);
|
|
@@ -596,7 +630,10 @@ class StoreService {
|
|
|
596
630
|
}
|
|
597
631
|
}
|
|
598
632
|
async setSchemas(schemas, appVersion) {
|
|
599
|
-
const params = {
|
|
633
|
+
const params = {
|
|
634
|
+
appId: appVersion.id,
|
|
635
|
+
appVersion: appVersion.version,
|
|
636
|
+
};
|
|
600
637
|
const key = this.mintKey(key_1.KeyType.SCHEMAS, params);
|
|
601
638
|
const _schemas = { ...schemas };
|
|
602
639
|
Object.entries(_schemas).forEach(([key, value]) => {
|
|
@@ -607,7 +644,10 @@ class StoreService {
|
|
|
607
644
|
return response;
|
|
608
645
|
}
|
|
609
646
|
async setSubscriptions(subscriptions, appVersion) {
|
|
610
|
-
const params = {
|
|
647
|
+
const params = {
|
|
648
|
+
appId: appVersion.id,
|
|
649
|
+
appVersion: appVersion.version,
|
|
650
|
+
};
|
|
611
651
|
const key = this.mintKey(key_1.KeyType.SUBSCRIPTIONS, params);
|
|
612
652
|
const _subscriptions = { ...subscriptions };
|
|
613
653
|
Object.entries(_subscriptions).forEach(([key, value]) => {
|
|
@@ -623,9 +663,13 @@ class StoreService {
|
|
|
623
663
|
return subscriptions;
|
|
624
664
|
}
|
|
625
665
|
else {
|
|
626
|
-
const params = {
|
|
666
|
+
const params = {
|
|
667
|
+
appId: appVersion.id,
|
|
668
|
+
appVersion: appVersion.version,
|
|
669
|
+
};
|
|
627
670
|
const key = this.mintKey(key_1.KeyType.SUBSCRIPTIONS, params);
|
|
628
|
-
subscriptions =
|
|
671
|
+
subscriptions =
|
|
672
|
+
await this.redisClient[this.commands.hgetall](key) || {};
|
|
629
673
|
Object.entries(subscriptions).forEach(([key, value]) => {
|
|
630
674
|
subscriptions[key] = JSON.parse(value);
|
|
631
675
|
});
|
|
@@ -638,7 +682,10 @@ class StoreService {
|
|
|
638
682
|
return subscriptions[topic];
|
|
639
683
|
}
|
|
640
684
|
async setTransitions(transitions, appVersion) {
|
|
641
|
-
const params = {
|
|
685
|
+
const params = {
|
|
686
|
+
appId: appVersion.id,
|
|
687
|
+
appVersion: appVersion.version,
|
|
688
|
+
};
|
|
642
689
|
const key = this.mintKey(key_1.KeyType.SUBSCRIPTION_PATTERNS, params);
|
|
643
690
|
const _subscriptions = { ...transitions };
|
|
644
691
|
Object.entries(_subscriptions).forEach(([key, value]) => {
|
|
@@ -656,7 +703,10 @@ class StoreService {
|
|
|
656
703
|
return transitions;
|
|
657
704
|
}
|
|
658
705
|
else {
|
|
659
|
-
const params = {
|
|
706
|
+
const params = {
|
|
707
|
+
appId: appVersion.id,
|
|
708
|
+
appVersion: appVersion.version,
|
|
709
|
+
};
|
|
660
710
|
const key = this.mintKey(key_1.KeyType.SUBSCRIPTION_PATTERNS, params);
|
|
661
711
|
transitions = {};
|
|
662
712
|
const hash = await this.redisClient[this.commands.hgetall](key);
|
|
@@ -743,7 +793,7 @@ class StoreService {
|
|
|
743
793
|
if (scrub) {
|
|
744
794
|
//indexes can be designed to be self-cleaning; `engine.hookAll` exposes this option
|
|
745
795
|
this.redisClient[this.commands.expire](processedKey, 0);
|
|
746
|
-
this.redisClient[this.commands.expire](key.split(
|
|
796
|
+
this.redisClient[this.commands.expire](key.split(':').slice(0, 5).join(':'), 0);
|
|
747
797
|
}
|
|
748
798
|
else {
|
|
749
799
|
await this.redisClient[this.commands.rename](processedKey, key);
|
|
@@ -756,7 +806,10 @@ class StoreService {
|
|
|
756
806
|
}
|
|
757
807
|
async expireJob(jobId, inSeconds) {
|
|
758
808
|
if (!isNaN(inSeconds) && inSeconds > 0) {
|
|
759
|
-
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
809
|
+
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
810
|
+
appId: this.appId,
|
|
811
|
+
jobId,
|
|
812
|
+
});
|
|
760
813
|
await this.redisClient[this.commands.expire](jobKey, inSeconds);
|
|
761
814
|
}
|
|
762
815
|
}
|
|
@@ -785,15 +838,12 @@ class StoreService {
|
|
|
785
838
|
* organized into 'n'-second blocks (LISTS))
|
|
786
839
|
*/
|
|
787
840
|
async registerTimeHook(jobId, gId, activityId, type, deletionTime, dad, multi) {
|
|
788
|
-
const listKey = this.mintKey(key_1.KeyType.TIME_RANGE, {
|
|
841
|
+
const listKey = this.mintKey(key_1.KeyType.TIME_RANGE, {
|
|
842
|
+
appId: this.appId,
|
|
843
|
+
timeValue: deletionTime,
|
|
844
|
+
});
|
|
789
845
|
//construct the composite key (the key has enough info to signal the hook)
|
|
790
|
-
const timeEvent = [
|
|
791
|
-
type,
|
|
792
|
-
activityId,
|
|
793
|
-
gId,
|
|
794
|
-
dad,
|
|
795
|
-
jobId
|
|
796
|
-
].join(key_1.VALSEP);
|
|
846
|
+
const timeEvent = [type, activityId, gId, dad, jobId].join(key_1.VALSEP);
|
|
797
847
|
const len = await (multi || this.redisClient)[this.commands.rpush](listKey, timeEvent);
|
|
798
848
|
if (multi || len === 1) {
|
|
799
849
|
const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
|
|
@@ -863,7 +913,10 @@ class StoreService {
|
|
|
863
913
|
}
|
|
864
914
|
//decrement job status (:) by 1bil
|
|
865
915
|
const amount = -1000000000;
|
|
866
|
-
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
916
|
+
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
917
|
+
appId: this.appId,
|
|
918
|
+
jobId,
|
|
919
|
+
});
|
|
867
920
|
const result = await this.redisClient[this.commands.hincrbyfloat](jobKey, ':', amount);
|
|
868
921
|
if (result <= amount) {
|
|
869
922
|
//verify active state; job already interrupted
|
|
@@ -881,7 +934,7 @@ class StoreService {
|
|
|
881
934
|
code: options.code ?? enums_1.HMSH_CODE_INTERRUPT,
|
|
882
935
|
message: options.reason ?? `job [${jobId}] interrupted`,
|
|
883
936
|
stack: options.stack ?? '',
|
|
884
|
-
job_id: jobId
|
|
937
|
+
job_id: jobId,
|
|
885
938
|
});
|
|
886
939
|
const payload = { [errKey]: amount.toString() };
|
|
887
940
|
const hashData = this.serializer.package(payload, symbolNames);
|
|
@@ -899,18 +952,24 @@ class StoreService {
|
|
|
899
952
|
}
|
|
900
953
|
}
|
|
901
954
|
async scrub(jobId) {
|
|
902
|
-
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
955
|
+
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
956
|
+
appId: this.appId,
|
|
957
|
+
jobId,
|
|
958
|
+
});
|
|
903
959
|
await this.redisClient[this.commands.del](jobKey);
|
|
904
960
|
}
|
|
905
961
|
async findJobs(queryString = '*', limit = 1000, batchSize = 1000, cursor = '0') {
|
|
906
|
-
const matchKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
962
|
+
const matchKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
963
|
+
appId: this.appId,
|
|
964
|
+
jobId: queryString,
|
|
965
|
+
});
|
|
907
966
|
let keys;
|
|
908
967
|
const matchingKeys = [];
|
|
909
968
|
do {
|
|
910
|
-
const output = await this.exec('SCAN', cursor, 'MATCH', matchKey, 'COUNT', batchSize.toString());
|
|
969
|
+
const output = (await this.exec('SCAN', cursor, 'MATCH', matchKey, 'COUNT', batchSize.toString()));
|
|
911
970
|
if (Array.isArray(output)) {
|
|
912
971
|
[cursor, keys] = output;
|
|
913
|
-
for (
|
|
972
|
+
for (const key of [...keys]) {
|
|
914
973
|
matchingKeys.push(key);
|
|
915
974
|
}
|
|
916
975
|
if (matchingKeys.length >= limit) {
|
|
@@ -926,10 +985,13 @@ class StoreService {
|
|
|
926
985
|
async findJobFields(jobId, fieldMatchPattern = '*', limit = 1000, batchSize = 1000, cursor = '0') {
|
|
927
986
|
let fields = [];
|
|
928
987
|
const matchingFields = {};
|
|
929
|
-
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
988
|
+
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
989
|
+
appId: this.appId,
|
|
990
|
+
jobId,
|
|
991
|
+
});
|
|
930
992
|
let len = 0;
|
|
931
993
|
do {
|
|
932
|
-
const output = await this.exec('HSCAN', jobKey, cursor, 'MATCH', fieldMatchPattern, 'COUNT', batchSize.toString());
|
|
994
|
+
const output = (await this.exec('HSCAN', jobKey, cursor, 'MATCH', fieldMatchPattern, 'COUNT', batchSize.toString()));
|
|
933
995
|
if (Array.isArray(output)) {
|
|
934
996
|
[cursor, fields] = output;
|
|
935
997
|
for (let i = 0; i < fields.length; i += 2) {
|
|
@@ -23,7 +23,7 @@ class IORedisStreamService extends index_1.StreamService {
|
|
|
23
23
|
async xgroup(command, key, groupName, id, mkStream) {
|
|
24
24
|
if (mkStream === 'MKSTREAM') {
|
|
25
25
|
try {
|
|
26
|
-
return (await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK';
|
|
26
|
+
return ((await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK');
|
|
27
27
|
}
|
|
28
28
|
catch (error) {
|
|
29
29
|
this.logger.info(`Consumer group not created with MKSTREAM for key: ${key} and group: ${groupName}`);
|
|
@@ -32,7 +32,7 @@ class IORedisStreamService extends index_1.StreamService {
|
|
|
32
32
|
}
|
|
33
33
|
else {
|
|
34
34
|
try {
|
|
35
|
-
return (await this.redisClient.xgroup(command, key, groupName, id)) === 'OK';
|
|
35
|
+
return ((await this.redisClient.xgroup(command, key, groupName, id)) === 'OK');
|
|
36
36
|
}
|
|
37
37
|
catch (error) {
|
|
38
38
|
this.logger.info(`Consumer group not created for key: ${key} and group: ${groupName}`);
|
|
@@ -73,7 +73,7 @@ class IORedisStreamService extends index_1.StreamService {
|
|
|
73
73
|
if (consumer)
|
|
74
74
|
args.push(consumer);
|
|
75
75
|
try {
|
|
76
|
-
return await this.redisClient.call('XPENDING', ...args);
|
|
76
|
+
return (await this.redisClient.call('XPENDING', ...args));
|
|
77
77
|
}
|
|
78
78
|
catch (error) {
|
|
79
79
|
this.logger.error('err, args', { ...error }, args);
|
|
@@ -86,7 +86,7 @@ class IORedisStreamService extends index_1.StreamService {
|
|
|
86
86
|
}
|
|
87
87
|
async xclaim(key, group, consumer, minIdleTime, id, ...args) {
|
|
88
88
|
try {
|
|
89
|
-
return await this.redisClient.xclaim(key, group, consumer, minIdleTime, id, ...args);
|
|
89
|
+
return (await this.redisClient.xclaim(key, group, consumer, minIdleTime, id, ...args));
|
|
90
90
|
}
|
|
91
91
|
catch (error) {
|
|
92
92
|
this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { ...error });
|
|
@@ -23,7 +23,14 @@ class RedisStreamService extends index_1.StreamService {
|
|
|
23
23
|
async xgroup(command, key, groupName, id, mkStream) {
|
|
24
24
|
const args = mkStream === 'MKSTREAM' ? ['MKSTREAM'] : [];
|
|
25
25
|
try {
|
|
26
|
-
return (await this.redisClient.sendCommand([
|
|
26
|
+
return ((await this.redisClient.sendCommand([
|
|
27
|
+
'XGROUP',
|
|
28
|
+
'CREATE',
|
|
29
|
+
key,
|
|
30
|
+
groupName,
|
|
31
|
+
id,
|
|
32
|
+
...args,
|
|
33
|
+
])) === 1);
|
|
27
34
|
}
|
|
28
35
|
catch (err) {
|
|
29
36
|
const streamType = mkStream === 'MKSTREAM' ? 'with MKSTREAM' : 'without MKSTREAM';
|
|
@@ -37,7 +44,9 @@ class RedisStreamService extends index_1.StreamService {
|
|
|
37
44
|
multi = args.pop();
|
|
38
45
|
}
|
|
39
46
|
try {
|
|
40
|
-
return await (multi || this.redisClient).XADD(key, id, {
|
|
47
|
+
return await (multi || this.redisClient).XADD(key, id, {
|
|
48
|
+
[args[0]]: args[1],
|
|
49
|
+
});
|
|
41
50
|
}
|
|
42
51
|
catch (err) {
|
|
43
52
|
this.logger.error(`Error publishing 'xadd'; key: ${key}`, err);
|
|
@@ -46,7 +55,17 @@ class RedisStreamService extends index_1.StreamService {
|
|
|
46
55
|
}
|
|
47
56
|
async xreadgroup(command, groupName, consumerName, blockOption, blockTime, streamsOption, streamName, id) {
|
|
48
57
|
try {
|
|
49
|
-
return await this.redisClient.sendCommand([
|
|
58
|
+
return await this.redisClient.sendCommand([
|
|
59
|
+
'XREADGROUP',
|
|
60
|
+
command,
|
|
61
|
+
groupName,
|
|
62
|
+
consumerName,
|
|
63
|
+
blockOption,
|
|
64
|
+
blockTime.toString(),
|
|
65
|
+
streamsOption,
|
|
66
|
+
streamName,
|
|
67
|
+
id,
|
|
68
|
+
]);
|
|
50
69
|
}
|
|
51
70
|
catch (err) {
|
|
52
71
|
this.logger.error(`Error in reading data from group: ${groupName} in stream: ${streamName}`, err);
|
|
@@ -78,7 +97,15 @@ class RedisStreamService extends index_1.StreamService {
|
|
|
78
97
|
}
|
|
79
98
|
async xclaim(key, group, consumer, minIdleTime, id, ...args) {
|
|
80
99
|
try {
|
|
81
|
-
return await this.redisClient.sendCommand([
|
|
100
|
+
return (await this.redisClient.sendCommand([
|
|
101
|
+
'XCLAIM',
|
|
102
|
+
key,
|
|
103
|
+
group,
|
|
104
|
+
consumer,
|
|
105
|
+
minIdleTime.toString(),
|
|
106
|
+
id,
|
|
107
|
+
...args,
|
|
108
|
+
]));
|
|
82
109
|
}
|
|
83
110
|
catch (err) {
|
|
84
111
|
this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, err);
|
|
@@ -37,14 +37,14 @@ class TaskService {
|
|
|
37
37
|
async registerJobForCleanup(jobId, inSeconds = enums_1.HMSH_EXPIRE_DURATION, options) {
|
|
38
38
|
if (inSeconds > 0) {
|
|
39
39
|
await this.store.expireJob(jobId, inSeconds);
|
|
40
|
-
const fromNow = Date.now() +
|
|
40
|
+
const fromNow = Date.now() + inSeconds * 1000;
|
|
41
41
|
const fidelityMS = enums_1.HMSH_FIDELITY_SECONDS * 1000;
|
|
42
42
|
const timeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
|
|
43
43
|
await this.store.registerDependenciesForCleanup(jobId, timeSlot, options);
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
46
|
async registerTimeHook(jobId, gId, activityId, type, inSeconds = enums_1.HMSH_FIDELITY_SECONDS, dad, multi) {
|
|
47
|
-
const fromNow = Date.now() +
|
|
47
|
+
const fromNow = Date.now() + inSeconds * 1000;
|
|
48
48
|
const fidelityMS = enums_1.HMSH_FIDELITY_SECONDS * 1000;
|
|
49
49
|
const awakenTimeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
|
|
50
50
|
await this.store.registerTimeHook(jobId, gId, activityId, type, awakenTimeSlot, dad, multi);
|
|
@@ -83,7 +83,9 @@ class TaskService {
|
|
|
83
83
|
}
|
|
84
84
|
else if (type === 'delist') {
|
|
85
85
|
//delist the signalKey (target)
|
|
86
|
-
const key = this.store.mintKey(hotmesh_1.KeyType.SIGNALS, {
|
|
86
|
+
const key = this.store.mintKey(hotmesh_1.KeyType.SIGNALS, {
|
|
87
|
+
appId: this.store.appId,
|
|
88
|
+
});
|
|
87
89
|
await this.store.redisClient[this.store.commands.hdel](key, target);
|
|
88
90
|
}
|
|
89
91
|
else {
|
|
@@ -102,7 +104,7 @@ class TaskService {
|
|
|
102
104
|
}
|
|
103
105
|
else {
|
|
104
106
|
//no worklists exist; sleep before checking
|
|
105
|
-
|
|
107
|
+
const sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_FIDELITY_SECONDS * 1000);
|
|
106
108
|
this.cleanupTimeout = sleep.timerId;
|
|
107
109
|
await sleep.promise;
|
|
108
110
|
this.errorCount = 0;
|
|
@@ -121,7 +123,7 @@ class TaskService {
|
|
|
121
123
|
}
|
|
122
124
|
else {
|
|
123
125
|
//didn't get the scout role; try again in 'one-ish' minutes
|
|
124
|
-
|
|
126
|
+
const sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_SCOUT_INTERVAL_SECONDS * 1000 * 2 * Math.random());
|
|
125
127
|
this.cleanupTimeout = sleep.timerId;
|
|
126
128
|
await sleep.promise;
|
|
127
129
|
this.processTimeHooks(timeEventCallback);
|
|
@@ -146,12 +148,7 @@ class TaskService {
|
|
|
146
148
|
const gId = context.metadata.gid;
|
|
147
149
|
const activityId = hookRule.to;
|
|
148
150
|
//composite keys are used to fully describe the task target
|
|
149
|
-
const compositeJobKey = [
|
|
150
|
-
activityId,
|
|
151
|
-
dad,
|
|
152
|
-
gId,
|
|
153
|
-
jobId
|
|
154
|
-
].join(key_1.WEBSEP);
|
|
151
|
+
const compositeJobKey = [activityId, dad, gId, jobId].join(key_1.WEBSEP);
|
|
155
152
|
const hook = {
|
|
156
153
|
topic,
|
|
157
154
|
resolved,
|