@hotmeshio/hotmesh 0.0.60 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/modules/enums.d.ts +2 -1
- package/build/modules/enums.js +12 -3
- package/build/modules/errors.d.ts +3 -3
- package/build/modules/errors.js +8 -8
- package/build/modules/key.d.ts +1 -1
- package/build/modules/key.js +3 -3
- package/build/modules/utils.d.ts +6 -5
- package/build/modules/utils.js +24 -16
- package/build/package.json +41 -38
- package/build/services/activities/activity.js +40 -23
- package/build/services/activities/await.d.ts +1 -1
- package/build/services/activities/await.js +15 -7
- package/build/services/activities/cycle.d.ts +1 -1
- package/build/services/activities/cycle.js +16 -8
- package/build/services/activities/hook.d.ts +1 -1
- package/build/services/activities/hook.js +8 -4
- package/build/services/activities/interrupt.d.ts +1 -1
- package/build/services/activities/interrupt.js +14 -6
- package/build/services/activities/signal.d.ts +1 -1
- package/build/services/activities/signal.js +12 -4
- package/build/services/activities/trigger.d.ts +1 -1
- package/build/services/activities/trigger.js +19 -12
- package/build/services/activities/worker.d.ts +1 -1
- package/build/services/activities/worker.js +15 -7
- package/build/services/collator/index.js +12 -12
- package/build/services/compiler/deployer.js +17 -12
- package/build/services/compiler/index.js +4 -4
- package/build/services/compiler/validator.d.ts +3 -3
- package/build/services/compiler/validator.js +12 -3
- package/build/services/durable/client.d.ts +1 -1
- package/build/services/durable/client.js +18 -12
- package/build/services/durable/connection.d.ts +1 -1
- package/build/services/durable/exporter.d.ts +1 -1
- package/build/services/durable/exporter.js +3 -4
- package/build/services/durable/handle.d.ts +1 -1
- package/build/services/durable/handle.js +4 -1
- package/build/services/durable/index.d.ts +1 -1
- package/build/services/durable/index.js +2 -2
- package/build/services/durable/schemas/factory.d.ts +1 -1
- package/build/services/durable/search.js +19 -11
- package/build/services/durable/worker.js +50 -30
- package/build/services/durable/workflow.d.ts +5 -5
- package/build/services/durable/workflow.js +34 -18
- package/build/services/engine/index.js +33 -26
- package/build/services/exporter/index.d.ts +1 -1
- package/build/services/exporter/index.js +3 -3
- package/build/services/hotmesh/index.js +1 -1
- package/build/services/logger/index.js +1 -1
- package/build/services/mapper/index.js +3 -1
- package/build/services/pipe/functions/date.js +1 -1
- package/build/services/pipe/index.js +37 -10
- package/build/services/quorum/index.js +14 -11
- package/build/services/reporter/index.js +15 -12
- package/build/services/router/index.d.ts +2 -2
- package/build/services/router/index.js +73 -23
- package/build/services/serializer/index.js +48 -26
- package/build/services/store/cache.d.ts +5 -5
- package/build/services/store/cache.js +2 -2
- package/build/services/store/clients/ioredis.d.ts +6 -0
- package/build/services/store/clients/ioredis.js +85 -3
- package/build/services/store/clients/redis.d.ts +6 -0
- package/build/services/store/clients/redis.js +140 -4
- package/build/services/store/index.d.ts +9 -3
- package/build/services/store/index.js +121 -60
- package/build/services/stream/clients/ioredis.js +4 -4
- package/build/services/stream/clients/redis.js +31 -4
- package/build/services/task/index.js +8 -11
- package/build/services/telemetry/index.js +21 -14
- package/build/services/worker/index.d.ts +6 -6
- package/build/services/worker/index.js +12 -7
- package/build/types/activity.d.ts +3 -3
- package/build/types/durable.d.ts +81 -77
- package/build/types/exporter.d.ts +2 -2
- package/build/types/exporter.js +0 -6
- package/build/types/hook.d.ts +1 -1
- package/build/types/hotmesh.js +0 -1
- package/build/types/index.d.ts +13 -13
- package/build/types/job.d.ts +1 -1
- package/build/types/logger.js +0 -1
- package/build/types/quorum.d.ts +1 -1
- package/build/types/redis.d.ts +6 -1
- package/build/types/stats.d.ts +1 -1
- package/build/types/stream.d.ts +1 -2
- package/build/types/telemetry.d.ts +1 -1
- package/build/types/transition.d.ts +1 -1
- package/package.json +41 -38
- package/types/activity.ts +56 -39
- package/types/async.ts +2 -3
- package/types/collator.ts +5 -5
- package/types/durable.ts +90 -86
- package/types/error.ts +37 -37
- package/types/exporter.ts +14 -9
- package/types/hook.ts +11 -4
- package/types/hotmesh.ts +26 -25
- package/types/index.ts +54 -53
- package/types/job.ts +33 -33
- package/types/logger.ts +1 -1
- package/types/map.ts +1 -1
- package/types/pipe.ts +10 -8
- package/types/quorum.ts +20 -13
- package/types/redis.ts +96 -16
- package/types/serializer.ts +8 -6
- package/types/stats.ts +22 -6
- package/types/stream.ts +9 -9
- package/types/task.ts +7 -1
- package/types/telemetry.ts +2 -1
- package/types/transition.ts +8 -8
|
@@ -26,10 +26,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
26
26
|
exports.StoreService = void 0;
|
|
27
27
|
const key_1 = require("../../modules/key");
|
|
28
28
|
const serializer_1 = require("../serializer");
|
|
29
|
-
const cache_1 = require("./cache");
|
|
30
29
|
const utils_1 = require("../../modules/utils");
|
|
31
30
|
const enums_1 = require("../../modules/enums");
|
|
32
31
|
const errors_1 = require("../../modules/errors");
|
|
32
|
+
const cache_1 = require("./cache");
|
|
33
33
|
class StoreService {
|
|
34
34
|
constructor(redisClient) {
|
|
35
35
|
this.commands = {
|
|
@@ -108,12 +108,18 @@ class StoreService {
|
|
|
108
108
|
* time and signal task queues.
|
|
109
109
|
*/
|
|
110
110
|
async reserveScoutRole(scoutType, delay = enums_1.HMSH_SCOUT_INTERVAL_SECONDS) {
|
|
111
|
-
const key = this.mintKey(key_1.KeyType.WORK_ITEMS, {
|
|
111
|
+
const key = this.mintKey(key_1.KeyType.WORK_ITEMS, {
|
|
112
|
+
appId: this.appId,
|
|
113
|
+
scoutType,
|
|
114
|
+
});
|
|
112
115
|
const success = await this.exec('SET', key, `${scoutType}:${(0, utils_1.formatISODate)(new Date())}`, 'NX', 'EX', `${delay - 1}`);
|
|
113
116
|
return this.isSuccessful(success);
|
|
114
117
|
}
|
|
115
118
|
async releaseScoutRole(scoutType) {
|
|
116
|
-
const key = this.mintKey(key_1.KeyType.WORK_ITEMS, {
|
|
119
|
+
const key = this.mintKey(key_1.KeyType.WORK_ITEMS, {
|
|
120
|
+
appId: this.appId,
|
|
121
|
+
scoutType,
|
|
122
|
+
});
|
|
117
123
|
const success = await this.exec('DEL', key);
|
|
118
124
|
return this.isSuccessful(success);
|
|
119
125
|
}
|
|
@@ -139,9 +145,12 @@ class StoreService {
|
|
|
139
145
|
const key = this.mintKey(key_1.KeyType.HOTMESH, params);
|
|
140
146
|
return await this.redisClient[this.commands.hset](key, manifest);
|
|
141
147
|
}
|
|
142
|
-
async reserveSymbolRange(target, size, type) {
|
|
148
|
+
async reserveSymbolRange(target, size, type, tryCount = 1) {
|
|
143
149
|
const rangeKey = this.mintKey(key_1.KeyType.SYMKEYS, { appId: this.appId });
|
|
144
|
-
const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, {
|
|
150
|
+
const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, {
|
|
151
|
+
activityId: target,
|
|
152
|
+
appId: this.appId,
|
|
153
|
+
});
|
|
145
154
|
//reserve the slot in a `pending` state (range will be established in the next step)
|
|
146
155
|
const response = await this.redisClient[this.commands.hsetnx](rangeKey, target, '?:?');
|
|
147
156
|
if (response) {
|
|
@@ -158,12 +167,23 @@ class StoreService {
|
|
|
158
167
|
//if the key already existed, get the lower limit and add the number of symbols
|
|
159
168
|
const range = await this.redisClient[this.commands.hget](rangeKey, target);
|
|
160
169
|
const [lowerLimitString] = range.split(':');
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
170
|
+
if (lowerLimitString === '?') {
|
|
171
|
+
await (0, utils_1.sleepFor)(tryCount * 1000);
|
|
172
|
+
if (tryCount < 5) {
|
|
173
|
+
return this.reserveSymbolRange(target, size, type, tryCount + 1);
|
|
174
|
+
}
|
|
175
|
+
else {
|
|
176
|
+
throw new Error('Symbol range reservation failed due to deployment contention');
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
else {
|
|
180
|
+
const lowerLimit = parseInt(lowerLimitString, 10);
|
|
181
|
+
const symbols = await this.redisClient[this.commands.hgetall](symbolKey);
|
|
182
|
+
const symbolCount = Object.keys(symbols).length;
|
|
183
|
+
const actualLowerLimit = lowerLimit + serializer_1.MDATA_SYMBOLS.SLOTS + symbolCount;
|
|
184
|
+
const upperLimit = Number(lowerLimit + size - 1);
|
|
185
|
+
return [actualLowerLimit, upperLimit, symbols];
|
|
186
|
+
}
|
|
167
187
|
}
|
|
168
188
|
}
|
|
169
189
|
async getAllSymbols() {
|
|
@@ -174,10 +194,13 @@ class StoreService {
|
|
|
174
194
|
delete rangeKeys[':cursor'];
|
|
175
195
|
const multi = this.getMulti();
|
|
176
196
|
for (const rangeKey of rangeKeys) {
|
|
177
|
-
const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, {
|
|
197
|
+
const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, {
|
|
198
|
+
activityId: rangeKey,
|
|
199
|
+
appId: this.appId,
|
|
200
|
+
});
|
|
178
201
|
multi[this.commands.hgetall](symbolKey);
|
|
179
202
|
}
|
|
180
|
-
const results = await multi.exec();
|
|
203
|
+
const results = (await multi.exec());
|
|
181
204
|
const symbolSets = {};
|
|
182
205
|
results.forEach((result, index) => {
|
|
183
206
|
if (result) {
|
|
@@ -189,7 +212,9 @@ class StoreService {
|
|
|
189
212
|
vals = result;
|
|
190
213
|
}
|
|
191
214
|
for (const [key, value] of Object.entries(vals)) {
|
|
192
|
-
symbolSets[value] = key.startsWith(rangeKeys[index])
|
|
215
|
+
symbolSets[value] = key.startsWith(rangeKeys[index])
|
|
216
|
+
? key
|
|
217
|
+
: `${rangeKeys[index]}/${key}`;
|
|
193
218
|
}
|
|
194
219
|
}
|
|
195
220
|
});
|
|
@@ -320,7 +345,7 @@ class StoreService {
|
|
|
320
345
|
id,
|
|
321
346
|
version: version.toString(),
|
|
322
347
|
[versionId]: `activated:${(0, utils_1.formatISODate)(new Date())}`,
|
|
323
|
-
active: true
|
|
348
|
+
active: true,
|
|
324
349
|
};
|
|
325
350
|
Object.entries(payload).forEach(([key, value]) => {
|
|
326
351
|
payload[key] = value.toString();
|
|
@@ -352,13 +377,7 @@ class StoreService {
|
|
|
352
377
|
jobId: originJobId,
|
|
353
378
|
};
|
|
354
379
|
const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
|
|
355
|
-
const expireTask = [
|
|
356
|
-
depType,
|
|
357
|
-
topic,
|
|
358
|
-
gId,
|
|
359
|
-
pd,
|
|
360
|
-
jobId,
|
|
361
|
-
].join(key_1.VALSEP);
|
|
380
|
+
const expireTask = [depType, topic, gId, pd, jobId].join(key_1.VALSEP);
|
|
362
381
|
privateMulti[this.commands.rpush](depKey, expireTask);
|
|
363
382
|
if (!multi) {
|
|
364
383
|
return await privateMulti.exec();
|
|
@@ -373,20 +392,19 @@ class StoreService {
|
|
|
373
392
|
const dependencyParams = { appId: this.appId, jobId };
|
|
374
393
|
const dependencyKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
|
|
375
394
|
//persiste dependency tasks as multi-segment composite keys
|
|
376
|
-
const delistTask = [
|
|
377
|
-
'delist',
|
|
378
|
-
'signal',
|
|
379
|
-
jobId,
|
|
380
|
-
dad,
|
|
381
|
-
signalKey
|
|
382
|
-
].join(key_1.VALSEP);
|
|
395
|
+
const delistTask = ['delist', 'signal', jobId, dad, signalKey].join(key_1.VALSEP);
|
|
383
396
|
privateMulti[this.commands.rpush](dependencyKey, delistTask);
|
|
384
397
|
if (!multi) {
|
|
385
398
|
return await privateMulti.exec();
|
|
386
399
|
}
|
|
387
400
|
}
|
|
388
401
|
async setStats(jobKey, jobId, dateTime, stats, appVersion, multi) {
|
|
389
|
-
const params = {
|
|
402
|
+
const params = {
|
|
403
|
+
appId: appVersion.id,
|
|
404
|
+
jobId,
|
|
405
|
+
jobKey,
|
|
406
|
+
dateTime,
|
|
407
|
+
};
|
|
390
408
|
const privateMulti = multi || this.getMulti();
|
|
391
409
|
if (stats.general.length) {
|
|
392
410
|
const generalStatsKey = this.mintKey(key_1.KeyType.JOB_STATS_GENERAL, params);
|
|
@@ -469,7 +487,10 @@ class StoreService {
|
|
|
469
487
|
}
|
|
470
488
|
async setState({ ...state }, status, jobId, symbolNames, dIds, multi) {
|
|
471
489
|
delete state['metadata/js'];
|
|
472
|
-
const hashKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
490
|
+
const hashKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
491
|
+
appId: this.appId,
|
|
492
|
+
jobId,
|
|
493
|
+
});
|
|
473
494
|
const symKeys = await this.getSymbolKeys(symbolNames);
|
|
474
495
|
const symVals = await this.getSymbolValues();
|
|
475
496
|
this.serializer.resetSymbols(symKeys, symVals, dIds);
|
|
@@ -489,7 +510,7 @@ class StoreService {
|
|
|
489
510
|
*/
|
|
490
511
|
async getQueryState(jobId, fields) {
|
|
491
512
|
const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
|
|
492
|
-
const _fields = fields.map(field => `_${field}`);
|
|
513
|
+
const _fields = fields.map((field) => `_${field}`);
|
|
493
514
|
const jobDataArray = await this.redisClient[this.commands.hmget](key, _fields);
|
|
494
515
|
const jobData = {};
|
|
495
516
|
fields.forEach((field, index) => {
|
|
@@ -530,7 +551,10 @@ class StoreService {
|
|
|
530
551
|
}
|
|
531
552
|
}
|
|
532
553
|
async getRaw(jobId) {
|
|
533
|
-
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
554
|
+
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
555
|
+
appId: this.appId,
|
|
556
|
+
jobId,
|
|
557
|
+
});
|
|
534
558
|
const job = await this.redisClient[this.commands.hgetall](jobKey);
|
|
535
559
|
if (!job) {
|
|
536
560
|
throw new errors_1.GetStateError(jobId);
|
|
@@ -542,7 +566,10 @@ class StoreService {
|
|
|
542
566
|
* in order to track their progress during processing.
|
|
543
567
|
*/
|
|
544
568
|
async collate(jobId, activityId, amount, dIds, multi) {
|
|
545
|
-
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
569
|
+
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
570
|
+
appId: this.appId,
|
|
571
|
+
jobId,
|
|
572
|
+
});
|
|
546
573
|
const collationKey = `${activityId}/output/metadata/as`; //activity state
|
|
547
574
|
const symbolNames = [activityId];
|
|
548
575
|
const symKeys = await this.getSymbolKeys(symbolNames);
|
|
@@ -560,7 +587,10 @@ class StoreService {
|
|
|
560
587
|
* purposeful re-entry.
|
|
561
588
|
*/
|
|
562
589
|
async collateSynthetic(jobId, guid, amount, multi) {
|
|
563
|
-
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
590
|
+
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
591
|
+
appId: this.appId,
|
|
592
|
+
jobId,
|
|
593
|
+
});
|
|
564
594
|
return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, guid, amount);
|
|
565
595
|
}
|
|
566
596
|
async setStateNX(jobId, appId) {
|
|
@@ -584,7 +614,10 @@ class StoreService {
|
|
|
584
614
|
return schemas;
|
|
585
615
|
}
|
|
586
616
|
else {
|
|
587
|
-
const params = {
|
|
617
|
+
const params = {
|
|
618
|
+
appId: appVersion.id,
|
|
619
|
+
appVersion: appVersion.version,
|
|
620
|
+
};
|
|
588
621
|
const key = this.mintKey(key_1.KeyType.SCHEMAS, params);
|
|
589
622
|
schemas = {};
|
|
590
623
|
const hash = await this.redisClient[this.commands.hgetall](key);
|
|
@@ -596,7 +629,10 @@ class StoreService {
|
|
|
596
629
|
}
|
|
597
630
|
}
|
|
598
631
|
async setSchemas(schemas, appVersion) {
|
|
599
|
-
const params = {
|
|
632
|
+
const params = {
|
|
633
|
+
appId: appVersion.id,
|
|
634
|
+
appVersion: appVersion.version,
|
|
635
|
+
};
|
|
600
636
|
const key = this.mintKey(key_1.KeyType.SCHEMAS, params);
|
|
601
637
|
const _schemas = { ...schemas };
|
|
602
638
|
Object.entries(_schemas).forEach(([key, value]) => {
|
|
@@ -607,7 +643,10 @@ class StoreService {
|
|
|
607
643
|
return response;
|
|
608
644
|
}
|
|
609
645
|
async setSubscriptions(subscriptions, appVersion) {
|
|
610
|
-
const params = {
|
|
646
|
+
const params = {
|
|
647
|
+
appId: appVersion.id,
|
|
648
|
+
appVersion: appVersion.version,
|
|
649
|
+
};
|
|
611
650
|
const key = this.mintKey(key_1.KeyType.SUBSCRIPTIONS, params);
|
|
612
651
|
const _subscriptions = { ...subscriptions };
|
|
613
652
|
Object.entries(_subscriptions).forEach(([key, value]) => {
|
|
@@ -623,9 +662,13 @@ class StoreService {
|
|
|
623
662
|
return subscriptions;
|
|
624
663
|
}
|
|
625
664
|
else {
|
|
626
|
-
const params = {
|
|
665
|
+
const params = {
|
|
666
|
+
appId: appVersion.id,
|
|
667
|
+
appVersion: appVersion.version,
|
|
668
|
+
};
|
|
627
669
|
const key = this.mintKey(key_1.KeyType.SUBSCRIPTIONS, params);
|
|
628
|
-
subscriptions =
|
|
670
|
+
subscriptions =
|
|
671
|
+
await this.redisClient[this.commands.hgetall](key) || {};
|
|
629
672
|
Object.entries(subscriptions).forEach(([key, value]) => {
|
|
630
673
|
subscriptions[key] = JSON.parse(value);
|
|
631
674
|
});
|
|
@@ -638,7 +681,10 @@ class StoreService {
|
|
|
638
681
|
return subscriptions[topic];
|
|
639
682
|
}
|
|
640
683
|
async setTransitions(transitions, appVersion) {
|
|
641
|
-
const params = {
|
|
684
|
+
const params = {
|
|
685
|
+
appId: appVersion.id,
|
|
686
|
+
appVersion: appVersion.version,
|
|
687
|
+
};
|
|
642
688
|
const key = this.mintKey(key_1.KeyType.SUBSCRIPTION_PATTERNS, params);
|
|
643
689
|
const _subscriptions = { ...transitions };
|
|
644
690
|
Object.entries(_subscriptions).forEach(([key, value]) => {
|
|
@@ -656,7 +702,10 @@ class StoreService {
|
|
|
656
702
|
return transitions;
|
|
657
703
|
}
|
|
658
704
|
else {
|
|
659
|
-
const params = {
|
|
705
|
+
const params = {
|
|
706
|
+
appId: appVersion.id,
|
|
707
|
+
appVersion: appVersion.version,
|
|
708
|
+
};
|
|
660
709
|
const key = this.mintKey(key_1.KeyType.SUBSCRIPTION_PATTERNS, params);
|
|
661
710
|
transitions = {};
|
|
662
711
|
const hash = await this.redisClient[this.commands.hgetall](key);
|
|
@@ -743,7 +792,7 @@ class StoreService {
|
|
|
743
792
|
if (scrub) {
|
|
744
793
|
//indexes can be designed to be self-cleaning; `engine.hookAll` exposes this option
|
|
745
794
|
this.redisClient[this.commands.expire](processedKey, 0);
|
|
746
|
-
this.redisClient[this.commands.expire](key.split(
|
|
795
|
+
this.redisClient[this.commands.expire](key.split(':').slice(0, 5).join(':'), 0);
|
|
747
796
|
}
|
|
748
797
|
else {
|
|
749
798
|
await this.redisClient[this.commands.rename](processedKey, key);
|
|
@@ -756,7 +805,10 @@ class StoreService {
|
|
|
756
805
|
}
|
|
757
806
|
async expireJob(jobId, inSeconds) {
|
|
758
807
|
if (!isNaN(inSeconds) && inSeconds > 0) {
|
|
759
|
-
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
808
|
+
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
809
|
+
appId: this.appId,
|
|
810
|
+
jobId,
|
|
811
|
+
});
|
|
760
812
|
await this.redisClient[this.commands.expire](jobKey, inSeconds);
|
|
761
813
|
}
|
|
762
814
|
}
|
|
@@ -785,15 +837,12 @@ class StoreService {
|
|
|
785
837
|
* organized into 'n'-second blocks (LISTS))
|
|
786
838
|
*/
|
|
787
839
|
async registerTimeHook(jobId, gId, activityId, type, deletionTime, dad, multi) {
|
|
788
|
-
const listKey = this.mintKey(key_1.KeyType.TIME_RANGE, {
|
|
840
|
+
const listKey = this.mintKey(key_1.KeyType.TIME_RANGE, {
|
|
841
|
+
appId: this.appId,
|
|
842
|
+
timeValue: deletionTime,
|
|
843
|
+
});
|
|
789
844
|
//construct the composite key (the key has enough info to signal the hook)
|
|
790
|
-
const timeEvent = [
|
|
791
|
-
type,
|
|
792
|
-
activityId,
|
|
793
|
-
gId,
|
|
794
|
-
dad,
|
|
795
|
-
jobId
|
|
796
|
-
].join(key_1.VALSEP);
|
|
845
|
+
const timeEvent = [type, activityId, gId, dad, jobId].join(key_1.VALSEP);
|
|
797
846
|
const len = await (multi || this.redisClient)[this.commands.rpush](listKey, timeEvent);
|
|
798
847
|
if (multi || len === 1) {
|
|
799
848
|
const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
|
|
@@ -863,7 +912,10 @@ class StoreService {
|
|
|
863
912
|
}
|
|
864
913
|
//decrement job status (:) by 1bil
|
|
865
914
|
const amount = -1000000000;
|
|
866
|
-
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
915
|
+
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
916
|
+
appId: this.appId,
|
|
917
|
+
jobId,
|
|
918
|
+
});
|
|
867
919
|
const result = await this.redisClient[this.commands.hincrbyfloat](jobKey, ':', amount);
|
|
868
920
|
if (result <= amount) {
|
|
869
921
|
//verify active state; job already interrupted
|
|
@@ -881,7 +933,7 @@ class StoreService {
|
|
|
881
933
|
code: options.code ?? enums_1.HMSH_CODE_INTERRUPT,
|
|
882
934
|
message: options.reason ?? `job [${jobId}] interrupted`,
|
|
883
935
|
stack: options.stack ?? '',
|
|
884
|
-
job_id: jobId
|
|
936
|
+
job_id: jobId,
|
|
885
937
|
});
|
|
886
938
|
const payload = { [errKey]: amount.toString() };
|
|
887
939
|
const hashData = this.serializer.package(payload, symbolNames);
|
|
@@ -899,18 +951,24 @@ class StoreService {
|
|
|
899
951
|
}
|
|
900
952
|
}
|
|
901
953
|
async scrub(jobId) {
|
|
902
|
-
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
954
|
+
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
955
|
+
appId: this.appId,
|
|
956
|
+
jobId,
|
|
957
|
+
});
|
|
903
958
|
await this.redisClient[this.commands.del](jobKey);
|
|
904
959
|
}
|
|
905
960
|
async findJobs(queryString = '*', limit = 1000, batchSize = 1000, cursor = '0') {
|
|
906
|
-
const matchKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
961
|
+
const matchKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
962
|
+
appId: this.appId,
|
|
963
|
+
jobId: queryString,
|
|
964
|
+
});
|
|
907
965
|
let keys;
|
|
908
966
|
const matchingKeys = [];
|
|
909
967
|
do {
|
|
910
|
-
const output = await this.exec('SCAN', cursor, 'MATCH', matchKey, 'COUNT', batchSize.toString());
|
|
968
|
+
const output = (await this.exec('SCAN', cursor, 'MATCH', matchKey, 'COUNT', batchSize.toString()));
|
|
911
969
|
if (Array.isArray(output)) {
|
|
912
970
|
[cursor, keys] = output;
|
|
913
|
-
for (
|
|
971
|
+
for (const key of [...keys]) {
|
|
914
972
|
matchingKeys.push(key);
|
|
915
973
|
}
|
|
916
974
|
if (matchingKeys.length >= limit) {
|
|
@@ -926,10 +984,13 @@ class StoreService {
|
|
|
926
984
|
async findJobFields(jobId, fieldMatchPattern = '*', limit = 1000, batchSize = 1000, cursor = '0') {
|
|
927
985
|
let fields = [];
|
|
928
986
|
const matchingFields = {};
|
|
929
|
-
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
987
|
+
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
988
|
+
appId: this.appId,
|
|
989
|
+
jobId,
|
|
990
|
+
});
|
|
930
991
|
let len = 0;
|
|
931
992
|
do {
|
|
932
|
-
const output = await this.exec('HSCAN', jobKey, cursor, 'MATCH', fieldMatchPattern, 'COUNT', batchSize.toString());
|
|
993
|
+
const output = (await this.exec('HSCAN', jobKey, cursor, 'MATCH', fieldMatchPattern, 'COUNT', batchSize.toString()));
|
|
933
994
|
if (Array.isArray(output)) {
|
|
934
995
|
[cursor, fields] = output;
|
|
935
996
|
for (let i = 0; i < fields.length; i += 2) {
|
|
@@ -23,7 +23,7 @@ class IORedisStreamService extends index_1.StreamService {
|
|
|
23
23
|
async xgroup(command, key, groupName, id, mkStream) {
|
|
24
24
|
if (mkStream === 'MKSTREAM') {
|
|
25
25
|
try {
|
|
26
|
-
return (await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK';
|
|
26
|
+
return ((await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK');
|
|
27
27
|
}
|
|
28
28
|
catch (error) {
|
|
29
29
|
this.logger.info(`Consumer group not created with MKSTREAM for key: ${key} and group: ${groupName}`);
|
|
@@ -32,7 +32,7 @@ class IORedisStreamService extends index_1.StreamService {
|
|
|
32
32
|
}
|
|
33
33
|
else {
|
|
34
34
|
try {
|
|
35
|
-
return (await this.redisClient.xgroup(command, key, groupName, id)) === 'OK';
|
|
35
|
+
return ((await this.redisClient.xgroup(command, key, groupName, id)) === 'OK');
|
|
36
36
|
}
|
|
37
37
|
catch (error) {
|
|
38
38
|
this.logger.info(`Consumer group not created for key: ${key} and group: ${groupName}`);
|
|
@@ -73,7 +73,7 @@ class IORedisStreamService extends index_1.StreamService {
|
|
|
73
73
|
if (consumer)
|
|
74
74
|
args.push(consumer);
|
|
75
75
|
try {
|
|
76
|
-
return await this.redisClient.call('XPENDING', ...args);
|
|
76
|
+
return (await this.redisClient.call('XPENDING', ...args));
|
|
77
77
|
}
|
|
78
78
|
catch (error) {
|
|
79
79
|
this.logger.error('err, args', { ...error }, args);
|
|
@@ -86,7 +86,7 @@ class IORedisStreamService extends index_1.StreamService {
|
|
|
86
86
|
}
|
|
87
87
|
async xclaim(key, group, consumer, minIdleTime, id, ...args) {
|
|
88
88
|
try {
|
|
89
|
-
return await this.redisClient.xclaim(key, group, consumer, minIdleTime, id, ...args);
|
|
89
|
+
return (await this.redisClient.xclaim(key, group, consumer, minIdleTime, id, ...args));
|
|
90
90
|
}
|
|
91
91
|
catch (error) {
|
|
92
92
|
this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { ...error });
|
|
@@ -23,7 +23,14 @@ class RedisStreamService extends index_1.StreamService {
|
|
|
23
23
|
async xgroup(command, key, groupName, id, mkStream) {
|
|
24
24
|
const args = mkStream === 'MKSTREAM' ? ['MKSTREAM'] : [];
|
|
25
25
|
try {
|
|
26
|
-
return (await this.redisClient.sendCommand([
|
|
26
|
+
return ((await this.redisClient.sendCommand([
|
|
27
|
+
'XGROUP',
|
|
28
|
+
'CREATE',
|
|
29
|
+
key,
|
|
30
|
+
groupName,
|
|
31
|
+
id,
|
|
32
|
+
...args,
|
|
33
|
+
])) === 1);
|
|
27
34
|
}
|
|
28
35
|
catch (err) {
|
|
29
36
|
const streamType = mkStream === 'MKSTREAM' ? 'with MKSTREAM' : 'without MKSTREAM';
|
|
@@ -37,7 +44,9 @@ class RedisStreamService extends index_1.StreamService {
|
|
|
37
44
|
multi = args.pop();
|
|
38
45
|
}
|
|
39
46
|
try {
|
|
40
|
-
return await (multi || this.redisClient).XADD(key, id, {
|
|
47
|
+
return await (multi || this.redisClient).XADD(key, id, {
|
|
48
|
+
[args[0]]: args[1],
|
|
49
|
+
});
|
|
41
50
|
}
|
|
42
51
|
catch (err) {
|
|
43
52
|
this.logger.error(`Error publishing 'xadd'; key: ${key}`, err);
|
|
@@ -46,7 +55,17 @@ class RedisStreamService extends index_1.StreamService {
|
|
|
46
55
|
}
|
|
47
56
|
async xreadgroup(command, groupName, consumerName, blockOption, blockTime, streamsOption, streamName, id) {
|
|
48
57
|
try {
|
|
49
|
-
return await this.redisClient.sendCommand([
|
|
58
|
+
return await this.redisClient.sendCommand([
|
|
59
|
+
'XREADGROUP',
|
|
60
|
+
command,
|
|
61
|
+
groupName,
|
|
62
|
+
consumerName,
|
|
63
|
+
blockOption,
|
|
64
|
+
blockTime.toString(),
|
|
65
|
+
streamsOption,
|
|
66
|
+
streamName,
|
|
67
|
+
id,
|
|
68
|
+
]);
|
|
50
69
|
}
|
|
51
70
|
catch (err) {
|
|
52
71
|
this.logger.error(`Error in reading data from group: ${groupName} in stream: ${streamName}`, err);
|
|
@@ -78,7 +97,15 @@ class RedisStreamService extends index_1.StreamService {
|
|
|
78
97
|
}
|
|
79
98
|
async xclaim(key, group, consumer, minIdleTime, id, ...args) {
|
|
80
99
|
try {
|
|
81
|
-
return await this.redisClient.sendCommand([
|
|
100
|
+
return (await this.redisClient.sendCommand([
|
|
101
|
+
'XCLAIM',
|
|
102
|
+
key,
|
|
103
|
+
group,
|
|
104
|
+
consumer,
|
|
105
|
+
minIdleTime.toString(),
|
|
106
|
+
id,
|
|
107
|
+
...args,
|
|
108
|
+
]));
|
|
82
109
|
}
|
|
83
110
|
catch (err) {
|
|
84
111
|
this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, err);
|
|
@@ -37,14 +37,14 @@ class TaskService {
|
|
|
37
37
|
async registerJobForCleanup(jobId, inSeconds = enums_1.HMSH_EXPIRE_DURATION, options) {
|
|
38
38
|
if (inSeconds > 0) {
|
|
39
39
|
await this.store.expireJob(jobId, inSeconds);
|
|
40
|
-
const fromNow = Date.now() +
|
|
40
|
+
const fromNow = Date.now() + inSeconds * 1000;
|
|
41
41
|
const fidelityMS = enums_1.HMSH_FIDELITY_SECONDS * 1000;
|
|
42
42
|
const timeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
|
|
43
43
|
await this.store.registerDependenciesForCleanup(jobId, timeSlot, options);
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
46
|
async registerTimeHook(jobId, gId, activityId, type, inSeconds = enums_1.HMSH_FIDELITY_SECONDS, dad, multi) {
|
|
47
|
-
const fromNow = Date.now() +
|
|
47
|
+
const fromNow = Date.now() + inSeconds * 1000;
|
|
48
48
|
const fidelityMS = enums_1.HMSH_FIDELITY_SECONDS * 1000;
|
|
49
49
|
const awakenTimeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
|
|
50
50
|
await this.store.registerTimeHook(jobId, gId, activityId, type, awakenTimeSlot, dad, multi);
|
|
@@ -83,7 +83,9 @@ class TaskService {
|
|
|
83
83
|
}
|
|
84
84
|
else if (type === 'delist') {
|
|
85
85
|
//delist the signalKey (target)
|
|
86
|
-
const key = this.store.mintKey(hotmesh_1.KeyType.SIGNALS, {
|
|
86
|
+
const key = this.store.mintKey(hotmesh_1.KeyType.SIGNALS, {
|
|
87
|
+
appId: this.store.appId,
|
|
88
|
+
});
|
|
87
89
|
await this.store.redisClient[this.store.commands.hdel](key, target);
|
|
88
90
|
}
|
|
89
91
|
else {
|
|
@@ -102,7 +104,7 @@ class TaskService {
|
|
|
102
104
|
}
|
|
103
105
|
else {
|
|
104
106
|
//no worklists exist; sleep before checking
|
|
105
|
-
|
|
107
|
+
const sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_FIDELITY_SECONDS * 1000);
|
|
106
108
|
this.cleanupTimeout = sleep.timerId;
|
|
107
109
|
await sleep.promise;
|
|
108
110
|
this.errorCount = 0;
|
|
@@ -121,7 +123,7 @@ class TaskService {
|
|
|
121
123
|
}
|
|
122
124
|
else {
|
|
123
125
|
//didn't get the scout role; try again in 'one-ish' minutes
|
|
124
|
-
|
|
126
|
+
const sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_SCOUT_INTERVAL_SECONDS * 1000 * 2 * Math.random());
|
|
125
127
|
this.cleanupTimeout = sleep.timerId;
|
|
126
128
|
await sleep.promise;
|
|
127
129
|
this.processTimeHooks(timeEventCallback);
|
|
@@ -146,12 +148,7 @@ class TaskService {
|
|
|
146
148
|
const gId = context.metadata.gid;
|
|
147
149
|
const activityId = hookRule.to;
|
|
148
150
|
//composite keys are used to fully describe the task target
|
|
149
|
-
const compositeJobKey = [
|
|
150
|
-
activityId,
|
|
151
|
-
dad,
|
|
152
|
-
gId,
|
|
153
|
-
jobId
|
|
154
|
-
].join(key_1.WEBSEP);
|
|
151
|
+
const compositeJobKey = [activityId, dad, gId, jobId].join(key_1.WEBSEP);
|
|
155
152
|
const hook = {
|
|
156
153
|
topic,
|
|
157
154
|
resolved,
|
|
@@ -60,7 +60,8 @@ class TelemetryService {
|
|
|
60
60
|
else if (role === stream_1.StreamRole.WORKER) {
|
|
61
61
|
type = 'EXECUTE';
|
|
62
62
|
}
|
|
63
|
-
else if (data.type === stream_1.StreamDataType.RESULT ||
|
|
63
|
+
else if (data.type === stream_1.StreamDataType.RESULT ||
|
|
64
|
+
data.type === stream_1.StreamDataType.RESPONSE) {
|
|
64
65
|
type = 'FANIN';
|
|
65
66
|
}
|
|
66
67
|
else {
|
|
@@ -77,7 +78,7 @@ class TelemetryService {
|
|
|
77
78
|
this.traceId = traceId;
|
|
78
79
|
this.spanId = spanId;
|
|
79
80
|
const tracer = telemetry_1.trace.getTracer(package_json_1.default.name, package_json_1.default.version);
|
|
80
|
-
|
|
81
|
+
const parentContext = this.getParentSpanContext();
|
|
81
82
|
const span = tracer.startSpan(spanName, { kind: telemetry_1.SpanKind.CLIENT, attributes, root: !parentContext }, parentContext);
|
|
82
83
|
return span;
|
|
83
84
|
}
|
|
@@ -91,7 +92,7 @@ class TelemetryService {
|
|
|
91
92
|
result[`app.activity.data.${key}`] = telemetryAtts[key];
|
|
92
93
|
}
|
|
93
94
|
return result;
|
|
94
|
-
}, {})
|
|
95
|
+
}, {}),
|
|
95
96
|
};
|
|
96
97
|
this.span?.setAttributes(namespacedAtts);
|
|
97
98
|
}
|
|
@@ -144,7 +145,6 @@ class TelemetryService {
|
|
|
144
145
|
'app.activity.leg': leg,
|
|
145
146
|
};
|
|
146
147
|
}
|
|
147
|
-
;
|
|
148
148
|
getStreamSpanAttrs(input) {
|
|
149
149
|
return {
|
|
150
150
|
...Object.keys(input.metadata).reduce((result, key) => {
|
|
@@ -152,10 +152,9 @@ class TelemetryService {
|
|
|
152
152
|
result[`app.stream.${key}`] = input.metadata[key];
|
|
153
153
|
}
|
|
154
154
|
return result;
|
|
155
|
-
}, {})
|
|
155
|
+
}, {}),
|
|
156
156
|
};
|
|
157
157
|
}
|
|
158
|
-
;
|
|
159
158
|
setTelemetryContext(span, leg) {
|
|
160
159
|
if (!this.context.metadata.trc) {
|
|
161
160
|
this.context.metadata.trc = span.spanContext().traceId;
|
|
@@ -208,22 +207,30 @@ class TelemetryService {
|
|
|
208
207
|
static bindActivityTelemetryToState(state, config, metadata, context, leg) {
|
|
209
208
|
if (config.type === 'trigger') {
|
|
210
209
|
//trigger activities run non-duplexed and only have a single leg (2)
|
|
211
|
-
state[`${metadata.aid}/output/metadata/l1s`] =
|
|
212
|
-
|
|
210
|
+
state[`${metadata.aid}/output/metadata/l1s`] =
|
|
211
|
+
context['$self'].output.metadata.l1s;
|
|
212
|
+
state[`${metadata.aid}/output/metadata/l2s`] =
|
|
213
|
+
context['$self'].output.metadata.l2s;
|
|
213
214
|
}
|
|
214
|
-
else if (utils_1.polyfill.resolveActivityType(config.type) === 'hook' &&
|
|
215
|
+
else if (utils_1.polyfill.resolveActivityType(config.type) === 'hook' &&
|
|
216
|
+
leg === 1) {
|
|
215
217
|
//hook activities run non-duplexed and only have a single leg (1)
|
|
216
|
-
state[`${metadata.aid}/output/metadata/l1s`] =
|
|
217
|
-
|
|
218
|
+
state[`${metadata.aid}/output/metadata/l1s`] =
|
|
219
|
+
context['$self'].output.metadata.l1s;
|
|
220
|
+
state[`${metadata.aid}/output/metadata/l2s`] =
|
|
221
|
+
context['$self'].output.metadata.l1s;
|
|
218
222
|
}
|
|
219
223
|
else if (config.type === 'signal' && leg === 1) {
|
|
220
224
|
//signal activities run non-duplexed and only have a single leg (1)
|
|
221
|
-
state[`${metadata.aid}/output/metadata/l1s`] =
|
|
222
|
-
|
|
225
|
+
state[`${metadata.aid}/output/metadata/l1s`] =
|
|
226
|
+
context['$self'].output.metadata.l1s;
|
|
227
|
+
state[`${metadata.aid}/output/metadata/l2s`] =
|
|
228
|
+
context['$self'].output.metadata.l1s;
|
|
223
229
|
}
|
|
224
230
|
else {
|
|
225
231
|
const target = `l${leg}s`;
|
|
226
|
-
state[`${metadata.aid}/output/metadata/${target}`] =
|
|
232
|
+
state[`${metadata.aid}/output/metadata/${target}`] =
|
|
233
|
+
context['$self'].output.metadata[target];
|
|
227
234
|
}
|
|
228
235
|
}
|
|
229
236
|
}
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { ILogger } from
|
|
3
|
-
import { Router } from
|
|
2
|
+
import { ILogger } from '../logger';
|
|
3
|
+
import { Router } from '../router';
|
|
4
4
|
import { StoreService } from '../store';
|
|
5
5
|
import { StreamService } from '../stream';
|
|
6
6
|
import { SubService } from '../sub';
|
|
7
|
-
import { HotMeshConfig, HotMeshWorker } from
|
|
8
|
-
import { RollCallMessage, SubscriptionCallback } from
|
|
9
|
-
import { RedisClient, RedisMulti } from
|
|
10
|
-
import { StreamData, StreamDataResponse } from
|
|
7
|
+
import { HotMeshConfig, HotMeshWorker } from '../../types/hotmesh';
|
|
8
|
+
import { RollCallMessage, SubscriptionCallback } from '../../types/quorum';
|
|
9
|
+
import { RedisClient, RedisMulti } from '../../types/redis';
|
|
10
|
+
import { StreamData, StreamDataResponse } from '../../types/stream';
|
|
11
11
|
declare class WorkerService {
|
|
12
12
|
namespace: string;
|
|
13
13
|
appId: string;
|