@hotmeshio/hotmesh 0.0.55 → 0.0.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/modules/enums.js +1 -10
- package/build/modules/key.d.ts +0 -38
- package/build/modules/key.js +4 -46
- package/build/modules/utils.d.ts +0 -8
- package/build/modules/utils.js +0 -14
- package/build/package.json +11 -4
- package/build/services/activities/activity.d.ts +0 -28
- package/build/services/activities/activity.js +1 -46
- package/build/services/activities/await.js +0 -4
- package/build/services/activities/cycle.d.ts +0 -7
- package/build/services/activities/cycle.js +1 -16
- package/build/services/activities/hook.d.ts +0 -6
- package/build/services/activities/hook.js +2 -12
- package/build/services/activities/interrupt.js +0 -8
- package/build/services/activities/signal.d.ts +0 -6
- package/build/services/activities/signal.js +0 -15
- package/build/services/activities/trigger.d.ts +0 -4
- package/build/services/activities/trigger.js +1 -7
- package/build/services/activities/worker.js +0 -4
- package/build/services/collator/index.d.ts +0 -70
- package/build/services/collator/index.js +1 -91
- package/build/services/compiler/deployer.js +6 -38
- package/build/services/compiler/index.d.ts +0 -15
- package/build/services/compiler/index.js +0 -20
- package/build/services/compiler/validator.d.ts +0 -3
- package/build/services/compiler/validator.js +0 -25
- package/build/services/connector/clients/ioredis.d.ts +2 -2
- package/build/services/connector/clients/ioredis.js +0 -2
- package/build/services/connector/clients/redis.d.ts +4 -4
- package/build/services/connector/clients/redis.js +1 -3
- package/build/services/connector/index.d.ts +1 -1
- package/build/services/connector/index.js +0 -2
- package/build/services/durable/client.d.ts +1 -26
- package/build/services/durable/client.js +0 -56
- package/build/services/durable/exporter.d.ts +0 -22
- package/build/services/durable/exporter.js +1 -30
- package/build/services/durable/handle.d.ts +0 -36
- package/build/services/durable/handle.js +0 -46
- package/build/services/durable/index.d.ts +0 -4
- package/build/services/durable/index.js +0 -4
- package/build/services/durable/schemas/factory.d.ts +0 -29
- package/build/services/durable/schemas/factory.js +0 -29
- package/build/services/durable/search.d.ts +1 -36
- package/build/services/durable/search.js +56 -56
- package/build/services/durable/worker.js +2 -22
- package/build/services/durable/workflow.d.ts +0 -114
- package/build/services/durable/workflow.js +1 -141
- package/build/services/engine/index.d.ts +1 -6
- package/build/services/engine/index.js +1 -43
- package/build/services/exporter/index.d.ts +0 -27
- package/build/services/exporter/index.js +0 -33
- package/build/services/hotmesh/index.d.ts +2 -2
- package/build/services/hotmesh/index.js +1 -9
- package/build/services/logger/index.js +0 -2
- package/build/services/mapper/index.d.ts +0 -14
- package/build/services/mapper/index.js +0 -14
- package/build/services/pipe/functions/date.d.ts +0 -7
- package/build/services/pipe/functions/date.js +0 -7
- package/build/services/pipe/functions/math.js +0 -2
- package/build/services/pipe/index.d.ts +0 -15
- package/build/services/pipe/index.js +2 -23
- package/build/services/quorum/index.d.ts +0 -7
- package/build/services/quorum/index.js +0 -21
- package/build/services/reporter/index.d.ts +0 -5
- package/build/services/reporter/index.js +0 -9
- package/build/services/router/index.d.ts +0 -9
- package/build/services/router/index.js +2 -38
- package/build/services/serializer/index.js +7 -26
- package/build/services/store/cache.d.ts +0 -18
- package/build/services/store/cache.js +0 -18
- package/build/services/store/clients/ioredis.d.ts +1 -1
- package/build/services/store/clients/ioredis.js +0 -1
- package/build/services/store/clients/redis.d.ts +1 -1
- package/build/services/store/index.d.ts +0 -55
- package/build/services/store/index.js +5 -81
- package/build/services/stream/clients/ioredis.d.ts +1 -1
- package/build/services/stream/clients/ioredis.js +1 -4
- package/build/services/stream/clients/redis.d.ts +1 -1
- package/build/services/sub/clients/ioredis.d.ts +1 -1
- package/build/services/sub/clients/redis.d.ts +1 -1
- package/build/services/task/index.d.ts +0 -9
- package/build/services/task/index.js +0 -31
- package/build/services/telemetry/index.d.ts +0 -7
- package/build/services/telemetry/index.js +1 -13
- package/build/services/worker/index.d.ts +0 -4
- package/build/services/worker/index.js +2 -6
- package/build/types/activity.d.ts +0 -81
- package/build/types/durable.d.ts +26 -177
- package/build/types/exporter.d.ts +0 -13
- package/build/types/hotmesh.d.ts +4 -16
- package/build/types/hotmesh.js +0 -3
- package/build/types/index.d.ts +4 -6
- package/build/types/index.js +4 -3
- package/build/types/job.d.ts +1 -86
- package/build/types/pipe.d.ts +0 -65
- package/build/types/quorum.d.ts +15 -10
- package/build/types/redis.d.ts +225 -7
- package/build/types/redis.js +9 -0
- package/build/types/stream.d.ts +0 -58
- package/build/types/stream.js +0 -4
- package/package.json +11 -4
- package/types/durable.ts +131 -4
- package/types/hotmesh.ts +3 -6
- package/types/index.ts +23 -10
- package/types/job.ts +1 -1
- package/types/quorum.ts +22 -0
- package/types/redis.ts +267 -18
- package/build/types/ioredisclient.d.ts +0 -5
- package/build/types/ioredisclient.js +0 -5
- package/build/types/redisclient.d.ts +0 -26
- package/build/types/redisclient.js +0 -2
- package/modules/enums.ts +0 -62
- package/modules/errors.ts +0 -280
- package/modules/key.ts +0 -101
- package/modules/storage.ts +0 -3
- package/modules/utils.ts +0 -242
- package/services/activities/activity.ts +0 -589
- package/services/activities/await.ts +0 -113
- package/services/activities/cycle.ts +0 -115
- package/services/activities/hook.ts +0 -197
- package/services/activities/index.ts +0 -19
- package/services/activities/interrupt.ts +0 -172
- package/services/activities/signal.ts +0 -148
- package/services/activities/trigger.ts +0 -295
- package/services/activities/worker.ts +0 -107
- package/services/collator/README.md +0 -102
- package/services/collator/index.ts +0 -291
- package/services/compiler/deployer.ts +0 -504
- package/services/compiler/index.ts +0 -98
- package/services/compiler/validator.ts +0 -158
- package/services/connector/clients/ioredis.ts +0 -57
- package/services/connector/clients/redis.ts +0 -72
- package/services/connector/index.ts +0 -42
- package/services/durable/client.ts +0 -266
- package/services/durable/connection.ts +0 -10
- package/services/durable/exporter.ts +0 -232
- package/services/durable/handle.ts +0 -160
- package/services/durable/index.ts +0 -27
- package/services/durable/schemas/factory.ts +0 -2358
- package/services/durable/search.ts +0 -196
- package/services/durable/worker.ts +0 -401
- package/services/durable/workflow.ts +0 -557
- package/services/engine/index.ts +0 -761
- package/services/exporter/index.ts +0 -146
- package/services/hotmesh/index.ts +0 -237
- package/services/logger/index.ts +0 -79
- package/services/mapper/index.ts +0 -89
- package/services/pipe/functions/array.ts +0 -78
- package/services/pipe/functions/bitwise.ts +0 -27
- package/services/pipe/functions/conditional.ts +0 -35
- package/services/pipe/functions/date.ts +0 -220
- package/services/pipe/functions/index.ts +0 -27
- package/services/pipe/functions/json.ts +0 -11
- package/services/pipe/functions/logical.ts +0 -11
- package/services/pipe/functions/math.ts +0 -217
- package/services/pipe/functions/number.ts +0 -75
- package/services/pipe/functions/object.ts +0 -98
- package/services/pipe/functions/string.ts +0 -86
- package/services/pipe/functions/symbol.ts +0 -39
- package/services/pipe/functions/unary.ts +0 -19
- package/services/pipe/index.ts +0 -216
- package/services/quorum/index.ts +0 -319
- package/services/reporter/index.ts +0 -387
- package/services/router/index.ts +0 -426
- package/services/serializer/README.md +0 -10
- package/services/serializer/index.ts +0 -285
- package/services/store/cache.ts +0 -172
- package/services/store/clients/ioredis.ts +0 -145
- package/services/store/clients/redis.ts +0 -191
- package/services/store/index.ts +0 -1091
- package/services/stream/clients/ioredis.ts +0 -157
- package/services/stream/clients/redis.ts +0 -158
- package/services/stream/index.ts +0 -58
- package/services/sub/clients/ioredis.ts +0 -83
- package/services/sub/clients/redis.ts +0 -74
- package/services/sub/index.ts +0 -25
- package/services/task/index.ts +0 -250
- package/services/telemetry/index.ts +0 -273
- package/services/worker/index.ts +0 -248
- package/types/ioredisclient.ts +0 -10
- package/types/redisclient.ts +0 -30
|
@@ -77,7 +77,6 @@ class StoreService {
|
|
|
77
77
|
return result > 0 || result === 'OK' || result === true;
|
|
78
78
|
}
|
|
79
79
|
async zAdd(key, score, value, redisMulti) {
|
|
80
|
-
//default call signature uses 'ioredis' NPM Package format
|
|
81
80
|
return await (redisMulti || this.redisClient)[this.commands.zadd](key, score, value);
|
|
82
81
|
}
|
|
83
82
|
async zRangeByScoreWithScores(key, score, value) {
|
|
@@ -102,11 +101,6 @@ class StoreService {
|
|
|
102
101
|
invalidateCache() {
|
|
103
102
|
this.cache.invalidate();
|
|
104
103
|
}
|
|
105
|
-
/**
|
|
106
|
-
* At any given time only a single engine will
|
|
107
|
-
* check for and process work items in the
|
|
108
|
-
* time and signal task queues.
|
|
109
|
-
*/
|
|
110
104
|
async reserveScoutRole(scoutType, delay = enums_1.HMSH_SCOUT_INTERVAL_SECONDS) {
|
|
111
105
|
const key = this.mintKey(key_1.KeyType.WORK_ITEMS, { appId: this.appId, scoutType });
|
|
112
106
|
const success = await this.exec('SET', key, `${scoutType}:${(0, utils_1.formatISODate)(new Date())}`, 'NX', 'EX', `${delay - 1}`);
|
|
@@ -134,7 +128,6 @@ class StoreService {
|
|
|
134
128
|
throw new Error('settings not found');
|
|
135
129
|
}
|
|
136
130
|
async setSettings(manifest) {
|
|
137
|
-
//HotMesh heartbeat. If a connection is made, the version will be set
|
|
138
131
|
const params = {};
|
|
139
132
|
const key = this.mintKey(key_1.KeyType.HOTMESH, params);
|
|
140
133
|
return await this.redisClient[this.commands.hset](key, manifest);
|
|
@@ -142,10 +135,8 @@ class StoreService {
|
|
|
142
135
|
async reserveSymbolRange(target, size, type) {
|
|
143
136
|
const rangeKey = this.mintKey(key_1.KeyType.SYMKEYS, { appId: this.appId });
|
|
144
137
|
const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, { activityId: target, appId: this.appId });
|
|
145
|
-
//reserve the slot in a `pending` state (range will be established in the next step)
|
|
146
138
|
const response = await this.redisClient[this.commands.hsetnx](rangeKey, target, '?:?');
|
|
147
139
|
if (response) {
|
|
148
|
-
//if the key didn't exist, set the inclusive range and seed metadata fields
|
|
149
140
|
const upperLimit = await this.redisClient[this.commands.hincrby](rangeKey, ':cursor', size);
|
|
150
141
|
const lowerLimit = upperLimit - size;
|
|
151
142
|
const inclusiveRange = `${lowerLimit}:${upperLimit - 1}`;
|
|
@@ -155,7 +146,6 @@ class StoreService {
|
|
|
155
146
|
return [lowerLimit + serializer_1.MDATA_SYMBOLS.SLOTS, upperLimit - 1, {}];
|
|
156
147
|
}
|
|
157
148
|
else {
|
|
158
|
-
//if the key already existed, get the lower limit and add the number of symbols
|
|
159
149
|
const range = await this.redisClient[this.commands.hget](rangeKey, target);
|
|
160
150
|
const [lowerLimitString] = range.split(':');
|
|
161
151
|
const lowerLimit = parseInt(lowerLimitString, 10);
|
|
@@ -167,7 +157,6 @@ class StoreService {
|
|
|
167
157
|
}
|
|
168
158
|
}
|
|
169
159
|
async getAllSymbols() {
|
|
170
|
-
//get hash with all reserved symbol ranges
|
|
171
160
|
const rangeKey = this.mintKey(key_1.KeyType.SYMKEYS, { appId: this.appId });
|
|
172
161
|
const ranges = await this.redisClient[this.commands.hgetall](rangeKey);
|
|
173
162
|
const rangeKeys = Object.keys(ranges).sort();
|
|
@@ -340,11 +329,6 @@ class StoreService {
|
|
|
340
329
|
};
|
|
341
330
|
return await this.redisClient[this.commands.hset](key, payload);
|
|
342
331
|
}
|
|
343
|
-
/**
|
|
344
|
-
* Registers the job, `jobId`, with `originJobId`. In the future,
|
|
345
|
-
* when `originJobId` is interrupted/expired, the items in the
|
|
346
|
-
* list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
|
|
347
|
-
*/
|
|
348
332
|
async registerJobDependency(depType, originJobId, topic, jobId, gId, pd = '', multi) {
|
|
349
333
|
const privateMulti = multi || this.getMulti();
|
|
350
334
|
const dependencyParams = {
|
|
@@ -364,15 +348,10 @@ class StoreService {
|
|
|
364
348
|
return await privateMulti.exec();
|
|
365
349
|
}
|
|
366
350
|
}
|
|
367
|
-
/**
|
|
368
|
-
* Ensures a `hook signal` is delisted when its parent activity/job
|
|
369
|
-
* is interrupted/expired.
|
|
370
|
-
*/
|
|
371
351
|
async registerSignalDependency(jobId, signalKey, dad, multi) {
|
|
372
352
|
const privateMulti = multi || this.getMulti();
|
|
373
353
|
const dependencyParams = { appId: this.appId, jobId };
|
|
374
354
|
const dependencyKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
|
|
375
|
-
//persiste dependency tasks as multi-segment composite keys
|
|
376
355
|
const delistTask = [
|
|
377
356
|
'delist',
|
|
378
357
|
'signal',
|
|
@@ -409,7 +388,6 @@ class StoreService {
|
|
|
409
388
|
}
|
|
410
389
|
}
|
|
411
390
|
hGetAllResult(result) {
|
|
412
|
-
//default response signature uses 'redis' NPM Package format
|
|
413
391
|
return result;
|
|
414
392
|
}
|
|
415
393
|
async getJobStats(jobKeys) {
|
|
@@ -438,13 +416,12 @@ class StoreService {
|
|
|
438
416
|
async getJobIds(indexKeys, idRange) {
|
|
439
417
|
const multi = this.getMulti();
|
|
440
418
|
for (const idsKey of indexKeys) {
|
|
441
|
-
multi[this.commands.lrange](idsKey, idRange[0], idRange[1]);
|
|
419
|
+
multi[this.commands.lrange](idsKey, idRange[0], idRange[1]);
|
|
442
420
|
}
|
|
443
421
|
const results = await multi.exec();
|
|
444
422
|
const output = {};
|
|
445
423
|
for (const [index, result] of results.entries()) {
|
|
446
424
|
const key = indexKeys[index];
|
|
447
|
-
//todo: resolve this discrepancy between redis/ioredis
|
|
448
425
|
const idsList = result[1] || result;
|
|
449
426
|
if (idsList && idsList.length > 0) {
|
|
450
427
|
output[key] = idsList;
|
|
@@ -483,10 +460,6 @@ class StoreService {
|
|
|
483
460
|
await (multi || this.redisClient)[this.commands.hset](hashKey, hashData);
|
|
484
461
|
return jobId;
|
|
485
462
|
}
|
|
486
|
-
/**
|
|
487
|
-
* Returns custom search fields and values.
|
|
488
|
-
* NOTE: The `fields` param should NOT prefix items with an underscore.
|
|
489
|
-
*/
|
|
490
463
|
async getQueryState(jobId, fields) {
|
|
491
464
|
const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
|
|
492
465
|
const _fields = fields.map(field => `_${field}`);
|
|
@@ -498,7 +471,6 @@ class StoreService {
|
|
|
498
471
|
return jobData;
|
|
499
472
|
}
|
|
500
473
|
async getState(jobId, consumes, dIds) {
|
|
501
|
-
//get abbreviated field list (the symbols for the paths)
|
|
502
474
|
const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
|
|
503
475
|
const symbolNames = Object.keys(consumes);
|
|
504
476
|
const symKeys = await this.getSymbolKeys(symbolNames);
|
|
@@ -506,7 +478,7 @@ class StoreService {
|
|
|
506
478
|
const fields = this.serializer.abbreviate(consumes, symbolNames, [':']);
|
|
507
479
|
const jobDataArray = await this.redisClient[this.commands.hmget](key, fields);
|
|
508
480
|
const jobData = {};
|
|
509
|
-
let atLeast1 = false;
|
|
481
|
+
let atLeast1 = false;
|
|
510
482
|
fields.forEach((field, index) => {
|
|
511
483
|
if (jobDataArray[index]) {
|
|
512
484
|
atLeast1 = true;
|
|
@@ -537,13 +509,9 @@ class StoreService {
|
|
|
537
509
|
}
|
|
538
510
|
return job;
|
|
539
511
|
}
|
|
540
|
-
/**
|
|
541
|
-
* collate is a generic method for incrementing a value in a hash
|
|
542
|
-
* in order to track their progress during processing.
|
|
543
|
-
*/
|
|
544
512
|
async collate(jobId, activityId, amount, dIds, multi) {
|
|
545
513
|
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
|
|
546
|
-
const collationKey = `${activityId}/output/metadata/as`;
|
|
514
|
+
const collationKey = `${activityId}/output/metadata/as`;
|
|
547
515
|
const symbolNames = [activityId];
|
|
548
516
|
const symKeys = await this.getSymbolKeys(symbolNames);
|
|
549
517
|
const symVals = await this.getSymbolValues();
|
|
@@ -553,12 +521,6 @@ class StoreService {
|
|
|
553
521
|
const targetId = Object.keys(hashData)[0];
|
|
554
522
|
return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, targetId, amount);
|
|
555
523
|
}
|
|
556
|
-
/**
|
|
557
|
-
* synthentic collation affects those activities in the graph
|
|
558
|
-
* that represent the synthetic DAG that was materialized during compilation;
|
|
559
|
-
* Synthetic targeting ensures that re-entry due to failure can be distinguished from
|
|
560
|
-
* purposeful re-entry.
|
|
561
|
-
*/
|
|
562
524
|
async collateSynthetic(jobId, guid, amount, multi) {
|
|
563
525
|
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
|
|
564
526
|
return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, guid, amount);
|
|
@@ -697,12 +659,10 @@ class StoreService {
|
|
|
697
659
|
}
|
|
698
660
|
async setHookSignal(hook, multi) {
|
|
699
661
|
const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
|
|
700
|
-
//destructure the hook key
|
|
701
662
|
const { topic, resolved, jobId } = hook;
|
|
702
663
|
const signalKey = `${topic}:${resolved}`;
|
|
703
664
|
const payload = { [signalKey]: jobId };
|
|
704
665
|
await (multi || this.redisClient)[this.commands.hset](key, payload);
|
|
705
|
-
//jobId needs even more destructuring
|
|
706
666
|
const [_aid, dad, _gid, jid] = jobId.split(key_1.VALSEP);
|
|
707
667
|
return await this.registerSignalDependency(jid, signalKey, dad, multi);
|
|
708
668
|
}
|
|
@@ -741,7 +701,6 @@ class StoreService {
|
|
|
741
701
|
const didRemove = await this.redisClient[this.commands.zrem](zsetKey, workItemKey);
|
|
742
702
|
if (didRemove) {
|
|
743
703
|
if (scrub) {
|
|
744
|
-
//indexes can be designed to be self-cleaning; `engine.hookAll` exposes this option
|
|
745
704
|
this.redisClient[this.commands.expire](processedKey, 0);
|
|
746
705
|
this.redisClient[this.commands.expire](key.split(":").slice(0, 5).join(":"), 0);
|
|
747
706
|
}
|
|
@@ -760,11 +719,6 @@ class StoreService {
|
|
|
760
719
|
await this.redisClient[this.commands.expire](jobKey, inSeconds);
|
|
761
720
|
}
|
|
762
721
|
}
|
|
763
|
-
/**
|
|
764
|
-
* register the descendants of an expired origin flow to be
|
|
765
|
-
* expired at a future date; options indicate whether this
|
|
766
|
-
* is a standard `expire` or an `interrupt`
|
|
767
|
-
*/
|
|
768
722
|
async registerDependenciesForCleanup(jobId, deletionTime, options) {
|
|
769
723
|
const depParams = { appId: this.appId, jobId };
|
|
770
724
|
const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
|
|
@@ -778,15 +732,8 @@ class StoreService {
|
|
|
778
732
|
const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
|
|
779
733
|
return this.redisClient[this.commands.lrange](depKey, 0, -1);
|
|
780
734
|
}
|
|
781
|
-
/**
|
|
782
|
-
* registers a hook activity to be awakened (uses ZSET to
|
|
783
|
-
* store the 'sleep group' and LIST to store the events
|
|
784
|
-
* for the given sleep group. Sleep groups are
|
|
785
|
-
* organized into 'n'-second blocks (LISTS))
|
|
786
|
-
*/
|
|
787
735
|
async registerTimeHook(jobId, gId, activityId, type, deletionTime, dad, multi) {
|
|
788
736
|
const listKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId, timeValue: deletionTime });
|
|
789
|
-
//construct the composite key (the key has enough info to signal the hook)
|
|
790
737
|
const timeEvent = [
|
|
791
738
|
type,
|
|
792
739
|
activityId,
|
|
@@ -807,7 +754,6 @@ class StoreService {
|
|
|
807
754
|
let [pType, pKey] = this.resolveTaskKeyContext(listKey);
|
|
808
755
|
const timeEvent = await this.redisClient[this.commands.lpop](pKey);
|
|
809
756
|
if (timeEvent) {
|
|
810
|
-
//deconstruct composite key
|
|
811
757
|
let [type, activityId, gId, _pd, ...jobId] = timeEvent.split(key_1.VALSEP);
|
|
812
758
|
const jid = jobId.join(key_1.VALSEP);
|
|
813
759
|
if (type === 'delist') {
|
|
@@ -826,14 +772,6 @@ class StoreService {
|
|
|
826
772
|
}
|
|
827
773
|
return false;
|
|
828
774
|
}
|
|
829
|
-
/**
|
|
830
|
-
* when processing time jobs, the target LIST ID returned
|
|
831
|
-
* from the ZSET query can be prefixed to denote what to
|
|
832
|
-
* do with the work list. (not everything is known in advance,
|
|
833
|
-
* so the ZSET key defines HOW to approach the work in the
|
|
834
|
-
* generic LIST (lists typically contain target job ids)
|
|
835
|
-
* @param {string} listKey - composite key
|
|
836
|
-
*/
|
|
837
775
|
resolveTaskKeyContext(listKey) {
|
|
838
776
|
if (listKey.startsWith(`${key_1.TYPSEP}INTERRUPT`)) {
|
|
839
777
|
return ['interrupt', listKey.split(key_1.TYPSEP)[2]];
|
|
@@ -845,38 +783,24 @@ class StoreService {
|
|
|
845
783
|
return ['sleep', listKey];
|
|
846
784
|
}
|
|
847
785
|
}
|
|
848
|
-
/**
|
|
849
|
-
* Interrupts a job and sets sets a job error (410), if 'throw'!=false.
|
|
850
|
-
* This method is called by the engine and not by an activity and is
|
|
851
|
-
* followed by a call to execute job completion/cleanup tasks
|
|
852
|
-
* associated with a job completion event.
|
|
853
|
-
*
|
|
854
|
-
* Todo: move most of this logic to the engine (too much logic for the store)
|
|
855
|
-
*/
|
|
856
786
|
async interrupt(topic, jobId, options = {}) {
|
|
857
787
|
try {
|
|
858
|
-
//verify job exists
|
|
859
788
|
const status = await this.getStatus(jobId, this.appId);
|
|
860
789
|
if (status <= 0) {
|
|
861
|
-
//verify still active; job already completed
|
|
862
790
|
throw new Error(`Job ${jobId} already completed`);
|
|
863
791
|
}
|
|
864
|
-
//decrement job status (:) by 1bil
|
|
865
792
|
const amount = -1000000000;
|
|
866
793
|
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
|
|
867
794
|
const result = await this.redisClient[this.commands.hincrbyfloat](jobKey, ':', amount);
|
|
868
795
|
if (result <= amount) {
|
|
869
|
-
//verify active state; job already interrupted
|
|
870
796
|
throw new Error(`Job ${jobId} already completed`);
|
|
871
797
|
}
|
|
872
|
-
//persist the error unless specifically told not to
|
|
873
798
|
if (options.throw !== false) {
|
|
874
|
-
const errKey = `metadata/err`;
|
|
875
|
-
const symbolNames = [`$${topic}`];
|
|
799
|
+
const errKey = `metadata/err`;
|
|
800
|
+
const symbolNames = [`$${topic}`];
|
|
876
801
|
const symKeys = await this.getSymbolKeys(symbolNames);
|
|
877
802
|
const symVals = await this.getSymbolValues();
|
|
878
803
|
this.serializer.resetSymbols(symKeys, symVals, {});
|
|
879
|
-
//persists the standard 410 error (job is `gone`)
|
|
880
804
|
const err = JSON.stringify({
|
|
881
805
|
code: options.code ?? enums_1.HMSH_CODE_INTERRUPT,
|
|
882
806
|
message: options.reason ?? `job [${jobId}] interrupted`,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { KeyStoreParams, KeyType } from '../../../modules/key';
|
|
2
2
|
import { ILogger } from '../../logger';
|
|
3
3
|
import { StreamService } from '../index';
|
|
4
|
-
import { RedisClientType, RedisMultiType } from '../../../types/
|
|
4
|
+
import { IORedisClientType as RedisClientType, IORedisMultiType as RedisMultiType } from '../../../types/redis';
|
|
5
5
|
import { ReclaimedMessageType } from '../../../types/stream';
|
|
6
6
|
declare class IORedisStreamService extends StreamService<RedisClientType, RedisMultiType> {
|
|
7
7
|
redisClient: RedisClientType;
|
|
@@ -51,10 +51,7 @@ class IORedisStreamService extends index_1.StreamService {
|
|
|
51
51
|
}
|
|
52
52
|
async xreadgroup(command, groupName, consumerName, blockOption, blockTime, streamsOption, streamName, id) {
|
|
53
53
|
try {
|
|
54
|
-
|
|
55
|
-
return await this.redisClient.xreadgroup(command, groupName, consumerName,
|
|
56
|
-
// @ts-ignore
|
|
57
|
-
blockOption, blockTime, streamsOption, streamName, id);
|
|
54
|
+
return await this.redisClient.xreadgroup(command, groupName, consumerName, blockOption, blockTime, streamsOption, streamName, id);
|
|
58
55
|
}
|
|
59
56
|
catch (error) {
|
|
60
57
|
this.logger.error(`Error reading stream data [Stream ${streamName}] [Group ${groupName}]`, { ...error });
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { KeyStoreParams, KeyType } from '../../../modules/key';
|
|
2
2
|
import { ILogger } from '../../logger';
|
|
3
3
|
import { StreamService } from '../index';
|
|
4
|
-
import { RedisClientType, RedisMultiType } from '../../../types/
|
|
4
|
+
import { RedisRedisClientType as RedisClientType, RedisRedisMultiType as RedisMultiType } from '../../../types/redis';
|
|
5
5
|
import { ReclaimedMessageType } from '../../../types/stream';
|
|
6
6
|
declare class RedisStreamService extends StreamService<RedisClientType, RedisMultiType> {
|
|
7
7
|
redisClient: RedisClientType;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { KeyStoreParams, KeyType } from '../../../modules/key';
|
|
2
2
|
import { ILogger } from '../../logger';
|
|
3
3
|
import { SubService } from '../index';
|
|
4
|
-
import { RedisClientType, RedisMultiType } from '../../../types/
|
|
4
|
+
import { IORedisClientType as RedisClientType, IORedisMultiType as RedisMultiType } from '../../../types/redis';
|
|
5
5
|
import { SubscriptionCallback } from '../../../types/quorum';
|
|
6
6
|
declare class IORedisSubService extends SubService<RedisClientType, RedisMultiType> {
|
|
7
7
|
redisClient: RedisClientType;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { KeyStoreParams, KeyType } from '../../../modules/key';
|
|
2
2
|
import { ILogger } from '../../logger';
|
|
3
3
|
import { SubService } from '../index';
|
|
4
|
-
import { RedisClientType, RedisMultiType } from '../../../types/
|
|
4
|
+
import { RedisRedisClientType as RedisClientType, RedisRedisMultiType as RedisMultiType } from '../../../types/redis';
|
|
5
5
|
import { SubscriptionCallback } from '../../../types/quorum';
|
|
6
6
|
declare class RedisSubService extends SubService<RedisClientType, RedisMultiType> {
|
|
7
7
|
redisClient: RedisClientType;
|
|
@@ -16,16 +16,7 @@ declare class TaskService {
|
|
|
16
16
|
enqueueWorkItems(keys: string[]): Promise<void>;
|
|
17
17
|
registerJobForCleanup(jobId: string, inSeconds: number, options: JobCompletionOptions): Promise<void>;
|
|
18
18
|
registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, inSeconds: number, dad: string, multi?: RedisMulti): Promise<void>;
|
|
19
|
-
/**
|
|
20
|
-
* Should this engine instance play the role of 'scout' on behalf
|
|
21
|
-
* of the entire quorum? The scout role is responsible for processing
|
|
22
|
-
* task lists on behalf of the collective.
|
|
23
|
-
*/
|
|
24
19
|
shouldScout(): Promise<boolean>;
|
|
25
|
-
/**
|
|
26
|
-
* Callback handler that takes an item from a work list and
|
|
27
|
-
* processes according to its type
|
|
28
|
-
*/
|
|
29
20
|
processTimeHooks(timeEventCallback: (jobId: string, gId: string, activityId: string, type: WorkListTaskType) => Promise<void>, listKey?: string): Promise<void>;
|
|
30
21
|
cancelCleanup(): void;
|
|
31
22
|
getHookRule(topic: string): Promise<HookRule | undefined>;
|
|
@@ -22,7 +22,6 @@ class TaskService {
|
|
|
22
22
|
const destinationKey = `${sourceKey}:processed`;
|
|
23
23
|
const jobId = await this.store.processTaskQueue(sourceKey, destinationKey);
|
|
24
24
|
if (jobId) {
|
|
25
|
-
//todo: don't use 'id', make configurable using hook rule
|
|
26
25
|
await hookEventCallback(topic, { ...data, id: jobId });
|
|
27
26
|
}
|
|
28
27
|
else {
|
|
@@ -49,11 +48,6 @@ class TaskService {
|
|
|
49
48
|
const awakenTimeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
|
|
50
49
|
await this.store.registerTimeHook(jobId, gId, activityId, type, awakenTimeSlot, dad, multi);
|
|
51
50
|
}
|
|
52
|
-
/**
|
|
53
|
-
* Should this engine instance play the role of 'scout' on behalf
|
|
54
|
-
* of the entire quorum? The scout role is responsible for processing
|
|
55
|
-
* task lists on behalf of the collective.
|
|
56
|
-
*/
|
|
57
51
|
async shouldScout() {
|
|
58
52
|
const wasScout = this.isScout;
|
|
59
53
|
const isScout = wasScout || (this.isScout = await this.store.reserveScoutRole('time'));
|
|
@@ -67,10 +61,6 @@ class TaskService {
|
|
|
67
61
|
}
|
|
68
62
|
return false;
|
|
69
63
|
}
|
|
70
|
-
/**
|
|
71
|
-
* Callback handler that takes an item from a work list and
|
|
72
|
-
* processes according to its type
|
|
73
|
-
*/
|
|
74
64
|
async processTimeHooks(timeEventCallback, listKey) {
|
|
75
65
|
if (await this.shouldScout()) {
|
|
76
66
|
try {
|
|
@@ -78,16 +68,12 @@ class TaskService {
|
|
|
78
68
|
if (Array.isArray(workListTask)) {
|
|
79
69
|
const [listKey, target, gId, activityId, type] = workListTask;
|
|
80
70
|
if (type === 'child') {
|
|
81
|
-
//continue; this child is listed here for convenience, but
|
|
82
|
-
// will be expired by an origin ancestor and is listed there
|
|
83
71
|
}
|
|
84
72
|
else if (type === 'delist') {
|
|
85
|
-
//delist the signalKey (target)
|
|
86
73
|
const key = this.store.mintKey(hotmesh_1.KeyType.SIGNALS, { appId: this.store.appId });
|
|
87
74
|
await this.store.redisClient[this.store.commands.hdel](key, target);
|
|
88
75
|
}
|
|
89
76
|
else {
|
|
90
|
-
//awaken/expire/interrupt
|
|
91
77
|
await timeEventCallback(target, gId, activityId, type);
|
|
92
78
|
}
|
|
93
79
|
await (0, utils_1.sleepFor)(0);
|
|
@@ -95,13 +81,11 @@ class TaskService {
|
|
|
95
81
|
this.processTimeHooks(timeEventCallback, listKey);
|
|
96
82
|
}
|
|
97
83
|
else if (workListTask) {
|
|
98
|
-
//a worklist was just emptied; try again immediately
|
|
99
84
|
await (0, utils_1.sleepFor)(0);
|
|
100
85
|
this.errorCount = 0;
|
|
101
86
|
this.processTimeHooks(timeEventCallback);
|
|
102
87
|
}
|
|
103
88
|
else {
|
|
104
|
-
//no worklists exist; sleep before checking
|
|
105
89
|
let sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_FIDELITY_SECONDS * 1000);
|
|
106
90
|
this.cleanupTimeout = sleep.timerId;
|
|
107
91
|
await sleep.promise;
|
|
@@ -110,8 +94,6 @@ class TaskService {
|
|
|
110
94
|
}
|
|
111
95
|
}
|
|
112
96
|
catch (err) {
|
|
113
|
-
//most common reasons: deleted job not found; container stopping; test stopping
|
|
114
|
-
//less common: redis/cluster down; retry with fallback (5s max main reassignment)
|
|
115
97
|
this.logger.warn('task-process-timehooks-error', err);
|
|
116
98
|
await (0, utils_1.sleepFor)(1000 * this.errorCount++);
|
|
117
99
|
if (this.errorCount < 5) {
|
|
@@ -120,7 +102,6 @@ class TaskService {
|
|
|
120
102
|
}
|
|
121
103
|
}
|
|
122
104
|
else {
|
|
123
|
-
//didn't get the scout role; try again in 'one-ish' minutes
|
|
124
105
|
let sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_SCOUT_INTERVAL_SECONDS * 1000 * 2 * Math.random());
|
|
125
106
|
this.cleanupTimeout = sleep.timerId;
|
|
126
107
|
await sleep.promise;
|
|
@@ -145,7 +126,6 @@ class TaskService {
|
|
|
145
126
|
const jobId = context.metadata.jid;
|
|
146
127
|
const gId = context.metadata.gid;
|
|
147
128
|
const activityId = hookRule.to;
|
|
148
|
-
//composite keys are used to fully describe the task target
|
|
149
129
|
const compositeJobKey = [
|
|
150
130
|
activityId,
|
|
151
131
|
dad,
|
|
@@ -167,22 +147,13 @@ class TaskService {
|
|
|
167
147
|
async processWebHookSignal(topic, data) {
|
|
168
148
|
const hookRule = await this.getHookRule(topic);
|
|
169
149
|
if (hookRule) {
|
|
170
|
-
//NOTE: both formats are supported by the mapping engine:
|
|
171
|
-
// `$self.hook.data` OR `$hook.data`
|
|
172
150
|
const context = { $self: { hook: { data } }, $hook: { data } };
|
|
173
151
|
const mapExpression = hookRule.conditions.match[0].actual;
|
|
174
152
|
const resolved = pipe_1.Pipe.resolve(mapExpression, context);
|
|
175
153
|
const hookSignalId = await this.store.getHookSignal(topic, resolved);
|
|
176
154
|
if (!hookSignalId) {
|
|
177
|
-
//messages can be double-processed; not an issue; return `undefined`
|
|
178
|
-
//users can also provide a bogus topic; not an issue; return `undefined`
|
|
179
155
|
return undefined;
|
|
180
156
|
}
|
|
181
|
-
//`aid` is part of composite key, but the hook `topic` is its public interface;
|
|
182
|
-
// this means that a new version of the graph can be deployed and the
|
|
183
|
-
// topic can be re-mapped to a different activity id. Outside callers
|
|
184
|
-
// can adhere to the unchanged contract (calling the same topic),
|
|
185
|
-
// while the internal system can be updated in real-time as necessary.
|
|
186
157
|
const [_aid, dad, gid, ...jid] = hookSignalId.split(key_1.WEBSEP);
|
|
187
158
|
return [jid.join(key_1.WEBSEP), hookRule.to, dad, gid];
|
|
188
159
|
}
|
|
@@ -193,8 +164,6 @@ class TaskService {
|
|
|
193
164
|
async deleteWebHookSignal(topic, data) {
|
|
194
165
|
const hookRule = await this.getHookRule(topic);
|
|
195
166
|
if (hookRule) {
|
|
196
|
-
//NOTE: both formats are supported by the mapping engine:
|
|
197
|
-
// `$self.hook.data` OR `$hook.data`
|
|
198
167
|
const context = { $self: { hook: { data } }, $hook: { data } };
|
|
199
168
|
const mapExpression = hookRule.conditions.match[0].actual;
|
|
200
169
|
const resolved = pipe_1.Pipe.resolve(mapExpression, context);
|
|
@@ -35,13 +35,6 @@ declare class TelemetryService {
|
|
|
35
35
|
setTelemetryContext(span: Span, leg: number): void;
|
|
36
36
|
setActivityError(message: string): void;
|
|
37
37
|
setStreamError(message: string): void;
|
|
38
|
-
/**
|
|
39
|
-
* Adds the paths (HGET) necessary to restore telemetry state for an activity
|
|
40
|
-
* @param consumes
|
|
41
|
-
* @param config
|
|
42
|
-
* @param metadata
|
|
43
|
-
* @param leg
|
|
44
|
-
*/
|
|
45
38
|
static addTargetTelemetryPaths(consumes: Consumes, config: ActivityType, metadata: ActivityMetadata, leg: number): void;
|
|
46
39
|
static bindJobTelemetryToState(state: StringStringType, config: ActivityType, context: JobState): void;
|
|
47
40
|
static bindActivityTelemetryToState(state: StringAnyType, config: ActivityType, metadata: ActivityMetadata, context: JobState, leg: number): void;
|
|
@@ -13,7 +13,6 @@ class TelemetryService {
|
|
|
13
13
|
constructor(appId, config, metadata, context) {
|
|
14
14
|
this.leg = 1;
|
|
15
15
|
this.appId = appId;
|
|
16
|
-
//these are REQUIRED for job and activity spans
|
|
17
16
|
this.config = config;
|
|
18
17
|
this.metadata = metadata;
|
|
19
18
|
this.context = context;
|
|
@@ -82,7 +81,6 @@ class TelemetryService {
|
|
|
82
81
|
return span;
|
|
83
82
|
}
|
|
84
83
|
mapActivityAttributes() {
|
|
85
|
-
//export user-defined span attributes (app.activity.data.*)
|
|
86
84
|
if (this.config.telemetry) {
|
|
87
85
|
const telemetryAtts = new mapper_1.MapperService(this.config.telemetry, this.context).mapRules();
|
|
88
86
|
const namespacedAtts = {
|
|
@@ -123,7 +121,7 @@ class TelemetryService {
|
|
|
123
121
|
traceId: this.traceId,
|
|
124
122
|
spanId: this.spanId,
|
|
125
123
|
isRemote: true,
|
|
126
|
-
traceFlags: 1,
|
|
124
|
+
traceFlags: 1,
|
|
127
125
|
};
|
|
128
126
|
const parentContext = telemetry_1.trace.setSpanContext(telemetry_1.context.active(), restoredSpanContext);
|
|
129
127
|
return parentContext;
|
|
@@ -179,13 +177,6 @@ class TelemetryService {
|
|
|
179
177
|
setStreamError(message) {
|
|
180
178
|
this.span?.setStatus({ code: telemetry_1.SpanStatusCode.ERROR, message });
|
|
181
179
|
}
|
|
182
|
-
/**
|
|
183
|
-
* Adds the paths (HGET) necessary to restore telemetry state for an activity
|
|
184
|
-
* @param consumes
|
|
185
|
-
* @param config
|
|
186
|
-
* @param metadata
|
|
187
|
-
* @param leg
|
|
188
|
-
*/
|
|
189
180
|
static addTargetTelemetryPaths(consumes, config, metadata, leg) {
|
|
190
181
|
if (leg === 1) {
|
|
191
182
|
if (!(config.parent in consumes)) {
|
|
@@ -207,17 +198,14 @@ class TelemetryService {
|
|
|
207
198
|
}
|
|
208
199
|
static bindActivityTelemetryToState(state, config, metadata, context, leg) {
|
|
209
200
|
if (config.type === 'trigger') {
|
|
210
|
-
//trigger activities run non-duplexed and only have a single leg (2)
|
|
211
201
|
state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
|
|
212
202
|
state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l2s;
|
|
213
203
|
}
|
|
214
204
|
else if (utils_1.polyfill.resolveActivityType(config.type) === 'hook' && leg === 1) {
|
|
215
|
-
//hook activities run non-duplexed and only have a single leg (1)
|
|
216
205
|
state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
|
|
217
206
|
state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l1s;
|
|
218
207
|
}
|
|
219
208
|
else if (config.type === 'signal' && leg === 1) {
|
|
220
|
-
//signal activities run non-duplexed and only have a single leg (1)
|
|
221
209
|
state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
|
|
222
210
|
state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l1s;
|
|
223
211
|
}
|
|
@@ -30,10 +30,6 @@ declare class WorkerService {
|
|
|
30
30
|
initStreamChannel(service: WorkerService, stream: RedisClient): Promise<void>;
|
|
31
31
|
initRouter(worker: HotMeshWorker, logger: ILogger): Router;
|
|
32
32
|
subscriptionHandler(): SubscriptionCallback;
|
|
33
|
-
/**
|
|
34
|
-
* A quorum-wide command to broadcaset system details.
|
|
35
|
-
*
|
|
36
|
-
*/
|
|
37
33
|
doRollCall(message: RollCallMessage): Promise<void>;
|
|
38
34
|
cancelRollCall(): void;
|
|
39
35
|
stop(): void;
|
|
@@ -97,7 +97,7 @@ class WorkerService {
|
|
|
97
97
|
return async (topic, message) => {
|
|
98
98
|
self.logger.debug('worker-event-received', { topic, type: message.type });
|
|
99
99
|
if (message.type === 'throttle') {
|
|
100
|
-
if (message.topic !== null) {
|
|
100
|
+
if (message.topic !== null) {
|
|
101
101
|
self.throttle(message.throttle);
|
|
102
102
|
}
|
|
103
103
|
}
|
|
@@ -105,16 +105,12 @@ class WorkerService {
|
|
|
105
105
|
self.sayPong(self.appId, self.guid, message.originator, message.details);
|
|
106
106
|
}
|
|
107
107
|
else if (message.type === 'rollcall') {
|
|
108
|
-
if (message.topic !== null) {
|
|
108
|
+
if (message.topic !== null) {
|
|
109
109
|
self.doRollCall(message);
|
|
110
110
|
}
|
|
111
111
|
}
|
|
112
112
|
};
|
|
113
113
|
}
|
|
114
|
-
/**
|
|
115
|
-
* A quorum-wide command to broadcaset system details.
|
|
116
|
-
*
|
|
117
|
-
*/
|
|
118
114
|
async doRollCall(message) {
|
|
119
115
|
let iteration = 0;
|
|
120
116
|
let max = !isNaN(message.max) ? message.max : enums_1.HMSH_QUORUM_ROLLCALL_CYCLES;
|