@hotmeshio/hotmesh 0.0.57 → 0.0.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -10
- package/build/modules/enums.js +10 -1
- package/build/modules/key.d.ts +38 -0
- package/build/modules/key.js +46 -4
- package/build/modules/utils.d.ts +9 -0
- package/build/modules/utils.js +19 -1
- package/build/package.json +1 -1
- package/build/services/activities/activity.d.ts +28 -0
- package/build/services/activities/activity.js +46 -1
- package/build/services/activities/await.js +4 -0
- package/build/services/activities/cycle.d.ts +7 -0
- package/build/services/activities/cycle.js +16 -1
- package/build/services/activities/hook.d.ts +6 -0
- package/build/services/activities/hook.js +12 -2
- package/build/services/activities/interrupt.js +8 -0
- package/build/services/activities/signal.d.ts +6 -0
- package/build/services/activities/signal.js +15 -0
- package/build/services/activities/trigger.d.ts +4 -0
- package/build/services/activities/trigger.js +7 -1
- package/build/services/activities/worker.js +4 -0
- package/build/services/collator/index.d.ts +70 -0
- package/build/services/collator/index.js +91 -1
- package/build/services/compiler/deployer.js +38 -6
- package/build/services/compiler/index.d.ts +15 -0
- package/build/services/compiler/index.js +20 -0
- package/build/services/compiler/validator.d.ts +3 -0
- package/build/services/compiler/validator.js +25 -0
- package/build/services/connector/clients/ioredis.js +2 -0
- package/build/services/connector/clients/redis.js +2 -0
- package/build/services/connector/index.js +2 -0
- package/build/services/durable/client.d.ts +20 -0
- package/build/services/durable/client.js +25 -0
- package/build/services/durable/exporter.d.ts +22 -0
- package/build/services/durable/exporter.js +30 -1
- package/build/services/durable/handle.d.ts +36 -0
- package/build/services/durable/handle.js +46 -0
- package/build/services/durable/index.d.ts +4 -0
- package/build/services/durable/index.js +4 -0
- package/build/services/durable/schemas/factory.d.ts +29 -0
- package/build/services/durable/schemas/factory.js +29 -0
- package/build/services/durable/search.d.ts +97 -0
- package/build/services/durable/search.js +99 -0
- package/build/services/durable/worker.js +35 -6
- package/build/services/durable/workflow.d.ts +118 -0
- package/build/services/durable/workflow.js +153 -6
- package/build/services/engine/index.d.ts +5 -0
- package/build/services/engine/index.js +43 -1
- package/build/services/exporter/index.d.ts +27 -0
- package/build/services/exporter/index.js +33 -0
- package/build/services/hotmesh/index.js +8 -0
- package/build/services/logger/index.js +2 -0
- package/build/services/mapper/index.d.ts +14 -0
- package/build/services/mapper/index.js +14 -0
- package/build/services/pipe/functions/date.d.ts +7 -0
- package/build/services/pipe/functions/date.js +7 -0
- package/build/services/pipe/functions/math.js +2 -0
- package/build/services/pipe/index.d.ts +15 -0
- package/build/services/pipe/index.js +23 -2
- package/build/services/quorum/index.d.ts +7 -0
- package/build/services/quorum/index.js +21 -0
- package/build/services/reporter/index.d.ts +5 -0
- package/build/services/reporter/index.js +9 -0
- package/build/services/router/index.d.ts +9 -0
- package/build/services/router/index.js +30 -2
- package/build/services/serializer/index.js +23 -6
- package/build/services/store/cache.d.ts +19 -0
- package/build/services/store/cache.js +19 -0
- package/build/services/store/clients/ioredis.js +1 -0
- package/build/services/store/index.d.ts +55 -0
- package/build/services/store/index.js +81 -5
- package/build/services/stream/clients/ioredis.js +4 -1
- package/build/services/task/index.d.ts +9 -0
- package/build/services/task/index.js +31 -0
- package/build/services/telemetry/index.d.ts +7 -0
- package/build/services/telemetry/index.js +13 -1
- package/build/services/worker/index.d.ts +4 -0
- package/build/services/worker/index.js +6 -2
- package/build/types/activity.d.ts +81 -0
- package/build/types/durable.d.ts +255 -0
- package/build/types/exporter.d.ts +13 -0
- package/build/types/hotmesh.d.ts +10 -1
- package/build/types/hotmesh.js +3 -0
- package/build/types/index.js +1 -1
- package/build/types/job.d.ts +85 -0
- package/build/types/pipe.d.ts +65 -0
- package/build/types/quorum.d.ts +14 -0
- package/build/types/redis.d.ts +6 -0
- package/build/types/stream.d.ts +58 -0
- package/build/types/stream.js +4 -0
- package/package.json +1 -1
|
@@ -77,6 +77,7 @@ class StoreService {
|
|
|
77
77
|
return result > 0 || result === 'OK' || result === true;
|
|
78
78
|
}
|
|
79
79
|
async zAdd(key, score, value, redisMulti) {
|
|
80
|
+
//default call signature uses 'ioredis' NPM Package format
|
|
80
81
|
return await (redisMulti || this.redisClient)[this.commands.zadd](key, score, value);
|
|
81
82
|
}
|
|
82
83
|
async zRangeByScoreWithScores(key, score, value) {
|
|
@@ -101,6 +102,11 @@ class StoreService {
|
|
|
101
102
|
invalidateCache() {
|
|
102
103
|
this.cache.invalidate();
|
|
103
104
|
}
|
|
105
|
+
/**
|
|
106
|
+
* At any given time only a single engine will
|
|
107
|
+
* check for and process work items in the
|
|
108
|
+
* time and signal task queues.
|
|
109
|
+
*/
|
|
104
110
|
async reserveScoutRole(scoutType, delay = enums_1.HMSH_SCOUT_INTERVAL_SECONDS) {
|
|
105
111
|
const key = this.mintKey(key_1.KeyType.WORK_ITEMS, { appId: this.appId, scoutType });
|
|
106
112
|
const success = await this.exec('SET', key, `${scoutType}:${(0, utils_1.formatISODate)(new Date())}`, 'NX', 'EX', `${delay - 1}`);
|
|
@@ -128,6 +134,7 @@ class StoreService {
|
|
|
128
134
|
throw new Error('settings not found');
|
|
129
135
|
}
|
|
130
136
|
async setSettings(manifest) {
|
|
137
|
+
//HotMesh heartbeat. If a connection is made, the version will be set
|
|
131
138
|
const params = {};
|
|
132
139
|
const key = this.mintKey(key_1.KeyType.HOTMESH, params);
|
|
133
140
|
return await this.redisClient[this.commands.hset](key, manifest);
|
|
@@ -135,8 +142,10 @@ class StoreService {
|
|
|
135
142
|
async reserveSymbolRange(target, size, type) {
|
|
136
143
|
const rangeKey = this.mintKey(key_1.KeyType.SYMKEYS, { appId: this.appId });
|
|
137
144
|
const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, { activityId: target, appId: this.appId });
|
|
145
|
+
//reserve the slot in a `pending` state (range will be established in the next step)
|
|
138
146
|
const response = await this.redisClient[this.commands.hsetnx](rangeKey, target, '?:?');
|
|
139
147
|
if (response) {
|
|
148
|
+
//if the key didn't exist, set the inclusive range and seed metadata fields
|
|
140
149
|
const upperLimit = await this.redisClient[this.commands.hincrby](rangeKey, ':cursor', size);
|
|
141
150
|
const lowerLimit = upperLimit - size;
|
|
142
151
|
const inclusiveRange = `${lowerLimit}:${upperLimit - 1}`;
|
|
@@ -146,6 +155,7 @@ class StoreService {
|
|
|
146
155
|
return [lowerLimit + serializer_1.MDATA_SYMBOLS.SLOTS, upperLimit - 1, {}];
|
|
147
156
|
}
|
|
148
157
|
else {
|
|
158
|
+
//if the key already existed, get the lower limit and add the number of symbols
|
|
149
159
|
const range = await this.redisClient[this.commands.hget](rangeKey, target);
|
|
150
160
|
const [lowerLimitString] = range.split(':');
|
|
151
161
|
const lowerLimit = parseInt(lowerLimitString, 10);
|
|
@@ -157,6 +167,7 @@ class StoreService {
|
|
|
157
167
|
}
|
|
158
168
|
}
|
|
159
169
|
async getAllSymbols() {
|
|
170
|
+
//get hash with all reserved symbol ranges
|
|
160
171
|
const rangeKey = this.mintKey(key_1.KeyType.SYMKEYS, { appId: this.appId });
|
|
161
172
|
const ranges = await this.redisClient[this.commands.hgetall](rangeKey);
|
|
162
173
|
const rangeKeys = Object.keys(ranges).sort();
|
|
@@ -329,6 +340,11 @@ class StoreService {
|
|
|
329
340
|
};
|
|
330
341
|
return await this.redisClient[this.commands.hset](key, payload);
|
|
331
342
|
}
|
|
343
|
+
/**
|
|
344
|
+
* Registers the job, `jobId`, with `originJobId`. In the future,
|
|
345
|
+
* when `originJobId` is interrupted/expired, the items in the
|
|
346
|
+
* list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
|
|
347
|
+
*/
|
|
332
348
|
async registerJobDependency(depType, originJobId, topic, jobId, gId, pd = '', multi) {
|
|
333
349
|
const privateMulti = multi || this.getMulti();
|
|
334
350
|
const dependencyParams = {
|
|
@@ -348,10 +364,15 @@ class StoreService {
|
|
|
348
364
|
return await privateMulti.exec();
|
|
349
365
|
}
|
|
350
366
|
}
|
|
367
|
+
/**
|
|
368
|
+
* Ensures a `hook signal` is delisted when its parent activity/job
|
|
369
|
+
* is interrupted/expired.
|
|
370
|
+
*/
|
|
351
371
|
async registerSignalDependency(jobId, signalKey, dad, multi) {
|
|
352
372
|
const privateMulti = multi || this.getMulti();
|
|
353
373
|
const dependencyParams = { appId: this.appId, jobId };
|
|
354
374
|
const dependencyKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
|
|
375
|
+
//persiste dependency tasks as multi-segment composite keys
|
|
355
376
|
const delistTask = [
|
|
356
377
|
'delist',
|
|
357
378
|
'signal',
|
|
@@ -388,6 +409,7 @@ class StoreService {
|
|
|
388
409
|
}
|
|
389
410
|
}
|
|
390
411
|
hGetAllResult(result) {
|
|
412
|
+
//default response signature uses 'redis' NPM Package format
|
|
391
413
|
return result;
|
|
392
414
|
}
|
|
393
415
|
async getJobStats(jobKeys) {
|
|
@@ -416,12 +438,13 @@ class StoreService {
|
|
|
416
438
|
async getJobIds(indexKeys, idRange) {
|
|
417
439
|
const multi = this.getMulti();
|
|
418
440
|
for (const idsKey of indexKeys) {
|
|
419
|
-
multi[this.commands.lrange](idsKey, idRange[0], idRange[1]);
|
|
441
|
+
multi[this.commands.lrange](idsKey, idRange[0], idRange[1]); //0,-1 returns all ids
|
|
420
442
|
}
|
|
421
443
|
const results = await multi.exec();
|
|
422
444
|
const output = {};
|
|
423
445
|
for (const [index, result] of results.entries()) {
|
|
424
446
|
const key = indexKeys[index];
|
|
447
|
+
//todo: resolve this discrepancy between redis/ioredis
|
|
425
448
|
const idsList = result[1] || result;
|
|
426
449
|
if (idsList && idsList.length > 0) {
|
|
427
450
|
output[key] = idsList;
|
|
@@ -460,6 +483,10 @@ class StoreService {
|
|
|
460
483
|
await (multi || this.redisClient)[this.commands.hset](hashKey, hashData);
|
|
461
484
|
return jobId;
|
|
462
485
|
}
|
|
486
|
+
/**
|
|
487
|
+
* Returns custom search fields and values.
|
|
488
|
+
* NOTE: The `fields` param should NOT prefix items with an underscore.
|
|
489
|
+
*/
|
|
463
490
|
async getQueryState(jobId, fields) {
|
|
464
491
|
const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
|
|
465
492
|
const _fields = fields.map(field => `_${field}`);
|
|
@@ -471,6 +498,7 @@ class StoreService {
|
|
|
471
498
|
return jobData;
|
|
472
499
|
}
|
|
473
500
|
async getState(jobId, consumes, dIds) {
|
|
501
|
+
//get abbreviated field list (the symbols for the paths)
|
|
474
502
|
const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
|
|
475
503
|
const symbolNames = Object.keys(consumes);
|
|
476
504
|
const symKeys = await this.getSymbolKeys(symbolNames);
|
|
@@ -478,7 +506,7 @@ class StoreService {
|
|
|
478
506
|
const fields = this.serializer.abbreviate(consumes, symbolNames, [':']);
|
|
479
507
|
const jobDataArray = await this.redisClient[this.commands.hmget](key, fields);
|
|
480
508
|
const jobData = {};
|
|
481
|
-
let atLeast1 = false;
|
|
509
|
+
let atLeast1 = false; //if status field (':') isn't present assume 404
|
|
482
510
|
fields.forEach((field, index) => {
|
|
483
511
|
if (jobDataArray[index]) {
|
|
484
512
|
atLeast1 = true;
|
|
@@ -509,9 +537,13 @@ class StoreService {
|
|
|
509
537
|
}
|
|
510
538
|
return job;
|
|
511
539
|
}
|
|
540
|
+
/**
|
|
541
|
+
* collate is a generic method for incrementing a value in a hash
|
|
542
|
+
* in order to track their progress during processing.
|
|
543
|
+
*/
|
|
512
544
|
async collate(jobId, activityId, amount, dIds, multi) {
|
|
513
545
|
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
|
|
514
|
-
const collationKey = `${activityId}/output/metadata/as`;
|
|
546
|
+
const collationKey = `${activityId}/output/metadata/as`; //activity state
|
|
515
547
|
const symbolNames = [activityId];
|
|
516
548
|
const symKeys = await this.getSymbolKeys(symbolNames);
|
|
517
549
|
const symVals = await this.getSymbolValues();
|
|
@@ -521,6 +553,12 @@ class StoreService {
|
|
|
521
553
|
const targetId = Object.keys(hashData)[0];
|
|
522
554
|
return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, targetId, amount);
|
|
523
555
|
}
|
|
556
|
+
/**
|
|
557
|
+
* synthentic collation affects those activities in the graph
|
|
558
|
+
* that represent the synthetic DAG that was materialized during compilation;
|
|
559
|
+
* Synthetic targeting ensures that re-entry due to failure can be distinguished from
|
|
560
|
+
* purposeful re-entry.
|
|
561
|
+
*/
|
|
524
562
|
async collateSynthetic(jobId, guid, amount, multi) {
|
|
525
563
|
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
|
|
526
564
|
return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, guid, amount);
|
|
@@ -659,10 +697,12 @@ class StoreService {
|
|
|
659
697
|
}
|
|
660
698
|
async setHookSignal(hook, multi) {
|
|
661
699
|
const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
|
|
700
|
+
//destructure the hook key
|
|
662
701
|
const { topic, resolved, jobId } = hook;
|
|
663
702
|
const signalKey = `${topic}:${resolved}`;
|
|
664
703
|
const payload = { [signalKey]: jobId };
|
|
665
704
|
await (multi || this.redisClient)[this.commands.hset](key, payload);
|
|
705
|
+
//jobId needs even more destructuring
|
|
666
706
|
const [_aid, dad, _gid, jid] = jobId.split(key_1.VALSEP);
|
|
667
707
|
return await this.registerSignalDependency(jid, signalKey, dad, multi);
|
|
668
708
|
}
|
|
@@ -701,6 +741,7 @@ class StoreService {
|
|
|
701
741
|
const didRemove = await this.redisClient[this.commands.zrem](zsetKey, workItemKey);
|
|
702
742
|
if (didRemove) {
|
|
703
743
|
if (scrub) {
|
|
744
|
+
//indexes can be designed to be self-cleaning; `engine.hookAll` exposes this option
|
|
704
745
|
this.redisClient[this.commands.expire](processedKey, 0);
|
|
705
746
|
this.redisClient[this.commands.expire](key.split(":").slice(0, 5).join(":"), 0);
|
|
706
747
|
}
|
|
@@ -719,6 +760,11 @@ class StoreService {
|
|
|
719
760
|
await this.redisClient[this.commands.expire](jobKey, inSeconds);
|
|
720
761
|
}
|
|
721
762
|
}
|
|
763
|
+
/**
|
|
764
|
+
* register the descendants of an expired origin flow to be
|
|
765
|
+
* expired at a future date; options indicate whether this
|
|
766
|
+
* is a standard `expire` or an `interrupt`
|
|
767
|
+
*/
|
|
722
768
|
async registerDependenciesForCleanup(jobId, deletionTime, options) {
|
|
723
769
|
const depParams = { appId: this.appId, jobId };
|
|
724
770
|
const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
|
|
@@ -732,8 +778,15 @@ class StoreService {
|
|
|
732
778
|
const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
|
|
733
779
|
return this.redisClient[this.commands.lrange](depKey, 0, -1);
|
|
734
780
|
}
|
|
781
|
+
/**
|
|
782
|
+
* registers a hook activity to be awakened (uses ZSET to
|
|
783
|
+
* store the 'sleep group' and LIST to store the events
|
|
784
|
+
* for the given sleep group. Sleep groups are
|
|
785
|
+
* organized into 'n'-second blocks (LISTS))
|
|
786
|
+
*/
|
|
735
787
|
async registerTimeHook(jobId, gId, activityId, type, deletionTime, dad, multi) {
|
|
736
788
|
const listKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId, timeValue: deletionTime });
|
|
789
|
+
//construct the composite key (the key has enough info to signal the hook)
|
|
737
790
|
const timeEvent = [
|
|
738
791
|
type,
|
|
739
792
|
activityId,
|
|
@@ -754,6 +807,7 @@ class StoreService {
|
|
|
754
807
|
let [pType, pKey] = this.resolveTaskKeyContext(listKey);
|
|
755
808
|
const timeEvent = await this.redisClient[this.commands.lpop](pKey);
|
|
756
809
|
if (timeEvent) {
|
|
810
|
+
//deconstruct composite key
|
|
757
811
|
let [type, activityId, gId, _pd, ...jobId] = timeEvent.split(key_1.VALSEP);
|
|
758
812
|
const jid = jobId.join(key_1.VALSEP);
|
|
759
813
|
if (type === 'delist') {
|
|
@@ -772,6 +826,14 @@ class StoreService {
|
|
|
772
826
|
}
|
|
773
827
|
return false;
|
|
774
828
|
}
|
|
829
|
+
/**
|
|
830
|
+
* when processing time jobs, the target LIST ID returned
|
|
831
|
+
* from the ZSET query can be prefixed to denote what to
|
|
832
|
+
* do with the work list. (not everything is known in advance,
|
|
833
|
+
* so the ZSET key defines HOW to approach the work in the
|
|
834
|
+
* generic LIST (lists typically contain target job ids)
|
|
835
|
+
* @param {string} listKey - composite key
|
|
836
|
+
*/
|
|
775
837
|
resolveTaskKeyContext(listKey) {
|
|
776
838
|
if (listKey.startsWith(`${key_1.TYPSEP}INTERRUPT`)) {
|
|
777
839
|
return ['interrupt', listKey.split(key_1.TYPSEP)[2]];
|
|
@@ -783,24 +845,38 @@ class StoreService {
|
|
|
783
845
|
return ['sleep', listKey];
|
|
784
846
|
}
|
|
785
847
|
}
|
|
848
|
+
/**
|
|
849
|
+
* Interrupts a job and sets sets a job error (410), if 'throw'!=false.
|
|
850
|
+
* This method is called by the engine and not by an activity and is
|
|
851
|
+
* followed by a call to execute job completion/cleanup tasks
|
|
852
|
+
* associated with a job completion event.
|
|
853
|
+
*
|
|
854
|
+
* Todo: move most of this logic to the engine (too much logic for the store)
|
|
855
|
+
*/
|
|
786
856
|
async interrupt(topic, jobId, options = {}) {
|
|
787
857
|
try {
|
|
858
|
+
//verify job exists
|
|
788
859
|
const status = await this.getStatus(jobId, this.appId);
|
|
789
860
|
if (status <= 0) {
|
|
861
|
+
//verify still active; job already completed
|
|
790
862
|
throw new Error(`Job ${jobId} already completed`);
|
|
791
863
|
}
|
|
864
|
+
//decrement job status (:) by 1bil
|
|
792
865
|
const amount = -1000000000;
|
|
793
866
|
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
|
|
794
867
|
const result = await this.redisClient[this.commands.hincrbyfloat](jobKey, ':', amount);
|
|
795
868
|
if (result <= amount) {
|
|
869
|
+
//verify active state; job already interrupted
|
|
796
870
|
throw new Error(`Job ${jobId} already completed`);
|
|
797
871
|
}
|
|
872
|
+
//persist the error unless specifically told not to
|
|
798
873
|
if (options.throw !== false) {
|
|
799
|
-
const errKey = `metadata/err`;
|
|
800
|
-
const symbolNames = [`$${topic}`];
|
|
874
|
+
const errKey = `metadata/err`; //job errors are stored at the path `metadata/err`
|
|
875
|
+
const symbolNames = [`$${topic}`]; //the symbol for `metadata/err` is in redis and stored using the job topic
|
|
801
876
|
const symKeys = await this.getSymbolKeys(symbolNames);
|
|
802
877
|
const symVals = await this.getSymbolValues();
|
|
803
878
|
this.serializer.resetSymbols(symKeys, symVals, {});
|
|
879
|
+
//persists the standard 410 error (job is `gone`)
|
|
804
880
|
const err = JSON.stringify({
|
|
805
881
|
code: options.code ?? enums_1.HMSH_CODE_INTERRUPT,
|
|
806
882
|
message: options.reason ?? `job [${jobId}] interrupted`,
|
|
@@ -51,7 +51,10 @@ class IORedisStreamService extends index_1.StreamService {
|
|
|
51
51
|
}
|
|
52
52
|
async xreadgroup(command, groupName, consumerName, blockOption, blockTime, streamsOption, streamName, id) {
|
|
53
53
|
try {
|
|
54
|
-
|
|
54
|
+
//@ts-ignore
|
|
55
|
+
return await this.redisClient.xreadgroup(command, groupName, consumerName,
|
|
56
|
+
// @ts-ignore
|
|
57
|
+
blockOption, blockTime, streamsOption, streamName, id);
|
|
55
58
|
}
|
|
56
59
|
catch (error) {
|
|
57
60
|
this.logger.error(`Error reading stream data [Stream ${streamName}] [Group ${groupName}]`, { ...error });
|
|
@@ -16,7 +16,16 @@ declare class TaskService {
|
|
|
16
16
|
enqueueWorkItems(keys: string[]): Promise<void>;
|
|
17
17
|
registerJobForCleanup(jobId: string, inSeconds: number, options: JobCompletionOptions): Promise<void>;
|
|
18
18
|
registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, inSeconds: number, dad: string, multi?: RedisMulti): Promise<void>;
|
|
19
|
+
/**
|
|
20
|
+
* Should this engine instance play the role of 'scout' on behalf
|
|
21
|
+
* of the entire quorum? The scout role is responsible for processing
|
|
22
|
+
* task lists on behalf of the collective.
|
|
23
|
+
*/
|
|
19
24
|
shouldScout(): Promise<boolean>;
|
|
25
|
+
/**
|
|
26
|
+
* Callback handler that takes an item from a work list and
|
|
27
|
+
* processes according to its type
|
|
28
|
+
*/
|
|
20
29
|
processTimeHooks(timeEventCallback: (jobId: string, gId: string, activityId: string, type: WorkListTaskType) => Promise<void>, listKey?: string): Promise<void>;
|
|
21
30
|
cancelCleanup(): void;
|
|
22
31
|
getHookRule(topic: string): Promise<HookRule | undefined>;
|
|
@@ -22,6 +22,7 @@ class TaskService {
|
|
|
22
22
|
const destinationKey = `${sourceKey}:processed`;
|
|
23
23
|
const jobId = await this.store.processTaskQueue(sourceKey, destinationKey);
|
|
24
24
|
if (jobId) {
|
|
25
|
+
//todo: don't use 'id', make configurable using hook rule
|
|
25
26
|
await hookEventCallback(topic, { ...data, id: jobId });
|
|
26
27
|
}
|
|
27
28
|
else {
|
|
@@ -48,6 +49,11 @@ class TaskService {
|
|
|
48
49
|
const awakenTimeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
|
|
49
50
|
await this.store.registerTimeHook(jobId, gId, activityId, type, awakenTimeSlot, dad, multi);
|
|
50
51
|
}
|
|
52
|
+
/**
|
|
53
|
+
* Should this engine instance play the role of 'scout' on behalf
|
|
54
|
+
* of the entire quorum? The scout role is responsible for processing
|
|
55
|
+
* task lists on behalf of the collective.
|
|
56
|
+
*/
|
|
51
57
|
async shouldScout() {
|
|
52
58
|
const wasScout = this.isScout;
|
|
53
59
|
const isScout = wasScout || (this.isScout = await this.store.reserveScoutRole('time'));
|
|
@@ -61,6 +67,10 @@ class TaskService {
|
|
|
61
67
|
}
|
|
62
68
|
return false;
|
|
63
69
|
}
|
|
70
|
+
/**
|
|
71
|
+
* Callback handler that takes an item from a work list and
|
|
72
|
+
* processes according to its type
|
|
73
|
+
*/
|
|
64
74
|
async processTimeHooks(timeEventCallback, listKey) {
|
|
65
75
|
if (await this.shouldScout()) {
|
|
66
76
|
try {
|
|
@@ -68,12 +78,16 @@ class TaskService {
|
|
|
68
78
|
if (Array.isArray(workListTask)) {
|
|
69
79
|
const [listKey, target, gId, activityId, type] = workListTask;
|
|
70
80
|
if (type === 'child') {
|
|
81
|
+
//continue; this child is listed here for convenience, but
|
|
82
|
+
// will be expired by an origin ancestor and is listed there
|
|
71
83
|
}
|
|
72
84
|
else if (type === 'delist') {
|
|
85
|
+
//delist the signalKey (target)
|
|
73
86
|
const key = this.store.mintKey(hotmesh_1.KeyType.SIGNALS, { appId: this.store.appId });
|
|
74
87
|
await this.store.redisClient[this.store.commands.hdel](key, target);
|
|
75
88
|
}
|
|
76
89
|
else {
|
|
90
|
+
//awaken/expire/interrupt
|
|
77
91
|
await timeEventCallback(target, gId, activityId, type);
|
|
78
92
|
}
|
|
79
93
|
await (0, utils_1.sleepFor)(0);
|
|
@@ -81,11 +95,13 @@ class TaskService {
|
|
|
81
95
|
this.processTimeHooks(timeEventCallback, listKey);
|
|
82
96
|
}
|
|
83
97
|
else if (workListTask) {
|
|
98
|
+
//a worklist was just emptied; try again immediately
|
|
84
99
|
await (0, utils_1.sleepFor)(0);
|
|
85
100
|
this.errorCount = 0;
|
|
86
101
|
this.processTimeHooks(timeEventCallback);
|
|
87
102
|
}
|
|
88
103
|
else {
|
|
104
|
+
//no worklists exist; sleep before checking
|
|
89
105
|
let sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_FIDELITY_SECONDS * 1000);
|
|
90
106
|
this.cleanupTimeout = sleep.timerId;
|
|
91
107
|
await sleep.promise;
|
|
@@ -94,6 +110,8 @@ class TaskService {
|
|
|
94
110
|
}
|
|
95
111
|
}
|
|
96
112
|
catch (err) {
|
|
113
|
+
//most common reasons: deleted job not found; container stopping; test stopping
|
|
114
|
+
//less common: redis/cluster down; retry with fallback (5s max main reassignment)
|
|
97
115
|
this.logger.warn('task-process-timehooks-error', err);
|
|
98
116
|
await (0, utils_1.sleepFor)(1000 * this.errorCount++);
|
|
99
117
|
if (this.errorCount < 5) {
|
|
@@ -102,6 +120,7 @@ class TaskService {
|
|
|
102
120
|
}
|
|
103
121
|
}
|
|
104
122
|
else {
|
|
123
|
+
//didn't get the scout role; try again in 'one-ish' minutes
|
|
105
124
|
let sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_SCOUT_INTERVAL_SECONDS * 1000 * 2 * Math.random());
|
|
106
125
|
this.cleanupTimeout = sleep.timerId;
|
|
107
126
|
await sleep.promise;
|
|
@@ -126,6 +145,7 @@ class TaskService {
|
|
|
126
145
|
const jobId = context.metadata.jid;
|
|
127
146
|
const gId = context.metadata.gid;
|
|
128
147
|
const activityId = hookRule.to;
|
|
148
|
+
//composite keys are used to fully describe the task target
|
|
129
149
|
const compositeJobKey = [
|
|
130
150
|
activityId,
|
|
131
151
|
dad,
|
|
@@ -147,13 +167,22 @@ class TaskService {
|
|
|
147
167
|
async processWebHookSignal(topic, data) {
|
|
148
168
|
const hookRule = await this.getHookRule(topic);
|
|
149
169
|
if (hookRule) {
|
|
170
|
+
//NOTE: both formats are supported by the mapping engine:
|
|
171
|
+
// `$self.hook.data` OR `$hook.data`
|
|
150
172
|
const context = { $self: { hook: { data } }, $hook: { data } };
|
|
151
173
|
const mapExpression = hookRule.conditions.match[0].actual;
|
|
152
174
|
const resolved = pipe_1.Pipe.resolve(mapExpression, context);
|
|
153
175
|
const hookSignalId = await this.store.getHookSignal(topic, resolved);
|
|
154
176
|
if (!hookSignalId) {
|
|
177
|
+
//messages can be double-processed; not an issue; return `undefined`
|
|
178
|
+
//users can also provide a bogus topic; not an issue; return `undefined`
|
|
155
179
|
return undefined;
|
|
156
180
|
}
|
|
181
|
+
//`aid` is part of composite key, but the hook `topic` is its public interface;
|
|
182
|
+
// this means that a new version of the graph can be deployed and the
|
|
183
|
+
// topic can be re-mapped to a different activity id. Outside callers
|
|
184
|
+
// can adhere to the unchanged contract (calling the same topic),
|
|
185
|
+
// while the internal system can be updated in real-time as necessary.
|
|
157
186
|
const [_aid, dad, gid, ...jid] = hookSignalId.split(key_1.WEBSEP);
|
|
158
187
|
return [jid.join(key_1.WEBSEP), hookRule.to, dad, gid];
|
|
159
188
|
}
|
|
@@ -164,6 +193,8 @@ class TaskService {
|
|
|
164
193
|
async deleteWebHookSignal(topic, data) {
|
|
165
194
|
const hookRule = await this.getHookRule(topic);
|
|
166
195
|
if (hookRule) {
|
|
196
|
+
//NOTE: both formats are supported by the mapping engine:
|
|
197
|
+
// `$self.hook.data` OR `$hook.data`
|
|
167
198
|
const context = { $self: { hook: { data } }, $hook: { data } };
|
|
168
199
|
const mapExpression = hookRule.conditions.match[0].actual;
|
|
169
200
|
const resolved = pipe_1.Pipe.resolve(mapExpression, context);
|
|
@@ -35,6 +35,13 @@ declare class TelemetryService {
|
|
|
35
35
|
setTelemetryContext(span: Span, leg: number): void;
|
|
36
36
|
setActivityError(message: string): void;
|
|
37
37
|
setStreamError(message: string): void;
|
|
38
|
+
/**
|
|
39
|
+
* Adds the paths (HGET) necessary to restore telemetry state for an activity
|
|
40
|
+
* @param consumes
|
|
41
|
+
* @param config
|
|
42
|
+
* @param metadata
|
|
43
|
+
* @param leg
|
|
44
|
+
*/
|
|
38
45
|
static addTargetTelemetryPaths(consumes: Consumes, config: ActivityType, metadata: ActivityMetadata, leg: number): void;
|
|
39
46
|
static bindJobTelemetryToState(state: StringStringType, config: ActivityType, context: JobState): void;
|
|
40
47
|
static bindActivityTelemetryToState(state: StringAnyType, config: ActivityType, metadata: ActivityMetadata, context: JobState, leg: number): void;
|
|
@@ -13,6 +13,7 @@ class TelemetryService {
|
|
|
13
13
|
constructor(appId, config, metadata, context) {
|
|
14
14
|
this.leg = 1;
|
|
15
15
|
this.appId = appId;
|
|
16
|
+
//these are REQUIRED for job and activity spans
|
|
16
17
|
this.config = config;
|
|
17
18
|
this.metadata = metadata;
|
|
18
19
|
this.context = context;
|
|
@@ -81,6 +82,7 @@ class TelemetryService {
|
|
|
81
82
|
return span;
|
|
82
83
|
}
|
|
83
84
|
mapActivityAttributes() {
|
|
85
|
+
//export user-defined span attributes (app.activity.data.*)
|
|
84
86
|
if (this.config.telemetry) {
|
|
85
87
|
const telemetryAtts = new mapper_1.MapperService(this.config.telemetry, this.context).mapRules();
|
|
86
88
|
const namespacedAtts = {
|
|
@@ -121,7 +123,7 @@ class TelemetryService {
|
|
|
121
123
|
traceId: this.traceId,
|
|
122
124
|
spanId: this.spanId,
|
|
123
125
|
isRemote: true,
|
|
124
|
-
traceFlags: 1,
|
|
126
|
+
traceFlags: 1, // (todo: revisit sampling strategy/config)
|
|
125
127
|
};
|
|
126
128
|
const parentContext = telemetry_1.trace.setSpanContext(telemetry_1.context.active(), restoredSpanContext);
|
|
127
129
|
return parentContext;
|
|
@@ -177,6 +179,13 @@ class TelemetryService {
|
|
|
177
179
|
setStreamError(message) {
|
|
178
180
|
this.span?.setStatus({ code: telemetry_1.SpanStatusCode.ERROR, message });
|
|
179
181
|
}
|
|
182
|
+
/**
|
|
183
|
+
* Adds the paths (HGET) necessary to restore telemetry state for an activity
|
|
184
|
+
* @param consumes
|
|
185
|
+
* @param config
|
|
186
|
+
* @param metadata
|
|
187
|
+
* @param leg
|
|
188
|
+
*/
|
|
180
189
|
static addTargetTelemetryPaths(consumes, config, metadata, leg) {
|
|
181
190
|
if (leg === 1) {
|
|
182
191
|
if (!(config.parent in consumes)) {
|
|
@@ -198,14 +207,17 @@ class TelemetryService {
|
|
|
198
207
|
}
|
|
199
208
|
static bindActivityTelemetryToState(state, config, metadata, context, leg) {
|
|
200
209
|
if (config.type === 'trigger') {
|
|
210
|
+
//trigger activities run non-duplexed and only have a single leg (2)
|
|
201
211
|
state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
|
|
202
212
|
state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l2s;
|
|
203
213
|
}
|
|
204
214
|
else if (utils_1.polyfill.resolveActivityType(config.type) === 'hook' && leg === 1) {
|
|
215
|
+
//hook activities run non-duplexed and only have a single leg (1)
|
|
205
216
|
state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
|
|
206
217
|
state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l1s;
|
|
207
218
|
}
|
|
208
219
|
else if (config.type === 'signal' && leg === 1) {
|
|
220
|
+
//signal activities run non-duplexed and only have a single leg (1)
|
|
209
221
|
state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
|
|
210
222
|
state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l1s;
|
|
211
223
|
}
|
|
@@ -30,6 +30,10 @@ declare class WorkerService {
|
|
|
30
30
|
initStreamChannel(service: WorkerService, stream: RedisClient): Promise<void>;
|
|
31
31
|
initRouter(worker: HotMeshWorker, logger: ILogger): Router;
|
|
32
32
|
subscriptionHandler(): SubscriptionCallback;
|
|
33
|
+
/**
|
|
34
|
+
* A quorum-wide command to broadcaset system details.
|
|
35
|
+
*
|
|
36
|
+
*/
|
|
33
37
|
doRollCall(message: RollCallMessage): Promise<void>;
|
|
34
38
|
cancelRollCall(): void;
|
|
35
39
|
stop(): void;
|
|
@@ -97,7 +97,7 @@ class WorkerService {
|
|
|
97
97
|
return async (topic, message) => {
|
|
98
98
|
self.logger.debug('worker-event-received', { topic, type: message.type });
|
|
99
99
|
if (message.type === 'throttle') {
|
|
100
|
-
if (message.topic !== null) {
|
|
100
|
+
if (message.topic !== null) { //undefined allows passthrough
|
|
101
101
|
self.throttle(message.throttle);
|
|
102
102
|
}
|
|
103
103
|
}
|
|
@@ -105,12 +105,16 @@ class WorkerService {
|
|
|
105
105
|
self.sayPong(self.appId, self.guid, message.originator, message.details);
|
|
106
106
|
}
|
|
107
107
|
else if (message.type === 'rollcall') {
|
|
108
|
-
if (message.topic !== null) {
|
|
108
|
+
if (message.topic !== null) { //undefined allows passthrough
|
|
109
109
|
self.doRollCall(message);
|
|
110
110
|
}
|
|
111
111
|
}
|
|
112
112
|
};
|
|
113
113
|
}
|
|
114
|
+
/**
|
|
115
|
+
* A quorum-wide command to broadcaset system details.
|
|
116
|
+
*
|
|
117
|
+
*/
|
|
114
118
|
async doRollCall(message) {
|
|
115
119
|
let iteration = 0;
|
|
116
120
|
let max = !isNaN(message.max) ? message.max : enums_1.HMSH_QUORUM_ROLLCALL_CYCLES;
|
|
@@ -32,7 +32,18 @@ interface Measure {
|
|
|
32
32
|
target: string;
|
|
33
33
|
}
|
|
34
34
|
interface TriggerActivityStats {
|
|
35
|
+
/**
|
|
36
|
+
* dependent parent job id; including this allows the parent's
|
|
37
|
+
* expiration/interruption events to cascade; set
|
|
38
|
+
* `expire` in the YAML for the dependent graph
|
|
39
|
+
* to 0 and provide the parent for dependent,
|
|
40
|
+
* cascading interruption and cleanup
|
|
41
|
+
*/
|
|
35
42
|
parent?: string;
|
|
43
|
+
/**
|
|
44
|
+
* adjacent parent job id; this is the actual adjacent
|
|
45
|
+
* parent in the graph, but it is not used for cascading expiration
|
|
46
|
+
*/
|
|
36
47
|
adjacent?: string;
|
|
37
48
|
id?: {
|
|
38
49
|
[key: string]: unknown;
|
|
@@ -40,7 +51,19 @@ interface TriggerActivityStats {
|
|
|
40
51
|
key?: {
|
|
41
52
|
[key: string]: unknown;
|
|
42
53
|
} | string;
|
|
54
|
+
/**
|
|
55
|
+
* @deprecated
|
|
56
|
+
* return 'infinity' to disable; default behavior
|
|
57
|
+
* is to always segment keys by time to ensure
|
|
58
|
+
* indexes (Redis LIST) never grow unbounded
|
|
59
|
+
* as a default behavior; for now, 5m is default
|
|
60
|
+
* and infinity can be set to override
|
|
61
|
+
*/
|
|
43
62
|
granularity?: string;
|
|
63
|
+
/**
|
|
64
|
+
* @deprecated
|
|
65
|
+
* what to capture
|
|
66
|
+
*/
|
|
44
67
|
measures?: Measure[];
|
|
45
68
|
}
|
|
46
69
|
interface TriggerActivity extends BaseActivity {
|
|
@@ -79,26 +102,84 @@ interface SignalActivity extends BaseActivity {
|
|
|
79
102
|
}
|
|
80
103
|
interface InterruptActivity extends BaseActivity {
|
|
81
104
|
type: 'interrupt';
|
|
105
|
+
/**
|
|
106
|
+
* Optional Reason; will be used as the error `message` when thrown
|
|
107
|
+
* @default 'Job Interrupted'
|
|
108
|
+
*/
|
|
82
109
|
reason?: string;
|
|
110
|
+
/**
|
|
111
|
+
* throw JobInterrupted error upon interrupting
|
|
112
|
+
* @default true
|
|
113
|
+
*/
|
|
83
114
|
throw?: boolean;
|
|
115
|
+
/**
|
|
116
|
+
* Interrupt child/descendant jobs
|
|
117
|
+
* @default false
|
|
118
|
+
*/
|
|
84
119
|
descend?: boolean;
|
|
120
|
+
/**
|
|
121
|
+
* Target job id (if not present the current job will be targeted)
|
|
122
|
+
*/
|
|
85
123
|
target?: string;
|
|
124
|
+
/**
|
|
125
|
+
* Optional topic to publish the interrupt message (if not present the current job topic will be used
|
|
126
|
+
*/
|
|
86
127
|
topic?: string;
|
|
128
|
+
/**
|
|
129
|
+
* Optional Error Code; will be used as the error `code` when thrown
|
|
130
|
+
* @default 410
|
|
131
|
+
*/
|
|
87
132
|
code?: number;
|
|
133
|
+
/**
|
|
134
|
+
* Optional stack trace
|
|
135
|
+
*/
|
|
88
136
|
stack?: string;
|
|
89
137
|
}
|
|
90
138
|
type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | InterruptActivity | HookActivity | SignalActivity | CycleActivity;
|
|
91
139
|
type ActivityData = Record<string, any>;
|
|
140
|
+
/**
|
|
141
|
+
* Type definition for activity metadata.
|
|
142
|
+
*/
|
|
92
143
|
type ActivityMetadata = {
|
|
144
|
+
/**
|
|
145
|
+
* Unique identifier for the activity.
|
|
146
|
+
*/
|
|
93
147
|
aid: string;
|
|
148
|
+
/**
|
|
149
|
+
* Type of the activity.
|
|
150
|
+
*/
|
|
94
151
|
atp: string;
|
|
152
|
+
/**
|
|
153
|
+
* Subtype of the activity.
|
|
154
|
+
*/
|
|
95
155
|
stp: string;
|
|
156
|
+
/**
|
|
157
|
+
* Timestamp when the activity was created, in GMT.
|
|
158
|
+
*/
|
|
96
159
|
ac: string;
|
|
160
|
+
/**
|
|
161
|
+
* Timestamp when the activity was last updated, in GMT.
|
|
162
|
+
*/
|
|
97
163
|
au: string;
|
|
164
|
+
/**
|
|
165
|
+
* Optional stringified JSON containing error details such as message, code, and an optional error object.
|
|
166
|
+
*/
|
|
98
167
|
err?: string;
|
|
168
|
+
/**
|
|
169
|
+
* OpenTelemetry span context for the first leg of the activity.
|
|
170
|
+
*/
|
|
99
171
|
l1s?: string;
|
|
172
|
+
/**
|
|
173
|
+
* OpenTelemetry span context for the second leg of the activity.
|
|
174
|
+
*/
|
|
100
175
|
l2s?: string;
|
|
176
|
+
/**
|
|
177
|
+
* Dimensional address, used for additional metadata.
|
|
178
|
+
*/
|
|
101
179
|
dad?: string;
|
|
180
|
+
/**
|
|
181
|
+
* Status of the activity, could be codes representing different states.
|
|
182
|
+
*/
|
|
102
183
|
as?: string;
|
|
103
184
|
};
|
|
104
185
|
type ActivityContext = {
|