@hotmeshio/hotmesh 0.0.57 → 0.0.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +1 -1
  2. package/build/modules/enums.js +10 -1
  3. package/build/modules/key.d.ts +38 -0
  4. package/build/modules/key.js +46 -4
  5. package/build/modules/utils.d.ts +9 -0
  6. package/build/modules/utils.js +19 -1
  7. package/build/package.json +2 -1
  8. package/build/services/activities/activity.d.ts +28 -0
  9. package/build/services/activities/activity.js +46 -1
  10. package/build/services/activities/await.js +4 -0
  11. package/build/services/activities/cycle.d.ts +7 -0
  12. package/build/services/activities/cycle.js +16 -1
  13. package/build/services/activities/hook.d.ts +6 -0
  14. package/build/services/activities/hook.js +12 -2
  15. package/build/services/activities/interrupt.js +8 -0
  16. package/build/services/activities/signal.d.ts +6 -0
  17. package/build/services/activities/signal.js +15 -0
  18. package/build/services/activities/trigger.d.ts +4 -0
  19. package/build/services/activities/trigger.js +7 -1
  20. package/build/services/activities/worker.js +4 -0
  21. package/build/services/collator/index.d.ts +70 -0
  22. package/build/services/collator/index.js +91 -1
  23. package/build/services/compiler/deployer.js +38 -6
  24. package/build/services/compiler/index.d.ts +15 -0
  25. package/build/services/compiler/index.js +20 -0
  26. package/build/services/compiler/validator.d.ts +3 -0
  27. package/build/services/compiler/validator.js +25 -0
  28. package/build/services/connector/clients/ioredis.js +2 -0
  29. package/build/services/connector/clients/redis.js +2 -0
  30. package/build/services/connector/index.js +2 -0
  31. package/build/services/durable/client.d.ts +20 -0
  32. package/build/services/durable/client.js +25 -0
  33. package/build/services/durable/exporter.d.ts +22 -0
  34. package/build/services/durable/exporter.js +30 -1
  35. package/build/services/durable/handle.d.ts +36 -0
  36. package/build/services/durable/handle.js +46 -0
  37. package/build/services/durable/index.d.ts +4 -0
  38. package/build/services/durable/index.js +4 -0
  39. package/build/services/durable/schemas/factory.d.ts +29 -0
  40. package/build/services/durable/schemas/factory.js +29 -0
  41. package/build/services/durable/search.d.ts +97 -0
  42. package/build/services/durable/search.js +99 -0
  43. package/build/services/durable/worker.js +35 -6
  44. package/build/services/durable/workflow.d.ts +118 -0
  45. package/build/services/durable/workflow.js +153 -6
  46. package/build/services/engine/index.d.ts +5 -0
  47. package/build/services/engine/index.js +43 -1
  48. package/build/services/exporter/index.d.ts +27 -0
  49. package/build/services/exporter/index.js +33 -0
  50. package/build/services/hotmesh/index.js +8 -0
  51. package/build/services/logger/index.js +2 -0
  52. package/build/services/mapper/index.d.ts +14 -0
  53. package/build/services/mapper/index.js +14 -0
  54. package/build/services/pipe/functions/date.d.ts +7 -0
  55. package/build/services/pipe/functions/date.js +7 -0
  56. package/build/services/pipe/functions/math.js +2 -0
  57. package/build/services/pipe/index.d.ts +16 -0
  58. package/build/services/pipe/index.js +45 -3
  59. package/build/services/quorum/index.d.ts +7 -0
  60. package/build/services/quorum/index.js +21 -0
  61. package/build/services/reporter/index.d.ts +5 -0
  62. package/build/services/reporter/index.js +9 -0
  63. package/build/services/router/index.d.ts +9 -0
  64. package/build/services/router/index.js +30 -2
  65. package/build/services/serializer/index.js +23 -6
  66. package/build/services/store/cache.d.ts +19 -0
  67. package/build/services/store/cache.js +19 -0
  68. package/build/services/store/clients/ioredis.js +1 -0
  69. package/build/services/store/index.d.ts +55 -0
  70. package/build/services/store/index.js +81 -5
  71. package/build/services/stream/clients/ioredis.js +4 -1
  72. package/build/services/task/index.d.ts +9 -0
  73. package/build/services/task/index.js +31 -0
  74. package/build/services/telemetry/index.d.ts +7 -0
  75. package/build/services/telemetry/index.js +13 -1
  76. package/build/services/worker/index.d.ts +4 -0
  77. package/build/services/worker/index.js +6 -2
  78. package/build/types/activity.d.ts +81 -0
  79. package/build/types/durable.d.ts +255 -0
  80. package/build/types/exporter.d.ts +13 -0
  81. package/build/types/hotmesh.d.ts +10 -1
  82. package/build/types/hotmesh.js +3 -0
  83. package/build/types/index.js +1 -1
  84. package/build/types/job.d.ts +85 -0
  85. package/build/types/pipe.d.ts +65 -0
  86. package/build/types/quorum.d.ts +14 -0
  87. package/build/types/redis.d.ts +6 -0
  88. package/build/types/stream.d.ts +58 -0
  89. package/build/types/stream.js +4 -0
  90. package/package.json +2 -1
@@ -77,6 +77,7 @@ class StoreService {
77
77
  return result > 0 || result === 'OK' || result === true;
78
78
  }
79
79
  async zAdd(key, score, value, redisMulti) {
80
+ //default call signature uses 'ioredis' NPM Package format
80
81
  return await (redisMulti || this.redisClient)[this.commands.zadd](key, score, value);
81
82
  }
82
83
  async zRangeByScoreWithScores(key, score, value) {
@@ -101,6 +102,11 @@ class StoreService {
101
102
  invalidateCache() {
102
103
  this.cache.invalidate();
103
104
  }
105
+ /**
106
+ * At any given time only a single engine will
107
+ * check for and process work items in the
108
+ * time and signal task queues.
109
+ */
104
110
  async reserveScoutRole(scoutType, delay = enums_1.HMSH_SCOUT_INTERVAL_SECONDS) {
105
111
  const key = this.mintKey(key_1.KeyType.WORK_ITEMS, { appId: this.appId, scoutType });
106
112
  const success = await this.exec('SET', key, `${scoutType}:${(0, utils_1.formatISODate)(new Date())}`, 'NX', 'EX', `${delay - 1}`);
@@ -128,6 +134,7 @@ class StoreService {
128
134
  throw new Error('settings not found');
129
135
  }
130
136
  async setSettings(manifest) {
137
+ //HotMesh heartbeat. If a connection is made, the version will be set
131
138
  const params = {};
132
139
  const key = this.mintKey(key_1.KeyType.HOTMESH, params);
133
140
  return await this.redisClient[this.commands.hset](key, manifest);
@@ -135,8 +142,10 @@ class StoreService {
135
142
  async reserveSymbolRange(target, size, type) {
136
143
  const rangeKey = this.mintKey(key_1.KeyType.SYMKEYS, { appId: this.appId });
137
144
  const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, { activityId: target, appId: this.appId });
145
+ //reserve the slot in a `pending` state (range will be established in the next step)
138
146
  const response = await this.redisClient[this.commands.hsetnx](rangeKey, target, '?:?');
139
147
  if (response) {
148
+ //if the key didn't exist, set the inclusive range and seed metadata fields
140
149
  const upperLimit = await this.redisClient[this.commands.hincrby](rangeKey, ':cursor', size);
141
150
  const lowerLimit = upperLimit - size;
142
151
  const inclusiveRange = `${lowerLimit}:${upperLimit - 1}`;
@@ -146,6 +155,7 @@ class StoreService {
146
155
  return [lowerLimit + serializer_1.MDATA_SYMBOLS.SLOTS, upperLimit - 1, {}];
147
156
  }
148
157
  else {
158
+ //if the key already existed, get the lower limit and add the number of symbols
149
159
  const range = await this.redisClient[this.commands.hget](rangeKey, target);
150
160
  const [lowerLimitString] = range.split(':');
151
161
  const lowerLimit = parseInt(lowerLimitString, 10);
@@ -157,6 +167,7 @@ class StoreService {
157
167
  }
158
168
  }
159
169
  async getAllSymbols() {
170
+ //get hash with all reserved symbol ranges
160
171
  const rangeKey = this.mintKey(key_1.KeyType.SYMKEYS, { appId: this.appId });
161
172
  const ranges = await this.redisClient[this.commands.hgetall](rangeKey);
162
173
  const rangeKeys = Object.keys(ranges).sort();
@@ -329,6 +340,11 @@ class StoreService {
329
340
  };
330
341
  return await this.redisClient[this.commands.hset](key, payload);
331
342
  }
343
+ /**
344
+ * Registers the job, `jobId`, with `originJobId`. In the future,
345
+ * when `originJobId` is interrupted/expired, the items in the
346
+ * list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
347
+ */
332
348
  async registerJobDependency(depType, originJobId, topic, jobId, gId, pd = '', multi) {
333
349
  const privateMulti = multi || this.getMulti();
334
350
  const dependencyParams = {
@@ -348,10 +364,15 @@ class StoreService {
348
364
  return await privateMulti.exec();
349
365
  }
350
366
  }
367
+ /**
368
+ * Ensures a `hook signal` is delisted when its parent activity/job
369
+ * is interrupted/expired.
370
+ */
351
371
  async registerSignalDependency(jobId, signalKey, dad, multi) {
352
372
  const privateMulti = multi || this.getMulti();
353
373
  const dependencyParams = { appId: this.appId, jobId };
354
374
  const dependencyKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
375
+ //persiste dependency tasks as multi-segment composite keys
355
376
  const delistTask = [
356
377
  'delist',
357
378
  'signal',
@@ -388,6 +409,7 @@ class StoreService {
388
409
  }
389
410
  }
390
411
  hGetAllResult(result) {
412
+ //default response signature uses 'redis' NPM Package format
391
413
  return result;
392
414
  }
393
415
  async getJobStats(jobKeys) {
@@ -416,12 +438,13 @@ class StoreService {
416
438
  async getJobIds(indexKeys, idRange) {
417
439
  const multi = this.getMulti();
418
440
  for (const idsKey of indexKeys) {
419
- multi[this.commands.lrange](idsKey, idRange[0], idRange[1]);
441
+ multi[this.commands.lrange](idsKey, idRange[0], idRange[1]); //0,-1 returns all ids
420
442
  }
421
443
  const results = await multi.exec();
422
444
  const output = {};
423
445
  for (const [index, result] of results.entries()) {
424
446
  const key = indexKeys[index];
447
+ //todo: resolve this discrepancy between redis/ioredis
425
448
  const idsList = result[1] || result;
426
449
  if (idsList && idsList.length > 0) {
427
450
  output[key] = idsList;
@@ -460,6 +483,10 @@ class StoreService {
460
483
  await (multi || this.redisClient)[this.commands.hset](hashKey, hashData);
461
484
  return jobId;
462
485
  }
486
+ /**
487
+ * Returns custom search fields and values.
488
+ * NOTE: The `fields` param should NOT prefix items with an underscore.
489
+ */
463
490
  async getQueryState(jobId, fields) {
464
491
  const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
465
492
  const _fields = fields.map(field => `_${field}`);
@@ -471,6 +498,7 @@ class StoreService {
471
498
  return jobData;
472
499
  }
473
500
  async getState(jobId, consumes, dIds) {
501
+ //get abbreviated field list (the symbols for the paths)
474
502
  const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
475
503
  const symbolNames = Object.keys(consumes);
476
504
  const symKeys = await this.getSymbolKeys(symbolNames);
@@ -478,7 +506,7 @@ class StoreService {
478
506
  const fields = this.serializer.abbreviate(consumes, symbolNames, [':']);
479
507
  const jobDataArray = await this.redisClient[this.commands.hmget](key, fields);
480
508
  const jobData = {};
481
- let atLeast1 = false;
509
+ let atLeast1 = false; //if status field (':') isn't present assume 404
482
510
  fields.forEach((field, index) => {
483
511
  if (jobDataArray[index]) {
484
512
  atLeast1 = true;
@@ -509,9 +537,13 @@ class StoreService {
509
537
  }
510
538
  return job;
511
539
  }
540
+ /**
541
+ * collate is a generic method for incrementing a value in a hash
542
+ * in order to track their progress during processing.
543
+ */
512
544
  async collate(jobId, activityId, amount, dIds, multi) {
513
545
  const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
514
- const collationKey = `${activityId}/output/metadata/as`;
546
+ const collationKey = `${activityId}/output/metadata/as`; //activity state
515
547
  const symbolNames = [activityId];
516
548
  const symKeys = await this.getSymbolKeys(symbolNames);
517
549
  const symVals = await this.getSymbolValues();
@@ -521,6 +553,12 @@ class StoreService {
521
553
  const targetId = Object.keys(hashData)[0];
522
554
  return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, targetId, amount);
523
555
  }
556
+ /**
557
+ * synthentic collation affects those activities in the graph
558
+ * that represent the synthetic DAG that was materialized during compilation;
559
+ * Synthetic targeting ensures that re-entry due to failure can be distinguished from
560
+ * purposeful re-entry.
561
+ */
524
562
  async collateSynthetic(jobId, guid, amount, multi) {
525
563
  const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
526
564
  return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, guid, amount);
@@ -659,10 +697,12 @@ class StoreService {
659
697
  }
660
698
  async setHookSignal(hook, multi) {
661
699
  const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
700
+ //destructure the hook key
662
701
  const { topic, resolved, jobId } = hook;
663
702
  const signalKey = `${topic}:${resolved}`;
664
703
  const payload = { [signalKey]: jobId };
665
704
  await (multi || this.redisClient)[this.commands.hset](key, payload);
705
+ //jobId needs even more destructuring
666
706
  const [_aid, dad, _gid, jid] = jobId.split(key_1.VALSEP);
667
707
  return await this.registerSignalDependency(jid, signalKey, dad, multi);
668
708
  }
@@ -701,6 +741,7 @@ class StoreService {
701
741
  const didRemove = await this.redisClient[this.commands.zrem](zsetKey, workItemKey);
702
742
  if (didRemove) {
703
743
  if (scrub) {
744
+ //indexes can be designed to be self-cleaning; `engine.hookAll` exposes this option
704
745
  this.redisClient[this.commands.expire](processedKey, 0);
705
746
  this.redisClient[this.commands.expire](key.split(":").slice(0, 5).join(":"), 0);
706
747
  }
@@ -719,6 +760,11 @@ class StoreService {
719
760
  await this.redisClient[this.commands.expire](jobKey, inSeconds);
720
761
  }
721
762
  }
763
+ /**
764
+ * register the descendants of an expired origin flow to be
765
+ * expired at a future date; options indicate whether this
766
+ * is a standard `expire` or an `interrupt`
767
+ */
722
768
  async registerDependenciesForCleanup(jobId, deletionTime, options) {
723
769
  const depParams = { appId: this.appId, jobId };
724
770
  const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
@@ -732,8 +778,15 @@ class StoreService {
732
778
  const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
733
779
  return this.redisClient[this.commands.lrange](depKey, 0, -1);
734
780
  }
781
+ /**
782
+ * registers a hook activity to be awakened (uses ZSET to
783
+ * store the 'sleep group' and LIST to store the events
784
+ * for the given sleep group. Sleep groups are
785
+ * organized into 'n'-second blocks (LISTS))
786
+ */
735
787
  async registerTimeHook(jobId, gId, activityId, type, deletionTime, dad, multi) {
736
788
  const listKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId, timeValue: deletionTime });
789
+ //construct the composite key (the key has enough info to signal the hook)
737
790
  const timeEvent = [
738
791
  type,
739
792
  activityId,
@@ -754,6 +807,7 @@ class StoreService {
754
807
  let [pType, pKey] = this.resolveTaskKeyContext(listKey);
755
808
  const timeEvent = await this.redisClient[this.commands.lpop](pKey);
756
809
  if (timeEvent) {
810
+ //deconstruct composite key
757
811
  let [type, activityId, gId, _pd, ...jobId] = timeEvent.split(key_1.VALSEP);
758
812
  const jid = jobId.join(key_1.VALSEP);
759
813
  if (type === 'delist') {
@@ -772,6 +826,14 @@ class StoreService {
772
826
  }
773
827
  return false;
774
828
  }
829
+ /**
830
+ * when processing time jobs, the target LIST ID returned
831
+ * from the ZSET query can be prefixed to denote what to
832
+ * do with the work list. (not everything is known in advance,
833
+ * so the ZSET key defines HOW to approach the work in the
834
+ * generic LIST (lists typically contain target job ids)
835
+ * @param {string} listKey - composite key
836
+ */
775
837
  resolveTaskKeyContext(listKey) {
776
838
  if (listKey.startsWith(`${key_1.TYPSEP}INTERRUPT`)) {
777
839
  return ['interrupt', listKey.split(key_1.TYPSEP)[2]];
@@ -783,24 +845,38 @@ class StoreService {
783
845
  return ['sleep', listKey];
784
846
  }
785
847
  }
848
+ /**
849
+ * Interrupts a job and sets sets a job error (410), if 'throw'!=false.
850
+ * This method is called by the engine and not by an activity and is
851
+ * followed by a call to execute job completion/cleanup tasks
852
+ * associated with a job completion event.
853
+ *
854
+ * Todo: move most of this logic to the engine (too much logic for the store)
855
+ */
786
856
  async interrupt(topic, jobId, options = {}) {
787
857
  try {
858
+ //verify job exists
788
859
  const status = await this.getStatus(jobId, this.appId);
789
860
  if (status <= 0) {
861
+ //verify still active; job already completed
790
862
  throw new Error(`Job ${jobId} already completed`);
791
863
  }
864
+ //decrement job status (:) by 1bil
792
865
  const amount = -1000000000;
793
866
  const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
794
867
  const result = await this.redisClient[this.commands.hincrbyfloat](jobKey, ':', amount);
795
868
  if (result <= amount) {
869
+ //verify active state; job already interrupted
796
870
  throw new Error(`Job ${jobId} already completed`);
797
871
  }
872
+ //persist the error unless specifically told not to
798
873
  if (options.throw !== false) {
799
- const errKey = `metadata/err`;
800
- const symbolNames = [`$${topic}`];
874
+ const errKey = `metadata/err`; //job errors are stored at the path `metadata/err`
875
+ const symbolNames = [`$${topic}`]; //the symbol for `metadata/err` is in redis and stored using the job topic
801
876
  const symKeys = await this.getSymbolKeys(symbolNames);
802
877
  const symVals = await this.getSymbolValues();
803
878
  this.serializer.resetSymbols(symKeys, symVals, {});
879
+ //persists the standard 410 error (job is `gone`)
804
880
  const err = JSON.stringify({
805
881
  code: options.code ?? enums_1.HMSH_CODE_INTERRUPT,
806
882
  message: options.reason ?? `job [${jobId}] interrupted`,
@@ -51,7 +51,10 @@ class IORedisStreamService extends index_1.StreamService {
51
51
  }
52
52
  async xreadgroup(command, groupName, consumerName, blockOption, blockTime, streamsOption, streamName, id) {
53
53
  try {
54
- return await this.redisClient.xreadgroup(command, groupName, consumerName, blockOption, blockTime, streamsOption, streamName, id);
54
+ //@ts-ignore
55
+ return await this.redisClient.xreadgroup(command, groupName, consumerName,
56
+ // @ts-ignore
57
+ blockOption, blockTime, streamsOption, streamName, id);
55
58
  }
56
59
  catch (error) {
57
60
  this.logger.error(`Error reading stream data [Stream ${streamName}] [Group ${groupName}]`, { ...error });
@@ -16,7 +16,16 @@ declare class TaskService {
16
16
  enqueueWorkItems(keys: string[]): Promise<void>;
17
17
  registerJobForCleanup(jobId: string, inSeconds: number, options: JobCompletionOptions): Promise<void>;
18
18
  registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, inSeconds: number, dad: string, multi?: RedisMulti): Promise<void>;
19
+ /**
20
+ * Should this engine instance play the role of 'scout' on behalf
21
+ * of the entire quorum? The scout role is responsible for processing
22
+ * task lists on behalf of the collective.
23
+ */
19
24
  shouldScout(): Promise<boolean>;
25
+ /**
26
+ * Callback handler that takes an item from a work list and
27
+ * processes according to its type
28
+ */
20
29
  processTimeHooks(timeEventCallback: (jobId: string, gId: string, activityId: string, type: WorkListTaskType) => Promise<void>, listKey?: string): Promise<void>;
21
30
  cancelCleanup(): void;
22
31
  getHookRule(topic: string): Promise<HookRule | undefined>;
@@ -22,6 +22,7 @@ class TaskService {
22
22
  const destinationKey = `${sourceKey}:processed`;
23
23
  const jobId = await this.store.processTaskQueue(sourceKey, destinationKey);
24
24
  if (jobId) {
25
+ //todo: don't use 'id', make configurable using hook rule
25
26
  await hookEventCallback(topic, { ...data, id: jobId });
26
27
  }
27
28
  else {
@@ -48,6 +49,11 @@ class TaskService {
48
49
  const awakenTimeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
49
50
  await this.store.registerTimeHook(jobId, gId, activityId, type, awakenTimeSlot, dad, multi);
50
51
  }
52
+ /**
53
+ * Should this engine instance play the role of 'scout' on behalf
54
+ * of the entire quorum? The scout role is responsible for processing
55
+ * task lists on behalf of the collective.
56
+ */
51
57
  async shouldScout() {
52
58
  const wasScout = this.isScout;
53
59
  const isScout = wasScout || (this.isScout = await this.store.reserveScoutRole('time'));
@@ -61,6 +67,10 @@ class TaskService {
61
67
  }
62
68
  return false;
63
69
  }
70
+ /**
71
+ * Callback handler that takes an item from a work list and
72
+ * processes according to its type
73
+ */
64
74
  async processTimeHooks(timeEventCallback, listKey) {
65
75
  if (await this.shouldScout()) {
66
76
  try {
@@ -68,12 +78,16 @@ class TaskService {
68
78
  if (Array.isArray(workListTask)) {
69
79
  const [listKey, target, gId, activityId, type] = workListTask;
70
80
  if (type === 'child') {
81
+ //continue; this child is listed here for convenience, but
82
+ // will be expired by an origin ancestor and is listed there
71
83
  }
72
84
  else if (type === 'delist') {
85
+ //delist the signalKey (target)
73
86
  const key = this.store.mintKey(hotmesh_1.KeyType.SIGNALS, { appId: this.store.appId });
74
87
  await this.store.redisClient[this.store.commands.hdel](key, target);
75
88
  }
76
89
  else {
90
+ //awaken/expire/interrupt
77
91
  await timeEventCallback(target, gId, activityId, type);
78
92
  }
79
93
  await (0, utils_1.sleepFor)(0);
@@ -81,11 +95,13 @@ class TaskService {
81
95
  this.processTimeHooks(timeEventCallback, listKey);
82
96
  }
83
97
  else if (workListTask) {
98
+ //a worklist was just emptied; try again immediately
84
99
  await (0, utils_1.sleepFor)(0);
85
100
  this.errorCount = 0;
86
101
  this.processTimeHooks(timeEventCallback);
87
102
  }
88
103
  else {
104
+ //no worklists exist; sleep before checking
89
105
  let sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_FIDELITY_SECONDS * 1000);
90
106
  this.cleanupTimeout = sleep.timerId;
91
107
  await sleep.promise;
@@ -94,6 +110,8 @@ class TaskService {
94
110
  }
95
111
  }
96
112
  catch (err) {
113
+ //most common reasons: deleted job not found; container stopping; test stopping
114
+ //less common: redis/cluster down; retry with fallback (5s max main reassignment)
97
115
  this.logger.warn('task-process-timehooks-error', err);
98
116
  await (0, utils_1.sleepFor)(1000 * this.errorCount++);
99
117
  if (this.errorCount < 5) {
@@ -102,6 +120,7 @@ class TaskService {
102
120
  }
103
121
  }
104
122
  else {
123
+ //didn't get the scout role; try again in 'one-ish' minutes
105
124
  let sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_SCOUT_INTERVAL_SECONDS * 1000 * 2 * Math.random());
106
125
  this.cleanupTimeout = sleep.timerId;
107
126
  await sleep.promise;
@@ -126,6 +145,7 @@ class TaskService {
126
145
  const jobId = context.metadata.jid;
127
146
  const gId = context.metadata.gid;
128
147
  const activityId = hookRule.to;
148
+ //composite keys are used to fully describe the task target
129
149
  const compositeJobKey = [
130
150
  activityId,
131
151
  dad,
@@ -147,13 +167,22 @@ class TaskService {
147
167
  async processWebHookSignal(topic, data) {
148
168
  const hookRule = await this.getHookRule(topic);
149
169
  if (hookRule) {
170
+ //NOTE: both formats are supported by the mapping engine:
171
+ // `$self.hook.data` OR `$hook.data`
150
172
  const context = { $self: { hook: { data } }, $hook: { data } };
151
173
  const mapExpression = hookRule.conditions.match[0].actual;
152
174
  const resolved = pipe_1.Pipe.resolve(mapExpression, context);
153
175
  const hookSignalId = await this.store.getHookSignal(topic, resolved);
154
176
  if (!hookSignalId) {
177
+ //messages can be double-processed; not an issue; return `undefined`
178
+ //users can also provide a bogus topic; not an issue; return `undefined`
155
179
  return undefined;
156
180
  }
181
+ //`aid` is part of composite key, but the hook `topic` is its public interface;
182
+ // this means that a new version of the graph can be deployed and the
183
+ // topic can be re-mapped to a different activity id. Outside callers
184
+ // can adhere to the unchanged contract (calling the same topic),
185
+ // while the internal system can be updated in real-time as necessary.
157
186
  const [_aid, dad, gid, ...jid] = hookSignalId.split(key_1.WEBSEP);
158
187
  return [jid.join(key_1.WEBSEP), hookRule.to, dad, gid];
159
188
  }
@@ -164,6 +193,8 @@ class TaskService {
164
193
  async deleteWebHookSignal(topic, data) {
165
194
  const hookRule = await this.getHookRule(topic);
166
195
  if (hookRule) {
196
+ //NOTE: both formats are supported by the mapping engine:
197
+ // `$self.hook.data` OR `$hook.data`
167
198
  const context = { $self: { hook: { data } }, $hook: { data } };
168
199
  const mapExpression = hookRule.conditions.match[0].actual;
169
200
  const resolved = pipe_1.Pipe.resolve(mapExpression, context);
@@ -35,6 +35,13 @@ declare class TelemetryService {
35
35
  setTelemetryContext(span: Span, leg: number): void;
36
36
  setActivityError(message: string): void;
37
37
  setStreamError(message: string): void;
38
+ /**
39
+ * Adds the paths (HGET) necessary to restore telemetry state for an activity
40
+ * @param consumes
41
+ * @param config
42
+ * @param metadata
43
+ * @param leg
44
+ */
38
45
  static addTargetTelemetryPaths(consumes: Consumes, config: ActivityType, metadata: ActivityMetadata, leg: number): void;
39
46
  static bindJobTelemetryToState(state: StringStringType, config: ActivityType, context: JobState): void;
40
47
  static bindActivityTelemetryToState(state: StringAnyType, config: ActivityType, metadata: ActivityMetadata, context: JobState, leg: number): void;
@@ -13,6 +13,7 @@ class TelemetryService {
13
13
  constructor(appId, config, metadata, context) {
14
14
  this.leg = 1;
15
15
  this.appId = appId;
16
+ //these are REQUIRED for job and activity spans
16
17
  this.config = config;
17
18
  this.metadata = metadata;
18
19
  this.context = context;
@@ -81,6 +82,7 @@ class TelemetryService {
81
82
  return span;
82
83
  }
83
84
  mapActivityAttributes() {
85
+ //export user-defined span attributes (app.activity.data.*)
84
86
  if (this.config.telemetry) {
85
87
  const telemetryAtts = new mapper_1.MapperService(this.config.telemetry, this.context).mapRules();
86
88
  const namespacedAtts = {
@@ -121,7 +123,7 @@ class TelemetryService {
121
123
  traceId: this.traceId,
122
124
  spanId: this.spanId,
123
125
  isRemote: true,
124
- traceFlags: 1,
126
+ traceFlags: 1, // (todo: revisit sampling strategy/config)
125
127
  };
126
128
  const parentContext = telemetry_1.trace.setSpanContext(telemetry_1.context.active(), restoredSpanContext);
127
129
  return parentContext;
@@ -177,6 +179,13 @@ class TelemetryService {
177
179
  setStreamError(message) {
178
180
  this.span?.setStatus({ code: telemetry_1.SpanStatusCode.ERROR, message });
179
181
  }
182
+ /**
183
+ * Adds the paths (HGET) necessary to restore telemetry state for an activity
184
+ * @param consumes
185
+ * @param config
186
+ * @param metadata
187
+ * @param leg
188
+ */
180
189
  static addTargetTelemetryPaths(consumes, config, metadata, leg) {
181
190
  if (leg === 1) {
182
191
  if (!(config.parent in consumes)) {
@@ -198,14 +207,17 @@ class TelemetryService {
198
207
  }
199
208
  static bindActivityTelemetryToState(state, config, metadata, context, leg) {
200
209
  if (config.type === 'trigger') {
210
+ //trigger activities run non-duplexed and only have a single leg (2)
201
211
  state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
202
212
  state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l2s;
203
213
  }
204
214
  else if (utils_1.polyfill.resolveActivityType(config.type) === 'hook' && leg === 1) {
215
+ //hook activities run non-duplexed and only have a single leg (1)
205
216
  state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
206
217
  state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l1s;
207
218
  }
208
219
  else if (config.type === 'signal' && leg === 1) {
220
+ //signal activities run non-duplexed and only have a single leg (1)
209
221
  state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
210
222
  state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l1s;
211
223
  }
@@ -30,6 +30,10 @@ declare class WorkerService {
30
30
  initStreamChannel(service: WorkerService, stream: RedisClient): Promise<void>;
31
31
  initRouter(worker: HotMeshWorker, logger: ILogger): Router;
32
32
  subscriptionHandler(): SubscriptionCallback;
33
+ /**
34
+ * A quorum-wide command to broadcaset system details.
35
+ *
36
+ */
33
37
  doRollCall(message: RollCallMessage): Promise<void>;
34
38
  cancelRollCall(): void;
35
39
  stop(): void;
@@ -97,7 +97,7 @@ class WorkerService {
97
97
  return async (topic, message) => {
98
98
  self.logger.debug('worker-event-received', { topic, type: message.type });
99
99
  if (message.type === 'throttle') {
100
- if (message.topic !== null) {
100
+ if (message.topic !== null) { //undefined allows passthrough
101
101
  self.throttle(message.throttle);
102
102
  }
103
103
  }
@@ -105,12 +105,16 @@ class WorkerService {
105
105
  self.sayPong(self.appId, self.guid, message.originator, message.details);
106
106
  }
107
107
  else if (message.type === 'rollcall') {
108
- if (message.topic !== null) {
108
+ if (message.topic !== null) { //undefined allows passthrough
109
109
  self.doRollCall(message);
110
110
  }
111
111
  }
112
112
  };
113
113
  }
114
+ /**
115
+ * A quorum-wide command to broadcaset system details.
116
+ *
117
+ */
114
118
  async doRollCall(message) {
115
119
  let iteration = 0;
116
120
  let max = !isNaN(message.max) ? message.max : enums_1.HMSH_QUORUM_ROLLCALL_CYCLES;
@@ -32,7 +32,18 @@ interface Measure {
32
32
  target: string;
33
33
  }
34
34
  interface TriggerActivityStats {
35
+ /**
36
+ * dependent parent job id; including this allows the parent's
37
+ * expiration/interruption events to cascade; set
38
+ * `expire` in the YAML for the dependent graph
39
+ * to 0 and provide the parent for dependent,
40
+ * cascading interruption and cleanup
41
+ */
35
42
  parent?: string;
43
+ /**
44
+ * adjacent parent job id; this is the actual adjacent
45
+ * parent in the graph, but it is not used for cascading expiration
46
+ */
36
47
  adjacent?: string;
37
48
  id?: {
38
49
  [key: string]: unknown;
@@ -40,7 +51,19 @@ interface TriggerActivityStats {
40
51
  key?: {
41
52
  [key: string]: unknown;
42
53
  } | string;
54
+ /**
55
+ * @deprecated
56
+ * return 'infinity' to disable; default behavior
57
+ * is to always segment keys by time to ensure
58
+ * indexes (Redis LIST) never grow unbounded
59
+ * as a default behavior; for now, 5m is default
60
+ * and infinity can be set to override
61
+ */
43
62
  granularity?: string;
63
+ /**
64
+ * @deprecated
65
+ * what to capture
66
+ */
44
67
  measures?: Measure[];
45
68
  }
46
69
  interface TriggerActivity extends BaseActivity {
@@ -79,26 +102,84 @@ interface SignalActivity extends BaseActivity {
79
102
  }
80
103
  interface InterruptActivity extends BaseActivity {
81
104
  type: 'interrupt';
105
+ /**
106
+ * Optional Reason; will be used as the error `message` when thrown
107
+ * @default 'Job Interrupted'
108
+ */
82
109
  reason?: string;
110
+ /**
111
+ * throw JobInterrupted error upon interrupting
112
+ * @default true
113
+ */
83
114
  throw?: boolean;
115
+ /**
116
+ * Interrupt child/descendant jobs
117
+ * @default false
118
+ */
84
119
  descend?: boolean;
120
+ /**
121
+ * Target job id (if not present the current job will be targeted)
122
+ */
85
123
  target?: string;
124
+ /**
125
+ * Optional topic to publish the interrupt message (if not present the current job topic will be used
126
+ */
86
127
  topic?: string;
128
+ /**
129
+ * Optional Error Code; will be used as the error `code` when thrown
130
+ * @default 410
131
+ */
87
132
  code?: number;
133
+ /**
134
+ * Optional stack trace
135
+ */
88
136
  stack?: string;
89
137
  }
90
138
  type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | InterruptActivity | HookActivity | SignalActivity | CycleActivity;
91
139
  type ActivityData = Record<string, any>;
140
+ /**
141
+ * Type definition for activity metadata.
142
+ */
92
143
  type ActivityMetadata = {
144
+ /**
145
+ * Unique identifier for the activity.
146
+ */
93
147
  aid: string;
148
+ /**
149
+ * Type of the activity.
150
+ */
94
151
  atp: string;
152
+ /**
153
+ * Subtype of the activity.
154
+ */
95
155
  stp: string;
156
+ /**
157
+ * Timestamp when the activity was created, in GMT.
158
+ */
96
159
  ac: string;
160
+ /**
161
+ * Timestamp when the activity was last updated, in GMT.
162
+ */
97
163
  au: string;
164
+ /**
165
+ * Optional stringified JSON containing error details such as message, code, and an optional error object.
166
+ */
98
167
  err?: string;
168
+ /**
169
+ * OpenTelemetry span context for the first leg of the activity.
170
+ */
99
171
  l1s?: string;
172
+ /**
173
+ * OpenTelemetry span context for the second leg of the activity.
174
+ */
100
175
  l2s?: string;
176
+ /**
177
+ * Dimensional address, used for additional metadata.
178
+ */
101
179
  dad?: string;
180
+ /**
181
+ * Status of the activity, could be codes representing different states.
182
+ */
102
183
  as?: string;
103
184
  };
104
185
  type ActivityContext = {