@hotmeshio/hotmesh 0.0.55 → 0.0.57

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/README.md +1 -1
  2. package/build/modules/enums.js +1 -10
  3. package/build/modules/key.d.ts +0 -38
  4. package/build/modules/key.js +4 -46
  5. package/build/modules/utils.d.ts +0 -8
  6. package/build/modules/utils.js +0 -14
  7. package/build/package.json +11 -4
  8. package/build/services/activities/activity.d.ts +0 -28
  9. package/build/services/activities/activity.js +1 -46
  10. package/build/services/activities/await.js +0 -4
  11. package/build/services/activities/cycle.d.ts +0 -7
  12. package/build/services/activities/cycle.js +1 -16
  13. package/build/services/activities/hook.d.ts +0 -6
  14. package/build/services/activities/hook.js +2 -12
  15. package/build/services/activities/interrupt.js +0 -8
  16. package/build/services/activities/signal.d.ts +0 -6
  17. package/build/services/activities/signal.js +0 -15
  18. package/build/services/activities/trigger.d.ts +0 -4
  19. package/build/services/activities/trigger.js +1 -7
  20. package/build/services/activities/worker.js +0 -4
  21. package/build/services/collator/index.d.ts +0 -70
  22. package/build/services/collator/index.js +1 -91
  23. package/build/services/compiler/deployer.js +6 -38
  24. package/build/services/compiler/index.d.ts +0 -15
  25. package/build/services/compiler/index.js +0 -20
  26. package/build/services/compiler/validator.d.ts +0 -3
  27. package/build/services/compiler/validator.js +0 -25
  28. package/build/services/connector/clients/ioredis.d.ts +2 -2
  29. package/build/services/connector/clients/ioredis.js +0 -2
  30. package/build/services/connector/clients/redis.d.ts +4 -4
  31. package/build/services/connector/clients/redis.js +1 -3
  32. package/build/services/connector/index.d.ts +1 -1
  33. package/build/services/connector/index.js +0 -2
  34. package/build/services/durable/client.d.ts +1 -26
  35. package/build/services/durable/client.js +0 -56
  36. package/build/services/durable/exporter.d.ts +0 -22
  37. package/build/services/durable/exporter.js +1 -30
  38. package/build/services/durable/handle.d.ts +0 -36
  39. package/build/services/durable/handle.js +0 -46
  40. package/build/services/durable/index.d.ts +0 -4
  41. package/build/services/durable/index.js +0 -4
  42. package/build/services/durable/schemas/factory.d.ts +0 -29
  43. package/build/services/durable/schemas/factory.js +0 -29
  44. package/build/services/durable/search.d.ts +1 -36
  45. package/build/services/durable/search.js +56 -56
  46. package/build/services/durable/worker.js +2 -22
  47. package/build/services/durable/workflow.d.ts +0 -114
  48. package/build/services/durable/workflow.js +1 -141
  49. package/build/services/engine/index.d.ts +1 -6
  50. package/build/services/engine/index.js +1 -43
  51. package/build/services/exporter/index.d.ts +0 -27
  52. package/build/services/exporter/index.js +0 -33
  53. package/build/services/hotmesh/index.d.ts +2 -2
  54. package/build/services/hotmesh/index.js +1 -9
  55. package/build/services/logger/index.js +0 -2
  56. package/build/services/mapper/index.d.ts +0 -14
  57. package/build/services/mapper/index.js +0 -14
  58. package/build/services/pipe/functions/date.d.ts +0 -7
  59. package/build/services/pipe/functions/date.js +0 -7
  60. package/build/services/pipe/functions/math.js +0 -2
  61. package/build/services/pipe/index.d.ts +0 -15
  62. package/build/services/pipe/index.js +2 -23
  63. package/build/services/quorum/index.d.ts +0 -7
  64. package/build/services/quorum/index.js +0 -21
  65. package/build/services/reporter/index.d.ts +0 -5
  66. package/build/services/reporter/index.js +0 -9
  67. package/build/services/router/index.d.ts +0 -9
  68. package/build/services/router/index.js +2 -38
  69. package/build/services/serializer/index.js +7 -26
  70. package/build/services/store/cache.d.ts +0 -18
  71. package/build/services/store/cache.js +0 -18
  72. package/build/services/store/clients/ioredis.d.ts +1 -1
  73. package/build/services/store/clients/ioredis.js +0 -1
  74. package/build/services/store/clients/redis.d.ts +1 -1
  75. package/build/services/store/index.d.ts +0 -55
  76. package/build/services/store/index.js +5 -81
  77. package/build/services/stream/clients/ioredis.d.ts +1 -1
  78. package/build/services/stream/clients/ioredis.js +1 -4
  79. package/build/services/stream/clients/redis.d.ts +1 -1
  80. package/build/services/sub/clients/ioredis.d.ts +1 -1
  81. package/build/services/sub/clients/redis.d.ts +1 -1
  82. package/build/services/task/index.d.ts +0 -9
  83. package/build/services/task/index.js +0 -31
  84. package/build/services/telemetry/index.d.ts +0 -7
  85. package/build/services/telemetry/index.js +1 -13
  86. package/build/services/worker/index.d.ts +0 -4
  87. package/build/services/worker/index.js +2 -6
  88. package/build/types/activity.d.ts +0 -81
  89. package/build/types/durable.d.ts +26 -177
  90. package/build/types/exporter.d.ts +0 -13
  91. package/build/types/hotmesh.d.ts +4 -16
  92. package/build/types/hotmesh.js +0 -3
  93. package/build/types/index.d.ts +4 -6
  94. package/build/types/index.js +4 -3
  95. package/build/types/job.d.ts +1 -86
  96. package/build/types/pipe.d.ts +0 -65
  97. package/build/types/quorum.d.ts +15 -10
  98. package/build/types/redis.d.ts +225 -7
  99. package/build/types/redis.js +9 -0
  100. package/build/types/stream.d.ts +0 -58
  101. package/build/types/stream.js +0 -4
  102. package/package.json +11 -4
  103. package/types/durable.ts +131 -4
  104. package/types/hotmesh.ts +3 -6
  105. package/types/index.ts +23 -10
  106. package/types/job.ts +1 -1
  107. package/types/quorum.ts +22 -0
  108. package/types/redis.ts +267 -18
  109. package/build/types/ioredisclient.d.ts +0 -5
  110. package/build/types/ioredisclient.js +0 -5
  111. package/build/types/redisclient.d.ts +0 -26
  112. package/build/types/redisclient.js +0 -2
  113. package/modules/enums.ts +0 -62
  114. package/modules/errors.ts +0 -280
  115. package/modules/key.ts +0 -101
  116. package/modules/storage.ts +0 -3
  117. package/modules/utils.ts +0 -242
  118. package/services/activities/activity.ts +0 -589
  119. package/services/activities/await.ts +0 -113
  120. package/services/activities/cycle.ts +0 -115
  121. package/services/activities/hook.ts +0 -197
  122. package/services/activities/index.ts +0 -19
  123. package/services/activities/interrupt.ts +0 -172
  124. package/services/activities/signal.ts +0 -148
  125. package/services/activities/trigger.ts +0 -295
  126. package/services/activities/worker.ts +0 -107
  127. package/services/collator/README.md +0 -102
  128. package/services/collator/index.ts +0 -291
  129. package/services/compiler/deployer.ts +0 -504
  130. package/services/compiler/index.ts +0 -98
  131. package/services/compiler/validator.ts +0 -158
  132. package/services/connector/clients/ioredis.ts +0 -57
  133. package/services/connector/clients/redis.ts +0 -72
  134. package/services/connector/index.ts +0 -42
  135. package/services/durable/client.ts +0 -266
  136. package/services/durable/connection.ts +0 -10
  137. package/services/durable/exporter.ts +0 -232
  138. package/services/durable/handle.ts +0 -160
  139. package/services/durable/index.ts +0 -27
  140. package/services/durable/schemas/factory.ts +0 -2358
  141. package/services/durable/search.ts +0 -196
  142. package/services/durable/worker.ts +0 -401
  143. package/services/durable/workflow.ts +0 -557
  144. package/services/engine/index.ts +0 -761
  145. package/services/exporter/index.ts +0 -146
  146. package/services/hotmesh/index.ts +0 -237
  147. package/services/logger/index.ts +0 -79
  148. package/services/mapper/index.ts +0 -89
  149. package/services/pipe/functions/array.ts +0 -78
  150. package/services/pipe/functions/bitwise.ts +0 -27
  151. package/services/pipe/functions/conditional.ts +0 -35
  152. package/services/pipe/functions/date.ts +0 -220
  153. package/services/pipe/functions/index.ts +0 -27
  154. package/services/pipe/functions/json.ts +0 -11
  155. package/services/pipe/functions/logical.ts +0 -11
  156. package/services/pipe/functions/math.ts +0 -217
  157. package/services/pipe/functions/number.ts +0 -75
  158. package/services/pipe/functions/object.ts +0 -98
  159. package/services/pipe/functions/string.ts +0 -86
  160. package/services/pipe/functions/symbol.ts +0 -39
  161. package/services/pipe/functions/unary.ts +0 -19
  162. package/services/pipe/index.ts +0 -216
  163. package/services/quorum/index.ts +0 -319
  164. package/services/reporter/index.ts +0 -387
  165. package/services/router/index.ts +0 -426
  166. package/services/serializer/README.md +0 -10
  167. package/services/serializer/index.ts +0 -285
  168. package/services/store/cache.ts +0 -172
  169. package/services/store/clients/ioredis.ts +0 -145
  170. package/services/store/clients/redis.ts +0 -191
  171. package/services/store/index.ts +0 -1091
  172. package/services/stream/clients/ioredis.ts +0 -157
  173. package/services/stream/clients/redis.ts +0 -158
  174. package/services/stream/index.ts +0 -58
  175. package/services/sub/clients/ioredis.ts +0 -83
  176. package/services/sub/clients/redis.ts +0 -74
  177. package/services/sub/index.ts +0 -25
  178. package/services/task/index.ts +0 -250
  179. package/services/telemetry/index.ts +0 -273
  180. package/services/worker/index.ts +0 -248
  181. package/types/ioredisclient.ts +0 -10
  182. package/types/redisclient.ts +0 -30
@@ -77,7 +77,6 @@ class StoreService {
77
77
  return result > 0 || result === 'OK' || result === true;
78
78
  }
79
79
  async zAdd(key, score, value, redisMulti) {
80
- //default call signature uses 'ioredis' NPM Package format
81
80
  return await (redisMulti || this.redisClient)[this.commands.zadd](key, score, value);
82
81
  }
83
82
  async zRangeByScoreWithScores(key, score, value) {
@@ -102,11 +101,6 @@ class StoreService {
102
101
  invalidateCache() {
103
102
  this.cache.invalidate();
104
103
  }
105
- /**
106
- * At any given time only a single engine will
107
- * check for and process work items in the
108
- * time and signal task queues.
109
- */
110
104
  async reserveScoutRole(scoutType, delay = enums_1.HMSH_SCOUT_INTERVAL_SECONDS) {
111
105
  const key = this.mintKey(key_1.KeyType.WORK_ITEMS, { appId: this.appId, scoutType });
112
106
  const success = await this.exec('SET', key, `${scoutType}:${(0, utils_1.formatISODate)(new Date())}`, 'NX', 'EX', `${delay - 1}`);
@@ -134,7 +128,6 @@ class StoreService {
134
128
  throw new Error('settings not found');
135
129
  }
136
130
  async setSettings(manifest) {
137
- //HotMesh heartbeat. If a connection is made, the version will be set
138
131
  const params = {};
139
132
  const key = this.mintKey(key_1.KeyType.HOTMESH, params);
140
133
  return await this.redisClient[this.commands.hset](key, manifest);
@@ -142,10 +135,8 @@ class StoreService {
142
135
  async reserveSymbolRange(target, size, type) {
143
136
  const rangeKey = this.mintKey(key_1.KeyType.SYMKEYS, { appId: this.appId });
144
137
  const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, { activityId: target, appId: this.appId });
145
- //reserve the slot in a `pending` state (range will be established in the next step)
146
138
  const response = await this.redisClient[this.commands.hsetnx](rangeKey, target, '?:?');
147
139
  if (response) {
148
- //if the key didn't exist, set the inclusive range and seed metadata fields
149
140
  const upperLimit = await this.redisClient[this.commands.hincrby](rangeKey, ':cursor', size);
150
141
  const lowerLimit = upperLimit - size;
151
142
  const inclusiveRange = `${lowerLimit}:${upperLimit - 1}`;
@@ -155,7 +146,6 @@ class StoreService {
155
146
  return [lowerLimit + serializer_1.MDATA_SYMBOLS.SLOTS, upperLimit - 1, {}];
156
147
  }
157
148
  else {
158
- //if the key already existed, get the lower limit and add the number of symbols
159
149
  const range = await this.redisClient[this.commands.hget](rangeKey, target);
160
150
  const [lowerLimitString] = range.split(':');
161
151
  const lowerLimit = parseInt(lowerLimitString, 10);
@@ -167,7 +157,6 @@ class StoreService {
167
157
  }
168
158
  }
169
159
  async getAllSymbols() {
170
- //get hash with all reserved symbol ranges
171
160
  const rangeKey = this.mintKey(key_1.KeyType.SYMKEYS, { appId: this.appId });
172
161
  const ranges = await this.redisClient[this.commands.hgetall](rangeKey);
173
162
  const rangeKeys = Object.keys(ranges).sort();
@@ -340,11 +329,6 @@ class StoreService {
340
329
  };
341
330
  return await this.redisClient[this.commands.hset](key, payload);
342
331
  }
343
- /**
344
- * Registers the job, `jobId`, with `originJobId`. In the future,
345
- * when `originJobId` is interrupted/expired, the items in the
346
- * list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
347
- */
348
332
  async registerJobDependency(depType, originJobId, topic, jobId, gId, pd = '', multi) {
349
333
  const privateMulti = multi || this.getMulti();
350
334
  const dependencyParams = {
@@ -364,15 +348,10 @@ class StoreService {
364
348
  return await privateMulti.exec();
365
349
  }
366
350
  }
367
- /**
368
- * Ensures a `hook signal` is delisted when its parent activity/job
369
- * is interrupted/expired.
370
- */
371
351
  async registerSignalDependency(jobId, signalKey, dad, multi) {
372
352
  const privateMulti = multi || this.getMulti();
373
353
  const dependencyParams = { appId: this.appId, jobId };
374
354
  const dependencyKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
375
- //persiste dependency tasks as multi-segment composite keys
376
355
  const delistTask = [
377
356
  'delist',
378
357
  'signal',
@@ -409,7 +388,6 @@ class StoreService {
409
388
  }
410
389
  }
411
390
  hGetAllResult(result) {
412
- //default response signature uses 'redis' NPM Package format
413
391
  return result;
414
392
  }
415
393
  async getJobStats(jobKeys) {
@@ -438,13 +416,12 @@ class StoreService {
438
416
  async getJobIds(indexKeys, idRange) {
439
417
  const multi = this.getMulti();
440
418
  for (const idsKey of indexKeys) {
441
- multi[this.commands.lrange](idsKey, idRange[0], idRange[1]); //0,-1 returns all ids
419
+ multi[this.commands.lrange](idsKey, idRange[0], idRange[1]);
442
420
  }
443
421
  const results = await multi.exec();
444
422
  const output = {};
445
423
  for (const [index, result] of results.entries()) {
446
424
  const key = indexKeys[index];
447
- //todo: resolve this discrepancy between redis/ioredis
448
425
  const idsList = result[1] || result;
449
426
  if (idsList && idsList.length > 0) {
450
427
  output[key] = idsList;
@@ -483,10 +460,6 @@ class StoreService {
483
460
  await (multi || this.redisClient)[this.commands.hset](hashKey, hashData);
484
461
  return jobId;
485
462
  }
486
- /**
487
- * Returns custom search fields and values.
488
- * NOTE: The `fields` param should NOT prefix items with an underscore.
489
- */
490
463
  async getQueryState(jobId, fields) {
491
464
  const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
492
465
  const _fields = fields.map(field => `_${field}`);
@@ -498,7 +471,6 @@ class StoreService {
498
471
  return jobData;
499
472
  }
500
473
  async getState(jobId, consumes, dIds) {
501
- //get abbreviated field list (the symbols for the paths)
502
474
  const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
503
475
  const symbolNames = Object.keys(consumes);
504
476
  const symKeys = await this.getSymbolKeys(symbolNames);
@@ -506,7 +478,7 @@ class StoreService {
506
478
  const fields = this.serializer.abbreviate(consumes, symbolNames, [':']);
507
479
  const jobDataArray = await this.redisClient[this.commands.hmget](key, fields);
508
480
  const jobData = {};
509
- let atLeast1 = false; //if status field (':') isn't present assume 404
481
+ let atLeast1 = false;
510
482
  fields.forEach((field, index) => {
511
483
  if (jobDataArray[index]) {
512
484
  atLeast1 = true;
@@ -537,13 +509,9 @@ class StoreService {
537
509
  }
538
510
  return job;
539
511
  }
540
- /**
541
- * collate is a generic method for incrementing a value in a hash
542
- * in order to track their progress during processing.
543
- */
544
512
  async collate(jobId, activityId, amount, dIds, multi) {
545
513
  const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
546
- const collationKey = `${activityId}/output/metadata/as`; //activity state
514
+ const collationKey = `${activityId}/output/metadata/as`;
547
515
  const symbolNames = [activityId];
548
516
  const symKeys = await this.getSymbolKeys(symbolNames);
549
517
  const symVals = await this.getSymbolValues();
@@ -553,12 +521,6 @@ class StoreService {
553
521
  const targetId = Object.keys(hashData)[0];
554
522
  return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, targetId, amount);
555
523
  }
556
- /**
557
- * synthentic collation affects those activities in the graph
558
- * that represent the synthetic DAG that was materialized during compilation;
559
- * Synthetic targeting ensures that re-entry due to failure can be distinguished from
560
- * purposeful re-entry.
561
- */
562
524
  async collateSynthetic(jobId, guid, amount, multi) {
563
525
  const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
564
526
  return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, guid, amount);
@@ -697,12 +659,10 @@ class StoreService {
697
659
  }
698
660
  async setHookSignal(hook, multi) {
699
661
  const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
700
- //destructure the hook key
701
662
  const { topic, resolved, jobId } = hook;
702
663
  const signalKey = `${topic}:${resolved}`;
703
664
  const payload = { [signalKey]: jobId };
704
665
  await (multi || this.redisClient)[this.commands.hset](key, payload);
705
- //jobId needs even more destructuring
706
666
  const [_aid, dad, _gid, jid] = jobId.split(key_1.VALSEP);
707
667
  return await this.registerSignalDependency(jid, signalKey, dad, multi);
708
668
  }
@@ -741,7 +701,6 @@ class StoreService {
741
701
  const didRemove = await this.redisClient[this.commands.zrem](zsetKey, workItemKey);
742
702
  if (didRemove) {
743
703
  if (scrub) {
744
- //indexes can be designed to be self-cleaning; `engine.hookAll` exposes this option
745
704
  this.redisClient[this.commands.expire](processedKey, 0);
746
705
  this.redisClient[this.commands.expire](key.split(":").slice(0, 5).join(":"), 0);
747
706
  }
@@ -760,11 +719,6 @@ class StoreService {
760
719
  await this.redisClient[this.commands.expire](jobKey, inSeconds);
761
720
  }
762
721
  }
763
- /**
764
- * register the descendants of an expired origin flow to be
765
- * expired at a future date; options indicate whether this
766
- * is a standard `expire` or an `interrupt`
767
- */
768
722
  async registerDependenciesForCleanup(jobId, deletionTime, options) {
769
723
  const depParams = { appId: this.appId, jobId };
770
724
  const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
@@ -778,15 +732,8 @@ class StoreService {
778
732
  const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
779
733
  return this.redisClient[this.commands.lrange](depKey, 0, -1);
780
734
  }
781
- /**
782
- * registers a hook activity to be awakened (uses ZSET to
783
- * store the 'sleep group' and LIST to store the events
784
- * for the given sleep group. Sleep groups are
785
- * organized into 'n'-second blocks (LISTS))
786
- */
787
735
  async registerTimeHook(jobId, gId, activityId, type, deletionTime, dad, multi) {
788
736
  const listKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId, timeValue: deletionTime });
789
- //construct the composite key (the key has enough info to signal the hook)
790
737
  const timeEvent = [
791
738
  type,
792
739
  activityId,
@@ -807,7 +754,6 @@ class StoreService {
807
754
  let [pType, pKey] = this.resolveTaskKeyContext(listKey);
808
755
  const timeEvent = await this.redisClient[this.commands.lpop](pKey);
809
756
  if (timeEvent) {
810
- //deconstruct composite key
811
757
  let [type, activityId, gId, _pd, ...jobId] = timeEvent.split(key_1.VALSEP);
812
758
  const jid = jobId.join(key_1.VALSEP);
813
759
  if (type === 'delist') {
@@ -826,14 +772,6 @@ class StoreService {
826
772
  }
827
773
  return false;
828
774
  }
829
- /**
830
- * when processing time jobs, the target LIST ID returned
831
- * from the ZSET query can be prefixed to denote what to
832
- * do with the work list. (not everything is known in advance,
833
- * so the ZSET key defines HOW to approach the work in the
834
- * generic LIST (lists typically contain target job ids)
835
- * @param {string} listKey - composite key
836
- */
837
775
  resolveTaskKeyContext(listKey) {
838
776
  if (listKey.startsWith(`${key_1.TYPSEP}INTERRUPT`)) {
839
777
  return ['interrupt', listKey.split(key_1.TYPSEP)[2]];
@@ -845,38 +783,24 @@ class StoreService {
845
783
  return ['sleep', listKey];
846
784
  }
847
785
  }
848
- /**
849
- * Interrupts a job and sets sets a job error (410), if 'throw'!=false.
850
- * This method is called by the engine and not by an activity and is
851
- * followed by a call to execute job completion/cleanup tasks
852
- * associated with a job completion event.
853
- *
854
- * Todo: move most of this logic to the engine (too much logic for the store)
855
- */
856
786
  async interrupt(topic, jobId, options = {}) {
857
787
  try {
858
- //verify job exists
859
788
  const status = await this.getStatus(jobId, this.appId);
860
789
  if (status <= 0) {
861
- //verify still active; job already completed
862
790
  throw new Error(`Job ${jobId} already completed`);
863
791
  }
864
- //decrement job status (:) by 1bil
865
792
  const amount = -1000000000;
866
793
  const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
867
794
  const result = await this.redisClient[this.commands.hincrbyfloat](jobKey, ':', amount);
868
795
  if (result <= amount) {
869
- //verify active state; job already interrupted
870
796
  throw new Error(`Job ${jobId} already completed`);
871
797
  }
872
- //persist the error unless specifically told not to
873
798
  if (options.throw !== false) {
874
- const errKey = `metadata/err`; //job errors are stored at the path `metadata/err`
875
- const symbolNames = [`$${topic}`]; //the symbol for `metadata/err` is in redis and stored using the job topic
799
+ const errKey = `metadata/err`;
800
+ const symbolNames = [`$${topic}`];
876
801
  const symKeys = await this.getSymbolKeys(symbolNames);
877
802
  const symVals = await this.getSymbolValues();
878
803
  this.serializer.resetSymbols(symKeys, symVals, {});
879
- //persists the standard 410 error (job is `gone`)
880
804
  const err = JSON.stringify({
881
805
  code: options.code ?? enums_1.HMSH_CODE_INTERRUPT,
882
806
  message: options.reason ?? `job [${jobId}] interrupted`,
@@ -1,7 +1,7 @@
1
1
  import { KeyStoreParams, KeyType } from '../../../modules/key';
2
2
  import { ILogger } from '../../logger';
3
3
  import { StreamService } from '../index';
4
- import { RedisClientType, RedisMultiType } from '../../../types/ioredisclient';
4
+ import { IORedisClientType as RedisClientType, IORedisMultiType as RedisMultiType } from '../../../types/redis';
5
5
  import { ReclaimedMessageType } from '../../../types/stream';
6
6
  declare class IORedisStreamService extends StreamService<RedisClientType, RedisMultiType> {
7
7
  redisClient: RedisClientType;
@@ -51,10 +51,7 @@ class IORedisStreamService extends index_1.StreamService {
51
51
  }
52
52
  async xreadgroup(command, groupName, consumerName, blockOption, blockTime, streamsOption, streamName, id) {
53
53
  try {
54
- //@ts-ignore
55
- return await this.redisClient.xreadgroup(command, groupName, consumerName,
56
- // @ts-ignore
57
- blockOption, blockTime, streamsOption, streamName, id);
54
+ return await this.redisClient.xreadgroup(command, groupName, consumerName, blockOption, blockTime, streamsOption, streamName, id);
58
55
  }
59
56
  catch (error) {
60
57
  this.logger.error(`Error reading stream data [Stream ${streamName}] [Group ${groupName}]`, { ...error });
@@ -1,7 +1,7 @@
1
1
  import { KeyStoreParams, KeyType } from '../../../modules/key';
2
2
  import { ILogger } from '../../logger';
3
3
  import { StreamService } from '../index';
4
- import { RedisClientType, RedisMultiType } from '../../../types/redisclient';
4
+ import { RedisRedisClientType as RedisClientType, RedisRedisMultiType as RedisMultiType } from '../../../types/redis';
5
5
  import { ReclaimedMessageType } from '../../../types/stream';
6
6
  declare class RedisStreamService extends StreamService<RedisClientType, RedisMultiType> {
7
7
  redisClient: RedisClientType;
@@ -1,7 +1,7 @@
1
1
  import { KeyStoreParams, KeyType } from '../../../modules/key';
2
2
  import { ILogger } from '../../logger';
3
3
  import { SubService } from '../index';
4
- import { RedisClientType, RedisMultiType } from '../../../types/ioredisclient';
4
+ import { IORedisClientType as RedisClientType, IORedisMultiType as RedisMultiType } from '../../../types/redis';
5
5
  import { SubscriptionCallback } from '../../../types/quorum';
6
6
  declare class IORedisSubService extends SubService<RedisClientType, RedisMultiType> {
7
7
  redisClient: RedisClientType;
@@ -1,7 +1,7 @@
1
1
  import { KeyStoreParams, KeyType } from '../../../modules/key';
2
2
  import { ILogger } from '../../logger';
3
3
  import { SubService } from '../index';
4
- import { RedisClientType, RedisMultiType } from '../../../types/redisclient';
4
+ import { RedisRedisClientType as RedisClientType, RedisRedisMultiType as RedisMultiType } from '../../../types/redis';
5
5
  import { SubscriptionCallback } from '../../../types/quorum';
6
6
  declare class RedisSubService extends SubService<RedisClientType, RedisMultiType> {
7
7
  redisClient: RedisClientType;
@@ -16,16 +16,7 @@ declare class TaskService {
16
16
  enqueueWorkItems(keys: string[]): Promise<void>;
17
17
  registerJobForCleanup(jobId: string, inSeconds: number, options: JobCompletionOptions): Promise<void>;
18
18
  registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, inSeconds: number, dad: string, multi?: RedisMulti): Promise<void>;
19
- /**
20
- * Should this engine instance play the role of 'scout' on behalf
21
- * of the entire quorum? The scout role is responsible for processing
22
- * task lists on behalf of the collective.
23
- */
24
19
  shouldScout(): Promise<boolean>;
25
- /**
26
- * Callback handler that takes an item from a work list and
27
- * processes according to its type
28
- */
29
20
  processTimeHooks(timeEventCallback: (jobId: string, gId: string, activityId: string, type: WorkListTaskType) => Promise<void>, listKey?: string): Promise<void>;
30
21
  cancelCleanup(): void;
31
22
  getHookRule(topic: string): Promise<HookRule | undefined>;
@@ -22,7 +22,6 @@ class TaskService {
22
22
  const destinationKey = `${sourceKey}:processed`;
23
23
  const jobId = await this.store.processTaskQueue(sourceKey, destinationKey);
24
24
  if (jobId) {
25
- //todo: don't use 'id', make configurable using hook rule
26
25
  await hookEventCallback(topic, { ...data, id: jobId });
27
26
  }
28
27
  else {
@@ -49,11 +48,6 @@ class TaskService {
49
48
  const awakenTimeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
50
49
  await this.store.registerTimeHook(jobId, gId, activityId, type, awakenTimeSlot, dad, multi);
51
50
  }
52
- /**
53
- * Should this engine instance play the role of 'scout' on behalf
54
- * of the entire quorum? The scout role is responsible for processing
55
- * task lists on behalf of the collective.
56
- */
57
51
  async shouldScout() {
58
52
  const wasScout = this.isScout;
59
53
  const isScout = wasScout || (this.isScout = await this.store.reserveScoutRole('time'));
@@ -67,10 +61,6 @@ class TaskService {
67
61
  }
68
62
  return false;
69
63
  }
70
- /**
71
- * Callback handler that takes an item from a work list and
72
- * processes according to its type
73
- */
74
64
  async processTimeHooks(timeEventCallback, listKey) {
75
65
  if (await this.shouldScout()) {
76
66
  try {
@@ -78,16 +68,12 @@ class TaskService {
78
68
  if (Array.isArray(workListTask)) {
79
69
  const [listKey, target, gId, activityId, type] = workListTask;
80
70
  if (type === 'child') {
81
- //continue; this child is listed here for convenience, but
82
- // will be expired by an origin ancestor and is listed there
83
71
  }
84
72
  else if (type === 'delist') {
85
- //delist the signalKey (target)
86
73
  const key = this.store.mintKey(hotmesh_1.KeyType.SIGNALS, { appId: this.store.appId });
87
74
  await this.store.redisClient[this.store.commands.hdel](key, target);
88
75
  }
89
76
  else {
90
- //awaken/expire/interrupt
91
77
  await timeEventCallback(target, gId, activityId, type);
92
78
  }
93
79
  await (0, utils_1.sleepFor)(0);
@@ -95,13 +81,11 @@ class TaskService {
95
81
  this.processTimeHooks(timeEventCallback, listKey);
96
82
  }
97
83
  else if (workListTask) {
98
- //a worklist was just emptied; try again immediately
99
84
  await (0, utils_1.sleepFor)(0);
100
85
  this.errorCount = 0;
101
86
  this.processTimeHooks(timeEventCallback);
102
87
  }
103
88
  else {
104
- //no worklists exist; sleep before checking
105
89
  let sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_FIDELITY_SECONDS * 1000);
106
90
  this.cleanupTimeout = sleep.timerId;
107
91
  await sleep.promise;
@@ -110,8 +94,6 @@ class TaskService {
110
94
  }
111
95
  }
112
96
  catch (err) {
113
- //most common reasons: deleted job not found; container stopping; test stopping
114
- //less common: redis/cluster down; retry with fallback (5s max main reassignment)
115
97
  this.logger.warn('task-process-timehooks-error', err);
116
98
  await (0, utils_1.sleepFor)(1000 * this.errorCount++);
117
99
  if (this.errorCount < 5) {
@@ -120,7 +102,6 @@ class TaskService {
120
102
  }
121
103
  }
122
104
  else {
123
- //didn't get the scout role; try again in 'one-ish' minutes
124
105
  let sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_SCOUT_INTERVAL_SECONDS * 1000 * 2 * Math.random());
125
106
  this.cleanupTimeout = sleep.timerId;
126
107
  await sleep.promise;
@@ -145,7 +126,6 @@ class TaskService {
145
126
  const jobId = context.metadata.jid;
146
127
  const gId = context.metadata.gid;
147
128
  const activityId = hookRule.to;
148
- //composite keys are used to fully describe the task target
149
129
  const compositeJobKey = [
150
130
  activityId,
151
131
  dad,
@@ -167,22 +147,13 @@ class TaskService {
167
147
  async processWebHookSignal(topic, data) {
168
148
  const hookRule = await this.getHookRule(topic);
169
149
  if (hookRule) {
170
- //NOTE: both formats are supported by the mapping engine:
171
- // `$self.hook.data` OR `$hook.data`
172
150
  const context = { $self: { hook: { data } }, $hook: { data } };
173
151
  const mapExpression = hookRule.conditions.match[0].actual;
174
152
  const resolved = pipe_1.Pipe.resolve(mapExpression, context);
175
153
  const hookSignalId = await this.store.getHookSignal(topic, resolved);
176
154
  if (!hookSignalId) {
177
- //messages can be double-processed; not an issue; return `undefined`
178
- //users can also provide a bogus topic; not an issue; return `undefined`
179
155
  return undefined;
180
156
  }
181
- //`aid` is part of composite key, but the hook `topic` is its public interface;
182
- // this means that a new version of the graph can be deployed and the
183
- // topic can be re-mapped to a different activity id. Outside callers
184
- // can adhere to the unchanged contract (calling the same topic),
185
- // while the internal system can be updated in real-time as necessary.
186
157
  const [_aid, dad, gid, ...jid] = hookSignalId.split(key_1.WEBSEP);
187
158
  return [jid.join(key_1.WEBSEP), hookRule.to, dad, gid];
188
159
  }
@@ -193,8 +164,6 @@ class TaskService {
193
164
  async deleteWebHookSignal(topic, data) {
194
165
  const hookRule = await this.getHookRule(topic);
195
166
  if (hookRule) {
196
- //NOTE: both formats are supported by the mapping engine:
197
- // `$self.hook.data` OR `$hook.data`
198
167
  const context = { $self: { hook: { data } }, $hook: { data } };
199
168
  const mapExpression = hookRule.conditions.match[0].actual;
200
169
  const resolved = pipe_1.Pipe.resolve(mapExpression, context);
@@ -35,13 +35,6 @@ declare class TelemetryService {
35
35
  setTelemetryContext(span: Span, leg: number): void;
36
36
  setActivityError(message: string): void;
37
37
  setStreamError(message: string): void;
38
- /**
39
- * Adds the paths (HGET) necessary to restore telemetry state for an activity
40
- * @param consumes
41
- * @param config
42
- * @param metadata
43
- * @param leg
44
- */
45
38
  static addTargetTelemetryPaths(consumes: Consumes, config: ActivityType, metadata: ActivityMetadata, leg: number): void;
46
39
  static bindJobTelemetryToState(state: StringStringType, config: ActivityType, context: JobState): void;
47
40
  static bindActivityTelemetryToState(state: StringAnyType, config: ActivityType, metadata: ActivityMetadata, context: JobState, leg: number): void;
@@ -13,7 +13,6 @@ class TelemetryService {
13
13
  constructor(appId, config, metadata, context) {
14
14
  this.leg = 1;
15
15
  this.appId = appId;
16
- //these are REQUIRED for job and activity spans
17
16
  this.config = config;
18
17
  this.metadata = metadata;
19
18
  this.context = context;
@@ -82,7 +81,6 @@ class TelemetryService {
82
81
  return span;
83
82
  }
84
83
  mapActivityAttributes() {
85
- //export user-defined span attributes (app.activity.data.*)
86
84
  if (this.config.telemetry) {
87
85
  const telemetryAtts = new mapper_1.MapperService(this.config.telemetry, this.context).mapRules();
88
86
  const namespacedAtts = {
@@ -123,7 +121,7 @@ class TelemetryService {
123
121
  traceId: this.traceId,
124
122
  spanId: this.spanId,
125
123
  isRemote: true,
126
- traceFlags: 1, // (todo: revisit sampling strategy/config)
124
+ traceFlags: 1,
127
125
  };
128
126
  const parentContext = telemetry_1.trace.setSpanContext(telemetry_1.context.active(), restoredSpanContext);
129
127
  return parentContext;
@@ -179,13 +177,6 @@ class TelemetryService {
179
177
  setStreamError(message) {
180
178
  this.span?.setStatus({ code: telemetry_1.SpanStatusCode.ERROR, message });
181
179
  }
182
- /**
183
- * Adds the paths (HGET) necessary to restore telemetry state for an activity
184
- * @param consumes
185
- * @param config
186
- * @param metadata
187
- * @param leg
188
- */
189
180
  static addTargetTelemetryPaths(consumes, config, metadata, leg) {
190
181
  if (leg === 1) {
191
182
  if (!(config.parent in consumes)) {
@@ -207,17 +198,14 @@ class TelemetryService {
207
198
  }
208
199
  static bindActivityTelemetryToState(state, config, metadata, context, leg) {
209
200
  if (config.type === 'trigger') {
210
- //trigger activities run non-duplexed and only have a single leg (2)
211
201
  state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
212
202
  state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l2s;
213
203
  }
214
204
  else if (utils_1.polyfill.resolveActivityType(config.type) === 'hook' && leg === 1) {
215
- //hook activities run non-duplexed and only have a single leg (1)
216
205
  state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
217
206
  state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l1s;
218
207
  }
219
208
  else if (config.type === 'signal' && leg === 1) {
220
- //signal activities run non-duplexed and only have a single leg (1)
221
209
  state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
222
210
  state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l1s;
223
211
  }
@@ -30,10 +30,6 @@ declare class WorkerService {
30
30
  initStreamChannel(service: WorkerService, stream: RedisClient): Promise<void>;
31
31
  initRouter(worker: HotMeshWorker, logger: ILogger): Router;
32
32
  subscriptionHandler(): SubscriptionCallback;
33
- /**
34
- * A quorum-wide command to broadcaset system details.
35
- *
36
- */
37
33
  doRollCall(message: RollCallMessage): Promise<void>;
38
34
  cancelRollCall(): void;
39
35
  stop(): void;
@@ -97,7 +97,7 @@ class WorkerService {
97
97
  return async (topic, message) => {
98
98
  self.logger.debug('worker-event-received', { topic, type: message.type });
99
99
  if (message.type === 'throttle') {
100
- if (message.topic !== null) { //undefined allows passthrough
100
+ if (message.topic !== null) {
101
101
  self.throttle(message.throttle);
102
102
  }
103
103
  }
@@ -105,16 +105,12 @@ class WorkerService {
105
105
  self.sayPong(self.appId, self.guid, message.originator, message.details);
106
106
  }
107
107
  else if (message.type === 'rollcall') {
108
- if (message.topic !== null) { //undefined allows passthrough
108
+ if (message.topic !== null) {
109
109
  self.doRollCall(message);
110
110
  }
111
111
  }
112
112
  };
113
113
  }
114
- /**
115
- * A quorum-wide command to broadcaset system details.
116
- *
117
- */
118
114
  async doRollCall(message) {
119
115
  let iteration = 0;
120
116
  let max = !isNaN(message.max) ? message.max : enums_1.HMSH_QUORUM_ROLLCALL_CYCLES;