@hotmeshio/hotmesh 0.1.14 → 0.1.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/README.md +623 -209
  2. package/build/index.d.ts +14 -3
  3. package/build/index.js +17 -4
  4. package/build/modules/enums.d.ts +12 -12
  5. package/build/modules/enums.js +15 -25
  6. package/build/modules/errors.d.ts +16 -16
  7. package/build/modules/errors.js +28 -28
  8. package/build/modules/key.d.ts +0 -37
  9. package/build/modules/key.js +4 -45
  10. package/build/modules/utils.d.ts +7 -15
  11. package/build/modules/utils.js +21 -44
  12. package/build/package.json +18 -15
  13. package/build/services/activities/activity.d.ts +0 -31
  14. package/build/services/activities/activity.js +1 -50
  15. package/build/services/activities/await.js +0 -4
  16. package/build/services/activities/cycle.d.ts +0 -7
  17. package/build/services/activities/cycle.js +1 -16
  18. package/build/services/activities/hook.d.ts +0 -6
  19. package/build/services/activities/hook.js +2 -12
  20. package/build/services/activities/interrupt.js +0 -8
  21. package/build/services/activities/signal.d.ts +0 -6
  22. package/build/services/activities/signal.js +0 -15
  23. package/build/services/activities/trigger.d.ts +4 -5
  24. package/build/services/activities/trigger.js +22 -16
  25. package/build/services/activities/worker.js +0 -4
  26. package/build/services/collator/index.d.ts +0 -70
  27. package/build/services/collator/index.js +1 -91
  28. package/build/services/compiler/deployer.js +6 -38
  29. package/build/services/compiler/index.d.ts +0 -15
  30. package/build/services/compiler/index.js +0 -20
  31. package/build/services/compiler/validator.d.ts +0 -3
  32. package/build/services/compiler/validator.js +0 -25
  33. package/build/services/connector/clients/ioredis.js +0 -2
  34. package/build/services/connector/clients/redis.js +0 -2
  35. package/build/services/connector/index.js +0 -2
  36. package/build/services/engine/index.d.ts +1 -10
  37. package/build/services/engine/index.js +1 -48
  38. package/build/services/exporter/index.d.ts +0 -27
  39. package/build/services/exporter/index.js +0 -33
  40. package/build/services/hotmesh/index.d.ts +8 -4
  41. package/build/services/hotmesh/index.js +20 -19
  42. package/build/services/logger/index.js +0 -2
  43. package/build/services/mapper/index.d.ts +0 -14
  44. package/build/services/mapper/index.js +0 -14
  45. package/build/services/meshcall/index.d.ts +21 -0
  46. package/build/services/meshcall/index.js +202 -0
  47. package/build/services/meshcall/schemas/factory.d.ts +2 -0
  48. package/build/services/meshcall/schemas/factory.js +179 -0
  49. package/build/services/meshdata/index.d.ts +75 -0
  50. package/build/services/meshdata/index.js +541 -0
  51. package/build/services/meshflow/client.d.ts +18 -0
  52. package/build/services/{durable → meshflow}/client.js +9 -40
  53. package/build/services/{durable → meshflow}/connection.d.ts +2 -1
  54. package/build/services/{durable → meshflow}/connection.js +1 -0
  55. package/build/services/meshflow/exporter.d.ts +29 -0
  56. package/build/services/{durable → meshflow}/exporter.js +0 -29
  57. package/build/services/meshflow/handle.d.ts +22 -0
  58. package/build/services/{durable → meshflow}/handle.js +0 -46
  59. package/build/services/meshflow/index.d.ts +17 -0
  60. package/build/services/meshflow/index.js +23 -0
  61. package/build/services/meshflow/schemas/factory.d.ts +4 -0
  62. package/build/services/{durable → meshflow}/schemas/factory.js +3 -31
  63. package/build/services/meshflow/search.d.ts +23 -0
  64. package/build/services/{durable → meshflow}/search.js +0 -99
  65. package/build/services/{durable → meshflow}/worker.d.ts +3 -2
  66. package/build/services/{durable → meshflow}/worker.js +23 -39
  67. package/build/services/meshflow/workflow.d.ts +27 -0
  68. package/build/services/{durable → meshflow}/workflow.js +27 -169
  69. package/build/services/pipe/functions/date.d.ts +0 -7
  70. package/build/services/pipe/functions/date.js +0 -7
  71. package/build/services/pipe/functions/math.js +0 -2
  72. package/build/services/pipe/index.d.ts +0 -15
  73. package/build/services/pipe/index.js +2 -23
  74. package/build/services/quorum/index.d.ts +1 -7
  75. package/build/services/quorum/index.js +0 -21
  76. package/build/services/reporter/index.d.ts +0 -5
  77. package/build/services/reporter/index.js +0 -9
  78. package/build/services/router/index.d.ts +0 -9
  79. package/build/services/router/index.js +2 -30
  80. package/build/services/serializer/index.js +6 -23
  81. package/build/services/store/cache.d.ts +0 -19
  82. package/build/services/store/cache.js +0 -19
  83. package/build/services/store/clients/ioredis.d.ts +0 -6
  84. package/build/services/store/clients/ioredis.js +0 -7
  85. package/build/services/store/clients/redis.d.ts +0 -6
  86. package/build/services/store/clients/redis.js +0 -6
  87. package/build/services/store/index.d.ts +0 -55
  88. package/build/services/store/index.js +14 -87
  89. package/build/services/stream/clients/ioredis.js +1 -4
  90. package/build/services/task/index.d.ts +0 -9
  91. package/build/services/task/index.js +0 -31
  92. package/build/services/telemetry/index.d.ts +0 -7
  93. package/build/services/telemetry/index.js +1 -13
  94. package/build/services/worker/index.d.ts +1 -4
  95. package/build/services/worker/index.js +0 -6
  96. package/build/types/activity.d.ts +0 -81
  97. package/build/types/error.d.ts +5 -5
  98. package/build/types/exporter.d.ts +1 -14
  99. package/build/types/hotmesh.d.ts +4 -12
  100. package/build/types/hotmesh.js +0 -3
  101. package/build/types/index.d.ts +5 -3
  102. package/build/types/index.js +1 -1
  103. package/build/types/job.d.ts +1 -95
  104. package/build/types/meshcall.d.ts +54 -0
  105. package/build/types/meshdata.d.ts +59 -0
  106. package/build/types/meshdata.js +2 -0
  107. package/build/types/meshflow.d.ts +202 -0
  108. package/build/types/meshflow.js +2 -0
  109. package/build/types/pipe.d.ts +0 -65
  110. package/build/types/quorum.d.ts +0 -12
  111. package/build/types/redis.d.ts +0 -6
  112. package/build/types/stream.d.ts +0 -59
  113. package/build/types/stream.js +0 -4
  114. package/index.ts +22 -3
  115. package/package.json +18 -15
  116. package/typedoc.json +38 -0
  117. package/types/error.ts +5 -5
  118. package/types/exporter.ts +1 -1
  119. package/types/hotmesh.ts +3 -2
  120. package/types/index.ts +25 -7
  121. package/types/job.ts +19 -1
  122. package/types/meshcall.ts +123 -0
  123. package/types/meshdata.ts +273 -0
  124. package/types/{durable.ts → meshflow.ts} +33 -9
  125. package/build/services/durable/client.d.ts +0 -49
  126. package/build/services/durable/exporter.d.ts +0 -51
  127. package/build/services/durable/handle.d.ts +0 -58
  128. package/build/services/durable/index.d.ts +0 -19
  129. package/build/services/durable/index.js +0 -25
  130. package/build/services/durable/schemas/factory.d.ts +0 -33
  131. package/build/services/durable/search.d.ts +0 -120
  132. package/build/services/durable/workflow.d.ts +0 -143
  133. package/build/types/durable.d.ts +0 -467
  134. /package/build/types/{durable.js → meshcall.js} +0 -0
@@ -42,11 +42,6 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
42
42
  zRangeByScore(key: string, score: number | string, value: string | number): Promise<string | null>;
43
43
  mintKey(type: KeyType, params: KeyStoreParams): string;
44
44
  invalidateCache(): void;
45
- /**
46
- * At any given time only a single engine will
47
- * check for and process work items in the
48
- * time and signal task queues.
49
- */
50
45
  reserveScoutRole(scoutType: 'time' | 'signal' | 'activate', delay?: number): Promise<boolean>;
51
46
  releaseScoutRole(scoutType: 'time' | 'signal' | 'activate'): Promise<boolean>;
52
47
  getSettings(bCreate?: boolean): Promise<HotMeshSettings>;
@@ -65,16 +60,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
65
60
  setApp(id: string, version: string): Promise<HotMeshApp>;
66
61
  activateAppVersion(id: string, version: string): Promise<boolean>;
67
62
  registerAppVersion(appId: string, version: string): Promise<any>;
68
- /**
69
- * Registers the job, `jobId`, with `originJobId`. In the future,
70
- * when `originJobId` is interrupted/expired, the items in the
71
- * list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
72
- */
73
63
  registerJobDependency(depType: WorkListTaskType, originJobId: string, topic: string, jobId: string, gId: string, pd?: string, multi?: U): Promise<any>;
74
- /**
75
- * Ensures a `hook signal` is delisted when its parent activity/job
76
- * is interrupted/expired.
77
- */
78
64
  registerSignalDependency(jobId: string, signalKey: string, dad: string, multi?: U): Promise<any>;
79
65
  setStats(jobKey: string, jobId: string, dateTime: string, stats: StatsType, appVersion: AppVID, multi?: U): Promise<any>;
80
66
  hGetAllResult(result: any): any;
@@ -83,24 +69,10 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
83
69
  setStatus(collationKeyStatus: number, jobId: string, appId: string, multi?: U): Promise<any>;
84
70
  getStatus(jobId: string, appId: string): Promise<number>;
85
71
  setState({ ...state }: StringAnyType, status: number | null, jobId: string, symbolNames: string[], dIds: StringStringType, multi?: U): Promise<string>;
86
- /**
87
- * Returns custom search fields and values.
88
- * NOTE: The `fields` param should NOT prefix items with an underscore.
89
- */
90
72
  getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
91
73
  getState(jobId: string, consumes: Consumes, dIds: StringStringType): Promise<[StringAnyType, number] | undefined>;
92
74
  getRaw(jobId: string): Promise<StringStringType>;
93
- /**
94
- * collate is a generic method for incrementing a value in a hash
95
- * in order to track their progress during processing.
96
- */
97
75
  collate(jobId: string, activityId: string, amount: number, dIds: StringStringType, multi?: U): Promise<number>;
98
- /**
99
- * synthentic collation affects those activities in the graph
100
- * that represent the synthetic DAG that was materialized during compilation;
101
- * Synthetic targeting ensures that re-entry due to failure can be distinguished from
102
- * purposeful re-entry.
103
- */
104
76
  collateSynthetic(jobId: string, guid: string, amount: number, multi?: U): Promise<number>;
105
77
  setStateNX(jobId: string, appId: string, status?: number): Promise<boolean>;
106
78
  getSchema(activityId: string, appVersion: AppVID): Promise<ActivityType>;
@@ -121,19 +93,8 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
121
93
  deleteProcessedTaskQueue(workItemKey: string, key: string, processedKey: string, scrub?: boolean): Promise<void>;
122
94
  processTaskQueue(sourceKey: string, destinationKey: string): Promise<any>;
123
95
  expireJob(jobId: string, inSeconds: number, redisMulti?: U): Promise<void>;
124
- /**
125
- * register the descendants of an expired origin flow to be
126
- * expired at a future date; options indicate whether this
127
- * is a standard `expire` or an `interrupt`
128
- */
129
96
  registerDependenciesForCleanup(jobId: string, deletionTime: number, options: JobCompletionOptions): Promise<void>;
130
97
  getDependencies(jobId: string): Promise<string[]>;
131
- /**
132
- * registers a hook activity to be awakened (uses ZSET to
133
- * store the 'sleep group' and LIST to store the events
134
- * for the given sleep group. Sleep groups are
135
- * organized into 'n'-second blocks (LISTS))
136
- */
137
98
  registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, deletionTime: number, dad: string, multi?: U): Promise<void>;
138
99
  getNextTask(listKey?: string): Promise<[
139
100
  listKey: string,
@@ -142,23 +103,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
142
103
  activityId: string,
143
104
  type: WorkListTaskType
144
105
  ] | boolean>;
145
- /**
146
- * when processing time jobs, the target LIST ID returned
147
- * from the ZSET query can be prefixed to denote what to
148
- * do with the work list. (not everything is known in advance,
149
- * so the ZSET key defines HOW to approach the work in the
150
- * generic LIST (lists typically contain target job ids)
151
- * @param {string} listKey - composite key
152
- */
153
106
  resolveTaskKeyContext(listKey: string): [WorkListTaskType, string];
154
- /**
155
- * Interrupts a job and sets sets a job error (410), if 'throw'!=false.
156
- * This method is called by the engine and not by an activity and is
157
- * followed by a call to execute job completion/cleanup tasks
158
- * associated with a job completion event.
159
- *
160
- * Todo: move most of this logic to the engine (too much logic for the store)
161
- */
162
107
  interrupt(topic: string, jobId: string, options?: JobInterruptOptions): Promise<void>;
163
108
  scrub(jobId: string): Promise<void>;
164
109
  findJobs(queryString?: string, limit?: number, batchSize?: number, cursor?: string): Promise<[string, string[]]>;
@@ -77,7 +77,6 @@ class StoreService {
77
77
  return result > 0 || result === 'OK' || result === true;
78
78
  }
79
79
  async zAdd(key, score, value, redisMulti) {
80
- //default call signature uses 'ioredis' NPM Package format
81
80
  return await (redisMulti || this.redisClient)[this.commands.zadd](key, score, value);
82
81
  }
83
82
  async zRangeByScoreWithScores(key, score, value) {
@@ -102,11 +101,6 @@ class StoreService {
102
101
  invalidateCache() {
103
102
  this.cache.invalidate();
104
103
  }
105
- /**
106
- * At any given time only a single engine will
107
- * check for and process work items in the
108
- * time and signal task queues.
109
- */
110
104
  async reserveScoutRole(scoutType, delay = enums_1.HMSH_SCOUT_INTERVAL_SECONDS) {
111
105
  const key = this.mintKey(key_1.KeyType.WORK_ITEMS, {
112
106
  appId: this.appId,
@@ -140,7 +134,6 @@ class StoreService {
140
134
  throw new Error('settings not found');
141
135
  }
142
136
  async setSettings(manifest) {
143
- //HotMesh heartbeat. If a connection is made, the version will be set
144
137
  const params = {};
145
138
  const key = this.mintKey(key_1.KeyType.HOTMESH, params);
146
139
  return await this.redisClient[this.commands.hset](key, manifest);
@@ -151,10 +144,8 @@ class StoreService {
151
144
  activityId: target,
152
145
  appId: this.appId,
153
146
  });
154
- //reserve the slot in a `pending` state (range will be established in the next step)
155
147
  const response = await this.redisClient[this.commands.hsetnx](rangeKey, target, '?:?');
156
148
  if (response) {
157
- //if the key didn't exist, set the inclusive range and seed metadata fields
158
149
  const upperLimit = await this.redisClient[this.commands.hincrby](rangeKey, ':cursor', size);
159
150
  const lowerLimit = upperLimit - size;
160
151
  const inclusiveRange = `${lowerLimit}:${upperLimit - 1}`;
@@ -164,7 +155,6 @@ class StoreService {
164
155
  return [lowerLimit + serializer_1.MDATA_SYMBOLS.SLOTS, upperLimit - 1, {}];
165
156
  }
166
157
  else {
167
- //if the key already existed, get the lower limit and add the number of symbols
168
158
  const range = await this.redisClient[this.commands.hget](rangeKey, target);
169
159
  const [lowerLimitString] = range.split(':');
170
160
  if (lowerLimitString === '?') {
@@ -187,7 +177,6 @@ class StoreService {
187
177
  }
188
178
  }
189
179
  async getAllSymbols() {
190
- //get hash with all reserved symbol ranges
191
180
  const rangeKey = this.mintKey(key_1.KeyType.SYMKEYS, { appId: this.appId });
192
181
  const ranges = await this.redisClient[this.commands.hgetall](rangeKey);
193
182
  const rangeKeys = Object.keys(ranges).sort();
@@ -365,11 +354,6 @@ class StoreService {
365
354
  };
366
355
  return await this.redisClient[this.commands.hset](key, payload);
367
356
  }
368
- /**
369
- * Registers the job, `jobId`, with `originJobId`. In the future,
370
- * when `originJobId` is interrupted/expired, the items in the
371
- * list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
372
- */
373
357
  async registerJobDependency(depType, originJobId, topic, jobId, gId, pd = '', multi) {
374
358
  const privateMulti = multi || this.getMulti();
375
359
  const dependencyParams = {
@@ -383,15 +367,10 @@ class StoreService {
383
367
  return await privateMulti.exec();
384
368
  }
385
369
  }
386
- /**
387
- * Ensures a `hook signal` is delisted when its parent activity/job
388
- * is interrupted/expired.
389
- */
390
370
  async registerSignalDependency(jobId, signalKey, dad, multi) {
391
371
  const privateMulti = multi || this.getMulti();
392
372
  const dependencyParams = { appId: this.appId, jobId };
393
373
  const dependencyKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
394
- //persiste dependency tasks as multi-segment composite keys
395
374
  const delistTask = ['delist', 'signal', jobId, dad, signalKey].join(key_1.VALSEP);
396
375
  privateMulti[this.commands.rpush](dependencyKey, delistTask);
397
376
  if (!multi) {
@@ -427,7 +406,6 @@ class StoreService {
427
406
  }
428
407
  }
429
408
  hGetAllResult(result) {
430
- //default response signature uses 'redis' NPM Package format
431
409
  return result;
432
410
  }
433
411
  async getJobStats(jobKeys) {
@@ -456,13 +434,12 @@ class StoreService {
456
434
  async getJobIds(indexKeys, idRange) {
457
435
  const multi = this.getMulti();
458
436
  for (const idsKey of indexKeys) {
459
- multi[this.commands.lrange](idsKey, idRange[0], idRange[1]); //0,-1 returns all ids
437
+ multi[this.commands.lrange](idsKey, idRange[0], idRange[1]);
460
438
  }
461
439
  const results = await multi.exec();
462
440
  const output = {};
463
441
  for (const [index, result] of results.entries()) {
464
442
  const key = indexKeys[index];
465
- //todo: resolve this discrepancy between redis/ioredis
466
443
  const idsList = result[1] || result;
467
444
  if (idsList && idsList.length > 0) {
468
445
  output[key] = idsList;
@@ -504,22 +481,25 @@ class StoreService {
504
481
  await (multi || this.redisClient)[this.commands.hset](hashKey, hashData);
505
482
  return jobId;
506
483
  }
507
- /**
508
- * Returns custom search fields and values.
509
- * NOTE: The `fields` param should NOT prefix items with an underscore.
510
- */
511
484
  async getQueryState(jobId, fields) {
512
485
  const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
513
- const _fields = fields.map((field) => `_${field}`);
486
+ const _fields = fields.map((field) => {
487
+ if (field.startsWith('"')) {
488
+ return field.slice(1, -1);
489
+ }
490
+ return `_${field}`;
491
+ });
514
492
  const jobDataArray = await this.redisClient[this.commands.hmget](key, _fields);
515
493
  const jobData = {};
516
494
  fields.forEach((field, index) => {
495
+ if (field.startsWith('"')) {
496
+ field = field.slice(1, -1);
497
+ }
517
498
  jobData[field] = jobDataArray[index];
518
499
  });
519
500
  return jobData;
520
501
  }
521
502
  async getState(jobId, consumes, dIds) {
522
- //get abbreviated field list (the symbols for the paths)
523
503
  const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
524
504
  const symbolNames = Object.keys(consumes);
525
505
  const symKeys = await this.getSymbolKeys(symbolNames);
@@ -527,7 +507,7 @@ class StoreService {
527
507
  const fields = this.serializer.abbreviate(consumes, symbolNames, [':']);
528
508
  const jobDataArray = await this.redisClient[this.commands.hmget](key, fields);
529
509
  const jobData = {};
530
- let atLeast1 = false; //if status field (':') isn't present assume 404
510
+ let atLeast1 = false;
531
511
  fields.forEach((field, index) => {
532
512
  if (jobDataArray[index]) {
533
513
  atLeast1 = true;
@@ -561,16 +541,12 @@ class StoreService {
561
541
  }
562
542
  return job;
563
543
  }
564
- /**
565
- * collate is a generic method for incrementing a value in a hash
566
- * in order to track their progress during processing.
567
- */
568
544
  async collate(jobId, activityId, amount, dIds, multi) {
569
545
  const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
570
546
  appId: this.appId,
571
547
  jobId,
572
548
  });
573
- const collationKey = `${activityId}/output/metadata/as`; //activity state
549
+ const collationKey = `${activityId}/output/metadata/as`;
574
550
  const symbolNames = [activityId];
575
551
  const symKeys = await this.getSymbolKeys(symbolNames);
576
552
  const symVals = await this.getSymbolValues();
@@ -580,12 +556,6 @@ class StoreService {
580
556
  const targetId = Object.keys(hashData)[0];
581
557
  return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, targetId, amount);
582
558
  }
583
- /**
584
- * synthentic collation affects those activities in the graph
585
- * that represent the synthetic DAG that was materialized during compilation;
586
- * Synthetic targeting ensures that re-entry due to failure can be distinguished from
587
- * purposeful re-entry.
588
- */
589
559
  async collateSynthetic(jobId, guid, amount, multi) {
590
560
  const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
591
561
  appId: this.appId,
@@ -746,12 +716,10 @@ class StoreService {
746
716
  }
747
717
  async setHookSignal(hook, multi) {
748
718
  const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
749
- //destructure the hook key
750
719
  const { topic, resolved, jobId } = hook;
751
720
  const signalKey = `${topic}:${resolved}`;
752
721
  const payload = { [signalKey]: jobId };
753
722
  await (multi || this.redisClient)[this.commands.hset](key, payload);
754
- //jobId needs even more destructuring
755
723
  const [_aid, dad, _gid, jid] = jobId.split(key_1.VALSEP);
756
724
  return await this.registerSignalDependency(jid, signalKey, dad, multi);
757
725
  }
@@ -790,7 +758,6 @@ class StoreService {
790
758
  const didRemove = await this.redisClient[this.commands.zrem](zsetKey, workItemKey);
791
759
  if (didRemove) {
792
760
  if (scrub) {
793
- //indexes can be designed to be self-cleaning; `engine.hookAll` exposes this option
794
761
  this.redisClient[this.commands.expire](processedKey, 0);
795
762
  this.redisClient[this.commands.expire](key.split(':').slice(0, 5).join(':'), 0);
796
763
  }
@@ -812,11 +779,6 @@ class StoreService {
812
779
  await (redisMulti || this.redisClient)[this.commands.expire](jobKey, inSeconds);
813
780
  }
814
781
  }
815
- /**
816
- * register the descendants of an expired origin flow to be
817
- * expired at a future date; options indicate whether this
818
- * is a standard `expire` or an `interrupt`
819
- */
820
782
  async registerDependenciesForCleanup(jobId, deletionTime, options) {
821
783
  const depParams = { appId: this.appId, jobId };
822
784
  const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
@@ -830,18 +792,11 @@ class StoreService {
830
792
  const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
831
793
  return this.redisClient[this.commands.lrange](depKey, 0, -1);
832
794
  }
833
- /**
834
- * registers a hook activity to be awakened (uses ZSET to
835
- * store the 'sleep group' and LIST to store the events
836
- * for the given sleep group. Sleep groups are
837
- * organized into 'n'-second blocks (LISTS))
838
- */
839
795
  async registerTimeHook(jobId, gId, activityId, type, deletionTime, dad, multi) {
840
796
  const listKey = this.mintKey(key_1.KeyType.TIME_RANGE, {
841
797
  appId: this.appId,
842
798
  timeValue: deletionTime,
843
799
  });
844
- //construct the composite key (the key has enough info to signal the hook)
845
800
  const timeEvent = [type, activityId, gId, dad, jobId].join(key_1.VALSEP);
846
801
  const len = await (multi || this.redisClient)[this.commands.rpush](listKey, timeEvent);
847
802
  if (multi || len === 1) {
@@ -856,7 +811,6 @@ class StoreService {
856
811
  let [pType, pKey] = this.resolveTaskKeyContext(listKey);
857
812
  const timeEvent = await this.redisClient[this.commands.lpop](pKey);
858
813
  if (timeEvent) {
859
- //deconstruct composite key
860
814
  let [type, activityId, gId, _pd, ...jobId] = timeEvent.split(key_1.VALSEP);
861
815
  const jid = jobId.join(key_1.VALSEP);
862
816
  if (type === 'delist') {
@@ -875,14 +829,6 @@ class StoreService {
875
829
  }
876
830
  return false;
877
831
  }
878
- /**
879
- * when processing time jobs, the target LIST ID returned
880
- * from the ZSET query can be prefixed to denote what to
881
- * do with the work list. (not everything is known in advance,
882
- * so the ZSET key defines HOW to approach the work in the
883
- * generic LIST (lists typically contain target job ids)
884
- * @param {string} listKey - composite key
885
- */
886
832
  resolveTaskKeyContext(listKey) {
887
833
  if (listKey.startsWith(`${key_1.TYPSEP}INTERRUPT`)) {
888
834
  return ['interrupt', listKey.split(key_1.TYPSEP)[2]];
@@ -894,23 +840,12 @@ class StoreService {
894
840
  return ['sleep', listKey];
895
841
  }
896
842
  }
897
- /**
898
- * Interrupts a job and sets sets a job error (410), if 'throw'!=false.
899
- * This method is called by the engine and not by an activity and is
900
- * followed by a call to execute job completion/cleanup tasks
901
- * associated with a job completion event.
902
- *
903
- * Todo: move most of this logic to the engine (too much logic for the store)
904
- */
905
843
  async interrupt(topic, jobId, options = {}) {
906
844
  try {
907
- //verify job exists
908
845
  const status = await this.getStatus(jobId, this.appId);
909
846
  if (status <= 0) {
910
- //verify still active; job already completed
911
847
  throw new Error(`Job ${jobId} already completed`);
912
848
  }
913
- //decrement job status (:) by 1bil
914
849
  const amount = -1000000000;
915
850
  const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
916
851
  appId: this.appId,
@@ -918,17 +853,14 @@ class StoreService {
918
853
  });
919
854
  const result = await this.redisClient[this.commands.hincrbyfloat](jobKey, ':', amount);
920
855
  if (result <= amount) {
921
- //verify active state; job already interrupted
922
856
  throw new Error(`Job ${jobId} already completed`);
923
857
  }
924
- //persist the error unless specifically told not to
925
858
  if (options.throw !== false) {
926
- const errKey = `metadata/err`; //job errors are stored at the path `metadata/err`
927
- const symbolNames = [`$${topic}`]; //the symbol for `metadata/err` is in redis and stored using the job topic
859
+ const errKey = `metadata/err`;
860
+ const symbolNames = [`$${topic}`];
928
861
  const symKeys = await this.getSymbolKeys(symbolNames);
929
862
  const symVals = await this.getSymbolValues();
930
863
  this.serializer.resetSymbols(symKeys, symVals, {});
931
- //persists the standard 410 error (job is `gone`)
932
864
  const err = JSON.stringify({
933
865
  code: options.code ?? enums_1.HMSH_CODE_INTERRUPT,
934
866
  message: options.reason ?? `job [${jobId}] interrupted`,
@@ -1006,11 +938,9 @@ class StoreService {
1006
938
  }
1007
939
  async setThrottleRate(options) {
1008
940
  const key = this.mintKey(key_1.KeyType.THROTTLE_RATE, { appId: this.appId });
1009
- //engine guids are session specific. no need to persist
1010
941
  if (options.guid) {
1011
942
  return;
1012
943
  }
1013
- //if a topic, update one
1014
944
  const rate = options.throttle.toString();
1015
945
  if (options.topic) {
1016
946
  await this.redisClient[this.commands.hset](key, {
@@ -1018,7 +948,6 @@ class StoreService {
1018
948
  });
1019
949
  }
1020
950
  else {
1021
- //if no topic, update all
1022
951
  const multi = this.getMulti();
1023
952
  multi[this.commands.del](key);
1024
953
  multi[this.commands.hset](key, { ':': rate });
@@ -1031,7 +960,6 @@ class StoreService {
1031
960
  return response ?? {};
1032
961
  }
1033
962
  async getThrottleRate(topic) {
1034
- //always return a valid number range
1035
963
  const resolveRate = (response, topic) => {
1036
964
  const rate = topic in response ? Number(response[topic]) : 0;
1037
965
  if (isNaN(rate))
@@ -1043,7 +971,6 @@ class StoreService {
1043
971
  const response = await this.getThrottleRates();
1044
972
  const globalRate = resolveRate(response, ':');
1045
973
  if (topic === ':' || !(topic in response)) {
1046
- //use global rate unless worker specifies rate
1047
974
  return globalRate;
1048
975
  }
1049
976
  return resolveRate(response, topic);
@@ -51,10 +51,7 @@ class IORedisStreamService extends index_1.StreamService {
51
51
  }
52
52
  async xreadgroup(command, groupName, consumerName, blockOption, blockTime, streamsOption, streamName, id) {
53
53
  try {
54
- //@ts-ignore
55
- return await this.redisClient.xreadgroup(command, groupName, consumerName,
56
- // @ts-ignore
57
- blockOption, blockTime, streamsOption, streamName, id);
54
+ return await this.redisClient.xreadgroup(command, groupName, consumerName, blockOption, blockTime, streamsOption, streamName, id);
58
55
  }
59
56
  catch (error) {
60
57
  this.logger.error(`Error reading stream data [Stream ${streamName}] [Group ${groupName}]`, { ...error });
@@ -16,16 +16,7 @@ declare class TaskService {
16
16
  enqueueWorkItems(keys: string[]): Promise<void>;
17
17
  registerJobForCleanup(jobId: string, inSeconds: number, options: JobCompletionOptions): Promise<void>;
18
18
  registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, inSeconds: number, dad: string, multi?: RedisMulti): Promise<void>;
19
- /**
20
- * Should this engine instance play the role of 'scout' on behalf
21
- * of the entire quorum? The scout role is responsible for processing
22
- * task lists on behalf of the collective.
23
- */
24
19
  shouldScout(): Promise<boolean>;
25
- /**
26
- * Callback handler that takes an item from a work list and
27
- * processes according to its type
28
- */
29
20
  processTimeHooks(timeEventCallback: (jobId: string, gId: string, activityId: string, type: WorkListTaskType) => Promise<void>, listKey?: string): Promise<void>;
30
21
  cancelCleanup(): void;
31
22
  getHookRule(topic: string): Promise<HookRule | undefined>;
@@ -22,7 +22,6 @@ class TaskService {
22
22
  const destinationKey = `${sourceKey}:processed`;
23
23
  const jobId = await this.store.processTaskQueue(sourceKey, destinationKey);
24
24
  if (jobId) {
25
- //todo: don't use 'id', make configurable using hook rule
26
25
  await hookEventCallback(topic, { ...data, id: jobId });
27
26
  }
28
27
  else {
@@ -49,11 +48,6 @@ class TaskService {
49
48
  const awakenTimeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
50
49
  await this.store.registerTimeHook(jobId, gId, activityId, type, awakenTimeSlot, dad, multi);
51
50
  }
52
- /**
53
- * Should this engine instance play the role of 'scout' on behalf
54
- * of the entire quorum? The scout role is responsible for processing
55
- * task lists on behalf of the collective.
56
- */
57
51
  async shouldScout() {
58
52
  const wasScout = this.isScout;
59
53
  const isScout = wasScout || (this.isScout = await this.store.reserveScoutRole('time'));
@@ -67,10 +61,6 @@ class TaskService {
67
61
  }
68
62
  return false;
69
63
  }
70
- /**
71
- * Callback handler that takes an item from a work list and
72
- * processes according to its type
73
- */
74
64
  async processTimeHooks(timeEventCallback, listKey) {
75
65
  if (await this.shouldScout()) {
76
66
  try {
@@ -78,18 +68,14 @@ class TaskService {
78
68
  if (Array.isArray(workListTask)) {
79
69
  const [listKey, target, gId, activityId, type] = workListTask;
80
70
  if (type === 'child') {
81
- //continue; this child is listed here for convenience, but
82
- // will be expired by an origin ancestor and is listed there
83
71
  }
84
72
  else if (type === 'delist') {
85
- //delist the signalKey (target)
86
73
  const key = this.store.mintKey(hotmesh_1.KeyType.SIGNALS, {
87
74
  appId: this.store.appId,
88
75
  });
89
76
  await this.store.redisClient[this.store.commands.hdel](key, target);
90
77
  }
91
78
  else {
92
- //awaken/expire/interrupt
93
79
  await timeEventCallback(target, gId, activityId, type);
94
80
  }
95
81
  await (0, utils_1.sleepFor)(0);
@@ -97,13 +83,11 @@ class TaskService {
97
83
  this.processTimeHooks(timeEventCallback, listKey);
98
84
  }
99
85
  else if (workListTask) {
100
- //a worklist was just emptied; try again immediately
101
86
  await (0, utils_1.sleepFor)(0);
102
87
  this.errorCount = 0;
103
88
  this.processTimeHooks(timeEventCallback);
104
89
  }
105
90
  else {
106
- //no worklists exist; sleep before checking
107
91
  const sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_FIDELITY_SECONDS * 1000);
108
92
  this.cleanupTimeout = sleep.timerId;
109
93
  await sleep.promise;
@@ -112,8 +96,6 @@ class TaskService {
112
96
  }
113
97
  }
114
98
  catch (err) {
115
- //most common reasons: deleted job not found; container stopping; test stopping
116
- //less common: redis/cluster down; retry with fallback (5s max main reassignment)
117
99
  this.logger.warn('task-process-timehooks-error', err);
118
100
  await (0, utils_1.sleepFor)(1000 * this.errorCount++);
119
101
  if (this.errorCount < 5) {
@@ -122,7 +104,6 @@ class TaskService {
122
104
  }
123
105
  }
124
106
  else {
125
- //didn't get the scout role; try again in 'one-ish' minutes
126
107
  const sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_SCOUT_INTERVAL_SECONDS * 1000 * 2 * Math.random());
127
108
  this.cleanupTimeout = sleep.timerId;
128
109
  await sleep.promise;
@@ -147,7 +128,6 @@ class TaskService {
147
128
  const jobId = context.metadata.jid;
148
129
  const gId = context.metadata.gid;
149
130
  const activityId = hookRule.to;
150
- //composite keys are used to fully describe the task target
151
131
  const compositeJobKey = [activityId, dad, gId, jobId].join(key_1.WEBSEP);
152
132
  const hook = {
153
133
  topic,
@@ -164,22 +144,13 @@ class TaskService {
164
144
  async processWebHookSignal(topic, data) {
165
145
  const hookRule = await this.getHookRule(topic);
166
146
  if (hookRule) {
167
- //NOTE: both formats are supported by the mapping engine:
168
- // `$self.hook.data` OR `$hook.data`
169
147
  const context = { $self: { hook: { data } }, $hook: { data } };
170
148
  const mapExpression = hookRule.conditions.match[0].actual;
171
149
  const resolved = pipe_1.Pipe.resolve(mapExpression, context);
172
150
  const hookSignalId = await this.store.getHookSignal(topic, resolved);
173
151
  if (!hookSignalId) {
174
- //messages can be double-processed; not an issue; return `undefined`
175
- //users can also provide a bogus topic; not an issue; return `undefined`
176
152
  return undefined;
177
153
  }
178
- //`aid` is part of composite key, but the hook `topic` is its public interface;
179
- // this means that a new version of the graph can be deployed and the
180
- // topic can be re-mapped to a different activity id. Outside callers
181
- // can adhere to the unchanged contract (calling the same topic),
182
- // while the internal system can be updated in real-time as necessary.
183
154
  const [_aid, dad, gid, ...jid] = hookSignalId.split(key_1.WEBSEP);
184
155
  return [jid.join(key_1.WEBSEP), hookRule.to, dad, gid];
185
156
  }
@@ -190,8 +161,6 @@ class TaskService {
190
161
  async deleteWebHookSignal(topic, data) {
191
162
  const hookRule = await this.getHookRule(topic);
192
163
  if (hookRule) {
193
- //NOTE: both formats are supported by the mapping engine:
194
- // `$self.hook.data` OR `$hook.data`
195
164
  const context = { $self: { hook: { data } }, $hook: { data } };
196
165
  const mapExpression = hookRule.conditions.match[0].actual;
197
166
  const resolved = pipe_1.Pipe.resolve(mapExpression, context);
@@ -35,13 +35,6 @@ declare class TelemetryService {
35
35
  setTelemetryContext(span: Span, leg: number): void;
36
36
  setActivityError(message: string): void;
37
37
  setStreamError(message: string): void;
38
- /**
39
- * Adds the paths (HGET) necessary to restore telemetry state for an activity
40
- * @param consumes
41
- * @param config
42
- * @param metadata
43
- * @param leg
44
- */
45
38
  static addTargetTelemetryPaths(consumes: Consumes, config: ActivityType, metadata: ActivityMetadata, leg: number): void;
46
39
  static bindJobTelemetryToState(state: StringStringType, config: ActivityType, context: JobState): void;
47
40
  static bindActivityTelemetryToState(state: StringAnyType, config: ActivityType, metadata: ActivityMetadata, context: JobState, leg: number): void;
@@ -13,7 +13,6 @@ class TelemetryService {
13
13
  constructor(appId, config, metadata, context) {
14
14
  this.leg = 1;
15
15
  this.appId = appId;
16
- //these are REQUIRED for job and activity spans
17
16
  this.config = config;
18
17
  this.metadata = metadata;
19
18
  this.context = context;
@@ -83,7 +82,6 @@ class TelemetryService {
83
82
  return span;
84
83
  }
85
84
  mapActivityAttributes() {
86
- //export user-defined span attributes (app.activity.data.*)
87
85
  if (this.config.telemetry) {
88
86
  const telemetryAtts = new mapper_1.MapperService(this.config.telemetry, this.context).mapRules();
89
87
  const namespacedAtts = {
@@ -124,7 +122,7 @@ class TelemetryService {
124
122
  traceId: this.traceId,
125
123
  spanId: this.spanId,
126
124
  isRemote: true,
127
- traceFlags: 1, // (todo: revisit sampling strategy/config)
125
+ traceFlags: 1,
128
126
  };
129
127
  const parentContext = telemetry_1.trace.setSpanContext(telemetry_1.context.active(), restoredSpanContext);
130
128
  return parentContext;
@@ -178,13 +176,6 @@ class TelemetryService {
178
176
  setStreamError(message) {
179
177
  this.span?.setStatus({ code: telemetry_1.SpanStatusCode.ERROR, message });
180
178
  }
181
- /**
182
- * Adds the paths (HGET) necessary to restore telemetry state for an activity
183
- * @param consumes
184
- * @param config
185
- * @param metadata
186
- * @param leg
187
- */
188
179
  static addTargetTelemetryPaths(consumes, config, metadata, leg) {
189
180
  if (leg === 1) {
190
181
  if (!(config.parent in consumes)) {
@@ -206,7 +197,6 @@ class TelemetryService {
206
197
  }
207
198
  static bindActivityTelemetryToState(state, config, metadata, context, leg) {
208
199
  if (config.type === 'trigger') {
209
- //trigger activities run non-duplexed and only have a single leg (2)
210
200
  state[`${metadata.aid}/output/metadata/l1s`] =
211
201
  context['$self'].output.metadata.l1s;
212
202
  state[`${metadata.aid}/output/metadata/l2s`] =
@@ -214,14 +204,12 @@ class TelemetryService {
214
204
  }
215
205
  else if (utils_1.polyfill.resolveActivityType(config.type) === 'hook' &&
216
206
  leg === 1) {
217
- //hook activities run non-duplexed and only have a single leg (1)
218
207
  state[`${metadata.aid}/output/metadata/l1s`] =
219
208
  context['$self'].output.metadata.l1s;
220
209
  state[`${metadata.aid}/output/metadata/l2s`] =
221
210
  context['$self'].output.metadata.l1s;
222
211
  }
223
212
  else if (config.type === 'signal' && leg === 1) {
224
- //signal activities run non-duplexed and only have a single leg (1)
225
213
  state[`${metadata.aid}/output/metadata/l1s`] =
226
214
  context['$self'].output.metadata.l1s;
227
215
  state[`${metadata.aid}/output/metadata/l2s`] =
@@ -23,6 +23,7 @@ declare class WorkerService {
23
23
  reporting: boolean;
24
24
  inited: string;
25
25
  rollCallInterval: NodeJS.Timeout;
26
+ constructor();
26
27
  static init(namespace: string, appId: string, guid: string, config: HotMeshConfig, logger: ILogger): Promise<WorkerService[]>;
27
28
  verifyWorkerFields(worker: HotMeshWorker): void;
28
29
  initStoreChannel(service: WorkerService, store: RedisClient): Promise<void>;
@@ -30,10 +31,6 @@ declare class WorkerService {
30
31
  initStreamChannel(service: WorkerService, stream: RedisClient): Promise<void>;
31
32
  initRouter(worker: HotMeshWorker, logger: ILogger): Promise<Router>;
32
33
  subscriptionHandler(): SubscriptionCallback;
33
- /**
34
- * A quorum-wide command to broadcaset system details.
35
- *
36
- */
37
34
  doRollCall(message: RollCallMessage): Promise<void>;
38
35
  cancelRollCall(): void;
39
36
  stop(): void;