@hotmeshio/hotmesh 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (283) hide show
  1. package/build/modules/enums.d.ts +110 -0
  2. package/build/modules/enums.js +134 -0
  3. package/build/modules/errors.d.ts +124 -0
  4. package/build/modules/errors.js +191 -0
  5. package/build/modules/key.d.ts +66 -0
  6. package/build/modules/key.js +190 -0
  7. package/build/modules/storage.d.ts +3 -0
  8. package/build/modules/storage.js +5 -0
  9. package/build/modules/utils.d.ts +119 -0
  10. package/build/modules/utils.js +374 -0
  11. package/build/package.json +1 -1
  12. package/build/services/activities/activity.d.ts +104 -0
  13. package/build/services/activities/activity.js +549 -0
  14. package/build/services/activities/await.d.ts +12 -0
  15. package/build/services/activities/await.js +114 -0
  16. package/build/services/activities/cycle.d.ts +19 -0
  17. package/build/services/activities/cycle.js +112 -0
  18. package/build/services/activities/hook.d.ts +27 -0
  19. package/build/services/activities/hook.js +168 -0
  20. package/build/services/activities/index.d.ts +19 -0
  21. package/build/services/activities/index.js +20 -0
  22. package/build/services/activities/interrupt.d.ts +16 -0
  23. package/build/services/activities/interrupt.js +158 -0
  24. package/build/services/activities/signal.d.ts +20 -0
  25. package/build/services/activities/signal.js +134 -0
  26. package/build/services/activities/trigger.d.ts +37 -0
  27. package/build/services/activities/trigger.js +246 -0
  28. package/build/services/activities/worker.d.ts +12 -0
  29. package/build/services/activities/worker.js +106 -0
  30. package/build/services/collator/index.d.ts +111 -0
  31. package/build/services/collator/index.js +293 -0
  32. package/build/services/compiler/deployer.d.ts +40 -0
  33. package/build/services/compiler/deployer.js +488 -0
  34. package/build/services/compiler/index.d.ts +32 -0
  35. package/build/services/compiler/index.js +112 -0
  36. package/build/services/compiler/validator.d.ts +34 -0
  37. package/build/services/compiler/validator.js +147 -0
  38. package/build/services/connector/factory.d.ts +22 -0
  39. package/build/services/connector/factory.js +99 -0
  40. package/build/services/connector/index.d.ts +30 -0
  41. package/build/services/connector/index.js +54 -0
  42. package/build/services/connector/providers/ioredis.d.ts +9 -0
  43. package/build/services/connector/providers/ioredis.js +26 -0
  44. package/build/services/connector/providers/nats.d.ts +9 -0
  45. package/build/services/connector/providers/nats.js +34 -0
  46. package/build/services/connector/providers/postgres.d.ts +20 -0
  47. package/build/services/connector/providers/postgres.js +102 -0
  48. package/build/services/connector/providers/redis.d.ts +9 -0
  49. package/build/services/connector/providers/redis.js +38 -0
  50. package/build/services/engine/index.d.ts +264 -0
  51. package/build/services/engine/index.js +761 -0
  52. package/build/services/exporter/index.d.ts +44 -0
  53. package/build/services/exporter/index.js +126 -0
  54. package/build/services/hotmesh/index.d.ts +483 -0
  55. package/build/services/hotmesh/index.js +622 -0
  56. package/build/services/logger/index.d.ts +16 -0
  57. package/build/services/logger/index.js +54 -0
  58. package/build/services/mapper/index.d.ts +28 -0
  59. package/build/services/mapper/index.js +81 -0
  60. package/build/services/memflow/client.d.ts +108 -0
  61. package/build/services/memflow/client.js +372 -0
  62. package/build/services/memflow/connection.d.ts +23 -0
  63. package/build/services/memflow/connection.js +33 -0
  64. package/build/services/memflow/context.d.ts +143 -0
  65. package/build/services/memflow/context.js +299 -0
  66. package/build/services/memflow/exporter.d.ts +51 -0
  67. package/build/services/memflow/exporter.js +215 -0
  68. package/build/services/memflow/handle.d.ts +90 -0
  69. package/build/services/memflow/handle.js +176 -0
  70. package/build/services/memflow/index.d.ts +116 -0
  71. package/build/services/memflow/index.js +122 -0
  72. package/build/services/memflow/schemas/factory.d.ts +29 -0
  73. package/build/services/memflow/schemas/factory.js +2492 -0
  74. package/build/services/memflow/search.d.ts +142 -0
  75. package/build/services/memflow/search.js +320 -0
  76. package/build/services/memflow/worker.d.ts +124 -0
  77. package/build/services/memflow/worker.js +514 -0
  78. package/build/services/memflow/workflow/all.d.ts +7 -0
  79. package/build/services/memflow/workflow/all.js +15 -0
  80. package/build/services/memflow/workflow/common.d.ts +20 -0
  81. package/build/services/memflow/workflow/common.js +47 -0
  82. package/build/services/memflow/workflow/context.d.ts +6 -0
  83. package/build/services/memflow/workflow/context.js +45 -0
  84. package/build/services/memflow/workflow/contextMethods.d.ts +14 -0
  85. package/build/services/memflow/workflow/contextMethods.js +33 -0
  86. package/build/services/memflow/workflow/didRun.d.ts +7 -0
  87. package/build/services/memflow/workflow/didRun.js +22 -0
  88. package/build/services/memflow/workflow/emit.d.ts +11 -0
  89. package/build/services/memflow/workflow/emit.js +29 -0
  90. package/build/services/memflow/workflow/enrich.d.ts +9 -0
  91. package/build/services/memflow/workflow/enrich.js +17 -0
  92. package/build/services/memflow/workflow/execChild.d.ts +18 -0
  93. package/build/services/memflow/workflow/execChild.js +102 -0
  94. package/build/services/memflow/workflow/execHook.d.ts +65 -0
  95. package/build/services/memflow/workflow/execHook.js +73 -0
  96. package/build/services/memflow/workflow/hook.d.ts +9 -0
  97. package/build/services/memflow/workflow/hook.js +56 -0
  98. package/build/services/memflow/workflow/index.d.ts +74 -0
  99. package/build/services/memflow/workflow/index.js +87 -0
  100. package/build/services/memflow/workflow/interrupt.d.ts +9 -0
  101. package/build/services/memflow/workflow/interrupt.js +24 -0
  102. package/build/services/memflow/workflow/isSideEffectAllowed.d.ts +10 -0
  103. package/build/services/memflow/workflow/isSideEffectAllowed.js +33 -0
  104. package/build/services/memflow/workflow/proxyActivities.d.ts +20 -0
  105. package/build/services/memflow/workflow/proxyActivities.js +97 -0
  106. package/build/services/memflow/workflow/random.d.ts +6 -0
  107. package/build/services/memflow/workflow/random.js +16 -0
  108. package/build/services/memflow/workflow/searchMethods.d.ts +6 -0
  109. package/build/services/memflow/workflow/searchMethods.js +25 -0
  110. package/build/services/memflow/workflow/signal.d.ts +7 -0
  111. package/build/services/memflow/workflow/signal.js +28 -0
  112. package/build/services/memflow/workflow/sleepFor.d.ts +8 -0
  113. package/build/services/memflow/workflow/sleepFor.js +35 -0
  114. package/build/services/memflow/workflow/trace.d.ts +14 -0
  115. package/build/services/memflow/workflow/trace.js +33 -0
  116. package/build/services/memflow/workflow/waitFor.d.ts +8 -0
  117. package/build/services/memflow/workflow/waitFor.js +35 -0
  118. package/build/services/meshcall/index.d.ts +194 -0
  119. package/build/services/meshcall/index.js +452 -0
  120. package/build/services/meshcall/schemas/factory.d.ts +9 -0
  121. package/build/services/meshcall/schemas/factory.js +189 -0
  122. package/build/services/meshdata/index.d.ts +795 -0
  123. package/build/services/meshdata/index.js +1235 -0
  124. package/build/services/meshos/index.d.ts +293 -0
  125. package/build/services/meshos/index.js +547 -0
  126. package/build/services/pipe/functions/array.d.ts +17 -0
  127. package/build/services/pipe/functions/array.js +74 -0
  128. package/build/services/pipe/functions/bitwise.d.ts +9 -0
  129. package/build/services/pipe/functions/bitwise.js +24 -0
  130. package/build/services/pipe/functions/conditional.d.ts +13 -0
  131. package/build/services/pipe/functions/conditional.js +36 -0
  132. package/build/services/pipe/functions/cron.d.ts +12 -0
  133. package/build/services/pipe/functions/cron.js +40 -0
  134. package/build/services/pipe/functions/date.d.ts +58 -0
  135. package/build/services/pipe/functions/date.js +171 -0
  136. package/build/services/pipe/functions/index.d.ts +29 -0
  137. package/build/services/pipe/functions/index.js +30 -0
  138. package/build/services/pipe/functions/json.d.ts +5 -0
  139. package/build/services/pipe/functions/json.js +12 -0
  140. package/build/services/pipe/functions/logical.d.ts +5 -0
  141. package/build/services/pipe/functions/logical.js +12 -0
  142. package/build/services/pipe/functions/math.d.ts +42 -0
  143. package/build/services/pipe/functions/math.js +184 -0
  144. package/build/services/pipe/functions/number.d.ts +21 -0
  145. package/build/services/pipe/functions/number.js +60 -0
  146. package/build/services/pipe/functions/object.d.ts +25 -0
  147. package/build/services/pipe/functions/object.js +81 -0
  148. package/build/services/pipe/functions/string.d.ts +23 -0
  149. package/build/services/pipe/functions/string.js +69 -0
  150. package/build/services/pipe/functions/symbol.d.ts +12 -0
  151. package/build/services/pipe/functions/symbol.js +33 -0
  152. package/build/services/pipe/functions/unary.d.ts +7 -0
  153. package/build/services/pipe/functions/unary.js +18 -0
  154. package/build/services/pipe/index.d.ts +48 -0
  155. package/build/services/pipe/index.js +242 -0
  156. package/build/services/quorum/index.d.ts +90 -0
  157. package/build/services/quorum/index.js +263 -0
  158. package/build/services/reporter/index.d.ts +50 -0
  159. package/build/services/reporter/index.js +348 -0
  160. package/build/services/router/config/index.d.ts +11 -0
  161. package/build/services/router/config/index.js +36 -0
  162. package/build/services/router/consumption/index.d.ts +34 -0
  163. package/build/services/router/consumption/index.js +395 -0
  164. package/build/services/router/error-handling/index.d.ts +8 -0
  165. package/build/services/router/error-handling/index.js +98 -0
  166. package/build/services/router/index.d.ts +57 -0
  167. package/build/services/router/index.js +121 -0
  168. package/build/services/router/lifecycle/index.d.ts +27 -0
  169. package/build/services/router/lifecycle/index.js +80 -0
  170. package/build/services/router/telemetry/index.d.ts +11 -0
  171. package/build/services/router/telemetry/index.js +32 -0
  172. package/build/services/router/throttling/index.d.ts +23 -0
  173. package/build/services/router/throttling/index.js +76 -0
  174. package/build/services/search/factory.d.ts +7 -0
  175. package/build/services/search/factory.js +24 -0
  176. package/build/services/search/index.d.ts +23 -0
  177. package/build/services/search/index.js +10 -0
  178. package/build/services/search/providers/postgres/postgres.d.ts +25 -0
  179. package/build/services/search/providers/postgres/postgres.js +149 -0
  180. package/build/services/search/providers/redis/ioredis.d.ts +19 -0
  181. package/build/services/search/providers/redis/ioredis.js +121 -0
  182. package/build/services/search/providers/redis/redis.d.ts +19 -0
  183. package/build/services/search/providers/redis/redis.js +134 -0
  184. package/build/services/serializer/index.d.ts +42 -0
  185. package/build/services/serializer/index.js +282 -0
  186. package/build/services/store/cache.d.ts +67 -0
  187. package/build/services/store/cache.js +128 -0
  188. package/build/services/store/factory.d.ts +8 -0
  189. package/build/services/store/factory.js +24 -0
  190. package/build/services/store/index.d.ts +89 -0
  191. package/build/services/store/index.js +9 -0
  192. package/build/services/store/providers/postgres/kvsql.d.ts +168 -0
  193. package/build/services/store/providers/postgres/kvsql.js +198 -0
  194. package/build/services/store/providers/postgres/kvtables.d.ts +20 -0
  195. package/build/services/store/providers/postgres/kvtables.js +441 -0
  196. package/build/services/store/providers/postgres/kvtransaction.d.ts +36 -0
  197. package/build/services/store/providers/postgres/kvtransaction.js +248 -0
  198. package/build/services/store/providers/postgres/kvtypes/hash.d.ts +60 -0
  199. package/build/services/store/providers/postgres/kvtypes/hash.js +1287 -0
  200. package/build/services/store/providers/postgres/kvtypes/list.d.ts +33 -0
  201. package/build/services/store/providers/postgres/kvtypes/list.js +194 -0
  202. package/build/services/store/providers/postgres/kvtypes/string.d.ts +20 -0
  203. package/build/services/store/providers/postgres/kvtypes/string.js +115 -0
  204. package/build/services/store/providers/postgres/kvtypes/zset.d.ts +41 -0
  205. package/build/services/store/providers/postgres/kvtypes/zset.js +214 -0
  206. package/build/services/store/providers/postgres/postgres.d.ts +145 -0
  207. package/build/services/store/providers/postgres/postgres.js +1036 -0
  208. package/build/services/store/providers/redis/_base.d.ts +137 -0
  209. package/build/services/store/providers/redis/_base.js +980 -0
  210. package/build/services/store/providers/redis/ioredis.d.ts +20 -0
  211. package/build/services/store/providers/redis/ioredis.js +180 -0
  212. package/build/services/store/providers/redis/redis.d.ts +18 -0
  213. package/build/services/store/providers/redis/redis.js +199 -0
  214. package/build/services/store/providers/store-initializable.d.ts +5 -0
  215. package/build/services/store/providers/store-initializable.js +2 -0
  216. package/build/services/stream/factory.d.ts +8 -0
  217. package/build/services/stream/factory.js +37 -0
  218. package/build/services/stream/index.d.ts +69 -0
  219. package/build/services/stream/index.js +11 -0
  220. package/build/services/stream/providers/nats/nats.d.ts +60 -0
  221. package/build/services/stream/providers/nats/nats.js +225 -0
  222. package/build/services/stream/providers/postgres/kvtables.d.ts +3 -0
  223. package/build/services/stream/providers/postgres/kvtables.js +146 -0
  224. package/build/services/stream/providers/postgres/postgres.d.ts +107 -0
  225. package/build/services/stream/providers/postgres/postgres.js +519 -0
  226. package/build/services/stream/providers/redis/ioredis.d.ts +61 -0
  227. package/build/services/stream/providers/redis/ioredis.js +272 -0
  228. package/build/services/stream/providers/redis/redis.d.ts +61 -0
  229. package/build/services/stream/providers/redis/redis.js +305 -0
  230. package/build/services/stream/providers/stream-initializable.d.ts +4 -0
  231. package/build/services/stream/providers/stream-initializable.js +2 -0
  232. package/build/services/sub/factory.d.ts +7 -0
  233. package/build/services/sub/factory.js +29 -0
  234. package/build/services/sub/index.d.ts +22 -0
  235. package/build/services/sub/index.js +10 -0
  236. package/build/services/sub/providers/nats/nats.d.ts +19 -0
  237. package/build/services/sub/providers/nats/nats.js +105 -0
  238. package/build/services/sub/providers/postgres/postgres.d.ts +19 -0
  239. package/build/services/sub/providers/postgres/postgres.js +92 -0
  240. package/build/services/sub/providers/redis/ioredis.d.ts +17 -0
  241. package/build/services/sub/providers/redis/ioredis.js +81 -0
  242. package/build/services/sub/providers/redis/redis.d.ts +17 -0
  243. package/build/services/sub/providers/redis/redis.js +72 -0
  244. package/build/services/task/index.d.ts +36 -0
  245. package/build/services/task/index.js +206 -0
  246. package/build/services/telemetry/index.d.ts +52 -0
  247. package/build/services/telemetry/index.js +306 -0
  248. package/build/services/worker/index.d.ts +77 -0
  249. package/build/services/worker/index.js +197 -0
  250. package/package.json +1 -1
  251. package/.github/ISSUE_TEMPLATE/bug_report.md +0 -38
  252. package/.github/ISSUE_TEMPLATE/feature_request.md +0 -20
  253. package/typedoc.json +0 -47
  254. package/types/activity.ts +0 -268
  255. package/types/app.ts +0 -20
  256. package/types/async.ts +0 -6
  257. package/types/cache.ts +0 -1
  258. package/types/collator.ts +0 -9
  259. package/types/error.ts +0 -56
  260. package/types/exporter.ts +0 -102
  261. package/types/hook.ts +0 -44
  262. package/types/hotmesh.ts +0 -314
  263. package/types/index.ts +0 -306
  264. package/types/job.ts +0 -233
  265. package/types/logger.ts +0 -8
  266. package/types/manifest.ts +0 -70
  267. package/types/map.ts +0 -5
  268. package/types/memflow.ts +0 -645
  269. package/types/meshcall.ts +0 -235
  270. package/types/meshdata.ts +0 -278
  271. package/types/ms.d.ts +0 -7
  272. package/types/nats.ts +0 -270
  273. package/types/pipe.ts +0 -90
  274. package/types/postgres.ts +0 -114
  275. package/types/provider.ts +0 -161
  276. package/types/quorum.ts +0 -167
  277. package/types/redis.ts +0 -404
  278. package/types/serializer.ts +0 -40
  279. package/types/stats.ts +0 -117
  280. package/types/stream.ts +0 -231
  281. package/types/task.ts +0 -7
  282. package/types/telemetry.ts +0 -16
  283. package/types/transition.ts +0 -20
@@ -0,0 +1,206 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.TaskService = void 0;
4
+ const enums_1 = require("../../modules/enums");
5
+ const utils_1 = require("../../modules/utils");
6
+ const pipe_1 = require("../pipe");
7
+ const hotmesh_1 = require("../../types/hotmesh");
8
+ const key_1 = require("../../modules/key");
9
+ class TaskService {
10
+ constructor(store, logger) {
11
+ this.cleanupTimeout = null;
12
+ this.isScout = false;
13
+ this.errorCount = 0;
14
+ this.logger = logger;
15
+ this.store = store;
16
+ }
17
+ async processWebHooks(hookEventCallback) {
18
+ const workItemKey = await this.store.getActiveTaskQueue();
19
+ if (workItemKey) {
20
+ const [topic, sourceKey, scrub, ...sdata] = workItemKey.split(key_1.WEBSEP);
21
+ const data = JSON.parse(sdata.join(key_1.WEBSEP));
22
+ const destinationKey = `${sourceKey}:processed`;
23
+ const jobId = await this.store.processTaskQueue(sourceKey, destinationKey);
24
+ if (jobId) {
25
+ //todo: don't use 'id', make configurable using hook rule
26
+ await hookEventCallback(topic, { ...data, id: jobId });
27
+ }
28
+ else {
29
+ await this.store.deleteProcessedTaskQueue(workItemKey, sourceKey, destinationKey, scrub === 'true');
30
+ }
31
+ setImmediate(() => this.processWebHooks(hookEventCallback));
32
+ }
33
+ }
34
+ async enqueueWorkItems(keys) {
35
+ await this.store.addTaskQueues(keys);
36
+ }
37
+ async registerJobForCleanup(jobId, inSeconds = enums_1.HMSH_EXPIRE_DURATION, options) {
38
+ if (inSeconds > 0) {
39
+ await this.store.expireJob(jobId, inSeconds);
40
+ // const fromNow = Date.now() + inSeconds * 1000;
41
+ // const fidelityMS = HMSH_FIDELITY_SECONDS * 1000;
42
+ // const timeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
43
+ // await this.store.registerDependenciesForCleanup(jobId, timeSlot, options);
44
+ }
45
+ }
46
+ async registerTimeHook(jobId, gId, activityId, type, inSeconds = enums_1.HMSH_FIDELITY_SECONDS, dad, transaction) {
47
+ const fromNow = Date.now() + inSeconds * 1000;
48
+ const fidelityMS = enums_1.HMSH_FIDELITY_SECONDS * 1000;
49
+ const awakenTimeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
50
+ await this.store.registerTimeHook(jobId, gId, activityId, type, awakenTimeSlot, dad, transaction);
51
+ }
52
+ /**
53
+ * Should this engine instance play the role of 'scout' on behalf
54
+ * of the entire quorum? The scout role is responsible for processing
55
+ * task lists on behalf of the collective.
56
+ */
57
+ async shouldScout() {
58
+ const wasScout = this.isScout;
59
+ const isScout = wasScout || (this.isScout = await this.store.reserveScoutRole('time'));
60
+ if (isScout) {
61
+ if (!wasScout) {
62
+ setTimeout(() => {
63
+ this.isScout = false;
64
+ }, enums_1.HMSH_SCOUT_INTERVAL_SECONDS * 1000);
65
+ }
66
+ return true;
67
+ }
68
+ return false;
69
+ }
70
+ /**
71
+ * Callback handler that takes an item from a work list and
72
+ * processes according to its type
73
+ */
74
+ async processTimeHooks(timeEventCallback, listKey) {
75
+ if (await this.shouldScout()) {
76
+ try {
77
+ const workListTask = await this.store.getNextTask(listKey);
78
+ if (Array.isArray(workListTask)) {
79
+ const [listKey, target, gId, activityId, type] = workListTask;
80
+ if (type === 'child') {
81
+ //continue; this child is listed here for convenience, but
82
+ // will be expired by an origin ancestor and is listed there
83
+ }
84
+ else if (type === 'delist') {
85
+ //delist the signalKey (target)
86
+ const key = this.store.mintKey(hotmesh_1.KeyType.SIGNALS, {
87
+ appId: this.store.appId,
88
+ });
89
+ await this.store.delistSignalKey(key, target);
90
+ }
91
+ else {
92
+ //awaken/expire/interrupt
93
+ await timeEventCallback(target, gId, activityId, type);
94
+ }
95
+ await (0, utils_1.sleepFor)(0);
96
+ this.errorCount = 0;
97
+ this.processTimeHooks(timeEventCallback, listKey);
98
+ }
99
+ else if (workListTask) {
100
+ //a worklist was just emptied; try again immediately
101
+ await (0, utils_1.sleepFor)(0);
102
+ this.errorCount = 0;
103
+ this.processTimeHooks(timeEventCallback);
104
+ }
105
+ else {
106
+ //no worklists exist; sleep before checking
107
+ const sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_FIDELITY_SECONDS * 1000);
108
+ this.cleanupTimeout = sleep.timerId;
109
+ await sleep.promise;
110
+ this.errorCount = 0;
111
+ this.processTimeHooks(timeEventCallback);
112
+ }
113
+ }
114
+ catch (err) {
115
+ //most common reasons: deleted job not found; container stopping; test stopping
116
+ //less common: redis/cluster down; retry with fallback (5s max main reassignment)
117
+ this.logger.warn('task-process-timehooks-error', err);
118
+ await (0, utils_1.sleepFor)(1000 * this.errorCount++);
119
+ if (this.errorCount < 5) {
120
+ this.processTimeHooks(timeEventCallback);
121
+ }
122
+ }
123
+ }
124
+ else {
125
+ //didn't get the scout role; try again in 'one-ish' minutes
126
+ const sleep = (0, utils_1.XSleepFor)(enums_1.HMSH_SCOUT_INTERVAL_SECONDS * 1000 * 2 * Math.random());
127
+ this.cleanupTimeout = sleep.timerId;
128
+ await sleep.promise;
129
+ this.processTimeHooks(timeEventCallback);
130
+ }
131
+ }
132
+ cancelCleanup() {
133
+ if (this.cleanupTimeout !== undefined) {
134
+ clearTimeout(this.cleanupTimeout);
135
+ this.cleanupTimeout = undefined;
136
+ }
137
+ }
138
+ async getHookRule(topic) {
139
+ const rules = await this.store.getHookRules();
140
+ return rules?.[topic]?.[0];
141
+ }
142
+ async registerWebHook(topic, context, dad, expire, transaction) {
143
+ const hookRule = await this.getHookRule(topic);
144
+ if (hookRule) {
145
+ const mapExpression = hookRule.conditions.match[0].expected;
146
+ const resolved = pipe_1.Pipe.resolve(mapExpression, context);
147
+ const jobId = context.metadata.jid;
148
+ const gId = context.metadata.gid;
149
+ const activityId = hookRule.to;
150
+ //composite keys are used to fully describe the task target
151
+ const compositeJobKey = [activityId, dad, gId, jobId].join(key_1.WEBSEP);
152
+ const hook = {
153
+ topic,
154
+ resolved,
155
+ jobId: compositeJobKey,
156
+ expire,
157
+ };
158
+ await this.store.setHookSignal(hook, transaction);
159
+ return jobId;
160
+ }
161
+ else {
162
+ throw new Error('signaler.registerWebHook:error: hook rule not found');
163
+ }
164
+ }
165
+ async processWebHookSignal(topic, data) {
166
+ const hookRule = await this.getHookRule(topic);
167
+ if (hookRule) {
168
+ //NOTE: both formats are supported by the mapping engine:
169
+ // `$self.hook.data` OR `$hook.data`
170
+ const context = { $self: { hook: { data } }, $hook: { data } };
171
+ const mapExpression = hookRule.conditions.match[0].actual;
172
+ const resolved = pipe_1.Pipe.resolve(mapExpression, context);
173
+ const hookSignalId = await this.store.getHookSignal(topic, resolved);
174
+ if (!hookSignalId) {
175
+ //messages can be double-processed; not an issue; return `undefined`
176
+ //users can also provide a bogus topic; not an issue; return `undefined`
177
+ return undefined;
178
+ }
179
+ //`aid` is part of composite key, but the hook `topic` is its public interface;
180
+ // this means that a new version of the graph can be deployed and the
181
+ // topic can be re-mapped to a different activity id. Outside callers
182
+ // can adhere to the unchanged contract (calling the same topic),
183
+ // while the internal system can be updated in real-time as necessary.
184
+ const [_aid, dad, gid, ...jid] = hookSignalId.split(key_1.WEBSEP);
185
+ return [jid.join(key_1.WEBSEP), hookRule.to, dad, gid];
186
+ }
187
+ else {
188
+ throw new Error('signal-not-found');
189
+ }
190
+ }
191
+ async deleteWebHookSignal(topic, data) {
192
+ const hookRule = await this.getHookRule(topic);
193
+ if (hookRule) {
194
+ //NOTE: both formats are supported by the mapping engine:
195
+ // `$self.hook.data` OR `$hook.data`
196
+ const context = { $self: { hook: { data } }, $hook: { data } };
197
+ const mapExpression = hookRule.conditions.match[0].actual;
198
+ const resolved = pipe_1.Pipe.resolve(mapExpression, context);
199
+ return await this.store.deleteHookSignal(topic, resolved);
200
+ }
201
+ else {
202
+ throw new Error('signaler.process:error: hook rule not found');
203
+ }
204
+ }
205
+ }
206
+ exports.TaskService = TaskService;
@@ -0,0 +1,52 @@
1
+ import { ActivityMetadata, ActivityType, Consumes } from '../../types/activity';
2
+ import { JobState } from '../../types/job';
3
+ import { StringAnyType, StringScalarType, StringStringType } from '../../types/serializer';
4
+ import { StreamData, StreamRole } from '../../types/stream';
5
+ import { Span, Context } from '../../types/telemetry';
6
+ declare class TelemetryService {
7
+ span: Span;
8
+ jobSpan: Span;
9
+ config: ActivityType;
10
+ traceId: string | null;
11
+ spanId: string | null;
12
+ appId: string;
13
+ metadata: ActivityMetadata;
14
+ context: JobState;
15
+ leg: number;
16
+ constructor(appId: string, config?: ActivityType, metadata?: ActivityMetadata, context?: JobState);
17
+ /**
18
+ * too chatty for production; only output traces, jobs, triggers and workers
19
+ */
20
+ private shouldCreateSpan;
21
+ private static createNoopSpan;
22
+ getJobParentSpanId(): string | undefined;
23
+ getActivityParentSpanId(leg: number): string | undefined;
24
+ getTraceId(): string | undefined;
25
+ startJobSpan(): TelemetryService;
26
+ /**
27
+ * Traces an activity.
28
+ * @private
29
+ */
30
+ static traceActivity(appId: string, attributes: StringScalarType, activityId: string, traceId: string, spanId: string, index?: number): Promise<boolean>;
31
+ startActivitySpan(leg?: number): TelemetryService;
32
+ startStreamSpan(data: StreamData, role: StreamRole): TelemetryService;
33
+ startSpan(traceId: string, spanId: string, spanName: string, attributes: StringScalarType, bCreate?: boolean): Span;
34
+ mapActivityAttributes(): void;
35
+ setActivityAttributes(attributes: StringScalarType): void;
36
+ setStreamAttributes(attributes: StringScalarType): void;
37
+ setJobAttributes(attributes: StringScalarType): void;
38
+ endJobSpan(): void;
39
+ endActivitySpan(): void;
40
+ endStreamSpan(): void;
41
+ endSpan(span: Span): void;
42
+ getParentSpanContext(): undefined | Context;
43
+ getSpanAttrs(leg: number): StringAnyType;
44
+ getStreamSpanAttrs(input: StreamData): StringAnyType;
45
+ setTelemetryContext(span: Span, leg: number): void;
46
+ setActivityError(message: string): void;
47
+ setStreamError(message: string): void;
48
+ static addTargetTelemetryPaths(consumes: Consumes, config: ActivityType, metadata: ActivityMetadata, leg: number): void;
49
+ static bindJobTelemetryToState(state: StringStringType, config: ActivityType, context: JobState): void;
50
+ static bindActivityTelemetryToState(state: StringAnyType, config: ActivityType, metadata: ActivityMetadata, context: JobState, leg: number): void;
51
+ }
52
+ export { TelemetryService };
@@ -0,0 +1,306 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.TelemetryService = void 0;
7
+ const package_json_1 = __importDefault(require("../../package.json"));
8
+ const mapper_1 = require("../mapper");
9
+ const stream_1 = require("../../types/stream");
10
+ const telemetry_1 = require("../../types/telemetry");
11
+ const enums_1 = require("../../modules/enums");
12
+ class TelemetryService {
13
+ constructor(appId, config, metadata, context) {
14
+ this.leg = 1;
15
+ this.appId = appId;
16
+ this.config = config;
17
+ this.metadata = metadata;
18
+ this.context = context;
19
+ }
20
+ /**
21
+ * too chatty for production; only output traces, jobs, triggers and workers
22
+ */
23
+ shouldCreateSpan() {
24
+ return (enums_1.HMSH_TELEMETRY === 'debug' ||
25
+ this.config?.type === 'trigger' ||
26
+ this.config?.type === 'worker');
27
+ }
28
+ static createNoopSpan(traceId, spanId) {
29
+ // A no-op span that returns the given spanContext and ignores all operations.
30
+ return {
31
+ spanContext() {
32
+ return {
33
+ traceId,
34
+ spanId,
35
+ isRemote: true,
36
+ traceFlags: 1,
37
+ };
38
+ },
39
+ addEvent(_name, _attributesOrStartTime, _startTime) {
40
+ return this;
41
+ },
42
+ setAttribute(_key, _value) {
43
+ return this;
44
+ },
45
+ setAttributes(_attributes) {
46
+ return this;
47
+ },
48
+ setStatus(_status) {
49
+ return this;
50
+ },
51
+ updateName(_name) {
52
+ return this;
53
+ },
54
+ end(_endTime) {
55
+ // no-op
56
+ },
57
+ isRecording() {
58
+ return false;
59
+ },
60
+ recordException(_exception, _time) {
61
+ // no-op
62
+ },
63
+ };
64
+ }
65
+ getJobParentSpanId() {
66
+ return this.context.metadata.spn;
67
+ }
68
+ getActivityParentSpanId(leg) {
69
+ if (leg === 1) {
70
+ return this.context[this.config.parent].output?.metadata?.l2s;
71
+ }
72
+ else {
73
+ return this.context['$self'].output?.metadata?.l1s;
74
+ }
75
+ }
76
+ getTraceId() {
77
+ return this.context.metadata.trc;
78
+ }
79
+ startJobSpan() {
80
+ const spanName = `JOB/${this.appId}/${this.config.subscribes}/1`;
81
+ const traceId = this.getTraceId();
82
+ const spanId = this.getJobParentSpanId();
83
+ const attributes = this.getSpanAttrs(1);
84
+ const span = this.startSpan(traceId, spanId, spanName, attributes);
85
+ this.jobSpan = span;
86
+ this.setTelemetryContext(span, 1);
87
+ return this;
88
+ }
89
+ /**
90
+ * Traces an activity.
91
+ * @private
92
+ */
93
+ static async traceActivity(appId, attributes, activityId, traceId, spanId, index = 0) {
94
+ const spanName = `TRACE/${appId}/${activityId}/${index}`;
95
+ const tracer = telemetry_1.trace.getTracer(package_json_1.default.name, package_json_1.default.version);
96
+ const restoredSpanContext = {
97
+ traceId,
98
+ spanId,
99
+ isRemote: true,
100
+ traceFlags: 1,
101
+ };
102
+ const parentContext = telemetry_1.trace.setSpanContext(telemetry_1.context.active(), restoredSpanContext);
103
+ return telemetry_1.context.with(parentContext, () => {
104
+ const span = tracer.startSpan(spanName, {
105
+ kind: telemetry_1.SpanKind.CLIENT,
106
+ attributes,
107
+ });
108
+ span.setAttributes(attributes);
109
+ span.end();
110
+ return true;
111
+ });
112
+ }
113
+ startActivitySpan(leg = this.leg) {
114
+ const spanName = `${this.config.type.toUpperCase()}/${this.appId}/${this.metadata.aid}/${leg}`;
115
+ const traceId = this.getTraceId();
116
+ const spanId = this.getActivityParentSpanId(leg);
117
+ const attributes = this.getSpanAttrs(leg);
118
+ const span = this.startSpan(traceId, spanId, spanName, attributes);
119
+ this.span = span;
120
+ this.setTelemetryContext(span, leg);
121
+ return this;
122
+ }
123
+ startStreamSpan(data, role) {
124
+ let type;
125
+ if (role === stream_1.StreamRole.SYSTEM) {
126
+ type = 'SYSTEM';
127
+ }
128
+ else if (role === stream_1.StreamRole.WORKER) {
129
+ type = 'EXECUTE';
130
+ }
131
+ else if (data.type === stream_1.StreamDataType.RESULT ||
132
+ data.type === stream_1.StreamDataType.RESPONSE) {
133
+ type = 'FANIN'; //re-entering engine router (from worker router)
134
+ }
135
+ else {
136
+ type = 'FANOUT'; //exiting engine router (to worker router)
137
+ }
138
+ // `EXECUTE` refers to the 'worker router' NOT the 'worker activity' run by the 'engine router'
139
+ // (Regardless, it's worker-related, so it matters and will be traced)
140
+ // `SYSTEM` refers to catastrophic errors, which are always traced
141
+ if (this.shouldCreateSpan() || type === 'EXECUTE' || type === 'SYSTEM') {
142
+ const topic = data.metadata.topic ? `/${data.metadata.topic}` : '';
143
+ const spanName = `${type}/${this.appId}/${data.metadata.aid}${topic}`;
144
+ const attributes = this.getStreamSpanAttrs(data);
145
+ this.span = this.startSpan(data.metadata.trc, data.metadata.spn, spanName, attributes, true);
146
+ }
147
+ else {
148
+ this.traceId = data.metadata.trc;
149
+ this.spanId = data.metadata.spn;
150
+ this.span = TelemetryService.createNoopSpan(data.metadata.trc, data.metadata.spn);
151
+ }
152
+ return this;
153
+ }
154
+ startSpan(traceId, spanId, spanName, attributes, bCreate = false) {
155
+ this.traceId = traceId;
156
+ this.spanId = spanId;
157
+ if (bCreate || this.shouldCreateSpan()) {
158
+ const tracer = telemetry_1.trace.getTracer(package_json_1.default.name, package_json_1.default.version);
159
+ const parentContext = this.getParentSpanContext();
160
+ const span = tracer.startSpan(spanName, { kind: telemetry_1.SpanKind.CLIENT, attributes, root: !parentContext }, parentContext);
161
+ return span;
162
+ }
163
+ return TelemetryService.createNoopSpan(traceId, spanId);
164
+ }
165
+ mapActivityAttributes() {
166
+ if (this.config.telemetry && this.span) {
167
+ const telemetryAtts = new mapper_1.MapperService(this.config.telemetry, this.context).mapRules();
168
+ const namespacedAtts = {
169
+ ...Object.keys(telemetryAtts).reduce((result, key) => {
170
+ if (['string', 'boolean', 'number'].includes(typeof telemetryAtts[key])) {
171
+ result[`app.activity.data.${key}`] = telemetryAtts[key];
172
+ }
173
+ return result;
174
+ }, {}),
175
+ };
176
+ this.span.setAttributes(namespacedAtts);
177
+ }
178
+ }
179
+ setActivityAttributes(attributes) {
180
+ this.span?.setAttributes(attributes);
181
+ }
182
+ setStreamAttributes(attributes) {
183
+ this.span?.setAttributes(attributes);
184
+ }
185
+ setJobAttributes(attributes) {
186
+ this.jobSpan?.setAttributes(attributes);
187
+ }
188
+ endJobSpan() {
189
+ this.endSpan(this.jobSpan);
190
+ }
191
+ endActivitySpan() {
192
+ this.endSpan(this.span);
193
+ }
194
+ endStreamSpan() {
195
+ this.endSpan(this.span);
196
+ }
197
+ endSpan(span) {
198
+ // For a no-op span, end() does nothing anyway
199
+ span && span.end();
200
+ }
201
+ getParentSpanContext() {
202
+ if (this.traceId && this.spanId) {
203
+ const restoredSpanContext = {
204
+ traceId: this.traceId,
205
+ spanId: this.spanId,
206
+ isRemote: true,
207
+ traceFlags: 1,
208
+ };
209
+ const parentContext = telemetry_1.trace.setSpanContext(telemetry_1.context.active(), restoredSpanContext);
210
+ return parentContext;
211
+ }
212
+ }
213
+ getSpanAttrs(leg) {
214
+ return {
215
+ ...Object.keys(this.context.metadata).reduce((result, key) => {
216
+ if (key !== 'trc') {
217
+ result[`app.job.${key}`] = this.context.metadata[key];
218
+ }
219
+ return result;
220
+ }, {}),
221
+ ...Object.keys(this.metadata).reduce((result, key) => {
222
+ result[`app.activity.${key}`] = this.metadata[key];
223
+ return result;
224
+ }, {}),
225
+ 'app.activity.leg': leg,
226
+ };
227
+ }
228
+ getStreamSpanAttrs(input) {
229
+ return {
230
+ ...Object.keys(input.metadata).reduce((result, key) => {
231
+ if (key !== 'trc' && key !== 'spn') {
232
+ result[`app.stream.${key}`] = input.metadata[key];
233
+ }
234
+ return result;
235
+ }, {}),
236
+ };
237
+ }
238
+ setTelemetryContext(span, leg) {
239
+ // Even if span is no-op, we still set context so that callers remain unaware
240
+ if (!this.context.metadata.trc) {
241
+ this.context.metadata.trc = span.spanContext().traceId;
242
+ }
243
+ if (!this.context['$self'].output.metadata) {
244
+ this.context['$self'].output.metadata = {};
245
+ }
246
+ // Echo the parent's or the newly created span's spanId
247
+ // This ensures the caller sees a consistent chain
248
+ if (leg === 1) {
249
+ this.context['$self'].output.metadata.l1s = span.spanContext().spanId;
250
+ }
251
+ else {
252
+ this.context['$self'].output.metadata.l2s = span.spanContext().spanId;
253
+ }
254
+ }
255
+ setActivityError(message) {
256
+ this.span?.setStatus({ code: telemetry_1.SpanStatusCode.ERROR, message });
257
+ }
258
+ setStreamError(message) {
259
+ this.span?.setStatus({ code: telemetry_1.SpanStatusCode.ERROR, message });
260
+ }
261
+ static addTargetTelemetryPaths(consumes, config, metadata, leg) {
262
+ if (leg === 1) {
263
+ if (!(config.parent in consumes)) {
264
+ consumes[config.parent] = [];
265
+ }
266
+ consumes[config.parent].push(`${config.parent}/output/metadata/l2s`);
267
+ }
268
+ else {
269
+ if (!(metadata.aid in consumes)) {
270
+ consumes[metadata.aid] = [];
271
+ }
272
+ consumes[metadata.aid].push(`${metadata.aid}/output/metadata/l1s`);
273
+ }
274
+ }
275
+ static bindJobTelemetryToState(state, config, context) {
276
+ if (config.type === 'trigger') {
277
+ state['metadata/trc'] = context.metadata.trc;
278
+ }
279
+ }
280
+ static bindActivityTelemetryToState(state, config, metadata, context, leg) {
281
+ if (config.type === 'trigger') {
282
+ state[`${metadata.aid}/output/metadata/l1s`] =
283
+ context['$self'].output.metadata.l1s;
284
+ state[`${metadata.aid}/output/metadata/l2s`] =
285
+ context['$self'].output.metadata.l2s;
286
+ }
287
+ else if (config.type === 'hook' && leg === 1) {
288
+ state[`${metadata.aid}/output/metadata/l1s`] =
289
+ context['$self'].output.metadata.l1s;
290
+ state[`${metadata.aid}/output/metadata/l2s`] =
291
+ context['$self'].output.metadata.l1s;
292
+ }
293
+ else if (config.type === 'signal' && leg === 1) {
294
+ state[`${metadata.aid}/output/metadata/l1s`] =
295
+ context['$self'].output.metadata.l1s;
296
+ state[`${metadata.aid}/output/metadata/l2s`] =
297
+ context['$self'].output.metadata.l1s;
298
+ }
299
+ else {
300
+ const target = `l${leg}s`;
301
+ state[`${metadata.aid}/output/metadata/${target}`] =
302
+ context['$self'].output.metadata[target];
303
+ }
304
+ }
305
+ }
306
+ exports.TelemetryService = TelemetryService;
@@ -0,0 +1,77 @@
1
+ /// <reference types="node" />
2
+ import { ILogger } from '../logger';
3
+ import { Router } from '../router';
4
+ import { StoreService } from '../store';
5
+ import { StreamService } from '../stream';
6
+ import { SubService } from '../sub';
7
+ import { HotMeshConfig, HotMeshWorker } from '../../types/hotmesh';
8
+ import { ProviderClient, ProviderTransaction } from '../../types/provider';
9
+ import { RollCallMessage, SubscriptionCallback } from '../../types/quorum';
10
+ import { StreamData, StreamDataResponse } from '../../types/stream';
11
+ declare class WorkerService {
12
+ namespace: string;
13
+ appId: string;
14
+ guid: string;
15
+ topic: string;
16
+ config: HotMeshConfig;
17
+ callback: (streamData: StreamData) => Promise<StreamDataResponse | void>;
18
+ store: StoreService<ProviderClient, ProviderTransaction> | null;
19
+ stream: StreamService<ProviderClient, ProviderTransaction> | null;
20
+ subscribe: SubService<ProviderClient> | null;
21
+ router: Router<typeof this.stream> | null;
22
+ logger: ILogger;
23
+ reporting: boolean;
24
+ inited: string;
25
+ rollCallInterval: NodeJS.Timeout;
26
+ /**
27
+ * @private
28
+ */
29
+ constructor();
30
+ /**
31
+ * @private
32
+ */
33
+ static init(namespace: string, appId: string, guid: string, config: HotMeshConfig, logger: ILogger): Promise<WorkerService[]>;
34
+ /**
35
+ * @private
36
+ */
37
+ verifyWorkerFields(worker: HotMeshWorker): void;
38
+ /**
39
+ * @private
40
+ */
41
+ initStoreChannel(service: WorkerService, store: ProviderClient): Promise<void>;
42
+ /**
43
+ * @private
44
+ */
45
+ initSubChannel(service: WorkerService, sub: ProviderClient, store: ProviderClient): Promise<void>;
46
+ /**
47
+ * @private
48
+ */
49
+ initStreamChannel(service: WorkerService, stream: ProviderClient, store: ProviderClient): Promise<void>;
50
+ /**
51
+ * @private
52
+ */
53
+ initRouter(worker: HotMeshWorker, logger: ILogger): Promise<Router<StreamService<ProviderClient, ProviderTransaction>>>;
54
+ /**
55
+ * @private
56
+ */
57
+ subscriptionHandler(): SubscriptionCallback;
58
+ /**
59
+ * A quorum-wide command to broadcaset system details.
60
+ *
61
+ */
62
+ doRollCall(message: RollCallMessage): Promise<void>;
63
+ cancelRollCall(): void;
64
+ /**
65
+ * @private
66
+ */
67
+ stop(): void;
68
+ /**
69
+ * @private
70
+ */
71
+ sayPong(appId: string, guid: string, originator?: string, details?: boolean, signature?: boolean): Promise<void>;
72
+ /**
73
+ * @private
74
+ */
75
+ throttle(delayInMillis: number): Promise<void>;
76
+ }
77
+ export { WorkerService };