@hotmeshio/hotmesh 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (283) hide show
  1. package/build/modules/enums.d.ts +110 -0
  2. package/build/modules/enums.js +134 -0
  3. package/build/modules/errors.d.ts +124 -0
  4. package/build/modules/errors.js +191 -0
  5. package/build/modules/key.d.ts +66 -0
  6. package/build/modules/key.js +190 -0
  7. package/build/modules/storage.d.ts +3 -0
  8. package/build/modules/storage.js +5 -0
  9. package/build/modules/utils.d.ts +119 -0
  10. package/build/modules/utils.js +374 -0
  11. package/build/package.json +1 -1
  12. package/build/services/activities/activity.d.ts +104 -0
  13. package/build/services/activities/activity.js +549 -0
  14. package/build/services/activities/await.d.ts +12 -0
  15. package/build/services/activities/await.js +114 -0
  16. package/build/services/activities/cycle.d.ts +19 -0
  17. package/build/services/activities/cycle.js +112 -0
  18. package/build/services/activities/hook.d.ts +27 -0
  19. package/build/services/activities/hook.js +168 -0
  20. package/build/services/activities/index.d.ts +19 -0
  21. package/build/services/activities/index.js +20 -0
  22. package/build/services/activities/interrupt.d.ts +16 -0
  23. package/build/services/activities/interrupt.js +158 -0
  24. package/build/services/activities/signal.d.ts +20 -0
  25. package/build/services/activities/signal.js +134 -0
  26. package/build/services/activities/trigger.d.ts +37 -0
  27. package/build/services/activities/trigger.js +246 -0
  28. package/build/services/activities/worker.d.ts +12 -0
  29. package/build/services/activities/worker.js +106 -0
  30. package/build/services/collator/index.d.ts +111 -0
  31. package/build/services/collator/index.js +293 -0
  32. package/build/services/compiler/deployer.d.ts +40 -0
  33. package/build/services/compiler/deployer.js +488 -0
  34. package/build/services/compiler/index.d.ts +32 -0
  35. package/build/services/compiler/index.js +112 -0
  36. package/build/services/compiler/validator.d.ts +34 -0
  37. package/build/services/compiler/validator.js +147 -0
  38. package/build/services/connector/factory.d.ts +22 -0
  39. package/build/services/connector/factory.js +99 -0
  40. package/build/services/connector/index.d.ts +30 -0
  41. package/build/services/connector/index.js +54 -0
  42. package/build/services/connector/providers/ioredis.d.ts +9 -0
  43. package/build/services/connector/providers/ioredis.js +26 -0
  44. package/build/services/connector/providers/nats.d.ts +9 -0
  45. package/build/services/connector/providers/nats.js +34 -0
  46. package/build/services/connector/providers/postgres.d.ts +20 -0
  47. package/build/services/connector/providers/postgres.js +102 -0
  48. package/build/services/connector/providers/redis.d.ts +9 -0
  49. package/build/services/connector/providers/redis.js +38 -0
  50. package/build/services/engine/index.d.ts +264 -0
  51. package/build/services/engine/index.js +761 -0
  52. package/build/services/exporter/index.d.ts +44 -0
  53. package/build/services/exporter/index.js +126 -0
  54. package/build/services/hotmesh/index.d.ts +483 -0
  55. package/build/services/hotmesh/index.js +622 -0
  56. package/build/services/logger/index.d.ts +16 -0
  57. package/build/services/logger/index.js +54 -0
  58. package/build/services/mapper/index.d.ts +28 -0
  59. package/build/services/mapper/index.js +81 -0
  60. package/build/services/memflow/client.d.ts +108 -0
  61. package/build/services/memflow/client.js +372 -0
  62. package/build/services/memflow/connection.d.ts +23 -0
  63. package/build/services/memflow/connection.js +33 -0
  64. package/build/services/memflow/context.d.ts +143 -0
  65. package/build/services/memflow/context.js +299 -0
  66. package/build/services/memflow/exporter.d.ts +51 -0
  67. package/build/services/memflow/exporter.js +215 -0
  68. package/build/services/memflow/handle.d.ts +90 -0
  69. package/build/services/memflow/handle.js +176 -0
  70. package/build/services/memflow/index.d.ts +116 -0
  71. package/build/services/memflow/index.js +122 -0
  72. package/build/services/memflow/schemas/factory.d.ts +29 -0
  73. package/build/services/memflow/schemas/factory.js +2492 -0
  74. package/build/services/memflow/search.d.ts +142 -0
  75. package/build/services/memflow/search.js +320 -0
  76. package/build/services/memflow/worker.d.ts +124 -0
  77. package/build/services/memflow/worker.js +514 -0
  78. package/build/services/memflow/workflow/all.d.ts +7 -0
  79. package/build/services/memflow/workflow/all.js +15 -0
  80. package/build/services/memflow/workflow/common.d.ts +20 -0
  81. package/build/services/memflow/workflow/common.js +47 -0
  82. package/build/services/memflow/workflow/context.d.ts +6 -0
  83. package/build/services/memflow/workflow/context.js +45 -0
  84. package/build/services/memflow/workflow/contextMethods.d.ts +14 -0
  85. package/build/services/memflow/workflow/contextMethods.js +33 -0
  86. package/build/services/memflow/workflow/didRun.d.ts +7 -0
  87. package/build/services/memflow/workflow/didRun.js +22 -0
  88. package/build/services/memflow/workflow/emit.d.ts +11 -0
  89. package/build/services/memflow/workflow/emit.js +29 -0
  90. package/build/services/memflow/workflow/enrich.d.ts +9 -0
  91. package/build/services/memflow/workflow/enrich.js +17 -0
  92. package/build/services/memflow/workflow/execChild.d.ts +18 -0
  93. package/build/services/memflow/workflow/execChild.js +102 -0
  94. package/build/services/memflow/workflow/execHook.d.ts +65 -0
  95. package/build/services/memflow/workflow/execHook.js +73 -0
  96. package/build/services/memflow/workflow/hook.d.ts +9 -0
  97. package/build/services/memflow/workflow/hook.js +56 -0
  98. package/build/services/memflow/workflow/index.d.ts +74 -0
  99. package/build/services/memflow/workflow/index.js +87 -0
  100. package/build/services/memflow/workflow/interrupt.d.ts +9 -0
  101. package/build/services/memflow/workflow/interrupt.js +24 -0
  102. package/build/services/memflow/workflow/isSideEffectAllowed.d.ts +10 -0
  103. package/build/services/memflow/workflow/isSideEffectAllowed.js +33 -0
  104. package/build/services/memflow/workflow/proxyActivities.d.ts +20 -0
  105. package/build/services/memflow/workflow/proxyActivities.js +97 -0
  106. package/build/services/memflow/workflow/random.d.ts +6 -0
  107. package/build/services/memflow/workflow/random.js +16 -0
  108. package/build/services/memflow/workflow/searchMethods.d.ts +6 -0
  109. package/build/services/memflow/workflow/searchMethods.js +25 -0
  110. package/build/services/memflow/workflow/signal.d.ts +7 -0
  111. package/build/services/memflow/workflow/signal.js +28 -0
  112. package/build/services/memflow/workflow/sleepFor.d.ts +8 -0
  113. package/build/services/memflow/workflow/sleepFor.js +35 -0
  114. package/build/services/memflow/workflow/trace.d.ts +14 -0
  115. package/build/services/memflow/workflow/trace.js +33 -0
  116. package/build/services/memflow/workflow/waitFor.d.ts +8 -0
  117. package/build/services/memflow/workflow/waitFor.js +35 -0
  118. package/build/services/meshcall/index.d.ts +194 -0
  119. package/build/services/meshcall/index.js +452 -0
  120. package/build/services/meshcall/schemas/factory.d.ts +9 -0
  121. package/build/services/meshcall/schemas/factory.js +189 -0
  122. package/build/services/meshdata/index.d.ts +795 -0
  123. package/build/services/meshdata/index.js +1235 -0
  124. package/build/services/meshos/index.d.ts +293 -0
  125. package/build/services/meshos/index.js +547 -0
  126. package/build/services/pipe/functions/array.d.ts +17 -0
  127. package/build/services/pipe/functions/array.js +74 -0
  128. package/build/services/pipe/functions/bitwise.d.ts +9 -0
  129. package/build/services/pipe/functions/bitwise.js +24 -0
  130. package/build/services/pipe/functions/conditional.d.ts +13 -0
  131. package/build/services/pipe/functions/conditional.js +36 -0
  132. package/build/services/pipe/functions/cron.d.ts +12 -0
  133. package/build/services/pipe/functions/cron.js +40 -0
  134. package/build/services/pipe/functions/date.d.ts +58 -0
  135. package/build/services/pipe/functions/date.js +171 -0
  136. package/build/services/pipe/functions/index.d.ts +29 -0
  137. package/build/services/pipe/functions/index.js +30 -0
  138. package/build/services/pipe/functions/json.d.ts +5 -0
  139. package/build/services/pipe/functions/json.js +12 -0
  140. package/build/services/pipe/functions/logical.d.ts +5 -0
  141. package/build/services/pipe/functions/logical.js +12 -0
  142. package/build/services/pipe/functions/math.d.ts +42 -0
  143. package/build/services/pipe/functions/math.js +184 -0
  144. package/build/services/pipe/functions/number.d.ts +21 -0
  145. package/build/services/pipe/functions/number.js +60 -0
  146. package/build/services/pipe/functions/object.d.ts +25 -0
  147. package/build/services/pipe/functions/object.js +81 -0
  148. package/build/services/pipe/functions/string.d.ts +23 -0
  149. package/build/services/pipe/functions/string.js +69 -0
  150. package/build/services/pipe/functions/symbol.d.ts +12 -0
  151. package/build/services/pipe/functions/symbol.js +33 -0
  152. package/build/services/pipe/functions/unary.d.ts +7 -0
  153. package/build/services/pipe/functions/unary.js +18 -0
  154. package/build/services/pipe/index.d.ts +48 -0
  155. package/build/services/pipe/index.js +242 -0
  156. package/build/services/quorum/index.d.ts +90 -0
  157. package/build/services/quorum/index.js +263 -0
  158. package/build/services/reporter/index.d.ts +50 -0
  159. package/build/services/reporter/index.js +348 -0
  160. package/build/services/router/config/index.d.ts +11 -0
  161. package/build/services/router/config/index.js +36 -0
  162. package/build/services/router/consumption/index.d.ts +34 -0
  163. package/build/services/router/consumption/index.js +395 -0
  164. package/build/services/router/error-handling/index.d.ts +8 -0
  165. package/build/services/router/error-handling/index.js +98 -0
  166. package/build/services/router/index.d.ts +57 -0
  167. package/build/services/router/index.js +121 -0
  168. package/build/services/router/lifecycle/index.d.ts +27 -0
  169. package/build/services/router/lifecycle/index.js +80 -0
  170. package/build/services/router/telemetry/index.d.ts +11 -0
  171. package/build/services/router/telemetry/index.js +32 -0
  172. package/build/services/router/throttling/index.d.ts +23 -0
  173. package/build/services/router/throttling/index.js +76 -0
  174. package/build/services/search/factory.d.ts +7 -0
  175. package/build/services/search/factory.js +24 -0
  176. package/build/services/search/index.d.ts +23 -0
  177. package/build/services/search/index.js +10 -0
  178. package/build/services/search/providers/postgres/postgres.d.ts +25 -0
  179. package/build/services/search/providers/postgres/postgres.js +149 -0
  180. package/build/services/search/providers/redis/ioredis.d.ts +19 -0
  181. package/build/services/search/providers/redis/ioredis.js +121 -0
  182. package/build/services/search/providers/redis/redis.d.ts +19 -0
  183. package/build/services/search/providers/redis/redis.js +134 -0
  184. package/build/services/serializer/index.d.ts +42 -0
  185. package/build/services/serializer/index.js +282 -0
  186. package/build/services/store/cache.d.ts +67 -0
  187. package/build/services/store/cache.js +128 -0
  188. package/build/services/store/factory.d.ts +8 -0
  189. package/build/services/store/factory.js +24 -0
  190. package/build/services/store/index.d.ts +89 -0
  191. package/build/services/store/index.js +9 -0
  192. package/build/services/store/providers/postgres/kvsql.d.ts +168 -0
  193. package/build/services/store/providers/postgres/kvsql.js +198 -0
  194. package/build/services/store/providers/postgres/kvtables.d.ts +20 -0
  195. package/build/services/store/providers/postgres/kvtables.js +441 -0
  196. package/build/services/store/providers/postgres/kvtransaction.d.ts +36 -0
  197. package/build/services/store/providers/postgres/kvtransaction.js +248 -0
  198. package/build/services/store/providers/postgres/kvtypes/hash.d.ts +60 -0
  199. package/build/services/store/providers/postgres/kvtypes/hash.js +1287 -0
  200. package/build/services/store/providers/postgres/kvtypes/list.d.ts +33 -0
  201. package/build/services/store/providers/postgres/kvtypes/list.js +194 -0
  202. package/build/services/store/providers/postgres/kvtypes/string.d.ts +20 -0
  203. package/build/services/store/providers/postgres/kvtypes/string.js +115 -0
  204. package/build/services/store/providers/postgres/kvtypes/zset.d.ts +41 -0
  205. package/build/services/store/providers/postgres/kvtypes/zset.js +214 -0
  206. package/build/services/store/providers/postgres/postgres.d.ts +145 -0
  207. package/build/services/store/providers/postgres/postgres.js +1036 -0
  208. package/build/services/store/providers/redis/_base.d.ts +137 -0
  209. package/build/services/store/providers/redis/_base.js +980 -0
  210. package/build/services/store/providers/redis/ioredis.d.ts +20 -0
  211. package/build/services/store/providers/redis/ioredis.js +180 -0
  212. package/build/services/store/providers/redis/redis.d.ts +18 -0
  213. package/build/services/store/providers/redis/redis.js +199 -0
  214. package/build/services/store/providers/store-initializable.d.ts +5 -0
  215. package/build/services/store/providers/store-initializable.js +2 -0
  216. package/build/services/stream/factory.d.ts +8 -0
  217. package/build/services/stream/factory.js +37 -0
  218. package/build/services/stream/index.d.ts +69 -0
  219. package/build/services/stream/index.js +11 -0
  220. package/build/services/stream/providers/nats/nats.d.ts +60 -0
  221. package/build/services/stream/providers/nats/nats.js +225 -0
  222. package/build/services/stream/providers/postgres/kvtables.d.ts +3 -0
  223. package/build/services/stream/providers/postgres/kvtables.js +146 -0
  224. package/build/services/stream/providers/postgres/postgres.d.ts +107 -0
  225. package/build/services/stream/providers/postgres/postgres.js +519 -0
  226. package/build/services/stream/providers/redis/ioredis.d.ts +61 -0
  227. package/build/services/stream/providers/redis/ioredis.js +272 -0
  228. package/build/services/stream/providers/redis/redis.d.ts +61 -0
  229. package/build/services/stream/providers/redis/redis.js +305 -0
  230. package/build/services/stream/providers/stream-initializable.d.ts +4 -0
  231. package/build/services/stream/providers/stream-initializable.js +2 -0
  232. package/build/services/sub/factory.d.ts +7 -0
  233. package/build/services/sub/factory.js +29 -0
  234. package/build/services/sub/index.d.ts +22 -0
  235. package/build/services/sub/index.js +10 -0
  236. package/build/services/sub/providers/nats/nats.d.ts +19 -0
  237. package/build/services/sub/providers/nats/nats.js +105 -0
  238. package/build/services/sub/providers/postgres/postgres.d.ts +19 -0
  239. package/build/services/sub/providers/postgres/postgres.js +92 -0
  240. package/build/services/sub/providers/redis/ioredis.d.ts +17 -0
  241. package/build/services/sub/providers/redis/ioredis.js +81 -0
  242. package/build/services/sub/providers/redis/redis.d.ts +17 -0
  243. package/build/services/sub/providers/redis/redis.js +72 -0
  244. package/build/services/task/index.d.ts +36 -0
  245. package/build/services/task/index.js +206 -0
  246. package/build/services/telemetry/index.d.ts +52 -0
  247. package/build/services/telemetry/index.js +306 -0
  248. package/build/services/worker/index.d.ts +77 -0
  249. package/build/services/worker/index.js +197 -0
  250. package/package.json +1 -1
  251. package/.github/ISSUE_TEMPLATE/bug_report.md +0 -38
  252. package/.github/ISSUE_TEMPLATE/feature_request.md +0 -20
  253. package/typedoc.json +0 -47
  254. package/types/activity.ts +0 -268
  255. package/types/app.ts +0 -20
  256. package/types/async.ts +0 -6
  257. package/types/cache.ts +0 -1
  258. package/types/collator.ts +0 -9
  259. package/types/error.ts +0 -56
  260. package/types/exporter.ts +0 -102
  261. package/types/hook.ts +0 -44
  262. package/types/hotmesh.ts +0 -314
  263. package/types/index.ts +0 -306
  264. package/types/job.ts +0 -233
  265. package/types/logger.ts +0 -8
  266. package/types/manifest.ts +0 -70
  267. package/types/map.ts +0 -5
  268. package/types/memflow.ts +0 -645
  269. package/types/meshcall.ts +0 -235
  270. package/types/meshdata.ts +0 -278
  271. package/types/ms.d.ts +0 -7
  272. package/types/nats.ts +0 -270
  273. package/types/pipe.ts +0 -90
  274. package/types/postgres.ts +0 -114
  275. package/types/provider.ts +0 -161
  276. package/types/quorum.ts +0 -167
  277. package/types/redis.ts +0 -404
  278. package/types/serializer.ts +0 -40
  279. package/types/stats.ts +0 -117
  280. package/types/stream.ts +0 -231
  281. package/types/task.ts +0 -7
  282. package/types/telemetry.ts +0 -16
  283. package/types/transition.ts +0 -20
@@ -0,0 +1,134 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Signal = void 0;
4
+ const errors_1 = require("../../modules/errors");
5
+ const collator_1 = require("../collator");
6
+ const mapper_1 = require("../mapper");
7
+ const pipe_1 = require("../pipe");
8
+ const telemetry_1 = require("../telemetry");
9
+ const activity_1 = require("./activity");
10
+ class Signal extends activity_1.Activity {
11
+ constructor(config, data, metadata, hook, engine, context) {
12
+ super(config, data, metadata, hook, engine, context);
13
+ }
14
+ //******** LEG 1 ENTRY ********//
15
+ async process() {
16
+ this.logger.debug('signal-process', {
17
+ jid: this.context.metadata.jid,
18
+ gid: this.context.metadata.gid,
19
+ aid: this.metadata.aid,
20
+ });
21
+ let telemetry;
22
+ try {
23
+ await this.verifyEntry();
24
+ telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
25
+ telemetry.startActivitySpan(this.leg);
26
+ //save state and notarize early completion (signals only run leg1)
27
+ const transaction = this.store.transact();
28
+ this.adjacencyList = await this.filterAdjacent();
29
+ this.mapOutputData();
30
+ this.mapJobData();
31
+ await this.setState(transaction);
32
+ await collator_1.CollatorService.notarizeEarlyCompletion(this, transaction);
33
+ await this.setStatus(this.adjacencyList.length - 1, transaction);
34
+ const multiResponse = (await transaction.exec());
35
+ //todo: this should execute BEFORE the status is decremented
36
+ if (this.config.subtype === 'all') {
37
+ await this.hookAll();
38
+ }
39
+ else {
40
+ await this.hookOne();
41
+ }
42
+ //transition to adjacent activities
43
+ const jobStatus = this.resolveStatus(multiResponse);
44
+ const attrs = { 'app.job.jss': jobStatus };
45
+ const messageIds = await this.transition(this.adjacencyList, jobStatus);
46
+ if (messageIds.length) {
47
+ attrs['app.activity.mids'] = messageIds.join(',');
48
+ }
49
+ telemetry.mapActivityAttributes();
50
+ telemetry.setActivityAttributes(attrs);
51
+ return this.context.metadata.aid;
52
+ }
53
+ catch (error) {
54
+ if (error instanceof errors_1.InactiveJobError) {
55
+ this.logger.error('signal-inactive-job-error', { error });
56
+ return;
57
+ }
58
+ else if (error instanceof errors_1.GenerationalError) {
59
+ this.logger.info('process-event-generational-job-error', { error });
60
+ return;
61
+ }
62
+ else if (error instanceof errors_1.GetStateError) {
63
+ this.logger.error('signal-get-state-error', { error });
64
+ return;
65
+ }
66
+ else if (error instanceof errors_1.CollationError) {
67
+ if (error.fault === 'duplicate') {
68
+ this.logger.info('signal-collation-overage', {
69
+ job_id: this.context.metadata.jid,
70
+ guid: this.context.metadata.guid,
71
+ });
72
+ return;
73
+ }
74
+ //unknown collation error
75
+ this.logger.error('signal-collation-error', { error });
76
+ }
77
+ else {
78
+ this.logger.error('signal-process-error', { error });
79
+ }
80
+ telemetry?.setActivityError(error.message);
81
+ throw error;
82
+ }
83
+ finally {
84
+ telemetry?.endActivitySpan();
85
+ this.logger.debug('signal-process-end', {
86
+ jid: this.context.metadata.jid,
87
+ gid: this.context.metadata.gid,
88
+ aid: this.metadata.aid,
89
+ });
90
+ }
91
+ }
92
+ mapSignalData() {
93
+ if (this.config.signal?.maps) {
94
+ const mapper = new mapper_1.MapperService(this.config.signal.maps, this.context);
95
+ return mapper.mapRules();
96
+ }
97
+ }
98
+ mapResolverData() {
99
+ if (this.config.resolver?.maps) {
100
+ const mapper = new mapper_1.MapperService(this.config.resolver.maps, this.context);
101
+ return mapper.mapRules();
102
+ }
103
+ }
104
+ /**
105
+ * The signal activity will hook one
106
+ */
107
+ async hookOne() {
108
+ const topic = pipe_1.Pipe.resolve(this.config.topic, this.context);
109
+ const signalInputData = this.mapSignalData();
110
+ const status = pipe_1.Pipe.resolve(this.config.status, this.context);
111
+ const code = pipe_1.Pipe.resolve(this.config.code, this.context);
112
+ return await this.engine.hook(topic, signalInputData, status, code);
113
+ }
114
+ /**
115
+ * The signal activity will hook all paused jobs that share the same job key.
116
+ */
117
+ async hookAll() {
118
+ //prep 1) generate `input signal data` (essentially the webhook payload)
119
+ const signalInputData = this.mapSignalData();
120
+ //prep 2) generate data that resolves the job key (per the YAML config)
121
+ const keyResolverData = this.mapResolverData();
122
+ if (this.config.scrub) {
123
+ //self-clean the indexes upon use if configured
124
+ keyResolverData.scrub = true;
125
+ }
126
+ //prep 3) jobKeys can contain multiple indexes (per the YAML config)
127
+ const key_name = pipe_1.Pipe.resolve(this.config.key_name, this.context);
128
+ const key_value = pipe_1.Pipe.resolve(this.config.key_value, this.context);
129
+ const indexQueryFacets = [`${key_name}:${key_value}`];
130
+ //execute: `hookAll` will now resume all paused jobs that share the same job key
131
+ return await this.engine.hookAll(this.config.topic, signalInputData, keyResolverData, indexQueryFacets);
132
+ }
133
+ }
134
+ exports.Signal = Signal;
@@ -0,0 +1,37 @@
1
+ import { EngineService } from '../engine';
2
+ import { ActivityData, ActivityMetadata, ActivityType, TriggerActivity } from '../../types/activity';
3
+ import { JobState, ExtensionType, JobStatus } from '../../types/job';
4
+ import { ProviderTransaction } from '../../types/provider';
5
+ import { StringScalarType } from '../../types/serializer';
6
+ import { Activity } from './activity';
7
+ declare class Trigger extends Activity {
8
+ config: TriggerActivity;
9
+ constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
10
+ process(options?: ExtensionType): Promise<string>;
11
+ transitionAndLogAdjacent(options: ExtensionType, jobStatus: JobStatus, attrs: StringScalarType): Promise<void>;
12
+ /**
13
+ * `pending` flows will not transition from the trigger to adjacent children until resumed
14
+ */
15
+ initStatus(options: ExtensionType, count: number): number;
16
+ setExpired(seconds: number, transaction: ProviderTransaction): Promise<void>;
17
+ safeKey(key: string): string;
18
+ bindSearchData(options?: ExtensionType): void;
19
+ bindMarkerData(options?: ExtensionType): void;
20
+ setStatus(amount: number): Promise<void>;
21
+ /**
22
+ * if the parent (spawner) chose not to await, emit the job_id
23
+ * as the data payload { job_id }
24
+ */
25
+ execAdjacentParent(): Promise<void>;
26
+ createInputContext(): Partial<JobState>;
27
+ getState(): Promise<void>;
28
+ bindJobMetadataPaths(): string[];
29
+ bindActivityMetadataPaths(): string[];
30
+ resolveGranularity(): string;
31
+ getJobStatus(): number;
32
+ resolveJobId(context: Partial<JobState>): string;
33
+ resolveJobKey(context: Partial<JobState>): string;
34
+ setStateNX(status?: number): Promise<void>;
35
+ setStats(transaction?: ProviderTransaction): Promise<void>;
36
+ }
37
+ export { Trigger };
@@ -0,0 +1,246 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Trigger = void 0;
4
+ const errors_1 = require("../../modules/errors");
5
+ const utils_1 = require("../../modules/utils");
6
+ const collator_1 = require("../collator");
7
+ const pipe_1 = require("../pipe");
8
+ const reporter_1 = require("../reporter");
9
+ const serializer_1 = require("../serializer");
10
+ const telemetry_1 = require("../telemetry");
11
+ const activity_1 = require("./activity");
12
+ class Trigger extends activity_1.Activity {
13
+ constructor(config, data, metadata, hook, engine, context) {
14
+ super(config, data, metadata, hook, engine, context);
15
+ }
16
+ async process(options) {
17
+ this.logger.debug('trigger-process', {
18
+ subscribes: this.config.subscribes,
19
+ });
20
+ let telemetry;
21
+ try {
22
+ this.setLeg(2);
23
+ await this.getState();
24
+ telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
25
+ telemetry.startJobSpan();
26
+ telemetry.startActivitySpan(this.leg);
27
+ this.mapJobData();
28
+ this.adjacencyList = await this.filterAdjacent();
29
+ const initialStatus = this.initStatus(options, this.adjacencyList.length);
30
+ await this.setStateNX(initialStatus);
31
+ await this.setStatus(initialStatus);
32
+ this.bindSearchData(options);
33
+ this.bindMarkerData(options);
34
+ const transaction = this.store.transact();
35
+ await this.setState(transaction);
36
+ await this.setStats(transaction);
37
+ if (options?.pending) {
38
+ await this.setExpired(options?.pending, transaction);
39
+ }
40
+ await collator_1.CollatorService.notarizeInception(this, this.context.metadata.guid, transaction);
41
+ await transaction.exec();
42
+ this.execAdjacentParent();
43
+ telemetry.mapActivityAttributes();
44
+ const jobStatus = Number(this.context.metadata.js);
45
+ telemetry.setJobAttributes({ 'app.job.jss': jobStatus });
46
+ const attrs = { 'app.job.jss': jobStatus };
47
+ await this.transitionAndLogAdjacent(options, jobStatus, attrs);
48
+ telemetry.setActivityAttributes(attrs);
49
+ return this.context.metadata.jid;
50
+ }
51
+ catch (error) {
52
+ telemetry?.setActivityError(error.message);
53
+ if (error instanceof errors_1.DuplicateJobError) {
54
+ //todo: verify baseline in x-AZ rollover
55
+ await (0, utils_1.sleepFor)(1000);
56
+ const isOverage = await collator_1.CollatorService.isInceptionOverage(this, this.context.metadata.guid);
57
+ if (isOverage) {
58
+ this.logger.info('trigger-collation-overage', {
59
+ job_id: error.jobId,
60
+ guid: this.context.metadata.guid,
61
+ });
62
+ return;
63
+ }
64
+ this.logger.error('duplicate-job-error', {
65
+ job_id: error.jobId,
66
+ guid: this.context.metadata.guid,
67
+ });
68
+ }
69
+ else {
70
+ this.logger.error('trigger-process-error', { error });
71
+ }
72
+ throw error;
73
+ }
74
+ finally {
75
+ telemetry?.endJobSpan();
76
+ telemetry?.endActivitySpan();
77
+ this.logger.debug('trigger-process-end', {
78
+ subscribes: this.config.subscribes,
79
+ jid: this.context.metadata.jid,
80
+ gid: this.context.metadata.gid,
81
+ });
82
+ }
83
+ }
84
+ async transitionAndLogAdjacent(options = {}, jobStatus, attrs) {
85
+ //todo: enable resume from pending state
86
+ if (isNaN(options.pending)) {
87
+ const messageIds = await this.transition(this.adjacencyList, jobStatus);
88
+ if (messageIds.length) {
89
+ attrs['app.activity.mids'] = messageIds.join(',');
90
+ }
91
+ }
92
+ }
93
+ /**
94
+ * `pending` flows will not transition from the trigger to adjacent children until resumed
95
+ */
96
+ initStatus(options = {}, count) {
97
+ if (options.pending) {
98
+ return -1;
99
+ }
100
+ return count;
101
+ }
102
+ async setExpired(seconds, transaction) {
103
+ await this.store.expireJob(this.context.metadata.jid, seconds, transaction);
104
+ }
105
+ safeKey(key) {
106
+ return `_${key}`;
107
+ }
108
+ bindSearchData(options) {
109
+ if (options?.search) {
110
+ Object.keys(options.search).forEach((key) => {
111
+ this.context.data[this.safeKey(key)] = options.search[key].toString();
112
+ });
113
+ }
114
+ }
115
+ bindMarkerData(options) {
116
+ if (options?.marker) {
117
+ Object.keys(options.marker).forEach((key) => {
118
+ if (key.startsWith('-')) {
119
+ this.context.data[key] = options.marker[key].toString();
120
+ }
121
+ });
122
+ }
123
+ }
124
+ async setStatus(amount) {
125
+ this.context.metadata.js = amount;
126
+ }
127
+ /**
128
+ * if the parent (spawner) chose not to await, emit the job_id
129
+ * as the data payload { job_id }
130
+ */
131
+ async execAdjacentParent() {
132
+ if (this.context.metadata.px) {
133
+ const timestamp = (0, utils_1.formatISODate)(new Date());
134
+ const jobStartedConfirmationMessage = {
135
+ metadata: this.context.metadata,
136
+ data: {
137
+ job_id: this.context.metadata.jid,
138
+ jc: timestamp,
139
+ ju: timestamp,
140
+ },
141
+ };
142
+ await this.engine.execAdjacentParent(this.context, jobStartedConfirmationMessage);
143
+ }
144
+ }
145
+ createInputContext() {
146
+ const input = {
147
+ [this.metadata.aid]: {
148
+ input: { data: this.data },
149
+ },
150
+ $self: {
151
+ input: { data: this.data },
152
+ output: { data: this.data },
153
+ },
154
+ };
155
+ return input;
156
+ }
157
+ async getState() {
158
+ const inputContext = this.createInputContext();
159
+ const jobId = this.resolveJobId(inputContext);
160
+ const jobKey = this.resolveJobKey(inputContext);
161
+ const utc = (0, utils_1.formatISODate)(new Date());
162
+ const { id, version } = await this.engine.getVID();
163
+ this.initDimensionalAddress(collator_1.CollatorService.getDimensionalSeed());
164
+ const activityMetadata = {
165
+ ...this.metadata,
166
+ jid: jobId,
167
+ key: jobKey,
168
+ as: collator_1.CollatorService.getTriggerSeed(),
169
+ };
170
+ this.context = {
171
+ metadata: {
172
+ ...this.metadata,
173
+ gid: (0, utils_1.guid)(),
174
+ ngn: this.context.metadata.ngn,
175
+ pj: this.context.metadata.pj,
176
+ pg: this.context.metadata.pg,
177
+ pd: this.context.metadata.pd,
178
+ pa: this.context.metadata.pa,
179
+ px: this.context.metadata.px,
180
+ app: id,
181
+ vrs: version,
182
+ tpc: this.config.subscribes,
183
+ trc: this.context.metadata.trc,
184
+ spn: this.context.metadata.spn,
185
+ guid: this.context.metadata.guid,
186
+ jid: jobId,
187
+ dad: collator_1.CollatorService.getDimensionalSeed(),
188
+ key: jobKey,
189
+ jc: utc,
190
+ ju: utc,
191
+ ts: (0, utils_1.getTimeSeries)(this.resolveGranularity()),
192
+ js: 0,
193
+ },
194
+ data: {},
195
+ [this.metadata.aid]: {
196
+ input: {
197
+ data: this.data,
198
+ metadata: activityMetadata,
199
+ },
200
+ output: {
201
+ data: this.data,
202
+ metadata: activityMetadata,
203
+ },
204
+ settings: { data: {} },
205
+ errors: { data: {} },
206
+ },
207
+ };
208
+ this.context['$self'] = this.context[this.metadata.aid];
209
+ this.context['$job'] = this.context; //NEVER call STRINGIFY! (circular)
210
+ }
211
+ bindJobMetadataPaths() {
212
+ return serializer_1.MDATA_SYMBOLS.JOB.KEYS.map((key) => `metadata/${key}`);
213
+ }
214
+ bindActivityMetadataPaths() {
215
+ return serializer_1.MDATA_SYMBOLS.ACTIVITY.KEYS.map((key) => `output/metadata/${key}`);
216
+ }
217
+ resolveGranularity() {
218
+ return (this.config.stats?.granularity || reporter_1.ReporterService.DEFAULT_GRANULARITY);
219
+ }
220
+ getJobStatus() {
221
+ return this.context.metadata.js;
222
+ }
223
+ resolveJobId(context) {
224
+ const jobId = this.config.stats?.id;
225
+ return jobId ? pipe_1.Pipe.resolve(jobId, context) : (0, utils_1.guid)();
226
+ }
227
+ resolveJobKey(context) {
228
+ const jobKey = this.config.stats?.key;
229
+ return jobKey ? pipe_1.Pipe.resolve(jobKey, context) : '';
230
+ }
231
+ async setStateNX(status) {
232
+ const jobId = this.context.metadata.jid;
233
+ if (!await this.store.setStateNX(jobId, this.engine.appId, status)) {
234
+ throw new errors_1.DuplicateJobError(jobId);
235
+ }
236
+ }
237
+ async setStats(transaction) {
238
+ const md = this.context.metadata;
239
+ if (md.key && this.config.stats?.measures) {
240
+ const config = await this.engine.getVID();
241
+ const reporter = new reporter_1.ReporterService(config, this.store, this.logger);
242
+ await this.store.setStats(md.key, md.jid, md.ts, reporter.resolveTriggerStatistics(this.config, this.context), config, transaction);
243
+ }
244
+ }
245
+ }
246
+ exports.Trigger = Trigger;
@@ -0,0 +1,12 @@
1
+ import { EngineService } from '../engine';
2
+ import { ActivityData, ActivityMetadata, ActivityType, WorkerActivity } from '../../types/activity';
3
+ import { JobState } from '../../types/job';
4
+ import { ProviderTransaction } from '../../types/provider';
5
+ import { Activity } from './activity';
6
+ declare class Worker extends Activity {
7
+ config: WorkerActivity;
8
+ constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
9
+ process(): Promise<string>;
10
+ execActivity(transaction: ProviderTransaction): Promise<string>;
11
+ }
12
+ export { Worker };
@@ -0,0 +1,106 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Worker = void 0;
4
+ const errors_1 = require("../../modules/errors");
5
+ const utils_1 = require("../../modules/utils");
6
+ const collator_1 = require("../collator");
7
+ const pipe_1 = require("../pipe");
8
+ const telemetry_1 = require("../telemetry");
9
+ const activity_1 = require("./activity");
10
+ class Worker extends activity_1.Activity {
11
+ constructor(config, data, metadata, hook, engine, context) {
12
+ super(config, data, metadata, hook, engine, context);
13
+ }
14
+ //******** INITIAL ENTRY POINT (A) ********//
15
+ async process() {
16
+ this.logger.debug('worker-process', {
17
+ jid: this.context.metadata.jid,
18
+ gid: this.context.metadata.gid,
19
+ aid: this.metadata.aid,
20
+ });
21
+ let telemetry;
22
+ try {
23
+ await this.verifyEntry();
24
+ telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
25
+ telemetry.startActivitySpan(this.leg);
26
+ this.mapInputData();
27
+ //save state and authorize reentry
28
+ const transaction = this.store.transact();
29
+ //todo: await this.registerTimeout();
30
+ const messageId = await this.execActivity(transaction);
31
+ await collator_1.CollatorService.authorizeReentry(this, transaction);
32
+ await this.setState(transaction);
33
+ await this.setStatus(0, transaction);
34
+ const txResponse = (await transaction.exec());
35
+ //telemetry
36
+ telemetry.mapActivityAttributes();
37
+ const jobStatus = this.resolveStatus(txResponse);
38
+ telemetry.setActivityAttributes({
39
+ 'app.activity.mid': messageId,
40
+ 'app.job.jss': jobStatus,
41
+ });
42
+ return this.context.metadata.aid;
43
+ }
44
+ catch (error) {
45
+ if (error instanceof errors_1.InactiveJobError) {
46
+ this.logger.error('await-inactive-job-error', { error });
47
+ return;
48
+ }
49
+ else if (error instanceof errors_1.GenerationalError) {
50
+ this.logger.info('process-event-generational-job-error', { error });
51
+ return;
52
+ }
53
+ else if (error instanceof errors_1.GetStateError) {
54
+ this.logger.error('worker-get-state-error', { error });
55
+ return;
56
+ }
57
+ else if (error instanceof errors_1.CollationError) {
58
+ if (error.fault === 'duplicate') {
59
+ this.logger.info('worker-collation-overage', {
60
+ job_id: this.context.metadata.jid,
61
+ guid: this.context.metadata.guid,
62
+ });
63
+ return;
64
+ }
65
+ //unknown collation error
66
+ this.logger.error('worker-collation-error', { error });
67
+ }
68
+ else {
69
+ this.logger.error('worker-process-error', { error });
70
+ }
71
+ telemetry?.setActivityError(error.message);
72
+ throw error;
73
+ }
74
+ finally {
75
+ telemetry?.endActivitySpan();
76
+ this.logger.debug('worker-process-end', {
77
+ jid: this.context.metadata.jid,
78
+ gid: this.context.metadata.gid,
79
+ aid: this.metadata.aid,
80
+ });
81
+ }
82
+ }
83
+ async execActivity(transaction) {
84
+ const topic = pipe_1.Pipe.resolve(this.config.subtype, this.context);
85
+ const streamData = {
86
+ metadata: {
87
+ guid: (0, utils_1.guid)(),
88
+ jid: this.context.metadata.jid,
89
+ gid: this.context.metadata.gid,
90
+ dad: this.metadata.dad,
91
+ aid: this.metadata.aid,
92
+ topic,
93
+ spn: this.context['$self'].output.metadata.l1s,
94
+ trc: this.context.metadata.trc,
95
+ },
96
+ data: this.context.data,
97
+ };
98
+ if (this.config.retry) {
99
+ streamData.policies = {
100
+ retry: this.config.retry,
101
+ };
102
+ }
103
+ return (await this.engine.router?.publishMessage(topic, streamData, transaction));
104
+ }
105
+ }
106
+ exports.Worker = Worker;
@@ -0,0 +1,111 @@
1
+ import { CollationStage } from '../../types/collator';
2
+ import { ActivityDuplex } from '../../types/activity';
3
+ import { HotMeshGraph } from '../../types/hotmesh';
4
+ import { ProviderTransaction } from '../../types/provider';
5
+ import { Activity } from '../activities/activity';
6
+ import { Cycle } from '../activities/cycle';
7
+ declare class CollatorService {
8
+ static targetLength: number;
9
+ /**
10
+ * Upon re/entry, verify that the job status is active
11
+ */
12
+ static assertJobActive(status: number, jobId: string, activityId: string, threshold?: number): void;
13
+ /**
14
+ * returns the dimensional address (dad) for the target; due
15
+ * to the nature of the notary system, the dad for leg 2 entry
16
+ * must target the `0` index while leg 2 exit must target the
17
+ * current index (0)
18
+ */
19
+ static getDimensionalAddress(activity: Activity, isEntry?: boolean): Record<string, string>;
20
+ /**
21
+ * resolves the dimensional address for the
22
+ * ancestor in the graph to go back to. this address
23
+ * is determined by trimming the last digits from
24
+ * the `dad` (including the target).
25
+ * the target activity index is then set to `0`, so that
26
+ * the origin node can be queried for approval/entry.
27
+ */
28
+ static resolveReentryDimension(activity: Cycle): string;
29
+ static notarizeEntry(activity: Activity, transaction?: ProviderTransaction): Promise<number>;
30
+ static authorizeReentry(activity: Activity, transaction?: ProviderTransaction): Promise<number>;
31
+ static notarizeEarlyExit(activity: Activity, transaction?: ProviderTransaction): Promise<number>;
32
+ static notarizeEarlyCompletion(activity: Activity, transaction?: ProviderTransaction): Promise<number>;
33
+ /**
34
+ * sets the synthetic inception key (in case of an overage occurs).
35
+ */
36
+ static notarizeInception(activity: Activity, guid: string, transaction: ProviderTransaction): Promise<void>;
37
+ /**
38
+ * ignore those ID collisions that are due to re-entry overages
39
+ */
40
+ static isInceptionOverage(activity: Activity, guid: string): Promise<boolean>;
41
+ /**
42
+ * verifies both the concrete and synthetic keys for the activity; concrete keys
43
+ * exist in the original model and are effectively the 'real' keys. In reality,
44
+ * hook activities are atomized during compilation to create a synthetic DAG that
45
+ * is used to track the status of the graph in a distributed environment. The
46
+ * synthetic key represents different dimensional realities and is used to
47
+ * track re-entry overages (it distinguishes between the original and re-entry).
48
+ * The essential challenge is: is this a re-entry that is purposeful in
49
+ * order to induce cycles, or is the re-entry due to a failure in the system?
50
+ */
51
+ static notarizeReentry(activity: Activity, guid: string, transaction?: ProviderTransaction): Promise<number>;
52
+ static notarizeContinuation(activity: Activity, transaction?: ProviderTransaction): Promise<number>;
53
+ static notarizeCompletion(activity: Activity, transaction?: ProviderTransaction): Promise<number>;
54
+ static getDigitAtIndex(num: number, targetDigitIndex: number): number | null;
55
+ static getDimensionalIndex(num: number): number | null;
56
+ static isDuplicate(num: number, targetDigitIndex: number): boolean;
57
+ static isInactive(num: number): boolean;
58
+ static isPrimed(amount: number, leg: ActivityDuplex): boolean;
59
+ /**
60
+ * During compilation, the graphs are compiled into structures necessary
61
+ * for distributed processing; these are referred to as 'synthetic DAGs',
62
+ * because they are not part of the original graph, but are used to track
63
+ * the status of the graph in a distributed environment. This check ensures
64
+ * that the 'synthetic key' is not a duplicate. (which is different than
65
+ * saying the 'key' is not a duplicate)
66
+ */
67
+ static verifySyntheticInteger(amount: number): void;
68
+ static verifyInteger(amount: number, leg: ActivityDuplex, stage: CollationStage): void;
69
+ static getDimensionsById(ancestors: string[], dad: string): Record<string, string>;
70
+ /**
71
+ * All non-trigger activities are assigned a status seed by their parent
72
+ */
73
+ static getSeed(): string;
74
+ /**
75
+ * All trigger activities are assigned a status seed in a completed state
76
+ */
77
+ static getTriggerSeed(): string;
78
+ /**
79
+ * entry point for compiler-type activities. This is called by the compiler
80
+ * to bind the sorted activity IDs to the trigger activity. These are then used
81
+ * at runtime by the activities to track job/activity status.
82
+ * @param graphs
83
+ */
84
+ static compile(graphs: HotMeshGraph[]): void;
85
+ /**
86
+ * binds the ancestor array to each activity.
87
+ * Used in conjunction with the dimensional
88
+ * address (dad). If dad is `,0,1,0,0` and the
89
+ * ancestor array is `['t1', 'a1', 'a2']` for
90
+ * activity 'a3', then the SAVED DAD
91
+ * will always have the trailing
92
+ * 0's removed. This ensures that the addressing
93
+ * remains consistent even if the graph changes.
94
+ * id DAD SAVED DAD
95
+ * * t1 => ,0 => [empty]
96
+ * * a1 => ,0,1 => ,0,1
97
+ * * a2 => ,0,1,0 => ,0,1
98
+ * * a3 => ,0,1,0,0 => ,0,1
99
+ *
100
+ */
101
+ static bindAncestorArray(graphs: HotMeshGraph[]): void;
102
+ /**
103
+ * All activities exist on a dimensional plane. Zero
104
+ * is the default. A value of
105
+ * `AxY,0,0,0,0,1,0,0` would reflect that
106
+ * an ancestor activity was dimensionalized beyond
107
+ * the default.
108
+ */
109
+ static getDimensionalSeed(index?: number): string;
110
+ }
111
+ export { CollatorService };