@hotmeshio/hotmesh 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (283) hide show
  1. package/build/modules/enums.d.ts +110 -0
  2. package/build/modules/enums.js +134 -0
  3. package/build/modules/errors.d.ts +124 -0
  4. package/build/modules/errors.js +191 -0
  5. package/build/modules/key.d.ts +66 -0
  6. package/build/modules/key.js +190 -0
  7. package/build/modules/storage.d.ts +3 -0
  8. package/build/modules/storage.js +5 -0
  9. package/build/modules/utils.d.ts +119 -0
  10. package/build/modules/utils.js +374 -0
  11. package/build/package.json +1 -1
  12. package/build/services/activities/activity.d.ts +104 -0
  13. package/build/services/activities/activity.js +549 -0
  14. package/build/services/activities/await.d.ts +12 -0
  15. package/build/services/activities/await.js +114 -0
  16. package/build/services/activities/cycle.d.ts +19 -0
  17. package/build/services/activities/cycle.js +112 -0
  18. package/build/services/activities/hook.d.ts +27 -0
  19. package/build/services/activities/hook.js +168 -0
  20. package/build/services/activities/index.d.ts +19 -0
  21. package/build/services/activities/index.js +20 -0
  22. package/build/services/activities/interrupt.d.ts +16 -0
  23. package/build/services/activities/interrupt.js +158 -0
  24. package/build/services/activities/signal.d.ts +20 -0
  25. package/build/services/activities/signal.js +134 -0
  26. package/build/services/activities/trigger.d.ts +37 -0
  27. package/build/services/activities/trigger.js +246 -0
  28. package/build/services/activities/worker.d.ts +12 -0
  29. package/build/services/activities/worker.js +106 -0
  30. package/build/services/collator/index.d.ts +111 -0
  31. package/build/services/collator/index.js +293 -0
  32. package/build/services/compiler/deployer.d.ts +40 -0
  33. package/build/services/compiler/deployer.js +488 -0
  34. package/build/services/compiler/index.d.ts +32 -0
  35. package/build/services/compiler/index.js +112 -0
  36. package/build/services/compiler/validator.d.ts +34 -0
  37. package/build/services/compiler/validator.js +147 -0
  38. package/build/services/connector/factory.d.ts +22 -0
  39. package/build/services/connector/factory.js +99 -0
  40. package/build/services/connector/index.d.ts +30 -0
  41. package/build/services/connector/index.js +54 -0
  42. package/build/services/connector/providers/ioredis.d.ts +9 -0
  43. package/build/services/connector/providers/ioredis.js +26 -0
  44. package/build/services/connector/providers/nats.d.ts +9 -0
  45. package/build/services/connector/providers/nats.js +34 -0
  46. package/build/services/connector/providers/postgres.d.ts +20 -0
  47. package/build/services/connector/providers/postgres.js +102 -0
  48. package/build/services/connector/providers/redis.d.ts +9 -0
  49. package/build/services/connector/providers/redis.js +38 -0
  50. package/build/services/engine/index.d.ts +264 -0
  51. package/build/services/engine/index.js +761 -0
  52. package/build/services/exporter/index.d.ts +44 -0
  53. package/build/services/exporter/index.js +126 -0
  54. package/build/services/hotmesh/index.d.ts +483 -0
  55. package/build/services/hotmesh/index.js +622 -0
  56. package/build/services/logger/index.d.ts +16 -0
  57. package/build/services/logger/index.js +54 -0
  58. package/build/services/mapper/index.d.ts +28 -0
  59. package/build/services/mapper/index.js +81 -0
  60. package/build/services/memflow/client.d.ts +108 -0
  61. package/build/services/memflow/client.js +372 -0
  62. package/build/services/memflow/connection.d.ts +23 -0
  63. package/build/services/memflow/connection.js +33 -0
  64. package/build/services/memflow/context.d.ts +143 -0
  65. package/build/services/memflow/context.js +299 -0
  66. package/build/services/memflow/exporter.d.ts +51 -0
  67. package/build/services/memflow/exporter.js +215 -0
  68. package/build/services/memflow/handle.d.ts +90 -0
  69. package/build/services/memflow/handle.js +176 -0
  70. package/build/services/memflow/index.d.ts +116 -0
  71. package/build/services/memflow/index.js +122 -0
  72. package/build/services/memflow/schemas/factory.d.ts +29 -0
  73. package/build/services/memflow/schemas/factory.js +2492 -0
  74. package/build/services/memflow/search.d.ts +142 -0
  75. package/build/services/memflow/search.js +320 -0
  76. package/build/services/memflow/worker.d.ts +124 -0
  77. package/build/services/memflow/worker.js +514 -0
  78. package/build/services/memflow/workflow/all.d.ts +7 -0
  79. package/build/services/memflow/workflow/all.js +15 -0
  80. package/build/services/memflow/workflow/common.d.ts +20 -0
  81. package/build/services/memflow/workflow/common.js +47 -0
  82. package/build/services/memflow/workflow/context.d.ts +6 -0
  83. package/build/services/memflow/workflow/context.js +45 -0
  84. package/build/services/memflow/workflow/contextMethods.d.ts +14 -0
  85. package/build/services/memflow/workflow/contextMethods.js +33 -0
  86. package/build/services/memflow/workflow/didRun.d.ts +7 -0
  87. package/build/services/memflow/workflow/didRun.js +22 -0
  88. package/build/services/memflow/workflow/emit.d.ts +11 -0
  89. package/build/services/memflow/workflow/emit.js +29 -0
  90. package/build/services/memflow/workflow/enrich.d.ts +9 -0
  91. package/build/services/memflow/workflow/enrich.js +17 -0
  92. package/build/services/memflow/workflow/execChild.d.ts +18 -0
  93. package/build/services/memflow/workflow/execChild.js +102 -0
  94. package/build/services/memflow/workflow/execHook.d.ts +65 -0
  95. package/build/services/memflow/workflow/execHook.js +73 -0
  96. package/build/services/memflow/workflow/hook.d.ts +9 -0
  97. package/build/services/memflow/workflow/hook.js +56 -0
  98. package/build/services/memflow/workflow/index.d.ts +74 -0
  99. package/build/services/memflow/workflow/index.js +87 -0
  100. package/build/services/memflow/workflow/interrupt.d.ts +9 -0
  101. package/build/services/memflow/workflow/interrupt.js +24 -0
  102. package/build/services/memflow/workflow/isSideEffectAllowed.d.ts +10 -0
  103. package/build/services/memflow/workflow/isSideEffectAllowed.js +33 -0
  104. package/build/services/memflow/workflow/proxyActivities.d.ts +20 -0
  105. package/build/services/memflow/workflow/proxyActivities.js +97 -0
  106. package/build/services/memflow/workflow/random.d.ts +6 -0
  107. package/build/services/memflow/workflow/random.js +16 -0
  108. package/build/services/memflow/workflow/searchMethods.d.ts +6 -0
  109. package/build/services/memflow/workflow/searchMethods.js +25 -0
  110. package/build/services/memflow/workflow/signal.d.ts +7 -0
  111. package/build/services/memflow/workflow/signal.js +28 -0
  112. package/build/services/memflow/workflow/sleepFor.d.ts +8 -0
  113. package/build/services/memflow/workflow/sleepFor.js +35 -0
  114. package/build/services/memflow/workflow/trace.d.ts +14 -0
  115. package/build/services/memflow/workflow/trace.js +33 -0
  116. package/build/services/memflow/workflow/waitFor.d.ts +8 -0
  117. package/build/services/memflow/workflow/waitFor.js +35 -0
  118. package/build/services/meshcall/index.d.ts +194 -0
  119. package/build/services/meshcall/index.js +452 -0
  120. package/build/services/meshcall/schemas/factory.d.ts +9 -0
  121. package/build/services/meshcall/schemas/factory.js +189 -0
  122. package/build/services/meshdata/index.d.ts +795 -0
  123. package/build/services/meshdata/index.js +1235 -0
  124. package/build/services/meshos/index.d.ts +293 -0
  125. package/build/services/meshos/index.js +547 -0
  126. package/build/services/pipe/functions/array.d.ts +17 -0
  127. package/build/services/pipe/functions/array.js +74 -0
  128. package/build/services/pipe/functions/bitwise.d.ts +9 -0
  129. package/build/services/pipe/functions/bitwise.js +24 -0
  130. package/build/services/pipe/functions/conditional.d.ts +13 -0
  131. package/build/services/pipe/functions/conditional.js +36 -0
  132. package/build/services/pipe/functions/cron.d.ts +12 -0
  133. package/build/services/pipe/functions/cron.js +40 -0
  134. package/build/services/pipe/functions/date.d.ts +58 -0
  135. package/build/services/pipe/functions/date.js +171 -0
  136. package/build/services/pipe/functions/index.d.ts +29 -0
  137. package/build/services/pipe/functions/index.js +30 -0
  138. package/build/services/pipe/functions/json.d.ts +5 -0
  139. package/build/services/pipe/functions/json.js +12 -0
  140. package/build/services/pipe/functions/logical.d.ts +5 -0
  141. package/build/services/pipe/functions/logical.js +12 -0
  142. package/build/services/pipe/functions/math.d.ts +42 -0
  143. package/build/services/pipe/functions/math.js +184 -0
  144. package/build/services/pipe/functions/number.d.ts +21 -0
  145. package/build/services/pipe/functions/number.js +60 -0
  146. package/build/services/pipe/functions/object.d.ts +25 -0
  147. package/build/services/pipe/functions/object.js +81 -0
  148. package/build/services/pipe/functions/string.d.ts +23 -0
  149. package/build/services/pipe/functions/string.js +69 -0
  150. package/build/services/pipe/functions/symbol.d.ts +12 -0
  151. package/build/services/pipe/functions/symbol.js +33 -0
  152. package/build/services/pipe/functions/unary.d.ts +7 -0
  153. package/build/services/pipe/functions/unary.js +18 -0
  154. package/build/services/pipe/index.d.ts +48 -0
  155. package/build/services/pipe/index.js +242 -0
  156. package/build/services/quorum/index.d.ts +90 -0
  157. package/build/services/quorum/index.js +263 -0
  158. package/build/services/reporter/index.d.ts +50 -0
  159. package/build/services/reporter/index.js +348 -0
  160. package/build/services/router/config/index.d.ts +11 -0
  161. package/build/services/router/config/index.js +36 -0
  162. package/build/services/router/consumption/index.d.ts +34 -0
  163. package/build/services/router/consumption/index.js +395 -0
  164. package/build/services/router/error-handling/index.d.ts +8 -0
  165. package/build/services/router/error-handling/index.js +98 -0
  166. package/build/services/router/index.d.ts +57 -0
  167. package/build/services/router/index.js +121 -0
  168. package/build/services/router/lifecycle/index.d.ts +27 -0
  169. package/build/services/router/lifecycle/index.js +80 -0
  170. package/build/services/router/telemetry/index.d.ts +11 -0
  171. package/build/services/router/telemetry/index.js +32 -0
  172. package/build/services/router/throttling/index.d.ts +23 -0
  173. package/build/services/router/throttling/index.js +76 -0
  174. package/build/services/search/factory.d.ts +7 -0
  175. package/build/services/search/factory.js +24 -0
  176. package/build/services/search/index.d.ts +23 -0
  177. package/build/services/search/index.js +10 -0
  178. package/build/services/search/providers/postgres/postgres.d.ts +25 -0
  179. package/build/services/search/providers/postgres/postgres.js +149 -0
  180. package/build/services/search/providers/redis/ioredis.d.ts +19 -0
  181. package/build/services/search/providers/redis/ioredis.js +121 -0
  182. package/build/services/search/providers/redis/redis.d.ts +19 -0
  183. package/build/services/search/providers/redis/redis.js +134 -0
  184. package/build/services/serializer/index.d.ts +42 -0
  185. package/build/services/serializer/index.js +282 -0
  186. package/build/services/store/cache.d.ts +67 -0
  187. package/build/services/store/cache.js +128 -0
  188. package/build/services/store/factory.d.ts +8 -0
  189. package/build/services/store/factory.js +24 -0
  190. package/build/services/store/index.d.ts +89 -0
  191. package/build/services/store/index.js +9 -0
  192. package/build/services/store/providers/postgres/kvsql.d.ts +168 -0
  193. package/build/services/store/providers/postgres/kvsql.js +198 -0
  194. package/build/services/store/providers/postgres/kvtables.d.ts +20 -0
  195. package/build/services/store/providers/postgres/kvtables.js +441 -0
  196. package/build/services/store/providers/postgres/kvtransaction.d.ts +36 -0
  197. package/build/services/store/providers/postgres/kvtransaction.js +248 -0
  198. package/build/services/store/providers/postgres/kvtypes/hash.d.ts +60 -0
  199. package/build/services/store/providers/postgres/kvtypes/hash.js +1287 -0
  200. package/build/services/store/providers/postgres/kvtypes/list.d.ts +33 -0
  201. package/build/services/store/providers/postgres/kvtypes/list.js +194 -0
  202. package/build/services/store/providers/postgres/kvtypes/string.d.ts +20 -0
  203. package/build/services/store/providers/postgres/kvtypes/string.js +115 -0
  204. package/build/services/store/providers/postgres/kvtypes/zset.d.ts +41 -0
  205. package/build/services/store/providers/postgres/kvtypes/zset.js +214 -0
  206. package/build/services/store/providers/postgres/postgres.d.ts +145 -0
  207. package/build/services/store/providers/postgres/postgres.js +1036 -0
  208. package/build/services/store/providers/redis/_base.d.ts +137 -0
  209. package/build/services/store/providers/redis/_base.js +980 -0
  210. package/build/services/store/providers/redis/ioredis.d.ts +20 -0
  211. package/build/services/store/providers/redis/ioredis.js +180 -0
  212. package/build/services/store/providers/redis/redis.d.ts +18 -0
  213. package/build/services/store/providers/redis/redis.js +199 -0
  214. package/build/services/store/providers/store-initializable.d.ts +5 -0
  215. package/build/services/store/providers/store-initializable.js +2 -0
  216. package/build/services/stream/factory.d.ts +8 -0
  217. package/build/services/stream/factory.js +37 -0
  218. package/build/services/stream/index.d.ts +69 -0
  219. package/build/services/stream/index.js +11 -0
  220. package/build/services/stream/providers/nats/nats.d.ts +60 -0
  221. package/build/services/stream/providers/nats/nats.js +225 -0
  222. package/build/services/stream/providers/postgres/kvtables.d.ts +3 -0
  223. package/build/services/stream/providers/postgres/kvtables.js +146 -0
  224. package/build/services/stream/providers/postgres/postgres.d.ts +107 -0
  225. package/build/services/stream/providers/postgres/postgres.js +519 -0
  226. package/build/services/stream/providers/redis/ioredis.d.ts +61 -0
  227. package/build/services/stream/providers/redis/ioredis.js +272 -0
  228. package/build/services/stream/providers/redis/redis.d.ts +61 -0
  229. package/build/services/stream/providers/redis/redis.js +305 -0
  230. package/build/services/stream/providers/stream-initializable.d.ts +4 -0
  231. package/build/services/stream/providers/stream-initializable.js +2 -0
  232. package/build/services/sub/factory.d.ts +7 -0
  233. package/build/services/sub/factory.js +29 -0
  234. package/build/services/sub/index.d.ts +22 -0
  235. package/build/services/sub/index.js +10 -0
  236. package/build/services/sub/providers/nats/nats.d.ts +19 -0
  237. package/build/services/sub/providers/nats/nats.js +105 -0
  238. package/build/services/sub/providers/postgres/postgres.d.ts +19 -0
  239. package/build/services/sub/providers/postgres/postgres.js +92 -0
  240. package/build/services/sub/providers/redis/ioredis.d.ts +17 -0
  241. package/build/services/sub/providers/redis/ioredis.js +81 -0
  242. package/build/services/sub/providers/redis/redis.d.ts +17 -0
  243. package/build/services/sub/providers/redis/redis.js +72 -0
  244. package/build/services/task/index.d.ts +36 -0
  245. package/build/services/task/index.js +206 -0
  246. package/build/services/telemetry/index.d.ts +52 -0
  247. package/build/services/telemetry/index.js +306 -0
  248. package/build/services/worker/index.d.ts +77 -0
  249. package/build/services/worker/index.js +197 -0
  250. package/package.json +1 -1
  251. package/.github/ISSUE_TEMPLATE/bug_report.md +0 -38
  252. package/.github/ISSUE_TEMPLATE/feature_request.md +0 -20
  253. package/typedoc.json +0 -47
  254. package/types/activity.ts +0 -268
  255. package/types/app.ts +0 -20
  256. package/types/async.ts +0 -6
  257. package/types/cache.ts +0 -1
  258. package/types/collator.ts +0 -9
  259. package/types/error.ts +0 -56
  260. package/types/exporter.ts +0 -102
  261. package/types/hook.ts +0 -44
  262. package/types/hotmesh.ts +0 -314
  263. package/types/index.ts +0 -306
  264. package/types/job.ts +0 -233
  265. package/types/logger.ts +0 -8
  266. package/types/manifest.ts +0 -70
  267. package/types/map.ts +0 -5
  268. package/types/memflow.ts +0 -645
  269. package/types/meshcall.ts +0 -235
  270. package/types/meshdata.ts +0 -278
  271. package/types/ms.d.ts +0 -7
  272. package/types/nats.ts +0 -270
  273. package/types/pipe.ts +0 -90
  274. package/types/postgres.ts +0 -114
  275. package/types/provider.ts +0 -161
  276. package/types/quorum.ts +0 -167
  277. package/types/redis.ts +0 -404
  278. package/types/serializer.ts +0 -40
  279. package/types/stats.ts +0 -117
  280. package/types/stream.ts +0 -231
  281. package/types/task.ts +0 -7
  282. package/types/telemetry.ts +0 -16
  283. package/types/transition.ts +0 -20
@@ -0,0 +1,488 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Deployer = void 0;
4
+ const key_1 = require("../../modules/key");
5
+ const utils_1 = require("../../modules/utils");
6
+ const collator_1 = require("../collator");
7
+ const serializer_1 = require("../serializer");
8
+ const pipe_1 = require("../pipe");
9
+ const validator_1 = require("./validator");
10
+ const DEFAULT_METADATA_RANGE_SIZE = 26; //metadata is 26 slots ([a-z] * 1)
11
+ const DEFAULT_DATA_RANGE_SIZE = 260; //data is 260 slots ([a-zA-Z] * 5)
12
+ const DEFAULT_RANGE_SIZE = DEFAULT_METADATA_RANGE_SIZE + DEFAULT_DATA_RANGE_SIZE;
13
+ class Deployer {
14
+ constructor(manifest) {
15
+ this.manifest = null;
16
+ this.manifest = manifest;
17
+ }
18
+ async deploy(store, stream) {
19
+ this.store = store;
20
+ this.stream = stream;
21
+ collator_1.CollatorService.compile(this.manifest.app.graphs);
22
+ this.convertActivitiesToHooks();
23
+ this.convertTopicsToTypes();
24
+ this.copyJobSchemas();
25
+ this.bindBackRefs();
26
+ this.bindParents();
27
+ this.bindCycleTarget();
28
+ this.resolveMappingDependencies();
29
+ this.resolveJobMapsPaths();
30
+ await this.generateSymKeys();
31
+ await this.generateSymVals();
32
+ await this.deployHookPatterns();
33
+ await this.deployActivitySchemas();
34
+ await this.deploySubscriptions();
35
+ await this.deployTransitions();
36
+ await this.deployConsumerGroups();
37
+ }
38
+ getVID() {
39
+ return {
40
+ id: this.manifest.app.id,
41
+ version: this.manifest.app.version,
42
+ };
43
+ }
44
+ async generateSymKeys() {
45
+ //note: symbol ranges are additive (per version); path assignments are immutable
46
+ for (const graph of this.manifest.app.graphs) {
47
+ //generate JOB symbols
48
+ const [, trigger] = this.findTrigger(graph);
49
+ const topic = trigger.subscribes;
50
+ const [lower, upper, symbols] = await this.store.reserveSymbolRange(`$${topic}`, DEFAULT_RANGE_SIZE, 'JOB');
51
+ const prefix = ''; //job meta/data is NOT namespaced
52
+ const newSymbols = this.bindSymbols(lower, upper, symbols, prefix, trigger.PRODUCES);
53
+ if (Object.keys(newSymbols).length) {
54
+ await this.store.addSymbols(`$${topic}`, newSymbols);
55
+ }
56
+ //generate ACTIVITY symbols
57
+ for (const [activityId, activity] of Object.entries(graph.activities)) {
58
+ const [lower, upper, symbols] = await this.store.reserveSymbolRange(activityId, DEFAULT_RANGE_SIZE, 'ACTIVITY');
59
+ const prefix = `${activityId}/`; //activity meta/data is namespaced
60
+ this.bindSelf(activity.consumes, activity.produces, activityId);
61
+ const newSymbols = this.bindSymbols(lower, upper, symbols, prefix, activity.produces);
62
+ if (Object.keys(newSymbols).length) {
63
+ await this.store.addSymbols(activityId, newSymbols);
64
+ }
65
+ }
66
+ }
67
+ }
68
+ bindSelf(consumes, produces, activityId) {
69
+ //bind self-referential mappings
70
+ for (const selfId of [activityId, '$self']) {
71
+ const selfConsumes = consumes[selfId];
72
+ if (selfConsumes) {
73
+ for (const path of selfConsumes) {
74
+ if (!produces.includes(path)) {
75
+ produces.push(path);
76
+ }
77
+ }
78
+ }
79
+ }
80
+ }
81
+ bindSymbols(startIndex, maxIndex, existingSymbols, prefix, produces) {
82
+ const newSymbols = {};
83
+ const currentSymbols = { ...existingSymbols };
84
+ for (const path of produces) {
85
+ const fullPath = `${prefix}${path}`;
86
+ if (!currentSymbols[fullPath]) {
87
+ if (startIndex > maxIndex) {
88
+ throw new Error('Symbol index out of bounds');
89
+ }
90
+ const symbol = (0, utils_1.getSymKey)(startIndex);
91
+ startIndex++;
92
+ newSymbols[fullPath] = symbol;
93
+ currentSymbols[fullPath] = symbol; // update the currentSymbols to include this new symbol
94
+ }
95
+ }
96
+ return newSymbols;
97
+ }
98
+ copyJobSchemas() {
99
+ const graphs = this.manifest.app.graphs;
100
+ for (const graph of graphs) {
101
+ const jobSchema = graph.output?.schema;
102
+ const outputSchema = graph.input?.schema;
103
+ if (!jobSchema && !outputSchema)
104
+ continue;
105
+ const activities = graph.activities;
106
+ // Find the trigger activity and bind the job schema to it
107
+ // at execution time, the trigger is a standin for the job
108
+ for (const activityKey in activities) {
109
+ if (activities[activityKey].type === 'trigger') {
110
+ const trigger = activities[activityKey];
111
+ if (jobSchema) {
112
+ //possible for trigger to have job mappings
113
+ if (!trigger.job) {
114
+ trigger.job = {};
115
+ }
116
+ trigger.job.schema = jobSchema;
117
+ }
118
+ if (outputSchema) {
119
+ //impossible for trigger to have output mappings.
120
+ trigger.output = { schema: outputSchema };
121
+ }
122
+ }
123
+ }
124
+ }
125
+ }
126
+ bindBackRefs() {
127
+ for (const graph of this.manifest.app.graphs) {
128
+ const activities = graph.activities;
129
+ const triggerId = this.findTrigger(graph)[0];
130
+ for (const activityKey in activities) {
131
+ activities[activityKey].trigger = triggerId;
132
+ activities[activityKey].subscribes = graph.subscribes;
133
+ if (graph.publishes) {
134
+ activities[activityKey].publishes = graph.publishes;
135
+ }
136
+ activities[activityKey].expire = graph.expire ?? undefined;
137
+ activities[activityKey].persistent = graph.persistent ?? undefined;
138
+ }
139
+ }
140
+ }
141
+ //the cycle/goto activity includes and ancestor target;
142
+ //update with the cycle flag, so it can be rerun
143
+ bindCycleTarget() {
144
+ for (const graph of this.manifest.app.graphs) {
145
+ const activities = graph.activities;
146
+ for (const activityKey in activities) {
147
+ const activity = activities[activityKey];
148
+ if (activity.type === 'cycle') {
149
+ activities[activity.ancestor].cycle = true;
150
+ }
151
+ }
152
+ }
153
+ }
154
+ //it's more intuitive for SDK users to use 'topic',
155
+ //but the compiler is desiged to be generic and uses the attribute, 'subtypes'
156
+ convertTopicsToTypes() {
157
+ for (const graph of this.manifest.app.graphs) {
158
+ const activities = graph.activities;
159
+ for (const activityKey in activities) {
160
+ const activity = activities[activityKey];
161
+ if (['worker', 'await'].includes(activity.type) &&
162
+ activity.topic &&
163
+ !activity.subtype) {
164
+ activity.subtype = activity.topic;
165
+ }
166
+ }
167
+ }
168
+ }
169
+ //legacy; remove at beta (assume no legacy refs to 'activity' at that point)
170
+ convertActivitiesToHooks() {
171
+ for (const graph of this.manifest.app.graphs) {
172
+ const activities = graph.activities;
173
+ for (const activityKey in activities) {
174
+ const activity = activities[activityKey];
175
+ if (['activity'].includes(activity.type)) {
176
+ activity.type = 'hook';
177
+ }
178
+ }
179
+ }
180
+ }
181
+ async bindParents() {
182
+ const graphs = this.manifest.app.graphs;
183
+ for (const graph of graphs) {
184
+ if (graph.transitions) {
185
+ for (const fromActivity in graph.transitions) {
186
+ const toTransitions = graph.transitions[fromActivity];
187
+ for (const transition of toTransitions) {
188
+ const to = transition.to;
189
+ //DAGs have one parent; easy to optimize for
190
+ graph.activities[to].parent = fromActivity;
191
+ }
192
+ //temporarily bind the transitions to the parent activity,
193
+ // so the consumer/producer registrar picks up the bindings
194
+ graph.activities[fromActivity].transitions = toTransitions;
195
+ }
196
+ }
197
+ }
198
+ }
199
+ collectValues(schema, values) {
200
+ for (const [key, value] of Object.entries(schema)) {
201
+ if (key === 'enum' || key === 'examples' || key === 'default') {
202
+ if (Array.isArray(value)) {
203
+ for (const v of value) {
204
+ if (typeof v === 'string' && v.length > 5) {
205
+ values.add(v);
206
+ }
207
+ }
208
+ }
209
+ else if (typeof value === 'string' && value.length > 5) {
210
+ values.add(value);
211
+ }
212
+ }
213
+ else if (typeof value === 'object') {
214
+ this.collectValues(value, values);
215
+ }
216
+ }
217
+ }
218
+ traverse(obj, values) {
219
+ for (const value of Object.values(obj)) {
220
+ if (typeof value === 'object') {
221
+ if ('schema' in value) {
222
+ this.collectValues(value.schema, values);
223
+ }
224
+ else {
225
+ this.traverse(value, values);
226
+ }
227
+ }
228
+ }
229
+ }
230
+ async generateSymVals() {
231
+ const uniqueStrings = new Set();
232
+ for (const graph of this.manifest.app.graphs) {
233
+ this.traverse(graph, uniqueStrings);
234
+ }
235
+ const existingSymbols = await this.store.getSymbolValues();
236
+ const startIndex = Object.keys(existingSymbols).length;
237
+ const maxIndex = Math.pow(52, 2) - 1;
238
+ const newSymbols = serializer_1.SerializerService.filterSymVals(startIndex, maxIndex, existingSymbols, uniqueStrings);
239
+ await this.store.addSymbolValues(newSymbols);
240
+ }
241
+ resolveJobMapsPaths() {
242
+ function parsePaths(obj) {
243
+ const result = [];
244
+ function traverse(obj, path = []) {
245
+ for (const key in obj) {
246
+ if (typeof obj[key] === 'object' &&
247
+ obj[key] !== null &&
248
+ !('@pipe' in obj[key])) {
249
+ const newPath = [...path, key];
250
+ traverse(obj[key], newPath);
251
+ }
252
+ else {
253
+ //wildcard mapping (e.g., 'friends[25]')
254
+ //when this is resolved, it will be expanded to
255
+ //`'friends/0', ..., 'friends/24'`, providing 25 dynamic
256
+ //slots in the flow's output data
257
+ const pathName = [...path, key].join('/');
258
+ if (!pathName.includes('[')) {
259
+ const finalPath = `data/${pathName}`;
260
+ if (!result.includes(finalPath)) {
261
+ result.push(finalPath);
262
+ }
263
+ }
264
+ else {
265
+ const [left, right] = pathName.split('[');
266
+ //check if this variable isLiteralKeyType (#, -, or _)
267
+ const [amount, _] = right.split(']');
268
+ if (!isNaN(parseInt(amount))) {
269
+ //loop to create all possible paths (0 to amount)
270
+ for (let i = 0; i < parseInt(amount); i++) {
271
+ const finalPath = `data/${left}/${i}`;
272
+ if (!result.includes(finalPath)) {
273
+ result.push(finalPath);
274
+ }
275
+ }
276
+ } //else ignore (amount might be '-' or '_') `-` is marker data; `_` is job data;
277
+ }
278
+ }
279
+ }
280
+ }
281
+ if (obj) {
282
+ traverse(obj);
283
+ }
284
+ return result;
285
+ }
286
+ for (const graph of this.manifest.app.graphs) {
287
+ let results = [];
288
+ const [, trigger] = this.findTrigger(graph);
289
+ for (const activityKey in graph.activities) {
290
+ const activity = graph.activities[activityKey];
291
+ results = results.concat(parsePaths(activity.job?.maps));
292
+ }
293
+ trigger.PRODUCES = results;
294
+ }
295
+ }
296
+ resolveMappingDependencies() {
297
+ const dynamicMappingRules = [];
298
+ //recursive function to descend into the object and find all dynamic mapping rules
299
+ function traverse(obj, consumes) {
300
+ for (const key in obj) {
301
+ if (typeof obj[key] === 'string') {
302
+ const stringValue = obj[key];
303
+ const dynamicMappingRuleMatch = stringValue.match(/^\{[^@].*}$/);
304
+ if (dynamicMappingRuleMatch &&
305
+ !validator_1.Validator.CONTEXT_VARS.includes(stringValue)) {
306
+ if (stringValue.split('.')[1] !== 'input') {
307
+ dynamicMappingRules.push(stringValue);
308
+ consumes.push(stringValue);
309
+ }
310
+ }
311
+ }
312
+ else if (typeof obj[key] === 'object' && obj[key] !== null) {
313
+ traverse(obj[key], consumes);
314
+ }
315
+ }
316
+ }
317
+ const graphs = this.manifest.app.graphs;
318
+ for (const graph of graphs) {
319
+ const activities = graph.activities;
320
+ for (const activityId in activities) {
321
+ const activity = activities[activityId];
322
+ activity.consumes = [];
323
+ traverse(activity, activity.consumes);
324
+ activity.consumes = this.groupMappingRules(activity.consumes);
325
+ }
326
+ }
327
+ const groupedRules = this.groupMappingRules(dynamicMappingRules);
328
+ // Iterate through the graph and add 'produces' field to each activity
329
+ for (const graph of graphs) {
330
+ const activities = graph.activities;
331
+ for (const activityId in activities) {
332
+ const activity = activities[activityId];
333
+ activity.produces = groupedRules[`${activityId}`] || [];
334
+ }
335
+ }
336
+ }
337
+ groupMappingRules(rules) {
338
+ rules = Array.from(new Set(rules)).sort();
339
+ // Group by the first symbol before the period (this is the activity name)
340
+ const groupedRules = {};
341
+ for (const rule of rules) {
342
+ const [group, resolved] = this.resolveMappableValue(rule);
343
+ if (!groupedRules[group]) {
344
+ groupedRules[group] = [];
345
+ }
346
+ groupedRules[group].push(resolved);
347
+ }
348
+ return groupedRules;
349
+ }
350
+ resolveMappableValue(mappable) {
351
+ mappable = mappable.substring(1, mappable.length - 1);
352
+ const parts = mappable.split('.');
353
+ if (parts[0] === '$job') {
354
+ const [group, ...path] = parts;
355
+ return [group, path.join('/')];
356
+ }
357
+ else {
358
+ //normalize paths to be relative to the activity
359
+ const [group, type, subtype, ...path] = parts;
360
+ const prefix = {
361
+ hook: 'hook/data',
362
+ input: 'input/data',
363
+ output: subtype === 'data' ? 'output/data' : 'output/metadata',
364
+ }[type];
365
+ return [group, `${prefix}/${path.join('/')}`];
366
+ }
367
+ }
368
+ async deployActivitySchemas() {
369
+ const graphs = this.manifest.app.graphs;
370
+ const activitySchemas = {};
371
+ for (const graph of graphs) {
372
+ const activities = graph.activities;
373
+ for (const activityKey in activities) {
374
+ const target = activities[activityKey];
375
+ //remove transitions; no longer necessary for runtime
376
+ delete target.transitions;
377
+ activitySchemas[activityKey] = target;
378
+ }
379
+ }
380
+ await this.store.setSchemas(activitySchemas, this.getVID());
381
+ }
382
+ async deploySubscriptions() {
383
+ const graphs = this.manifest.app.graphs;
384
+ const publicSubscriptions = {};
385
+ for (const graph of graphs) {
386
+ const activities = graph.activities;
387
+ const subscribesTopic = graph.subscribes;
388
+ // Find the activity ID associated with the subscribes topic
389
+ for (const activityKey in activities) {
390
+ if (activities[activityKey].type === 'trigger') {
391
+ publicSubscriptions[subscribesTopic] = activityKey;
392
+ break;
393
+ }
394
+ }
395
+ }
396
+ await this.store.setSubscriptions(publicSubscriptions, this.getVID());
397
+ }
398
+ findTrigger(graph) {
399
+ for (const activityKey in graph.activities) {
400
+ const activity = graph.activities[activityKey];
401
+ if (activity.type === 'trigger') {
402
+ return [activityKey, activity];
403
+ }
404
+ }
405
+ return null;
406
+ }
407
+ async deployTransitions() {
408
+ const graphs = this.manifest.app.graphs;
409
+ const privateSubscriptions = {};
410
+ for (const graph of graphs) {
411
+ if (graph.subscribes && graph.subscribes.startsWith('.')) {
412
+ const [triggerId] = this.findTrigger(graph);
413
+ if (triggerId) {
414
+ privateSubscriptions[graph.subscribes] = { [triggerId]: true };
415
+ }
416
+ }
417
+ if (graph.transitions) {
418
+ for (const fromActivity in graph.transitions) {
419
+ const toTransitions = graph.transitions[fromActivity];
420
+ const toValues = {};
421
+ for (const transition of toTransitions) {
422
+ const to = transition.to;
423
+ if (transition.conditions) {
424
+ toValues[to] = transition.conditions;
425
+ }
426
+ else {
427
+ toValues[to] = true;
428
+ }
429
+ }
430
+ if (Object.keys(toValues).length > 0) {
431
+ privateSubscriptions['.' + fromActivity] = toValues;
432
+ }
433
+ }
434
+ }
435
+ }
436
+ await this.store.setTransitions(privateSubscriptions, this.getVID());
437
+ }
438
+ async deployHookPatterns() {
439
+ const graphs = this.manifest.app.graphs;
440
+ const hookRules = {};
441
+ for (const graph of graphs) {
442
+ if (graph.hooks) {
443
+ for (const topic in graph.hooks) {
444
+ hookRules[topic] = graph.hooks[topic];
445
+ const activityId = graph.hooks[topic][0].to;
446
+ const targetActivity = graph.activities[activityId];
447
+ if (targetActivity) {
448
+ if (!targetActivity.hook) {
449
+ targetActivity.hook = {};
450
+ }
451
+ //create back-reference to the hook topic
452
+ targetActivity.hook.topic = topic;
453
+ }
454
+ }
455
+ }
456
+ }
457
+ await this.store.setHookRules(hookRules);
458
+ }
459
+ async deployConsumerGroups() {
460
+ //create one engine group
461
+ const params = { appId: this.manifest.app.id };
462
+ const key = this.store.mintKey(key_1.KeyType.STREAMS, params);
463
+ await this.deployConsumerGroup(key, 'ENGINE');
464
+ for (const graph of this.manifest.app.graphs) {
465
+ const activities = graph.activities;
466
+ for (const activityKey in activities) {
467
+ const activity = activities[activityKey];
468
+ //only precreate if the topic is concrete and not `mappable`
469
+ if (activity.type === 'worker' &&
470
+ pipe_1.Pipe.resolve(activity.subtype, {}) === activity.subtype) {
471
+ params.topic = activity.subtype;
472
+ const key = this.store.mintKey(key_1.KeyType.STREAMS, params);
473
+ //create one worker group per unique activity subtype (the topic)
474
+ await this.deployConsumerGroup(key, 'WORKER');
475
+ }
476
+ }
477
+ }
478
+ }
479
+ async deployConsumerGroup(stream, group) {
480
+ try {
481
+ await this.stream.createConsumerGroup(stream, group);
482
+ }
483
+ catch (err) {
484
+ this.store.logger.info('router-stream-group-exists', { stream, group });
485
+ }
486
+ }
487
+ }
488
+ exports.Deployer = Deployer;
@@ -0,0 +1,32 @@
1
+ import { ILogger } from '../logger';
2
+ import { StoreService } from '../store';
3
+ import { HotMeshManifest } from '../../types/hotmesh';
4
+ import { ProviderClient, ProviderTransaction } from '../../types/provider';
5
+ import { StreamService } from '../stream';
6
+ /**
7
+ * The compiler service converts a graph into a executable program.
8
+ */
9
+ declare class CompilerService {
10
+ store: StoreService<ProviderClient, ProviderTransaction> | null;
11
+ stream: StreamService<ProviderClient, ProviderTransaction> | null;
12
+ logger: ILogger;
13
+ constructor(store: StoreService<ProviderClient, ProviderTransaction>, stream: StreamService<ProviderClient, ProviderTransaction>, logger: ILogger);
14
+ /**
15
+ * verifies and plans the deployment of an app to Redis; the app is not deployed yet
16
+ * @param path
17
+ */
18
+ plan(mySchemaOrPath: string): Promise<HotMeshManifest>;
19
+ isPath(input: string): boolean;
20
+ /**
21
+ * deploys an app to Redis but does NOT activate it.
22
+ */
23
+ deploy(mySchemaOrPath: string): Promise<HotMeshManifest>;
24
+ /**
25
+ * activates a deployed version of an app;
26
+ * @param appId
27
+ * @param appVersion
28
+ */
29
+ activate(appId: string, appVersion: string): Promise<boolean>;
30
+ saveAsJSON(originalPath: string, schema: HotMeshManifest): Promise<void>;
31
+ }
32
+ export { CompilerService };
@@ -0,0 +1,112 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ var __importDefault = (this && this.__importDefault) || function (mod) {
26
+ return (mod && mod.__esModule) ? mod : { "default": mod };
27
+ };
28
+ Object.defineProperty(exports, "__esModule", { value: true });
29
+ exports.CompilerService = void 0;
30
+ const fs = __importStar(require("fs/promises"));
31
+ const path = __importStar(require("path"));
32
+ const json_schema_ref_parser_1 = __importDefault(require("@apidevtools/json-schema-ref-parser"));
33
+ const js_yaml_1 = __importDefault(require("js-yaml"));
34
+ const deployer_1 = require("./deployer");
35
+ const validator_1 = require("./validator");
36
+ /**
37
+ * The compiler service converts a graph into a executable program.
38
+ */
39
+ class CompilerService {
40
+ constructor(store, stream, logger) {
41
+ this.store = store;
42
+ this.stream = stream;
43
+ this.logger = logger;
44
+ }
45
+ /**
46
+ * verifies and plans the deployment of an app to Redis; the app is not deployed yet
47
+ * @param path
48
+ */
49
+ async plan(mySchemaOrPath) {
50
+ try {
51
+ let schema;
52
+ if (this.isPath(mySchemaOrPath)) {
53
+ schema = (await json_schema_ref_parser_1.default.dereference(mySchemaOrPath));
54
+ }
55
+ else {
56
+ schema = js_yaml_1.default.load(mySchemaOrPath);
57
+ }
58
+ // 1) validate the manifest file
59
+ const validator = new validator_1.Validator(schema);
60
+ validator.validate(this.store);
61
+ // 2) todo: add a PlannerService module that will plan the deployment (what might break, drift, etc)
62
+ return schema;
63
+ }
64
+ catch (err) {
65
+ this.logger.error('compiler-plan-error', err);
66
+ }
67
+ }
68
+ isPath(input) {
69
+ return !input.trim().startsWith('app:');
70
+ }
71
+ /**
72
+ * deploys an app to Redis but does NOT activate it.
73
+ */
74
+ async deploy(mySchemaOrPath) {
75
+ try {
76
+ let schema;
77
+ if (this.isPath(mySchemaOrPath)) {
78
+ schema = (await json_schema_ref_parser_1.default.dereference(mySchemaOrPath));
79
+ await this.saveAsJSON(mySchemaOrPath, schema);
80
+ }
81
+ else {
82
+ schema = js_yaml_1.default.load(mySchemaOrPath);
83
+ }
84
+ // 2) validate the manifest file (synchronous operation...no callbacks)
85
+ const validator = new validator_1.Validator(schema);
86
+ validator.validate(this.store);
87
+ // 3) deploy the schema (segment, optimize, etc; save to Redis)
88
+ const deployer = new deployer_1.Deployer(schema);
89
+ await deployer.deploy(this.store, this.stream);
90
+ // 4) save the app version to Redis (so it can be activated later)
91
+ await this.store.setApp(schema.app.id, schema.app.version);
92
+ return schema;
93
+ }
94
+ catch (err) {
95
+ this.logger.error('compiler-deploy-error', err);
96
+ }
97
+ }
98
+ /**
99
+ * activates a deployed version of an app;
100
+ * @param appId
101
+ * @param appVersion
102
+ */
103
+ async activate(appId, appVersion) {
104
+ return await this.store.activateAppVersion(appId, appVersion);
105
+ }
106
+ async saveAsJSON(originalPath, schema) {
107
+ const json = JSON.stringify(schema, null, 2);
108
+ const newPath = path.join(path.dirname(originalPath), `.hotmesh.${schema.app.id}.${schema.app.version}.json`);
109
+ await fs.writeFile(newPath, json, 'utf8');
110
+ }
111
+ }
112
+ exports.CompilerService = CompilerService;
@@ -0,0 +1,34 @@
1
+ import { StoreService } from '../store';
2
+ import { MappingStatements } from '../../types/map';
3
+ import { HotMeshManifest } from '../../types/hotmesh';
4
+ import { ProviderClient, ProviderTransaction } from '../../types/provider';
5
+ declare class Validator {
6
+ manifest: HotMeshManifest | null;
7
+ activityIds: string[];
8
+ mappingStatements: MappingStatements;
9
+ store: StoreService<ProviderClient, ProviderTransaction> | null;
10
+ static SYS_VARS: string[];
11
+ static CONTEXT_VARS: string[];
12
+ constructor(manifest: HotMeshManifest);
13
+ /**
14
+ * validate the manifest file
15
+ */
16
+ validate(store: StoreService<ProviderClient, ProviderTransaction>): Promise<void>;
17
+ validateActivityIds(): void;
18
+ isMappingStatement(value: string): boolean;
19
+ extractMappingStatements(obj: any, result: MappingStatements, currentActivityId: string): void;
20
+ getMappingStatements(): void;
21
+ validateReferencedActivityIds(): void;
22
+ isFunction(value: string): boolean;
23
+ isContextVariable(value: string): boolean;
24
+ validateMappingStatements(): void;
25
+ validateTransitions(): void;
26
+ validateTransitionConditions(): void;
27
+ validateStats(): void;
28
+ validateSchemas(): void;
29
+ validateUniqueHandledTopics(): void;
30
+ validateGraphPublishSubscribe(): void;
31
+ validateHooks(): void;
32
+ validateConditionalStatements(): void;
33
+ }
34
+ export { Validator };