@hotmeshio/hotmesh 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (263) hide show
  1. package/LICENSE +214 -0
  2. package/README.md +241 -0
  3. package/build/index.d.ts +4 -0
  4. package/build/index.js +7 -0
  5. package/build/modules/errors.d.ts +28 -0
  6. package/build/modules/errors.js +50 -0
  7. package/build/modules/key.d.ts +75 -0
  8. package/build/modules/key.js +116 -0
  9. package/build/modules/utils.d.ts +34 -0
  10. package/build/modules/utils.js +173 -0
  11. package/build/package.json +73 -0
  12. package/build/services/activities/activity.d.ts +59 -0
  13. package/build/services/activities/activity.js +396 -0
  14. package/build/services/activities/await.d.ts +16 -0
  15. package/build/services/activities/await.js +143 -0
  16. package/build/services/activities/emit.d.ts +9 -0
  17. package/build/services/activities/emit.js +13 -0
  18. package/build/services/activities/index.d.ts +15 -0
  19. package/build/services/activities/index.js +16 -0
  20. package/build/services/activities/iterate.d.ts +9 -0
  21. package/build/services/activities/iterate.js +13 -0
  22. package/build/services/activities/trigger.d.ts +22 -0
  23. package/build/services/activities/trigger.js +161 -0
  24. package/build/services/activities/worker.d.ts +17 -0
  25. package/build/services/activities/worker.js +164 -0
  26. package/build/services/collator/index.d.ts +54 -0
  27. package/build/services/collator/index.js +171 -0
  28. package/build/services/compiler/deployer.d.ts +35 -0
  29. package/build/services/compiler/deployer.js +412 -0
  30. package/build/services/compiler/index.d.ts +30 -0
  31. package/build/services/compiler/index.js +111 -0
  32. package/build/services/compiler/validator.d.ts +32 -0
  33. package/build/services/compiler/validator.js +134 -0
  34. package/build/services/connector/clients/ioredis.d.ts +13 -0
  35. package/build/services/connector/clients/ioredis.js +50 -0
  36. package/build/services/connector/clients/redis.d.ts +13 -0
  37. package/build/services/connector/clients/redis.js +62 -0
  38. package/build/services/connector/index.d.ts +5 -0
  39. package/build/services/connector/index.js +31 -0
  40. package/build/services/dimension/index.d.ts +29 -0
  41. package/build/services/dimension/index.js +35 -0
  42. package/build/services/durable/asyncLocalStorage.d.ts +3 -0
  43. package/build/services/durable/asyncLocalStorage.js +5 -0
  44. package/build/services/durable/client.d.ts +15 -0
  45. package/build/services/durable/client.js +108 -0
  46. package/build/services/durable/connection.d.ts +4 -0
  47. package/build/services/durable/connection.js +51 -0
  48. package/build/services/durable/factory.d.ts +3 -0
  49. package/build/services/durable/factory.js +123 -0
  50. package/build/services/durable/handle.d.ts +8 -0
  51. package/build/services/durable/handle.js +38 -0
  52. package/build/services/durable/index.d.ts +57 -0
  53. package/build/services/durable/index.js +58 -0
  54. package/build/services/durable/native.d.ts +4 -0
  55. package/build/services/durable/native.js +47 -0
  56. package/build/services/durable/worker.d.ts +36 -0
  57. package/build/services/durable/worker.js +266 -0
  58. package/build/services/durable/workflow.d.ts +6 -0
  59. package/build/services/durable/workflow.js +135 -0
  60. package/build/services/engine/index.d.ts +82 -0
  61. package/build/services/engine/index.js +511 -0
  62. package/build/services/hotmesh/index.d.ts +45 -0
  63. package/build/services/hotmesh/index.js +134 -0
  64. package/build/services/logger/index.d.ts +17 -0
  65. package/build/services/logger/index.js +73 -0
  66. package/build/services/mapper/index.d.ts +24 -0
  67. package/build/services/mapper/index.js +72 -0
  68. package/build/services/pipe/functions/array.d.ts +24 -0
  69. package/build/services/pipe/functions/array.js +69 -0
  70. package/build/services/pipe/functions/bitwise.d.ts +9 -0
  71. package/build/services/pipe/functions/bitwise.js +24 -0
  72. package/build/services/pipe/functions/conditional.d.ts +10 -0
  73. package/build/services/pipe/functions/conditional.js +27 -0
  74. package/build/services/pipe/functions/date.d.ts +57 -0
  75. package/build/services/pipe/functions/date.js +167 -0
  76. package/build/services/pipe/functions/index.d.ts +25 -0
  77. package/build/services/pipe/functions/index.js +26 -0
  78. package/build/services/pipe/functions/json.d.ts +5 -0
  79. package/build/services/pipe/functions/json.js +12 -0
  80. package/build/services/pipe/functions/math.d.ts +38 -0
  81. package/build/services/pipe/functions/math.js +111 -0
  82. package/build/services/pipe/functions/number.d.ts +25 -0
  83. package/build/services/pipe/functions/number.js +133 -0
  84. package/build/services/pipe/functions/object.d.ts +22 -0
  85. package/build/services/pipe/functions/object.js +63 -0
  86. package/build/services/pipe/functions/string.d.ts +23 -0
  87. package/build/services/pipe/functions/string.js +69 -0
  88. package/build/services/pipe/functions/symbol.d.ts +12 -0
  89. package/build/services/pipe/functions/symbol.js +33 -0
  90. package/build/services/pipe/functions/unary.d.ts +7 -0
  91. package/build/services/pipe/functions/unary.js +18 -0
  92. package/build/services/pipe/index.d.ts +30 -0
  93. package/build/services/pipe/index.js +128 -0
  94. package/build/services/quorum/index.d.ts +34 -0
  95. package/build/services/quorum/index.js +147 -0
  96. package/build/services/reporter/index.d.ts +47 -0
  97. package/build/services/reporter/index.js +330 -0
  98. package/build/services/serializer/index.d.ts +36 -0
  99. package/build/services/serializer/index.js +222 -0
  100. package/build/services/signaler/store.d.ts +15 -0
  101. package/build/services/signaler/store.js +53 -0
  102. package/build/services/signaler/stream.d.ts +43 -0
  103. package/build/services/signaler/stream.js +317 -0
  104. package/build/services/store/cache.d.ts +66 -0
  105. package/build/services/store/cache.js +127 -0
  106. package/build/services/store/clients/ioredis.d.ts +27 -0
  107. package/build/services/store/clients/ioredis.js +96 -0
  108. package/build/services/store/clients/redis.d.ts +29 -0
  109. package/build/services/store/clients/redis.js +143 -0
  110. package/build/services/store/index.d.ts +88 -0
  111. package/build/services/store/index.js +657 -0
  112. package/build/services/stream/clients/ioredis.d.ts +23 -0
  113. package/build/services/stream/clients/ioredis.js +115 -0
  114. package/build/services/stream/clients/redis.d.ts +23 -0
  115. package/build/services/stream/clients/redis.js +119 -0
  116. package/build/services/stream/index.d.ts +21 -0
  117. package/build/services/stream/index.js +9 -0
  118. package/build/services/sub/clients/ioredis.d.ts +20 -0
  119. package/build/services/sub/clients/ioredis.js +72 -0
  120. package/build/services/sub/clients/redis.d.ts +20 -0
  121. package/build/services/sub/clients/redis.js +63 -0
  122. package/build/services/sub/index.d.ts +18 -0
  123. package/build/services/sub/index.js +9 -0
  124. package/build/services/task/index.d.ts +18 -0
  125. package/build/services/task/index.js +73 -0
  126. package/build/services/telemetry/index.d.ts +49 -0
  127. package/build/services/telemetry/index.js +223 -0
  128. package/build/services/worker/index.d.ts +30 -0
  129. package/build/services/worker/index.js +105 -0
  130. package/build/types/activity.d.ts +86 -0
  131. package/build/types/activity.js +2 -0
  132. package/build/types/app.d.ts +16 -0
  133. package/build/types/app.js +2 -0
  134. package/build/types/async.d.ts +5 -0
  135. package/build/types/async.js +2 -0
  136. package/build/types/cache.d.ts +1 -0
  137. package/build/types/cache.js +2 -0
  138. package/build/types/collator.d.ts +8 -0
  139. package/build/types/collator.js +11 -0
  140. package/build/types/durable.d.ts +59 -0
  141. package/build/types/durable.js +2 -0
  142. package/build/types/hook.d.ts +31 -0
  143. package/build/types/hook.js +9 -0
  144. package/build/types/hotmesh.d.ts +82 -0
  145. package/build/types/hotmesh.js +2 -0
  146. package/build/types/index.d.ts +20 -0
  147. package/build/types/index.js +21 -0
  148. package/build/types/ioredisclient.d.ts +5 -0
  149. package/build/types/ioredisclient.js +5 -0
  150. package/build/types/job.d.ts +50 -0
  151. package/build/types/job.js +2 -0
  152. package/build/types/logger.d.ts +6 -0
  153. package/build/types/logger.js +2 -0
  154. package/build/types/map.d.ts +4 -0
  155. package/build/types/map.js +2 -0
  156. package/build/types/pipe.d.ts +4 -0
  157. package/build/types/pipe.js +2 -0
  158. package/build/types/quorum.d.ts +46 -0
  159. package/build/types/quorum.js +2 -0
  160. package/build/types/redis.d.ts +8 -0
  161. package/build/types/redis.js +2 -0
  162. package/build/types/redisclient.d.ts +25 -0
  163. package/build/types/redisclient.js +2 -0
  164. package/build/types/serializer.d.ts +33 -0
  165. package/build/types/serializer.js +2 -0
  166. package/build/types/stats.d.ts +83 -0
  167. package/build/types/stats.js +2 -0
  168. package/build/types/stream.d.ts +67 -0
  169. package/build/types/stream.js +25 -0
  170. package/build/types/telemetry.d.ts +1 -0
  171. package/build/types/telemetry.js +11 -0
  172. package/build/types/transition.d.ts +17 -0
  173. package/build/types/transition.js +2 -0
  174. package/index.ts +5 -0
  175. package/modules/errors.ts +55 -0
  176. package/modules/key.ts +129 -0
  177. package/modules/utils.ts +170 -0
  178. package/package.json +73 -0
  179. package/services/activities/activity.ts +473 -0
  180. package/services/activities/await.ts +172 -0
  181. package/services/activities/emit.ts +25 -0
  182. package/services/activities/index.ts +15 -0
  183. package/services/activities/iterate.ts +26 -0
  184. package/services/activities/trigger.ts +196 -0
  185. package/services/activities/worker.ts +190 -0
  186. package/services/collator/README.md +102 -0
  187. package/services/collator/index.ts +182 -0
  188. package/services/compiler/deployer.ts +432 -0
  189. package/services/compiler/index.ts +98 -0
  190. package/services/compiler/validator.ts +154 -0
  191. package/services/connector/clients/ioredis.ts +57 -0
  192. package/services/connector/clients/redis.ts +72 -0
  193. package/services/connector/index.ts +44 -0
  194. package/services/dimension/README.md +73 -0
  195. package/services/dimension/index.ts +39 -0
  196. package/services/durable/asyncLocalStorage.ts +3 -0
  197. package/services/durable/client.ts +116 -0
  198. package/services/durable/connection.ts +50 -0
  199. package/services/durable/factory.ts +124 -0
  200. package/services/durable/handle.ts +43 -0
  201. package/services/durable/index.ts +60 -0
  202. package/services/durable/native.ts +46 -0
  203. package/services/durable/worker.ts +254 -0
  204. package/services/durable/workflow.ts +136 -0
  205. package/services/engine/index.ts +615 -0
  206. package/services/hotmesh/index.ts +182 -0
  207. package/services/logger/index.ts +79 -0
  208. package/services/mapper/index.ts +84 -0
  209. package/services/pipe/functions/array.ts +87 -0
  210. package/services/pipe/functions/bitwise.ts +27 -0
  211. package/services/pipe/functions/conditional.ts +31 -0
  212. package/services/pipe/functions/date.ts +214 -0
  213. package/services/pipe/functions/index.ts +25 -0
  214. package/services/pipe/functions/json.ts +11 -0
  215. package/services/pipe/functions/math.ts +143 -0
  216. package/services/pipe/functions/number.ts +150 -0
  217. package/services/pipe/functions/object.ts +79 -0
  218. package/services/pipe/functions/string.ts +86 -0
  219. package/services/pipe/functions/symbol.ts +39 -0
  220. package/services/pipe/functions/unary.ts +19 -0
  221. package/services/pipe/index.ts +138 -0
  222. package/services/quorum/index.ts +200 -0
  223. package/services/reporter/index.ts +379 -0
  224. package/services/serializer/README.md +10 -0
  225. package/services/serializer/index.ts +243 -0
  226. package/services/signaler/store.ts +61 -0
  227. package/services/signaler/stream.ts +354 -0
  228. package/services/store/cache.ts +172 -0
  229. package/services/store/clients/ioredis.ts +123 -0
  230. package/services/store/clients/redis.ts +169 -0
  231. package/services/store/index.ts +757 -0
  232. package/services/stream/clients/ioredis.ts +148 -0
  233. package/services/stream/clients/redis.ts +144 -0
  234. package/services/stream/index.ts +57 -0
  235. package/services/sub/clients/ioredis.ts +83 -0
  236. package/services/sub/clients/redis.ts +74 -0
  237. package/services/sub/index.ts +25 -0
  238. package/services/task/index.ts +86 -0
  239. package/services/telemetry/index.ts +267 -0
  240. package/services/worker/index.ts +165 -0
  241. package/types/activity.ts +115 -0
  242. package/types/app.ts +20 -0
  243. package/types/async.ts +7 -0
  244. package/types/cache.ts +1 -0
  245. package/types/collator.ts +9 -0
  246. package/types/durable.ts +81 -0
  247. package/types/hook.ts +32 -0
  248. package/types/hotmesh.ts +102 -0
  249. package/types/index.ts +138 -0
  250. package/types/ioredisclient.ts +10 -0
  251. package/types/job.ts +59 -0
  252. package/types/logger.ts +6 -0
  253. package/types/map.ts +5 -0
  254. package/types/ms.d.ts +7 -0
  255. package/types/pipe.ts +7 -0
  256. package/types/quorum.ts +59 -0
  257. package/types/redis.ts +27 -0
  258. package/types/redisclient.ts +29 -0
  259. package/types/serializer.ts +38 -0
  260. package/types/stats.ts +100 -0
  261. package/types/stream.ts +75 -0
  262. package/types/telemetry.ts +15 -0
  263. package/types/transition.ts +20 -0
@@ -0,0 +1,26 @@
1
+ import { EngineService } from '../engine';
2
+ import { Activity, ActivityType } from './activity';
3
+ import {
4
+ ActivityData,
5
+ ActivityMetadata,
6
+ IterateActivity } from '../../types/activity';
7
+
8
+ class Iterate extends Activity {
9
+ config: IterateActivity;
10
+
11
+ constructor(
12
+ config: ActivityType,
13
+ data: ActivityData,
14
+ metadata: ActivityMetadata,
15
+ hook: ActivityData | null,
16
+ engine: EngineService
17
+ ) {
18
+ super(config, data, metadata, hook, engine);
19
+ }
20
+
21
+ async mapInputData(): Promise<void> {
22
+ this.logger.info('iterate-map-input-data');
23
+ }
24
+ }
25
+
26
+ export { Iterate };
@@ -0,0 +1,196 @@
1
+ import { nanoid } from 'nanoid';
2
+ import { DuplicateJobError } from '../../modules/errors';
3
+ import { formatISODate, getTimeSeries } from '../../modules/utils';
4
+ import { Activity } from './activity';
5
+ import { CollatorService } from '../collator';
6
+ import { DimensionService } from '../dimension';
7
+ import { EngineService } from '../engine';
8
+ import { Pipe } from '../pipe';
9
+ import { ReporterService } from '../reporter';
10
+ import { MDATA_SYMBOLS } from '../serializer';
11
+ import { TelemetryService } from '../telemetry';
12
+ import {
13
+ ActivityData,
14
+ ActivityMetadata,
15
+ ActivityType,
16
+ TriggerActivity } from '../../types/activity';
17
+ import { JobState } from '../../types/job';
18
+ import { RedisMulti } from '../../types/redis';
19
+ import { StringScalarType } from '../../types/serializer';
20
+
21
+ class Trigger extends Activity {
22
+ config: TriggerActivity;
23
+
24
+ constructor(
25
+ config: ActivityType,
26
+ data: ActivityData,
27
+ metadata: ActivityMetadata,
28
+ hook: ActivityData | null,
29
+ engine: EngineService,
30
+ context?: JobState) {
31
+ super(config, data, metadata, hook, engine, context);
32
+ }
33
+
34
+ async process(): Promise<string> {
35
+ this.logger.debug('trigger-process', { subscribes: this.config.subscribes });
36
+ let telemetry: TelemetryService;
37
+ try {
38
+ this.setLeg(2);
39
+ await this.getState();
40
+ telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
41
+ telemetry.startJobSpan();
42
+ telemetry.startActivitySpan(this.leg);
43
+ this.mapJobData();
44
+ await this.setStateNX();
45
+ this.adjacencyList = await this.filterAdjacent();
46
+ await this.setStatus(this.adjacencyList.length);
47
+
48
+ const multi = this.store.getMulti();
49
+ await this.setState(multi);
50
+ await this.setStats(multi);
51
+ await multi.exec();
52
+
53
+ telemetry.mapActivityAttributes();
54
+ const jobStatus = Number(this.context.metadata.js);
55
+ telemetry.setJobAttributes({ 'app.job.jss': jobStatus });
56
+ const attrs: StringScalarType = { 'app.job.jss': jobStatus };
57
+ const messageIds = await this.transition(this.adjacencyList, jobStatus);
58
+ if (messageIds.length) {
59
+ attrs['app.activity.mids'] = messageIds.join(',')
60
+ }
61
+ telemetry.setActivityAttributes(attrs);
62
+ return this.context.metadata.jid;
63
+ } catch (error) {
64
+ if (error instanceof DuplicateJobError) {
65
+ this.logger.error('duplicate-job-error', error);
66
+ } else {
67
+ this.logger.error('trigger-process-error', error);
68
+ }
69
+ telemetry.setActivityError(error.message);
70
+ throw error;
71
+ } finally {
72
+ telemetry.endJobSpan();
73
+ telemetry.endActivitySpan();
74
+ this.logger.debug('trigger-process-end', { subscribes: this.config.subscribes, jid: this.context.metadata.jid });
75
+ }
76
+ }
77
+
78
+ async setStatus(amount: number): Promise<void> {
79
+ this.context.metadata.js = amount;
80
+ }
81
+
82
+ createInputContext(): Partial<JobState> {
83
+ const input = {
84
+ [this.metadata.aid]: {
85
+ input: { data: this.data }
86
+ },
87
+ '$self': {
88
+ input: { data: this.data },
89
+ output: { data: this.data }
90
+ },
91
+ } as Partial<JobState>;
92
+ return input
93
+ }
94
+
95
+ async getState(): Promise<void> {
96
+ const inputContext = this.createInputContext();
97
+ const jobId = this.resolveJobId(inputContext);
98
+ const jobKey = this.resolveJobKey(inputContext);
99
+
100
+ const utc = formatISODate(new Date());
101
+ const { id, version } = await this.engine.getVID();
102
+ this.initDimensionalAddress(DimensionService.getSeed());
103
+ const activityMetadata = {
104
+ ...this.metadata,
105
+ jid: jobId,
106
+ key: jobKey,
107
+ as: CollatorService.getTriggerSeed(),
108
+ };
109
+ this.context = {
110
+ metadata: {
111
+ ...this.metadata,
112
+ ngn: this.context.metadata.ngn,
113
+ pj: this.context.metadata.pj,
114
+ pd: this.context.metadata.pd,
115
+ pa: this.context.metadata.pa,
116
+ app: id,
117
+ vrs: version,
118
+ tpc: this.config.subscribes,
119
+ trc: this.context.metadata.trc,
120
+ spn: this.context.metadata.spn,
121
+ jid: jobId,
122
+ dad: DimensionService.getSeed(), //top-level job implicitly uses `,0`
123
+ key: jobKey,
124
+ jc: utc,
125
+ ju: utc,
126
+ ts: getTimeSeries(this.resolveGranularity()),
127
+ js: 0,
128
+ },
129
+ data: {},
130
+ [this.metadata.aid]: {
131
+ input: {
132
+ data: this.data,
133
+ metadata: activityMetadata
134
+ },
135
+ output: {
136
+ data: this.data,
137
+ metadata: activityMetadata
138
+ },
139
+ settings: { data: {} },
140
+ errors: { data: {} },
141
+ },
142
+ };
143
+ this.context['$self'] = this.context[this.metadata.aid];
144
+ }
145
+
146
+ bindJobMetadataPaths(): string[] {
147
+ return MDATA_SYMBOLS.JOB.KEYS.map((key) => `metadata/${key}`);
148
+ }
149
+
150
+ bindActivityMetadataPaths(): string[] {
151
+ return MDATA_SYMBOLS.ACTIVITY.KEYS.map((key) => `output/metadata/${key}`);
152
+ }
153
+
154
+ resolveGranularity(): string {
155
+ return ReporterService.DEFAULT_GRANULARITY;
156
+ }
157
+
158
+ getJobStatus(): number {
159
+ return this.context.metadata.js;
160
+ }
161
+
162
+ resolveJobId(context: Partial<JobState>): string {
163
+ const jobId = this.config.stats?.id;
164
+ return jobId ? Pipe.resolve(jobId, context) : nanoid();
165
+ }
166
+
167
+ resolveJobKey(context: Partial<JobState>): string {
168
+ const jobKey = this.config.stats?.key;
169
+ return jobKey ? Pipe.resolve(jobKey, context) : '';
170
+ }
171
+
172
+ async setStateNX(): Promise<void> {
173
+ const jobId = this.context.metadata.jid;
174
+ if (!await this.store.setStateNX(jobId, this.engine.appId)) {
175
+ throw new DuplicateJobError(jobId);
176
+ }
177
+ }
178
+
179
+ async setStats(multi?: RedisMulti): Promise<void> {
180
+ const md = this.context.metadata;
181
+ if (this.config.stats?.measures) {
182
+ const config = await this.engine.getVID();
183
+ const reporter = new ReporterService(config, this.store, this.logger);
184
+ await this.store.setStats(
185
+ md.key,
186
+ md.jid,
187
+ md.ts,
188
+ reporter.resolveTriggerStatistics(this.config, this.context),
189
+ config,
190
+ multi
191
+ );
192
+ }
193
+ }
194
+ }
195
+
196
+ export { Trigger };
@@ -0,0 +1,190 @@
1
+ import { GetStateError } from '../../modules/errors';
2
+ import { Activity } from './activity';
3
+ import { CollatorService } from '../collator';
4
+ import { EngineService } from '../engine';
5
+ import {
6
+ ActivityData,
7
+ ActivityMetadata,
8
+ ActivityType,
9
+ WorkerActivity } from '../../types/activity';
10
+ import { JobState } from '../../types/job';
11
+ import { MultiResponseFlags } from '../../types/redis';
12
+ import { StringScalarType } from '../../types/serializer';
13
+ import {
14
+ StreamCode,
15
+ StreamData,
16
+ StreamStatus } from '../../types/stream';
17
+ import { TelemetryService } from '../telemetry';
18
+
19
+ class Worker extends Activity {
20
+ config: WorkerActivity;
21
+
22
+ constructor(
23
+ config: ActivityType,
24
+ data: ActivityData,
25
+ metadata: ActivityMetadata,
26
+ hook: ActivityData | null,
27
+ engine: EngineService,
28
+ context?: JobState) {
29
+ super(config, data, metadata, hook, engine, context);
30
+ }
31
+
32
+ //******** INITIAL ENTRY POINT (A) ********//
33
+ async process(): Promise<string> {
34
+ this.logger.debug('worker-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
35
+ let telemetry: TelemetryService;
36
+ try {
37
+ this.setLeg(1);
38
+ await CollatorService.notarizeEntry(this);
39
+
40
+ await this.getState();
41
+ telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
42
+ telemetry.startActivitySpan(this.leg);
43
+ this.mapInputData();
44
+
45
+ const multi = this.store.getMulti();
46
+ //await this.registerTimeout();
47
+ await CollatorService.authorizeReentry(this, multi);
48
+ await this.setState(multi);
49
+ await this.setStatus(0, multi);
50
+ const multiResponse = await multi.exec() as MultiResponseFlags;
51
+
52
+ telemetry.mapActivityAttributes();
53
+ const jobStatus = this.resolveStatus(multiResponse);
54
+ const messageId = await this.execActivity();
55
+ telemetry.setActivityAttributes({
56
+ 'app.activity.mid': messageId,
57
+ 'app.job.jss': jobStatus
58
+ });
59
+ //TODO: UPDATE ACTIVITY STATE (LEG 1 EXIT)
60
+ return this.context.metadata.aid;
61
+ } catch (error) {
62
+ if (error instanceof GetStateError) {
63
+ this.logger.error('worker-get-state-error', error);
64
+ } else {
65
+ this.logger.error('worker-process-error', error);
66
+ }
67
+ telemetry.setActivityError(error.message);
68
+ throw error;
69
+ } finally {
70
+ telemetry.endActivitySpan();
71
+ this.logger.debug('worker-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
72
+ }
73
+ }
74
+
75
+ async execActivity(): Promise<string> {
76
+ const streamData: StreamData = {
77
+ metadata: {
78
+ jid: this.context.metadata.jid,
79
+ dad: this.metadata.dad,
80
+ aid: this.metadata.aid,
81
+ topic: this.config.subtype,
82
+ spn: this.context['$self'].output.metadata.l1s,
83
+ trc: this.context.metadata.trc,
84
+ },
85
+ data: this.context.data
86
+ };
87
+ if (this.config.retry) {
88
+ streamData.policies = {
89
+ retry: this.config.retry
90
+ };
91
+ }
92
+ return (await this.engine.streamSignaler?.publishMessage(this.config.subtype, streamData)) as string;
93
+ }
94
+
95
+
96
+ //******** SIGNAL RE-ENTRY POINT (DUPLEX LEG 2 of 2) ********//
97
+ async processEvent(status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200): Promise<void> {
98
+ this.setLeg(2);
99
+ const jid = this.context.metadata.jid;
100
+ const aid = this.metadata.aid;
101
+ this.status = status;
102
+ this.code = code;
103
+ this.logger.debug('worker-process-event', { topic: this.config.subtype, jid, aid, status, code });
104
+ let telemetry: TelemetryService;
105
+ try {
106
+ await this.getState();
107
+ const aState = await CollatorService.notarizeReentry(this);
108
+ this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
109
+
110
+ telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
111
+ let isComplete = CollatorService.isActivityComplete(this.context.metadata.js);
112
+ if (isComplete) {
113
+ this.logger.warn('worker-process-event-duplicate', { jid, aid });
114
+ this.logger.debug('worker-process-event-duplicate-resolution', { resolution: 'Increase HotMesh config `reclaimDelay` timeout.' });
115
+ return;
116
+ }
117
+ telemetry.startActivitySpan(this.leg);
118
+ if (status === StreamStatus.PENDING) {
119
+ await this.processPending(telemetry);
120
+ } else if (status === StreamStatus.SUCCESS) {
121
+ await this.processSuccess(telemetry);
122
+ } else {
123
+ await this.processError(telemetry);
124
+ }
125
+ } catch (error) {
126
+ this.logger.error('worker-process-event-error', error);
127
+ telemetry.setActivityError(error.message);
128
+ throw error;
129
+ } finally {
130
+ telemetry.endActivitySpan();
131
+ this.logger.debug('worker-process-event-end', { jid, aid });
132
+ }
133
+ }
134
+
135
+ async processPending(telemetry: TelemetryService): Promise<void> {
136
+ this.bindActivityData('output');
137
+ this.adjacencyList = await this.filterAdjacent();
138
+ this.mapJobData();
139
+ const multi = this.store.getMulti();
140
+ await this.setState(multi);
141
+ await CollatorService.notarizeContinuation(this, multi);
142
+
143
+ await this.setStatus(0, multi);
144
+ const multiResponse = await multi.exec() as MultiResponseFlags;
145
+ telemetry.mapActivityAttributes();
146
+ const jobStatus = this.resolveStatus(multiResponse);
147
+ telemetry.setActivityAttributes({ 'app.job.jss': jobStatus });
148
+ }
149
+
150
+ async processSuccess(telemetry: TelemetryService): Promise<void> {
151
+ this.bindActivityData('output');
152
+ this.adjacencyList = await this.filterAdjacent();
153
+ this.mapJobData();
154
+ const multi = this.store.getMulti();
155
+ await this.setState(multi);
156
+ await CollatorService.notarizeCompletion(this, multi);
157
+
158
+ await this.setStatus(this.adjacencyList.length - 1, multi);
159
+ const multiResponse = await multi.exec() as MultiResponseFlags;
160
+ telemetry.mapActivityAttributes();
161
+ const jobStatus = this.resolveStatus(multiResponse);
162
+ const attrs: StringScalarType = { 'app.job.jss': jobStatus };
163
+ const messageIds = await this.transition(this.adjacencyList, jobStatus);
164
+ if (messageIds.length) {
165
+ attrs['app.activity.mids'] = messageIds.join(',')
166
+ }
167
+ telemetry.setActivityAttributes(attrs);
168
+ }
169
+
170
+ async processError(telemetry: TelemetryService): Promise<void> {
171
+ this.bindActivityError(this.data);
172
+ this.adjacencyList = await this.filterAdjacent();
173
+ const multi = this.store.getMulti();
174
+ await this.setState(multi);
175
+ await CollatorService.notarizeCompletion(this, multi);
176
+
177
+ await this.setStatus(this.adjacencyList.length - 1, multi);
178
+ const multiResponse = await multi.exec() as MultiResponseFlags;
179
+ telemetry.mapActivityAttributes();
180
+ const jobStatus = this.resolveStatus(multiResponse);
181
+ const attrs: StringScalarType = { 'app.job.jss': jobStatus };
182
+ const messageIds = await this.transition(this.adjacencyList, jobStatus);
183
+ if (messageIds.length) {
184
+ attrs['app.activity.mids'] = messageIds.join(',')
185
+ }
186
+ telemetry.setActivityAttributes(attrs);
187
+ }
188
+ }
189
+
190
+ export { Worker };
@@ -0,0 +1,102 @@
1
+ # Activity State Tracking
2
+
3
+ This document explains how HotMesh guarantees activity state management using a 15-digit integer. This mechanism is crucial for monitoring and controlling the activity's lifecycle across two legs (Leg1 and Leg2) and ensures that duplicate activities are never processed. The process further ensures that catastrophic, in-process failures will be automatically resolved with respect to idempotency, makig it possible to build 100% fault-tolerant workflows.
4
+
5
+ ## Overview
6
+
7
+ The 15-digit integer serves as a semaphore, starting with the value `999000000000000`. Each digit has a specific purpose:
8
+
9
+ - The first three digits track the lifecycle status for Legs 1 and 2.
10
+ - The last 12 digits represent 1 million possible dimensional threads that can be spawned by this activity.
11
+
12
+ ### Digit Details
13
+
14
+ 999000000000000
15
+ ^-------------- Leg1 Entry Status
16
+ ^------------- Leg1 Exit Status
17
+ ^^^^^^------ Leg2 Dimensional Thread Entry Count
18
+ ^^^^^^ Leg2 Dimensional Thread Exit Count
19
+ ^------------ Leg2 Exit Status
20
+
21
+ >*Dimensional Threads* isolate and track those activities in the workflow that run in a *cycle*. They ensure that no naming collisions occur, even if the same activity is run multiple times.
22
+
23
+ ## Leg1 Lifecycle
24
+
25
+ ### Initialization
26
+
27
+ The parent activity initializes the integer value (`999000000000000`) and saves it to Redis upon saving its own state. The parent then adds the child activity to the proper stream channel for eventual processing.
28
+
29
+ >Streams are used when executing an activity (such as transitioning to a child activity) as they guarantee that the child activity will be fully created and initialized before the request is marked for deletion. Even if the system has a catastrophic failure, the chain of custody can be guaranteed through the use of streams when the system comes online.
30
+
31
+ ### Beginning of Leg1
32
+
33
+ When Leg1 begins (when an activity is dequeued from its stream channel), the integer is decremented by 100 trillion:
34
+
35
+ HINCRBY -100000000000000
36
+
37
+ Result:
38
+
39
+ 899000000000000
40
+
41
+ ### Conclusion of Leg1
42
+
43
+ At the conclusion of Leg1, the integer is decremented by 10 trillion:
44
+
45
+ HINCRBY -10000000000000
46
+
47
+ Result:
48
+
49
+ 889000000000000
50
+
51
+ ### Error Handling
52
+
53
+ - If the value upon entering is `799############` (after decrementing the integer), Leg1 began but crashed before completion. The activity will perform the necessary cleanup and re-run the activity.
54
+ - If the value upon entering is `789############` (after decrementing the integer), Leg1 completed successfully and the activity should end. It is likely that the system crashed last time before acking and deleting. Verify transitions succeeded and resolve as necessary.
55
+
56
+ ## Leg2 Lifecycle
57
+
58
+ Leg2 supports multiple inputs and repeated responses. A worker can respond with a 'pending' status, allowing multiple inputs.
59
+
60
+ ### Beginning of Leg2
61
+
62
+ On the first Leg2 input, the integer is incremented by 1 million:
63
+
64
+ HINCRBY 1000000
65
+
66
+ Result:
67
+
68
+ 889000001000000
69
+
70
+ ### Pending Status
71
+
72
+ If the call status is 'pending', the digit is updated to reflect the successful completion of Leg2 but the third digit from the left will be untouched as the activity is pending and Leg2 should not end quite yet:
73
+
74
+ HINCRBY 1
75
+
76
+ Result:
77
+
78
+ 889000001000001
79
+
80
+ ### Success Status
81
+
82
+ If the next message call to Leg2 entry is 'success', the process will begin as before, and the integer will be incremented by 1 million:
83
+
84
+ HINCRBY 1000000
85
+
86
+ Result:
87
+
88
+ 889000002000001
89
+
90
+ But because this is not a 'pending' message, the integer should be updated differently, targeting both the dimensional thread counter as well as the third digit from the left that tracks Leg2 status (e.g., `1 - 1000000000 = -999999999`).
91
+
92
+ HINCRBY -999999999999
93
+
94
+ Result:
95
+
96
+ 888000002000002
97
+
98
+ >If an additional Leg2 call were to be received after this point, the result would be `888000003000002`. However, becuase the third digit is `8`, Leg2 access is not allowed and the system will conclude by decrementing the value it just set and returning immediately, so that the calling worker can mark and delete the stream entry.
99
+
100
+ ## Conclusion
101
+
102
+ This 15-digit integer allows sophisticated tracking of complex activities across two legs, with error handling and support for cycles using multiple dimensional threads.
@@ -0,0 +1,182 @@
1
+ import { CollationError } from "../../modules/errors";
2
+ import { RedisMulti } from "../../types/redis";
3
+ import { CollationFaultType, CollationStage } from "../../types/collator";
4
+ import { ActivityDuplex } from "../../types/activity";
5
+ import { HotMeshGraph } from "../../types/hotmesh";
6
+ import { Activity } from "../activities/activity";
7
+
8
+ class CollatorService {
9
+
10
+ //max int digit count that supports `hincrby`
11
+ static targetLength = 15;
12
+
13
+ static async notarizeEntry(activity: Activity, multi?: RedisMulti): Promise<number> {
14
+ //decrement by -100_000_000_000_000
15
+ const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -100_000_000_000_000, multi);
16
+ this.verifyInteger(amount, 1, 'enter');
17
+ return amount;
18
+ };
19
+
20
+ static async authorizeReentry(activity: Activity, multi?: RedisMulti): Promise<number> {
21
+ //set second digit to 8, allowing for re-entry
22
+ //decrement by -10_000_000_000_000
23
+ const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -10_000_000_000_000, multi);
24
+ //this.verifyInteger(amount, 1, 'exit');
25
+ return amount;
26
+ }
27
+
28
+ static async notarizeEarlyCompletion(activity: Activity, multi?: RedisMulti): Promise<number> {
29
+ //initialize both `possible` (1m) and `actualized` (1) zero dimension, while decrementing the 2nd and 3rd digits to deactivate the activity
30
+ return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1_000_001 - 11_000_000_000_000, multi);
31
+ };
32
+
33
+ static async notarizeReentry(activity: Activity, multi?: RedisMulti): Promise<number> {
34
+ //increment by 1_000_000 (indicates re-entry and is used to drive the 'dimensional address' for adjacent activities (minus 1))
35
+ const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1_000_000, multi);
36
+ this.verifyInteger(amount, 2, 'enter');
37
+ return amount;
38
+ };
39
+
40
+ static async notarizeContinuation(activity: Activity, multi?: RedisMulti): Promise<number> {
41
+ //keep open; actualize the leg2 dimension (+1)
42
+ return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1, multi);
43
+ };
44
+
45
+ static async notarizeCompletion(activity: Activity, multi?: RedisMulti): Promise<number> {
46
+ //close out; actualize leg2 dimension (+1) and decrement the 3rd digit (-1_000_000_000_000)
47
+ return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1 - 1_000_000_000_000, multi);
48
+ };
49
+
50
+ static getDigitAtIndex(num: number, targetDigitIndex: number): number | null {
51
+ const numStr = num.toString();
52
+ if (targetDigitIndex < 0 || targetDigitIndex >= numStr.length) {
53
+ return null;
54
+ }
55
+ const digit = parseInt(numStr[targetDigitIndex], 10);
56
+ return digit;
57
+ }
58
+
59
+ static getDimensionalIndex(num: number): number | null {
60
+ const numStr = num.toString();
61
+ if (numStr.length < 9) {
62
+ return null;
63
+ }
64
+ const extractedStr = numStr.substring(3, 9);
65
+ const extractedInt = parseInt(extractedStr, 10);
66
+ return extractedInt - 1;
67
+ }
68
+
69
+ static isDuplicate(num: number, targetDigitIndex: number): boolean {
70
+ return this.getDigitAtIndex(num, targetDigitIndex) < 8;
71
+ }
72
+
73
+ static isInactive(num: number): boolean {
74
+ return this.getDigitAtIndex(num, 2) < 9;
75
+ }
76
+
77
+ static isPrimed(amount: number, leg: ActivityDuplex): boolean {
78
+ //activity entry is not allowed if paths not properly pre-set
79
+ if (leg == 1) {
80
+ return amount != -100_000_000_000_000;
81
+ } else {
82
+ return this.getDigitAtIndex(amount, 0) < 9 &&
83
+ this.getDigitAtIndex(amount, 1) < 9;
84
+ }
85
+ }
86
+
87
+ static verifyInteger(amount: number, leg: ActivityDuplex, stage: CollationStage): void {
88
+ let faultType: CollationFaultType | undefined;
89
+ if (leg === 1 && stage === 'enter') {
90
+ if (!this.isPrimed(amount, 1)) {
91
+ faultType = CollationFaultType.MISSING;
92
+ } else if (this.isDuplicate(amount, 0)) {
93
+ faultType = CollationFaultType.DUPLICATE;
94
+ } else if (amount != 899_000_000_000_000) {
95
+ faultType = CollationFaultType.INVALID;
96
+ }
97
+ } else if (leg === 1 && stage === 'exit') {
98
+ if (amount === -10_000_000_000_000) {
99
+ faultType = CollationFaultType.MISSING;
100
+ } else if (this.isDuplicate(amount, 1)) {
101
+ faultType = CollationFaultType.DUPLICATE;
102
+ }
103
+ } else if (leg === 2 && stage === 'enter') {
104
+ if (!this.isPrimed(amount, 2)) {
105
+ faultType = CollationFaultType.FORBIDDEN;
106
+ } else if (this.isInactive(amount)) {
107
+ faultType = CollationFaultType.INACTIVE;
108
+ }
109
+ }
110
+ if (faultType) {
111
+ throw new CollationError(amount, leg, stage, faultType);
112
+ }
113
+ }
114
+
115
+ /**
116
+ * All non-trigger activities are assigned a status seed by their parent
117
+ */
118
+ static getSeed(): string {
119
+ return '999000000000000';
120
+ }
121
+
122
+ /**
123
+ * All trigger activities are assigned a status seed in a completed state
124
+ */
125
+ static getTriggerSeed(): string {
126
+ return '888000001000001';
127
+ }
128
+
129
+ /**
130
+ * entry point for compiler-type activities. This is called by the compiler
131
+ * to bind the sorted activity IDs to the trigger activity. These are then used
132
+ * at runtime by the activities to track job/activity status.
133
+ * @param graphs
134
+ */
135
+ static compile(graphs: HotMeshGraph[]) {
136
+ CollatorService.bindAncestorArray(graphs);
137
+ }
138
+
139
+ /**
140
+ * binds the ancestor array to each activity.
141
+ * Used in conjunction with the dimensional
142
+ * address (dad). If dad is `,0,1,0,0` and the
143
+ * ancestor array is `['t1', 'a1', 'a2']` for
144
+ * activity 'a3', then the SAVED DAD
145
+ * will always have the trailing
146
+ * 0's removed. This ensures that the addressing
147
+ * remains consistent even if the graph changes.
148
+ * id DAD SAVED DAD
149
+ * * t1 => ,0 => [empty]
150
+ * * a1 => ,0,1 => ,0,1
151
+ * * a2 => ,0,1,0 => ,0,1
152
+ * * a3 => ,0,1,0,0 => ,0,1
153
+ *
154
+ */
155
+ static bindAncestorArray(graphs: HotMeshGraph[]) {
156
+ graphs.forEach((graph) => {
157
+ const ancestors: Record<string, string[]> = {};
158
+ const startingNode = Object.keys(graph.activities).find(
159
+ (activity) => graph.activities[activity].type === 'trigger'
160
+ );
161
+ if (!startingNode) {
162
+ throw new Error('collator-trigger-activity-not-found');
163
+ }
164
+ const dfs = (node: string, parentList: string[]) => {
165
+ ancestors[node] = parentList;
166
+ graph.activities[node]['ancestors'] = parentList;
167
+ const transitions = graph.transitions?.[node] || [];
168
+ transitions.forEach((transition) => {
169
+ dfs(transition.to, [...parentList, node]);
170
+ });
171
+ };
172
+ // Start the DFS traversal
173
+ dfs(startingNode, []);
174
+ });
175
+ }
176
+
177
+ static isActivityComplete(status: number): boolean {
178
+ return (status - 0) <= 0;
179
+ }
180
+ }
181
+
182
+ export { CollatorService };