@hotmeshio/hotmesh 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (263) hide show
  1. package/LICENSE +214 -0
  2. package/README.md +241 -0
  3. package/build/index.d.ts +4 -0
  4. package/build/index.js +7 -0
  5. package/build/modules/errors.d.ts +28 -0
  6. package/build/modules/errors.js +50 -0
  7. package/build/modules/key.d.ts +75 -0
  8. package/build/modules/key.js +116 -0
  9. package/build/modules/utils.d.ts +34 -0
  10. package/build/modules/utils.js +173 -0
  11. package/build/package.json +73 -0
  12. package/build/services/activities/activity.d.ts +59 -0
  13. package/build/services/activities/activity.js +396 -0
  14. package/build/services/activities/await.d.ts +16 -0
  15. package/build/services/activities/await.js +143 -0
  16. package/build/services/activities/emit.d.ts +9 -0
  17. package/build/services/activities/emit.js +13 -0
  18. package/build/services/activities/index.d.ts +15 -0
  19. package/build/services/activities/index.js +16 -0
  20. package/build/services/activities/iterate.d.ts +9 -0
  21. package/build/services/activities/iterate.js +13 -0
  22. package/build/services/activities/trigger.d.ts +22 -0
  23. package/build/services/activities/trigger.js +161 -0
  24. package/build/services/activities/worker.d.ts +17 -0
  25. package/build/services/activities/worker.js +164 -0
  26. package/build/services/collator/index.d.ts +54 -0
  27. package/build/services/collator/index.js +171 -0
  28. package/build/services/compiler/deployer.d.ts +35 -0
  29. package/build/services/compiler/deployer.js +412 -0
  30. package/build/services/compiler/index.d.ts +30 -0
  31. package/build/services/compiler/index.js +111 -0
  32. package/build/services/compiler/validator.d.ts +32 -0
  33. package/build/services/compiler/validator.js +134 -0
  34. package/build/services/connector/clients/ioredis.d.ts +13 -0
  35. package/build/services/connector/clients/ioredis.js +50 -0
  36. package/build/services/connector/clients/redis.d.ts +13 -0
  37. package/build/services/connector/clients/redis.js +62 -0
  38. package/build/services/connector/index.d.ts +5 -0
  39. package/build/services/connector/index.js +31 -0
  40. package/build/services/dimension/index.d.ts +29 -0
  41. package/build/services/dimension/index.js +35 -0
  42. package/build/services/durable/asyncLocalStorage.d.ts +3 -0
  43. package/build/services/durable/asyncLocalStorage.js +5 -0
  44. package/build/services/durable/client.d.ts +15 -0
  45. package/build/services/durable/client.js +108 -0
  46. package/build/services/durable/connection.d.ts +4 -0
  47. package/build/services/durable/connection.js +51 -0
  48. package/build/services/durable/factory.d.ts +3 -0
  49. package/build/services/durable/factory.js +123 -0
  50. package/build/services/durable/handle.d.ts +8 -0
  51. package/build/services/durable/handle.js +38 -0
  52. package/build/services/durable/index.d.ts +57 -0
  53. package/build/services/durable/index.js +58 -0
  54. package/build/services/durable/native.d.ts +4 -0
  55. package/build/services/durable/native.js +47 -0
  56. package/build/services/durable/worker.d.ts +36 -0
  57. package/build/services/durable/worker.js +266 -0
  58. package/build/services/durable/workflow.d.ts +6 -0
  59. package/build/services/durable/workflow.js +135 -0
  60. package/build/services/engine/index.d.ts +82 -0
  61. package/build/services/engine/index.js +511 -0
  62. package/build/services/hotmesh/index.d.ts +45 -0
  63. package/build/services/hotmesh/index.js +134 -0
  64. package/build/services/logger/index.d.ts +17 -0
  65. package/build/services/logger/index.js +73 -0
  66. package/build/services/mapper/index.d.ts +24 -0
  67. package/build/services/mapper/index.js +72 -0
  68. package/build/services/pipe/functions/array.d.ts +24 -0
  69. package/build/services/pipe/functions/array.js +69 -0
  70. package/build/services/pipe/functions/bitwise.d.ts +9 -0
  71. package/build/services/pipe/functions/bitwise.js +24 -0
  72. package/build/services/pipe/functions/conditional.d.ts +10 -0
  73. package/build/services/pipe/functions/conditional.js +27 -0
  74. package/build/services/pipe/functions/date.d.ts +57 -0
  75. package/build/services/pipe/functions/date.js +167 -0
  76. package/build/services/pipe/functions/index.d.ts +25 -0
  77. package/build/services/pipe/functions/index.js +26 -0
  78. package/build/services/pipe/functions/json.d.ts +5 -0
  79. package/build/services/pipe/functions/json.js +12 -0
  80. package/build/services/pipe/functions/math.d.ts +38 -0
  81. package/build/services/pipe/functions/math.js +111 -0
  82. package/build/services/pipe/functions/number.d.ts +25 -0
  83. package/build/services/pipe/functions/number.js +133 -0
  84. package/build/services/pipe/functions/object.d.ts +22 -0
  85. package/build/services/pipe/functions/object.js +63 -0
  86. package/build/services/pipe/functions/string.d.ts +23 -0
  87. package/build/services/pipe/functions/string.js +69 -0
  88. package/build/services/pipe/functions/symbol.d.ts +12 -0
  89. package/build/services/pipe/functions/symbol.js +33 -0
  90. package/build/services/pipe/functions/unary.d.ts +7 -0
  91. package/build/services/pipe/functions/unary.js +18 -0
  92. package/build/services/pipe/index.d.ts +30 -0
  93. package/build/services/pipe/index.js +128 -0
  94. package/build/services/quorum/index.d.ts +34 -0
  95. package/build/services/quorum/index.js +147 -0
  96. package/build/services/reporter/index.d.ts +47 -0
  97. package/build/services/reporter/index.js +330 -0
  98. package/build/services/serializer/index.d.ts +36 -0
  99. package/build/services/serializer/index.js +222 -0
  100. package/build/services/signaler/store.d.ts +15 -0
  101. package/build/services/signaler/store.js +53 -0
  102. package/build/services/signaler/stream.d.ts +43 -0
  103. package/build/services/signaler/stream.js +317 -0
  104. package/build/services/store/cache.d.ts +66 -0
  105. package/build/services/store/cache.js +127 -0
  106. package/build/services/store/clients/ioredis.d.ts +27 -0
  107. package/build/services/store/clients/ioredis.js +96 -0
  108. package/build/services/store/clients/redis.d.ts +29 -0
  109. package/build/services/store/clients/redis.js +143 -0
  110. package/build/services/store/index.d.ts +88 -0
  111. package/build/services/store/index.js +657 -0
  112. package/build/services/stream/clients/ioredis.d.ts +23 -0
  113. package/build/services/stream/clients/ioredis.js +115 -0
  114. package/build/services/stream/clients/redis.d.ts +23 -0
  115. package/build/services/stream/clients/redis.js +119 -0
  116. package/build/services/stream/index.d.ts +21 -0
  117. package/build/services/stream/index.js +9 -0
  118. package/build/services/sub/clients/ioredis.d.ts +20 -0
  119. package/build/services/sub/clients/ioredis.js +72 -0
  120. package/build/services/sub/clients/redis.d.ts +20 -0
  121. package/build/services/sub/clients/redis.js +63 -0
  122. package/build/services/sub/index.d.ts +18 -0
  123. package/build/services/sub/index.js +9 -0
  124. package/build/services/task/index.d.ts +18 -0
  125. package/build/services/task/index.js +73 -0
  126. package/build/services/telemetry/index.d.ts +49 -0
  127. package/build/services/telemetry/index.js +223 -0
  128. package/build/services/worker/index.d.ts +30 -0
  129. package/build/services/worker/index.js +105 -0
  130. package/build/types/activity.d.ts +86 -0
  131. package/build/types/activity.js +2 -0
  132. package/build/types/app.d.ts +16 -0
  133. package/build/types/app.js +2 -0
  134. package/build/types/async.d.ts +5 -0
  135. package/build/types/async.js +2 -0
  136. package/build/types/cache.d.ts +1 -0
  137. package/build/types/cache.js +2 -0
  138. package/build/types/collator.d.ts +8 -0
  139. package/build/types/collator.js +11 -0
  140. package/build/types/durable.d.ts +59 -0
  141. package/build/types/durable.js +2 -0
  142. package/build/types/hook.d.ts +31 -0
  143. package/build/types/hook.js +9 -0
  144. package/build/types/hotmesh.d.ts +82 -0
  145. package/build/types/hotmesh.js +2 -0
  146. package/build/types/index.d.ts +20 -0
  147. package/build/types/index.js +21 -0
  148. package/build/types/ioredisclient.d.ts +5 -0
  149. package/build/types/ioredisclient.js +5 -0
  150. package/build/types/job.d.ts +50 -0
  151. package/build/types/job.js +2 -0
  152. package/build/types/logger.d.ts +6 -0
  153. package/build/types/logger.js +2 -0
  154. package/build/types/map.d.ts +4 -0
  155. package/build/types/map.js +2 -0
  156. package/build/types/pipe.d.ts +4 -0
  157. package/build/types/pipe.js +2 -0
  158. package/build/types/quorum.d.ts +46 -0
  159. package/build/types/quorum.js +2 -0
  160. package/build/types/redis.d.ts +8 -0
  161. package/build/types/redis.js +2 -0
  162. package/build/types/redisclient.d.ts +25 -0
  163. package/build/types/redisclient.js +2 -0
  164. package/build/types/serializer.d.ts +33 -0
  165. package/build/types/serializer.js +2 -0
  166. package/build/types/stats.d.ts +83 -0
  167. package/build/types/stats.js +2 -0
  168. package/build/types/stream.d.ts +67 -0
  169. package/build/types/stream.js +25 -0
  170. package/build/types/telemetry.d.ts +1 -0
  171. package/build/types/telemetry.js +11 -0
  172. package/build/types/transition.d.ts +17 -0
  173. package/build/types/transition.js +2 -0
  174. package/index.ts +5 -0
  175. package/modules/errors.ts +55 -0
  176. package/modules/key.ts +129 -0
  177. package/modules/utils.ts +170 -0
  178. package/package.json +73 -0
  179. package/services/activities/activity.ts +473 -0
  180. package/services/activities/await.ts +172 -0
  181. package/services/activities/emit.ts +25 -0
  182. package/services/activities/index.ts +15 -0
  183. package/services/activities/iterate.ts +26 -0
  184. package/services/activities/trigger.ts +196 -0
  185. package/services/activities/worker.ts +190 -0
  186. package/services/collator/README.md +102 -0
  187. package/services/collator/index.ts +182 -0
  188. package/services/compiler/deployer.ts +432 -0
  189. package/services/compiler/index.ts +98 -0
  190. package/services/compiler/validator.ts +154 -0
  191. package/services/connector/clients/ioredis.ts +57 -0
  192. package/services/connector/clients/redis.ts +72 -0
  193. package/services/connector/index.ts +44 -0
  194. package/services/dimension/README.md +73 -0
  195. package/services/dimension/index.ts +39 -0
  196. package/services/durable/asyncLocalStorage.ts +3 -0
  197. package/services/durable/client.ts +116 -0
  198. package/services/durable/connection.ts +50 -0
  199. package/services/durable/factory.ts +124 -0
  200. package/services/durable/handle.ts +43 -0
  201. package/services/durable/index.ts +60 -0
  202. package/services/durable/native.ts +46 -0
  203. package/services/durable/worker.ts +254 -0
  204. package/services/durable/workflow.ts +136 -0
  205. package/services/engine/index.ts +615 -0
  206. package/services/hotmesh/index.ts +182 -0
  207. package/services/logger/index.ts +79 -0
  208. package/services/mapper/index.ts +84 -0
  209. package/services/pipe/functions/array.ts +87 -0
  210. package/services/pipe/functions/bitwise.ts +27 -0
  211. package/services/pipe/functions/conditional.ts +31 -0
  212. package/services/pipe/functions/date.ts +214 -0
  213. package/services/pipe/functions/index.ts +25 -0
  214. package/services/pipe/functions/json.ts +11 -0
  215. package/services/pipe/functions/math.ts +143 -0
  216. package/services/pipe/functions/number.ts +150 -0
  217. package/services/pipe/functions/object.ts +79 -0
  218. package/services/pipe/functions/string.ts +86 -0
  219. package/services/pipe/functions/symbol.ts +39 -0
  220. package/services/pipe/functions/unary.ts +19 -0
  221. package/services/pipe/index.ts +138 -0
  222. package/services/quorum/index.ts +200 -0
  223. package/services/reporter/index.ts +379 -0
  224. package/services/serializer/README.md +10 -0
  225. package/services/serializer/index.ts +243 -0
  226. package/services/signaler/store.ts +61 -0
  227. package/services/signaler/stream.ts +354 -0
  228. package/services/store/cache.ts +172 -0
  229. package/services/store/clients/ioredis.ts +123 -0
  230. package/services/store/clients/redis.ts +169 -0
  231. package/services/store/index.ts +757 -0
  232. package/services/stream/clients/ioredis.ts +148 -0
  233. package/services/stream/clients/redis.ts +144 -0
  234. package/services/stream/index.ts +57 -0
  235. package/services/sub/clients/ioredis.ts +83 -0
  236. package/services/sub/clients/redis.ts +74 -0
  237. package/services/sub/index.ts +25 -0
  238. package/services/task/index.ts +86 -0
  239. package/services/telemetry/index.ts +267 -0
  240. package/services/worker/index.ts +165 -0
  241. package/types/activity.ts +115 -0
  242. package/types/app.ts +20 -0
  243. package/types/async.ts +7 -0
  244. package/types/cache.ts +1 -0
  245. package/types/collator.ts +9 -0
  246. package/types/durable.ts +81 -0
  247. package/types/hook.ts +32 -0
  248. package/types/hotmesh.ts +102 -0
  249. package/types/index.ts +138 -0
  250. package/types/ioredisclient.ts +10 -0
  251. package/types/job.ts +59 -0
  252. package/types/logger.ts +6 -0
  253. package/types/map.ts +5 -0
  254. package/types/ms.d.ts +7 -0
  255. package/types/pipe.ts +7 -0
  256. package/types/quorum.ts +59 -0
  257. package/types/redis.ts +27 -0
  258. package/types/redisclient.ts +29 -0
  259. package/types/serializer.ts +38 -0
  260. package/types/stats.ts +100 -0
  261. package/types/stream.ts +75 -0
  262. package/types/telemetry.ts +15 -0
  263. package/types/transition.ts +20 -0
@@ -0,0 +1,57 @@
1
+ import {
2
+ RedisClientOptions,
3
+ RedisClassType,
4
+ RedisClientType } from '../../../types/ioredisclient';
5
+
6
+ class RedisConnection {
7
+ private connection: any | null = null;
8
+ private static instances: Map<string, RedisConnection> = new Map();
9
+ private id: string | null = null;
10
+
11
+ private static clientOptions: RedisClientOptions = {
12
+ host: 'localhost',
13
+ port: 6379,
14
+ //password: config.REDIS_PASSWORD,
15
+ //db: config.REDIS_DATABASE,
16
+ };
17
+
18
+ private async createConnection(Redis: RedisClassType, options: RedisClientOptions): Promise<any> {
19
+ return new Redis(options);
20
+ }
21
+
22
+ public getClient(): RedisClientType {
23
+ if (!this.connection) {
24
+ throw new Error('Redis client is not connected');
25
+ }
26
+ return this.connection;
27
+ }
28
+
29
+ public async disconnect(): Promise<void> {
30
+ if (this.connection) {
31
+ await this.connection.quit();
32
+ this.connection = null;
33
+ }
34
+ if (this.id) {
35
+ RedisConnection.instances.delete(this.id);
36
+ }
37
+ }
38
+
39
+ public static async connect(id: string, Redis: RedisClassType, options?: RedisClientOptions): Promise<RedisConnection> {
40
+ if (this.instances.has(id)) {
41
+ return this.instances.get(id) as RedisConnection;
42
+ }
43
+ const instance = new RedisConnection();
44
+ const opts = options ? { ...options } : { ...this.clientOptions };
45
+ instance.connection = await instance.createConnection(Redis, opts);
46
+ instance.id = id;
47
+ this.instances.set(id, instance);
48
+ return instance;
49
+ }
50
+
51
+ public static async disconnectAll(): Promise<void> {
52
+ await Promise.all(Array.from(this.instances.values()).map((instance) => instance.disconnect()));
53
+ this.instances.clear();
54
+ }
55
+ }
56
+
57
+ export { RedisConnection };
@@ -0,0 +1,72 @@
1
+ import {
2
+ RedisClientType,
3
+ RedisClientOptions,
4
+ RedisClassType } from '../../../types/redisclient';
5
+
6
+ class RedisConnection {
7
+ private connection: RedisClientType | null = null;
8
+ private static instances: Map<string, RedisConnection> = new Map();
9
+ private id: string | null = null;
10
+
11
+ private static clientOptions: RedisClientOptions = {
12
+ socket: {
13
+ host: 'localhost',
14
+ port: 6379,
15
+ tls: false,
16
+ },
17
+ //password: config.REDIS_PASSWORD,
18
+ //database: config.REDIS_DATABASE,
19
+ };
20
+
21
+ private async createConnection(Redis: RedisClassType, options: RedisClientOptions): Promise<RedisClientType> {
22
+ return new Promise((resolve, reject) => {
23
+ const client = Redis.createClient(options);
24
+
25
+ client.on('error', (error: any) => {
26
+ reject(error);
27
+ });
28
+
29
+ client.on('ready', () => {
30
+ resolve(client);
31
+ });
32
+
33
+ client.connect();
34
+ });
35
+ }
36
+
37
+ public getClient(): RedisClientType {
38
+ if (!this.connection) {
39
+ throw new Error('Redis client is not connected');
40
+ }
41
+ return this.connection;
42
+ }
43
+
44
+ public async disconnect(): Promise<void> {
45
+ if (this.connection) {
46
+ await this.connection.quit();
47
+ this.connection = null;
48
+ }
49
+ if (this.id) {
50
+ RedisConnection.instances.delete(this.id);
51
+ }
52
+ }
53
+
54
+ public static async connect(id: string, Redis: RedisClassType, options?: RedisClientOptions): Promise<RedisConnection> {
55
+ if (this.instances.has(id)) {
56
+ return this.instances.get(id)!;
57
+ }
58
+ const instance = new RedisConnection();
59
+ const opts = options ? { ...options } : { ...this.clientOptions };
60
+ instance.connection = await instance.createConnection(Redis, opts);
61
+ instance.id = id;
62
+ this.instances.set(id, instance);
63
+ return instance;
64
+ }
65
+
66
+ public static async disconnectAll(): Promise<void> {
67
+ await Promise.all(Array.from(this.instances.values()).map((instance) => instance.disconnect()));
68
+ this.instances.clear();
69
+ }
70
+ }
71
+
72
+ export { RedisConnection, RedisClientType };
@@ -0,0 +1,44 @@
1
+ import { nanoid } from 'nanoid';
2
+
3
+ import { identifyRedisTypeFromClass } from '../../modules/utils';
4
+ import { RedisConnection as IORedisConnection } from '../connector/clients/ioredis';
5
+ import { RedisConnection } from '../connector/clients/redis';
6
+ import {
7
+ RedisClassType as IORedisClassType,
8
+ RedisClientOptions as IORedisClientOptions } from '../../types/ioredisclient';
9
+ import {
10
+ HotMeshEngine,
11
+ HotMeshWorker } from '../../types/hotmesh';
12
+ import { RedisClass, RedisOptions } from '../../types/redis';
13
+ import {
14
+ RedisClassType,
15
+ RedisClientOptions } from '../../types/redisclient';
16
+
17
+ export class ConnectorService {
18
+ //1) Initialize `store`, `stream`, and `subscription` Redis clients.
19
+ //2) Bind to the target if not already present
20
+ static async initRedisClients(Redis: RedisClass, options: RedisOptions, target: HotMeshEngine | HotMeshWorker): Promise<void> {
21
+ if (!target.store || !target.stream || !target.sub) {
22
+ const instances = [];
23
+ if (identifyRedisTypeFromClass(Redis) === 'redis') {
24
+ for (let i = 1; i <= 3; i++) {
25
+ instances.push(RedisConnection.connect(
26
+ nanoid(),
27
+ Redis as RedisClassType,
28
+ options as RedisClientOptions));
29
+ }
30
+ } else {
31
+ for (let i = 1; i <= 3; i++) {
32
+ instances.push(IORedisConnection.connect(
33
+ nanoid(),
34
+ Redis as IORedisClassType,
35
+ options as IORedisClientOptions));
36
+ }
37
+ }
38
+ const [store, stream, sub] = await Promise.all(instances);
39
+ target.store = target.store || store.getClient();
40
+ target.stream = target.stream || stream.getClient();
41
+ target.sub = target.sub || sub.getClient();
42
+ }
43
+ }
44
+ }
@@ -0,0 +1,73 @@
1
+ # Optimizing Cycle Management Through Shared Collation State
2
+ Activity state is managed using a 15-digit integer. For all non-trigger activities, this value is initialized by the parent activity/preceding in the DAG. (Triggers are initialized in a 'completed' state, since their execution only reflects the completion of the Leg 2 Duplex.)
3
+
4
+ >Presetting the value for subsequent generations is critical to establishing return receipt checks, so that the system can guarantee durability and idempotency in the result of system failure during handoff.
5
+
6
+ The first three digits of the 15-digit integer track and convey the activity lifecycle status. Supporting 3 states using any 3 integers is the only critical requirement for the chosen integers. As long as the backend system supports decrement and increment commands that return the modified integer value, the system can durably track state using the strategy described here. In the reference implementation, the digit `9` is “pending” and `8` is “complete”. Additional digits can be employed to convey additional states.
7
+
8
+ The remaining 12 digits offer 1 million distinct dimensional threads for activity expansion. Dimensional Threads isolate and track those activities in the workflow that run in a cycle. They ensure that no naming collisions occur, even if the same activity is run multiple times. Each time duplex leg 2 of an activity returns with its payload, it can traverse the primary execution tree that remains (the remaining nodes in the graph); however, if the message includes a ‘pending’ status, it means that the channel will remain open, necessitating a dimensional execution thread be added to the flow, so that subsequent incoming messages can be tracked. This pattern likewise exists for `iterator` and `mark/goto` activities in that every adjacent activity that follows in the DAG will be uniquely addressed using a sequential dimensional thread that reflects its location in the collection being iterated.
9
+
10
+ In the following series, [S] represents interactions with the STREAM while [D] represents interactions with the backend DATA STORE. The job begins with the trigger ( the entry point for the rooted tree). Every activity runs as a duplexed exchange with the request and response channels fully decoupled.
11
+
12
+ The trigger is unique, however, in that it does not execute Leg 1. This is because the outside caller is responsible for Leg 1, including generating the input data to be sent to the trigger to kick off the workflow.
13
+
14
+ TRIGGER LEG 2
15
+
16
+ ```bash
17
+ [D] a A | SET IF UNIQUE <JOBID> (ENSURE UNIQUE JOB ID)
18
+ 888000001000001 [D] a B | SET TRIGGER STATE (SET TO 888000001000001)
19
+ [S] b C | [SPAWN ADJACENT (CHILD) NODES]
20
+ 999000000000000 [D] a E | PRESET SPAWNED ADJACENT NODE(S) ACTIVITY STATE
21
+ F => RETURN JOB ID TO CALLER
22
+ WORKER ACTIVITY LEG 1
23
+ [S] a a | [DEQUEUE ACTIVITY]
24
+ 999000000000000 *ACTIVITY STATUS WAS PRESET BY THE PARENT NODE*
25
+ ^-------------- [D] b c | UPDATE ACTIVITY STATUS (DECREMENT to 899*)
26
+ => FIX or CANCEL `IF < /^8\d+$/` [ACK/DELETE]
27
+ [S] c d | [ENQUEUE WORKER REQUEST]
28
+ ^------------- [D] b e | UPDATE ACTIVITY STATUS (DECREMENT to 889*)
29
+ [S] a f | [ACK/DELETE]
30
+ WORKER (SYNCHRONOUS | VARIANT 1 OF 2)
31
+ [S] a g | [DEQUEUE WORKER REQUEST]
32
+ ^^^------------ [D] b h | QUERY ACTIVITY Status (CHECK IF JOB ACTIVE)
33
+ => CANCEL `IF != /^889\d+$/` => [ACK/DELETE]
34
+ X X => EXEC WORKER
35
+ [S] d j | [ENQUEUE WORKER RESPONSE]
36
+ [S] a k | [ACK/DELETE]
37
+ WORKER (ASYNCHRONOUS | VARIANT 2 OF 2)
38
+ [S] a g | [DEQUEUE WORKER REQUEST]
39
+ ^^^------------ [D] b h | QUERY ACTIVITY Status (CHECK IF JOB ACTIVE)
40
+ => CANCEL `IF != /^889\d+$/` => [ACK/DELETE]
41
+ X X => SAVE JOB METADATA | START WORKER
42
+ [S] a k | [ACK/DELETE]
43
+ => => =>
44
+ X X => => => STOP WORKER (ALL DONE)
45
+ [S] d j | [ENQUEUE WORKER RESPONSE]
46
+ WORKER ACTIVITY LEG 2 (PROCESS PENDING | VARIANT 1 of 3)
47
+ [S] a l | [DEQUEUE WORKER RESPONSE]
48
+ ^^^^^^------ [D] b m | INCREMENT DIMENSIONAL THREAD ENTRY COUNT by 1
49
+ => CANCEL `IF != /^889\d+$/` => [ACK/DELETE]
50
+ [S] c n | [ENQUEUE ADJACENT (CHILD) NODES]
51
+ ^^^^^^ [D] b o | INCREMENT DIMENSIONAL THREAD EXIT COUNT by 1
52
+ 999000000000000 [D] b q | PRESET SPAWNED ADJACENT NODE(S) STATE
53
+ [S] a s | [ACK/DELETE]
54
+ WORKER ACTIVITY LEG 2 (PROCESS SUCCESS [OR CAUGHT ERROR] | VARIANT 2 of 3)
55
+ [S] a l | [DEQUEUE WORKER RESPONSE]
56
+ ^^^^^^------ [D] b m | INCREMENT DIMENSIONAL THREAD ENTRY COUNT by 1
57
+ => CANCEL `IF != /^889\d+$/` => [ACK/DELETE]
58
+ [S] c n | [ENQUEUE ADJACENT (CHILD) NODES]
59
+ ^^^^^^ [D] b o | INCREMENT DIMENSIONAL THREAD EXIT COUNT by 1
60
+ ^------------ [D] b p | UPDATE ACTIVITY STATUS (DECREMENT to 888*)
61
+ 999000000000000 [D] b q | PRESET SPAWNED ADJACENT NODE(S) STATE
62
+ [S] d r | [ENQUEUE `JOB CLEANUP TASKS` IF JOB STATE = 0]
63
+ [S] a s | [ACK/DELETE]
64
+ WORKER ACTIVITY LEG 2 (PROCESS UNCAUGHT ERROR | VARIANT 3 of 3)
65
+ [S] a l | [DEQUEUE WORKER RESPONSE]
66
+ ^^^^^^------ [D] b m | INCREMENT DIMENSIONAL THREAD ENTRY COUNT by 1
67
+ => CANCEL `IF != /^\d{2}9\d+$/` => [ACK/DELETE]
68
+ ^------------ [D] b p | UPDATE ACTIVITY STATUS (DECREMENT to 887*)
69
+ [S] d r | [ENQUEUE `JOB CLEANUP TASKS`]
70
+ [S] a s | [ACK/DELETE]
71
+ ```
72
+
73
+ Streams are used when executing an activity (such as transitioning to a child activity) as they guarantee that the child activity will be fully created and initialized before the request is marked for deletion. Even if the system has a catastrophic failure, the chain of custody can be guaranteed through the use of streams when the system comes online. If, for example, the system crashed during activity processing, the event history will reveal exactly where in the process it occurred, so that it is possible to restore state while still guaranteeing idempotency.
@@ -0,0 +1,39 @@
1
+ import { HotMeshGraph } from "../../types/hotmesh";
2
+
3
+ class DimensionService {
4
+
5
+ //max int digit count that supports `hincrby`
6
+ static targetLength = 15;
7
+
8
+ /**
9
+ * entry point for compiler-type activities. This is called by the compiler
10
+ * to bind the sorted activity IDs to the trigger activity. These are then used
11
+ * at runtime by the activities to track job/activity status.
12
+ * @param graphs
13
+ */
14
+ static compile(graphs: HotMeshGraph[]) {
15
+
16
+ }
17
+
18
+ /**
19
+ * All activities exist on a dimensional plane. Zero
20
+ * is the default and is implied if no dimension is
21
+ * present in the hash item key. EVERY value in the
22
+ * job ledger is dimensionalized even if the dimension
23
+ * is not present. The key, `AaA`, might not contain
24
+ * a dimensional index, but it is still implicitly
25
+ * dimensionalized as `AaA,0` (assuming a trigger).
26
+ * A value of `AxY,0,0,0,0,1,0,0` would reflect that
27
+ * an ancestor activity was dimensionalized beyond
28
+ * the default. The dimensional string must
29
+ * be included if not zero. There is likely a preceding
30
+ * sibling dimension, so it would not need to include
31
+ * the suffix, so these addresses are equivalent:
32
+ * `AxY,0,0,0,0,0,0,0` == `AxY` for said sibling.
33
+ */
34
+ static getSeed(index = 0): string {
35
+ return `,${index}`;
36
+ }
37
+ }
38
+
39
+ export { DimensionService };
@@ -0,0 +1,3 @@
1
+ import { AsyncLocalStorage } from 'async_hooks';
2
+
3
+ export const asyncLocalStorage = new AsyncLocalStorage<Map<string, any>>();
@@ -0,0 +1,116 @@
1
+ import { WorkflowHandleService } from "./handle";
2
+ import { HotMeshService as HotMesh } from "../hotmesh";
3
+ import { ClientConfig, Connection, WorkflowOptions } from "../../types/durable";
4
+ import { getWorkflowYAML } from "./factory";
5
+ import { JobState } from "../../types/job";
6
+
7
+ /*
8
+ Here is an example of how the methods in this file are used:
9
+
10
+ ./client.ts
11
+
12
+ import { Durable } from '@hotmeshio/hotmesh';
13
+ import Redis from 'ioredis';
14
+ import { example } from './workflows';
15
+ import { nanoid } from 'nanoid';
16
+
17
+ async function run() {
18
+ const connection = await Durable.Connection.connect({
19
+ class: Redis,
20
+ options: {
21
+ host: 'localhost',
22
+ port: 6379,
23
+ },
24
+ });
25
+
26
+ const client = new Durable.Client({
27
+ connection,
28
+ });
29
+
30
+ const handle = await client.workflow.start({
31
+ args: ['HotMesh'],
32
+ taskQueue: 'hello-world',
33
+ workflowName: 'example',
34
+ workflowId: 'workflow-' + nanoid(),
35
+ });
36
+
37
+ console.log(`Started workflow ${handle.workflowId}`);
38
+ console.log(await handle.result());
39
+ }
40
+
41
+ run().catch((err) => {
42
+ console.error(err);
43
+ process.exit(1);
44
+ });
45
+
46
+ */
47
+
48
+ export class ClientService {
49
+
50
+ connection: Connection;
51
+ options: WorkflowOptions;
52
+ static instances = new Map<string, HotMesh | Promise<HotMesh>>();
53
+
54
+ constructor(config: ClientConfig) {
55
+ this.connection = config.connection;
56
+ }
57
+
58
+ getHotMesh = async (worflowTopic: string) => {
59
+ if (ClientService.instances.has(worflowTopic)) {
60
+ return await ClientService.instances.get(worflowTopic);
61
+ }
62
+
63
+ const hotMesh = HotMesh.init({
64
+ appId: worflowTopic,
65
+ engine: {
66
+ redis: {
67
+ class: this.connection.class,
68
+ options: this.connection.options,
69
+ }
70
+ }
71
+ });
72
+ ClientService.instances.set(worflowTopic, hotMesh);
73
+ await this.activateWorkflow(await hotMesh, worflowTopic);
74
+ return hotMesh;
75
+ }
76
+
77
+ workflow = {
78
+ start: async (options: WorkflowOptions): Promise<WorkflowHandleService> => {
79
+ const taskQueueName = options.taskQueue;
80
+ const workflowName = options.workflowName;
81
+ const trc = options.workflowTrace;
82
+ const spn = options.workflowSpan;
83
+ const workflowTopic = `${taskQueueName}-${workflowName}`;
84
+ const hotMesh = await this.getHotMesh(workflowTopic);
85
+ const payload = {
86
+ arguments: [...options.args],
87
+ workflowId: options.workflowId,
88
+ }
89
+ const context = { metadata: { trc, spn }, data: {}};
90
+ const jobId = await hotMesh.pub(workflowTopic, payload, context as JobState);
91
+ return new WorkflowHandleService(hotMesh, workflowTopic, jobId);
92
+ },
93
+ };
94
+
95
+ async activateWorkflow(hotMesh: HotMesh, workflowTopic: string): Promise<void> {
96
+ const version = '1';
97
+ const app = await hotMesh.engine.store.getApp(workflowTopic);
98
+ const appVersion = app?.version as unknown as number;
99
+ if(isNaN(appVersion)) {
100
+ try {
101
+ await hotMesh.deploy(getWorkflowYAML(workflowTopic, version));
102
+ await hotMesh.activate(version);
103
+ } catch (err) {
104
+ hotMesh.engine.logger.error('durable-client-workflow-activation-err', err);
105
+ throw err;
106
+ }
107
+ }
108
+ }
109
+
110
+ static async shutdown(): Promise<void> {
111
+ for (const [key, value] of ClientService.instances) {
112
+ const hotMesh = await value;
113
+ await hotMesh.stop();
114
+ }
115
+ }
116
+ }
@@ -0,0 +1,50 @@
1
+ import { Connection, ConnectionConfig } from "../../types/durable";
2
+
3
+ /*
4
+ Here is an example of how the methods in this file are used:
5
+
6
+ ./client.ts
7
+
8
+ import { Durable } from '@hotmeshio/hotmesh';
9
+ import Redis from 'ioredis';
10
+ import { nanoid } from 'nanoid';
11
+
12
+ async function run() {
13
+ const connection = await Durable.Connection.connect({
14
+ class: Redis,
15
+ options: {
16
+ host: 'localhost',
17
+ port: 6379,
18
+ },
19
+ });
20
+
21
+ const client = new Durable.Client({
22
+ connection,
23
+ });
24
+
25
+ const handle = await client.workflow.start(example, {
26
+ taskQueue: 'hello-world',
27
+ args: ['HotMesh'],
28
+ workflowName: 'example',
29
+ workflowId: nanoid(),
30
+ });
31
+
32
+ console.log(`Started workflow ${handle.workflowId}`);
33
+ console.log(await handle.result());
34
+ }
35
+
36
+ run().catch((err) => {
37
+ console.error(err);
38
+ process.exit(1);
39
+ });
40
+
41
+ */
42
+
43
+ export class ConnectionService {
44
+ static async connect(config: ConnectionConfig): Promise<Connection> {
45
+ return {
46
+ class: config.class,
47
+ options: { ...config.options },
48
+ } as Connection;
49
+ }
50
+ }
@@ -0,0 +1,124 @@
1
+ const getWorkflowYAML = (topic: string, version = '1') => {
2
+ return `app:
3
+ id: ${topic}
4
+ version: '${version}'
5
+ graphs:
6
+ - subscribes: ${topic}
7
+ publishes: ${topic}
8
+ expire: 120
9
+ input:
10
+ schema:
11
+ type: object
12
+ properties:
13
+ workflowId:
14
+ type: string
15
+ arguments:
16
+ type: array
17
+ output:
18
+ schema:
19
+ type: object
20
+ properties:
21
+ response:
22
+ type: any
23
+
24
+ activities:
25
+ t1:
26
+ type: trigger
27
+ stats:
28
+ id: '{$self.input.data.workflowId}'
29
+ a1:
30
+ type: worker
31
+ topic: ${topic}
32
+ input:
33
+ schema:
34
+ type: object
35
+ properties:
36
+ workflowId:
37
+ type: string
38
+ arguments:
39
+ type: array
40
+ maps:
41
+ workflowId: '{t1.output.data.workflowId}'
42
+ arguments: '{t1.output.data.arguments}'
43
+ output:
44
+ schema:
45
+ type: object
46
+ properties:
47
+ response:
48
+ type: any
49
+ job:
50
+ maps:
51
+ response: '{$self.output.data.response}'
52
+ transitions:
53
+ t1:
54
+ - to: a1`;
55
+ }
56
+
57
+ const getActivityYAML = (topic: string, version = '1') => {
58
+ return `app:
59
+ id: ${topic}
60
+ version: '${version}'
61
+ graphs:
62
+ - subscribes: ${topic}
63
+ input:
64
+ schema:
65
+ type: object
66
+ properties:
67
+ workflowId:
68
+ type: string
69
+ workflowTopic:
70
+ type: string
71
+ activityName:
72
+ type: array
73
+ arguments:
74
+ type: array
75
+ output:
76
+ schema:
77
+ type: object
78
+ properties:
79
+ response:
80
+ type: any
81
+
82
+ activities:
83
+ t1:
84
+ type: trigger
85
+ stats:
86
+ id: '{$self.input.data.workflowId}'
87
+ a1:
88
+ type: worker
89
+ topic: ${topic}
90
+ input:
91
+ schema:
92
+ type: object
93
+ properties:
94
+ workflowId:
95
+ type: string
96
+ workflowTopic:
97
+ type: string
98
+ activityName:
99
+ type: array
100
+ arguments:
101
+ type: array
102
+ maps:
103
+ workflowId: '{t1.output.data.workflowId}'
104
+ workflowTopic: '{t1.output.data.workflowTopic}'
105
+ activityName: '{t1.output.data.activityName}'
106
+ arguments: '{t1.output.data.arguments}'
107
+ output:
108
+ schema:
109
+ type: object
110
+ properties:
111
+ response:
112
+ type: any
113
+ job:
114
+ maps:
115
+ response: '{$self.output.data.response}'
116
+ transitions:
117
+ t1:
118
+ - to: a1`;
119
+ }
120
+
121
+ export {
122
+ getActivityYAML,
123
+ getWorkflowYAML
124
+ };
@@ -0,0 +1,43 @@
1
+ import { JobOutput } from "../../types/job";
2
+ import { HotMeshService as HotMesh } from "../hotmesh";
3
+
4
+ export class WorkflowHandleService {
5
+ hotMesh: HotMesh;
6
+ workflowTopic: string;
7
+ workflowId: string;
8
+
9
+ constructor(hotMesh: HotMesh, workflowTopic: string, workflowId: string) {
10
+ this.workflowTopic = workflowTopic;
11
+ this.workflowId = workflowId;
12
+ this.hotMesh = hotMesh;
13
+ }
14
+
15
+ async result(): Promise<any> {
16
+ let status = await this.hotMesh.getStatus(this.workflowId);
17
+ const topic = `${this.workflowTopic}.${this.workflowId}`;
18
+
19
+ if (status == 0) {
20
+ return (await this.hotMesh.getState(this.workflowTopic, this.workflowId)).data?.response;
21
+ }
22
+
23
+ return new Promise((resolve, reject) => {
24
+ let isResolved = false;
25
+ //common fulfill/unsubscribe
26
+ const complete = async (response?: any) => {
27
+ if (isResolved) return;
28
+ isResolved = true;
29
+ this.hotMesh.unsub(topic);
30
+ resolve(response || (await this.hotMesh.getState(this.workflowTopic, this.workflowId)).data?.response);
31
+ };
32
+ this.hotMesh.sub(topic, async (topic: string, message: JobOutput) => {
33
+ await complete(message.data?.response);
34
+ });
35
+ setTimeout(async () => {
36
+ status = await this.hotMesh.getStatus(this.workflowId);
37
+ if (status == 0) {
38
+ await complete();
39
+ }
40
+ }, 0);
41
+ });
42
+ }
43
+ }
@@ -0,0 +1,60 @@
1
+ import { ClientService } from './client';
2
+ import { ConnectionService } from './connection';
3
+ import { NativeConnectionService } from './native';
4
+ import { WorkerService } from './worker';
5
+ import { WorkflowService } from './workflow';
6
+ import { ContextType } from '../../types/durable';
7
+
8
+ /**
9
+ * As a durable integration platform, HotMesh
10
+ * can model and emulate other durable systems
11
+ * (like Temporal). As you review the code in
12
+ * this file, note the following:
13
+ *
14
+ * 1) There is no central governing server.
15
+ * HotMesh is a client-side SDK that connects to Redis
16
+ * using CQRS principles to implicitly drive
17
+ * orchestrations using a headless quorum. Stream
18
+ * semantics guarantee that all events are
19
+ * processed by the quorum of connected clients.
20
+ *
21
+ * 2) Every developer-defined `workflow` function
22
+ * is assigned a HotMesh workflow (which runs in
23
+ * the background) to support it.
24
+ *
25
+ * If the HotMesh workflow is not yet defined,
26
+ * it will be deployed and activated on-the-fly.
27
+ * (The generated DAG will have one Trigger Activity
28
+ * and one Worker Activity.) The Worker Activity
29
+ * is configured to catch execution errors and
30
+ * return them, using a 'pending' status to
31
+ * indicate that the workflow is still running.
32
+ * It is possible for workflow activities to throw
33
+ * errors that will force the entire workflow to
34
+ * fail. This is not the case here. The workflow will
35
+ * continue to run until it is completed or cancelled.
36
+ *
37
+ * 2) Every developer-defined `activity` function
38
+ * is assigned a HotMesh workflow (which runs in
39
+ * the background) to support it.
40
+ *
41
+ * (The generated DAG will have one Trigger Activity and one
42
+ * Worker Activity.) The JOB ID for Worker Activity executions
43
+ * is derived from the containing JOB ID,
44
+ * allowing Activity state to be 'replayed' when the workflow
45
+ * is run again. The Activity Function Runner (activity proxy)
46
+ * is configured similar to how the workflow worker is and will
47
+ * catch execution errors and return them to the caller, using a
48
+ * 'pending' status to indicate that the activity is still running.
49
+ * This allows the activity to be retried until it succeeds.
50
+ */
51
+
52
+ export const Durable = {
53
+ Client: ClientService,
54
+ Connection: ConnectionService,
55
+ NativeConnection: NativeConnectionService,
56
+ Worker: WorkerService,
57
+ workflow: WorkflowService,
58
+ };
59
+
60
+ export type { ContextType };