@hotmeshio/hotmesh 0.4.0 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (284) hide show
  1. package/README.md +39 -14
  2. package/build/modules/enums.d.ts +110 -0
  3. package/build/modules/enums.js +134 -0
  4. package/build/modules/errors.d.ts +124 -0
  5. package/build/modules/errors.js +191 -0
  6. package/build/modules/key.d.ts +66 -0
  7. package/build/modules/key.js +190 -0
  8. package/build/modules/storage.d.ts +3 -0
  9. package/build/modules/storage.js +5 -0
  10. package/build/modules/utils.d.ts +119 -0
  11. package/build/modules/utils.js +374 -0
  12. package/build/package.json +1 -1
  13. package/build/services/activities/activity.d.ts +104 -0
  14. package/build/services/activities/activity.js +549 -0
  15. package/build/services/activities/await.d.ts +12 -0
  16. package/build/services/activities/await.js +114 -0
  17. package/build/services/activities/cycle.d.ts +19 -0
  18. package/build/services/activities/cycle.js +112 -0
  19. package/build/services/activities/hook.d.ts +27 -0
  20. package/build/services/activities/hook.js +168 -0
  21. package/build/services/activities/index.d.ts +19 -0
  22. package/build/services/activities/index.js +20 -0
  23. package/build/services/activities/interrupt.d.ts +16 -0
  24. package/build/services/activities/interrupt.js +158 -0
  25. package/build/services/activities/signal.d.ts +20 -0
  26. package/build/services/activities/signal.js +134 -0
  27. package/build/services/activities/trigger.d.ts +37 -0
  28. package/build/services/activities/trigger.js +246 -0
  29. package/build/services/activities/worker.d.ts +12 -0
  30. package/build/services/activities/worker.js +106 -0
  31. package/build/services/collator/index.d.ts +111 -0
  32. package/build/services/collator/index.js +293 -0
  33. package/build/services/compiler/deployer.d.ts +40 -0
  34. package/build/services/compiler/deployer.js +488 -0
  35. package/build/services/compiler/index.d.ts +32 -0
  36. package/build/services/compiler/index.js +112 -0
  37. package/build/services/compiler/validator.d.ts +34 -0
  38. package/build/services/compiler/validator.js +147 -0
  39. package/build/services/connector/factory.d.ts +22 -0
  40. package/build/services/connector/factory.js +99 -0
  41. package/build/services/connector/index.d.ts +30 -0
  42. package/build/services/connector/index.js +54 -0
  43. package/build/services/connector/providers/ioredis.d.ts +9 -0
  44. package/build/services/connector/providers/ioredis.js +26 -0
  45. package/build/services/connector/providers/nats.d.ts +9 -0
  46. package/build/services/connector/providers/nats.js +34 -0
  47. package/build/services/connector/providers/postgres.d.ts +20 -0
  48. package/build/services/connector/providers/postgres.js +102 -0
  49. package/build/services/connector/providers/redis.d.ts +9 -0
  50. package/build/services/connector/providers/redis.js +38 -0
  51. package/build/services/engine/index.d.ts +264 -0
  52. package/build/services/engine/index.js +761 -0
  53. package/build/services/exporter/index.d.ts +44 -0
  54. package/build/services/exporter/index.js +126 -0
  55. package/build/services/hotmesh/index.d.ts +483 -0
  56. package/build/services/hotmesh/index.js +622 -0
  57. package/build/services/logger/index.d.ts +16 -0
  58. package/build/services/logger/index.js +54 -0
  59. package/build/services/mapper/index.d.ts +28 -0
  60. package/build/services/mapper/index.js +81 -0
  61. package/build/services/memflow/client.d.ts +108 -0
  62. package/build/services/memflow/client.js +372 -0
  63. package/build/services/memflow/connection.d.ts +23 -0
  64. package/build/services/memflow/connection.js +33 -0
  65. package/build/services/memflow/context.d.ts +143 -0
  66. package/build/services/memflow/context.js +299 -0
  67. package/build/services/memflow/exporter.d.ts +51 -0
  68. package/build/services/memflow/exporter.js +215 -0
  69. package/build/services/memflow/handle.d.ts +90 -0
  70. package/build/services/memflow/handle.js +176 -0
  71. package/build/services/memflow/index.d.ts +116 -0
  72. package/build/services/memflow/index.js +122 -0
  73. package/build/services/memflow/schemas/factory.d.ts +29 -0
  74. package/build/services/memflow/schemas/factory.js +2492 -0
  75. package/build/services/memflow/search.d.ts +142 -0
  76. package/build/services/memflow/search.js +320 -0
  77. package/build/services/memflow/worker.d.ts +124 -0
  78. package/build/services/memflow/worker.js +514 -0
  79. package/build/services/memflow/workflow/all.d.ts +7 -0
  80. package/build/services/memflow/workflow/all.js +15 -0
  81. package/build/services/memflow/workflow/common.d.ts +20 -0
  82. package/build/services/memflow/workflow/common.js +47 -0
  83. package/build/services/memflow/workflow/context.d.ts +6 -0
  84. package/build/services/memflow/workflow/context.js +45 -0
  85. package/build/services/memflow/workflow/contextMethods.d.ts +14 -0
  86. package/build/services/memflow/workflow/contextMethods.js +33 -0
  87. package/build/services/memflow/workflow/didRun.d.ts +7 -0
  88. package/build/services/memflow/workflow/didRun.js +22 -0
  89. package/build/services/memflow/workflow/emit.d.ts +11 -0
  90. package/build/services/memflow/workflow/emit.js +29 -0
  91. package/build/services/memflow/workflow/enrich.d.ts +9 -0
  92. package/build/services/memflow/workflow/enrich.js +17 -0
  93. package/build/services/memflow/workflow/execChild.d.ts +18 -0
  94. package/build/services/memflow/workflow/execChild.js +102 -0
  95. package/build/services/memflow/workflow/execHook.d.ts +65 -0
  96. package/build/services/memflow/workflow/execHook.js +73 -0
  97. package/build/services/memflow/workflow/hook.d.ts +9 -0
  98. package/build/services/memflow/workflow/hook.js +56 -0
  99. package/build/services/memflow/workflow/index.d.ts +74 -0
  100. package/build/services/memflow/workflow/index.js +87 -0
  101. package/build/services/memflow/workflow/interrupt.d.ts +9 -0
  102. package/build/services/memflow/workflow/interrupt.js +24 -0
  103. package/build/services/memflow/workflow/isSideEffectAllowed.d.ts +10 -0
  104. package/build/services/memflow/workflow/isSideEffectAllowed.js +33 -0
  105. package/build/services/memflow/workflow/proxyActivities.d.ts +20 -0
  106. package/build/services/memflow/workflow/proxyActivities.js +97 -0
  107. package/build/services/memflow/workflow/random.d.ts +6 -0
  108. package/build/services/memflow/workflow/random.js +16 -0
  109. package/build/services/memflow/workflow/searchMethods.d.ts +6 -0
  110. package/build/services/memflow/workflow/searchMethods.js +25 -0
  111. package/build/services/memflow/workflow/signal.d.ts +29 -0
  112. package/build/services/memflow/workflow/signal.js +50 -0
  113. package/build/services/memflow/workflow/sleepFor.d.ts +24 -0
  114. package/build/services/memflow/workflow/sleepFor.js +51 -0
  115. package/build/services/memflow/workflow/trace.d.ts +14 -0
  116. package/build/services/memflow/workflow/trace.js +33 -0
  117. package/build/services/memflow/workflow/waitFor.d.ts +29 -0
  118. package/build/services/memflow/workflow/waitFor.js +56 -0
  119. package/build/services/meshcall/index.d.ts +194 -0
  120. package/build/services/meshcall/index.js +452 -0
  121. package/build/services/meshcall/schemas/factory.d.ts +9 -0
  122. package/build/services/meshcall/schemas/factory.js +189 -0
  123. package/build/services/meshdata/index.d.ts +795 -0
  124. package/build/services/meshdata/index.js +1235 -0
  125. package/build/services/meshos/index.d.ts +293 -0
  126. package/build/services/meshos/index.js +547 -0
  127. package/build/services/pipe/functions/array.d.ts +17 -0
  128. package/build/services/pipe/functions/array.js +74 -0
  129. package/build/services/pipe/functions/bitwise.d.ts +9 -0
  130. package/build/services/pipe/functions/bitwise.js +24 -0
  131. package/build/services/pipe/functions/conditional.d.ts +13 -0
  132. package/build/services/pipe/functions/conditional.js +36 -0
  133. package/build/services/pipe/functions/cron.d.ts +12 -0
  134. package/build/services/pipe/functions/cron.js +40 -0
  135. package/build/services/pipe/functions/date.d.ts +58 -0
  136. package/build/services/pipe/functions/date.js +171 -0
  137. package/build/services/pipe/functions/index.d.ts +29 -0
  138. package/build/services/pipe/functions/index.js +30 -0
  139. package/build/services/pipe/functions/json.d.ts +5 -0
  140. package/build/services/pipe/functions/json.js +12 -0
  141. package/build/services/pipe/functions/logical.d.ts +5 -0
  142. package/build/services/pipe/functions/logical.js +12 -0
  143. package/build/services/pipe/functions/math.d.ts +42 -0
  144. package/build/services/pipe/functions/math.js +184 -0
  145. package/build/services/pipe/functions/number.d.ts +21 -0
  146. package/build/services/pipe/functions/number.js +60 -0
  147. package/build/services/pipe/functions/object.d.ts +25 -0
  148. package/build/services/pipe/functions/object.js +81 -0
  149. package/build/services/pipe/functions/string.d.ts +23 -0
  150. package/build/services/pipe/functions/string.js +69 -0
  151. package/build/services/pipe/functions/symbol.d.ts +12 -0
  152. package/build/services/pipe/functions/symbol.js +33 -0
  153. package/build/services/pipe/functions/unary.d.ts +7 -0
  154. package/build/services/pipe/functions/unary.js +18 -0
  155. package/build/services/pipe/index.d.ts +48 -0
  156. package/build/services/pipe/index.js +242 -0
  157. package/build/services/quorum/index.d.ts +90 -0
  158. package/build/services/quorum/index.js +263 -0
  159. package/build/services/reporter/index.d.ts +50 -0
  160. package/build/services/reporter/index.js +348 -0
  161. package/build/services/router/config/index.d.ts +11 -0
  162. package/build/services/router/config/index.js +36 -0
  163. package/build/services/router/consumption/index.d.ts +34 -0
  164. package/build/services/router/consumption/index.js +395 -0
  165. package/build/services/router/error-handling/index.d.ts +8 -0
  166. package/build/services/router/error-handling/index.js +98 -0
  167. package/build/services/router/index.d.ts +57 -0
  168. package/build/services/router/index.js +121 -0
  169. package/build/services/router/lifecycle/index.d.ts +27 -0
  170. package/build/services/router/lifecycle/index.js +80 -0
  171. package/build/services/router/telemetry/index.d.ts +11 -0
  172. package/build/services/router/telemetry/index.js +32 -0
  173. package/build/services/router/throttling/index.d.ts +23 -0
  174. package/build/services/router/throttling/index.js +76 -0
  175. package/build/services/search/factory.d.ts +7 -0
  176. package/build/services/search/factory.js +24 -0
  177. package/build/services/search/index.d.ts +23 -0
  178. package/build/services/search/index.js +10 -0
  179. package/build/services/search/providers/postgres/postgres.d.ts +25 -0
  180. package/build/services/search/providers/postgres/postgres.js +149 -0
  181. package/build/services/search/providers/redis/ioredis.d.ts +19 -0
  182. package/build/services/search/providers/redis/ioredis.js +121 -0
  183. package/build/services/search/providers/redis/redis.d.ts +19 -0
  184. package/build/services/search/providers/redis/redis.js +134 -0
  185. package/build/services/serializer/index.d.ts +42 -0
  186. package/build/services/serializer/index.js +282 -0
  187. package/build/services/store/cache.d.ts +67 -0
  188. package/build/services/store/cache.js +128 -0
  189. package/build/services/store/factory.d.ts +8 -0
  190. package/build/services/store/factory.js +24 -0
  191. package/build/services/store/index.d.ts +89 -0
  192. package/build/services/store/index.js +9 -0
  193. package/build/services/store/providers/postgres/kvsql.d.ts +168 -0
  194. package/build/services/store/providers/postgres/kvsql.js +198 -0
  195. package/build/services/store/providers/postgres/kvtables.d.ts +20 -0
  196. package/build/services/store/providers/postgres/kvtables.js +441 -0
  197. package/build/services/store/providers/postgres/kvtransaction.d.ts +36 -0
  198. package/build/services/store/providers/postgres/kvtransaction.js +248 -0
  199. package/build/services/store/providers/postgres/kvtypes/hash.d.ts +60 -0
  200. package/build/services/store/providers/postgres/kvtypes/hash.js +1287 -0
  201. package/build/services/store/providers/postgres/kvtypes/list.d.ts +33 -0
  202. package/build/services/store/providers/postgres/kvtypes/list.js +194 -0
  203. package/build/services/store/providers/postgres/kvtypes/string.d.ts +20 -0
  204. package/build/services/store/providers/postgres/kvtypes/string.js +115 -0
  205. package/build/services/store/providers/postgres/kvtypes/zset.d.ts +41 -0
  206. package/build/services/store/providers/postgres/kvtypes/zset.js +214 -0
  207. package/build/services/store/providers/postgres/postgres.d.ts +178 -0
  208. package/build/services/store/providers/postgres/postgres.js +1244 -0
  209. package/build/services/store/providers/redis/_base.d.ts +137 -0
  210. package/build/services/store/providers/redis/_base.js +980 -0
  211. package/build/services/store/providers/redis/ioredis.d.ts +20 -0
  212. package/build/services/store/providers/redis/ioredis.js +180 -0
  213. package/build/services/store/providers/redis/redis.d.ts +18 -0
  214. package/build/services/store/providers/redis/redis.js +199 -0
  215. package/build/services/store/providers/store-initializable.d.ts +5 -0
  216. package/build/services/store/providers/store-initializable.js +2 -0
  217. package/build/services/stream/factory.d.ts +8 -0
  218. package/build/services/stream/factory.js +37 -0
  219. package/build/services/stream/index.d.ts +69 -0
  220. package/build/services/stream/index.js +11 -0
  221. package/build/services/stream/providers/nats/nats.d.ts +60 -0
  222. package/build/services/stream/providers/nats/nats.js +225 -0
  223. package/build/services/stream/providers/postgres/kvtables.d.ts +3 -0
  224. package/build/services/stream/providers/postgres/kvtables.js +146 -0
  225. package/build/services/stream/providers/postgres/postgres.d.ts +107 -0
  226. package/build/services/stream/providers/postgres/postgres.js +519 -0
  227. package/build/services/stream/providers/redis/ioredis.d.ts +61 -0
  228. package/build/services/stream/providers/redis/ioredis.js +272 -0
  229. package/build/services/stream/providers/redis/redis.d.ts +61 -0
  230. package/build/services/stream/providers/redis/redis.js +305 -0
  231. package/build/services/stream/providers/stream-initializable.d.ts +4 -0
  232. package/build/services/stream/providers/stream-initializable.js +2 -0
  233. package/build/services/sub/factory.d.ts +7 -0
  234. package/build/services/sub/factory.js +29 -0
  235. package/build/services/sub/index.d.ts +22 -0
  236. package/build/services/sub/index.js +10 -0
  237. package/build/services/sub/providers/nats/nats.d.ts +19 -0
  238. package/build/services/sub/providers/nats/nats.js +105 -0
  239. package/build/services/sub/providers/postgres/postgres.d.ts +19 -0
  240. package/build/services/sub/providers/postgres/postgres.js +92 -0
  241. package/build/services/sub/providers/redis/ioredis.d.ts +17 -0
  242. package/build/services/sub/providers/redis/ioredis.js +81 -0
  243. package/build/services/sub/providers/redis/redis.d.ts +17 -0
  244. package/build/services/sub/providers/redis/redis.js +72 -0
  245. package/build/services/task/index.d.ts +48 -0
  246. package/build/services/task/index.js +253 -0
  247. package/build/services/telemetry/index.d.ts +52 -0
  248. package/build/services/telemetry/index.js +306 -0
  249. package/build/services/worker/index.d.ts +77 -0
  250. package/build/services/worker/index.js +197 -0
  251. package/package.json +1 -1
  252. package/.github/ISSUE_TEMPLATE/bug_report.md +0 -38
  253. package/.github/ISSUE_TEMPLATE/feature_request.md +0 -20
  254. package/typedoc.json +0 -47
  255. package/types/activity.ts +0 -268
  256. package/types/app.ts +0 -20
  257. package/types/async.ts +0 -6
  258. package/types/cache.ts +0 -1
  259. package/types/collator.ts +0 -9
  260. package/types/error.ts +0 -56
  261. package/types/exporter.ts +0 -102
  262. package/types/hook.ts +0 -44
  263. package/types/hotmesh.ts +0 -314
  264. package/types/index.ts +0 -306
  265. package/types/job.ts +0 -233
  266. package/types/logger.ts +0 -8
  267. package/types/manifest.ts +0 -70
  268. package/types/map.ts +0 -5
  269. package/types/memflow.ts +0 -645
  270. package/types/meshcall.ts +0 -235
  271. package/types/meshdata.ts +0 -278
  272. package/types/ms.d.ts +0 -7
  273. package/types/nats.ts +0 -270
  274. package/types/pipe.ts +0 -90
  275. package/types/postgres.ts +0 -114
  276. package/types/provider.ts +0 -161
  277. package/types/quorum.ts +0 -167
  278. package/types/redis.ts +0 -404
  279. package/types/serializer.ts +0 -40
  280. package/types/stats.ts +0 -117
  281. package/types/stream.ts +0 -231
  282. package/types/task.ts +0 -7
  283. package/types/telemetry.ts +0 -16
  284. package/types/transition.ts +0 -20
@@ -0,0 +1,395 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.ConsumptionManager = void 0;
4
+ const utils_1 = require("../../../modules/utils");
5
+ const telemetry_1 = require("../telemetry");
6
+ const config_1 = require("../config");
7
+ const stream_1 = require("../../../types/stream");
8
+ const key_1 = require("../../../modules/key");
9
+ class ConsumptionManager {
10
+ constructor(stream, logger, throttleManager, errorHandler, lifecycleManager, reclaimDelay, reclaimCount, appId, role, router) {
11
+ this.errorCount = 0;
12
+ this.counts = {};
13
+ this.stream = stream;
14
+ this.logger = logger;
15
+ this.throttleManager = throttleManager;
16
+ this.errorHandler = errorHandler;
17
+ this.lifecycleManager = lifecycleManager;
18
+ this.reclaimDelay = reclaimDelay;
19
+ this.reclaimCount = reclaimCount;
20
+ this.appId = appId;
21
+ this.role = role;
22
+ this.router = router;
23
+ }
24
+ async createGroup(stream, group) {
25
+ try {
26
+ await this.stream.createConsumerGroup(stream, group);
27
+ }
28
+ catch (err) {
29
+ this.logger.debug('router-stream-group-exists', { stream, group });
30
+ }
31
+ }
32
+ async publishMessage(topic, streamData, transaction) {
33
+ const code = streamData?.code || '200';
34
+ this.counts[code] = (this.counts[code] || 0) + 1;
35
+ const stream = this.stream.mintKey(key_1.KeyType.STREAMS, { topic });
36
+ const responses = await this.stream.publishMessages(stream, [JSON.stringify(streamData)], { transaction });
37
+ return responses[0];
38
+ }
39
+ async consumeMessages(stream, group, consumer, callback) {
40
+ // exit early if readonly
41
+ if (this.lifecycleManager.isReadonly()) {
42
+ this.logger.info(`router-stream-readonly`, { group, consumer, stream });
43
+ return;
44
+ }
45
+ this.logger.info(`router-stream-starting`, { group, consumer, stream });
46
+ await this.lifecycleManager.startConsuming(this.router);
47
+ await this.createGroup(stream, group);
48
+ // Check if notifications are supported
49
+ const features = this.stream.getProviderSpecificFeatures();
50
+ const supportsNotifications = features.supportsNotifications;
51
+ if (supportsNotifications) {
52
+ this.logger.info(`router-stream-using-notifications`, { group, consumer, stream });
53
+ this.lifecycleManager.setIsUsingNotifications(true);
54
+ return this.consumeWithNotifications(stream, group, consumer, callback);
55
+ }
56
+ else {
57
+ this.logger.info(`router-stream-using-polling`, { group, consumer, stream });
58
+ this.lifecycleManager.setIsUsingNotifications(false);
59
+ return this.consumeWithPolling(stream, group, consumer, callback);
60
+ }
61
+ }
62
+ async consumeWithNotifications(stream, group, consumer, callback) {
63
+ let lastCheckedPendingMessagesAt = Date.now();
64
+ // Set up notification-based consumption
65
+ const notificationCallback = async (messages) => {
66
+ if (this.lifecycleManager.isStopped(group, consumer, stream)) {
67
+ return;
68
+ }
69
+ await this.throttleManager.customSleep(); // respect throttle
70
+ if (this.lifecycleManager.isStopped(group, consumer, stream) || this.throttleManager.isPaused()) {
71
+ return;
72
+ }
73
+ // Process messages - use parallel processing for PostgreSQL
74
+ const features = this.stream.getProviderSpecificFeatures();
75
+ const isPostgres = features.supportsNotifications; // Only PostgreSQL supports notifications currently
76
+ if (isPostgres && messages.length > 1) {
77
+ // Parallel processing for PostgreSQL batches
78
+ this.logger.debug('postgres-stream-parallel-processing', {
79
+ streamName: stream,
80
+ groupName: group,
81
+ messageCount: messages.length
82
+ });
83
+ const processingStart = Date.now();
84
+ const processingPromises = messages.map(async (message) => {
85
+ if (this.lifecycleManager.isStopped(group, consumer, stream)) {
86
+ return;
87
+ }
88
+ return this.consumeOne(stream, group, message.id, message.data, callback);
89
+ });
90
+ // Process all messages in parallel
91
+ await Promise.allSettled(processingPromises);
92
+ this.logger.debug('postgres-stream-parallel-processing-complete', {
93
+ streamName: stream,
94
+ groupName: group,
95
+ messageCount: messages.length,
96
+ processingDuration: Date.now() - processingStart
97
+ });
98
+ }
99
+ else {
100
+ // Sequential processing for other providers or single messages
101
+ for (const message of messages) {
102
+ if (this.lifecycleManager.isStopped(group, consumer, stream)) {
103
+ return;
104
+ }
105
+ await this.consumeOne(stream, group, message.id, message.data, callback);
106
+ }
107
+ }
108
+ // Check for pending messages if supported and enough time has passed
109
+ const now = Date.now();
110
+ if (this.stream.getProviderSpecificFeatures().supportsRetry &&
111
+ now - lastCheckedPendingMessagesAt > this.reclaimDelay) {
112
+ lastCheckedPendingMessagesAt = now;
113
+ const pendingMessages = await this.stream.retryMessages(stream, group, {
114
+ consumerName: consumer,
115
+ minIdleTime: this.reclaimDelay,
116
+ limit: config_1.HMSH_XPENDING_COUNT,
117
+ });
118
+ // Process reclaimed messages (also parallel for PostgreSQL)
119
+ if (isPostgres && pendingMessages.length > 1) {
120
+ const reclaimPromises = pendingMessages.map(async (message) => {
121
+ if (this.lifecycleManager.isStopped(group, consumer, stream)) {
122
+ return;
123
+ }
124
+ return this.consumeOne(stream, group, message.id, message.data, callback);
125
+ });
126
+ await Promise.allSettled(reclaimPromises);
127
+ }
128
+ else {
129
+ for (const message of pendingMessages) {
130
+ if (this.lifecycleManager.isStopped(group, consumer, stream)) {
131
+ return;
132
+ }
133
+ await this.consumeOne(stream, group, message.id, message.data, callback);
134
+ }
135
+ }
136
+ }
137
+ };
138
+ try {
139
+ // Start notification-based consumption - this should return immediately after setup
140
+ await this.stream.consumeMessages(stream, group, consumer, {
141
+ enableNotifications: true,
142
+ notificationCallback,
143
+ blockTimeout: config_1.HMSH_BLOCK_TIME_MS,
144
+ });
145
+ // Don't block here - let the worker initialization complete
146
+ // The notification system will handle message processing asynchronously
147
+ }
148
+ catch (error) {
149
+ this.logger.error(`router-stream-notification-error`, {
150
+ error,
151
+ stream,
152
+ group,
153
+ consumer,
154
+ });
155
+ // Fall back to polling if notifications fail
156
+ this.logger.info(`router-stream-fallback-to-polling`, { group, consumer, stream });
157
+ this.lifecycleManager.setIsUsingNotifications(false);
158
+ return this.consumeWithPolling(stream, group, consumer, callback);
159
+ }
160
+ }
161
+ async consumeWithPolling(stream, group, consumer, callback) {
162
+ let lastCheckedPendingMessagesAt = Date.now();
163
+ // Track if we've hit maximum backoff. Initially false.
164
+ // When true, we use the fallback mode: single attempt, no backoff, then sleep 3s if empty.
165
+ if (typeof this.hasReachedMaxBackoff === 'undefined') {
166
+ this.hasReachedMaxBackoff = false;
167
+ }
168
+ const sleepFor = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
169
+ const consume = async () => {
170
+ await this.throttleManager.customSleep(); // always respect the global throttle
171
+ if (this.lifecycleManager.isStopped(group, consumer, stream)) {
172
+ return;
173
+ }
174
+ else if (this.throttleManager.isPaused()) {
175
+ setImmediate(consume.bind(this));
176
+ return;
177
+ }
178
+ // Randomizer that asymptotes at 150% of `HMSH_BLOCK_TIME_MS`
179
+ const streamDuration = config_1.HMSH_BLOCK_TIME_MS + Math.round(config_1.HMSH_BLOCK_TIME_MS * Math.random());
180
+ try {
181
+ let messages = [];
182
+ if (!this.hasReachedMaxBackoff) {
183
+ // Normal mode: try with backoff and finite retries
184
+ const features = this.stream.getProviderSpecificFeatures();
185
+ const isPostgres = features.supportsNotifications; // Only PostgreSQL supports notifications
186
+ const batchSize = isPostgres ? 10 : 1; // Use batch size of 10 for PostgreSQL, 1 for others
187
+ messages = await this.stream.consumeMessages(stream, group, consumer, {
188
+ blockTimeout: streamDuration,
189
+ batchSize,
190
+ enableBackoff: true,
191
+ initialBackoff: config_1.INITIAL_STREAM_BACKOFF,
192
+ maxBackoff: config_1.MAX_STREAM_BACKOFF,
193
+ maxRetries: config_1.MAX_STREAM_RETRIES,
194
+ });
195
+ }
196
+ else {
197
+ // Fallback mode: just try once, no backoff
198
+ const features = this.stream.getProviderSpecificFeatures();
199
+ const isPostgres = features.supportsNotifications; // Only PostgreSQL supports notifications
200
+ const batchSize = isPostgres ? 10 : 1; // Use batch size of 10 for PostgreSQL, 1 for others
201
+ messages = await this.stream.consumeMessages(stream, group, consumer, {
202
+ blockTimeout: streamDuration,
203
+ batchSize,
204
+ enableBackoff: false,
205
+ maxRetries: 1,
206
+ });
207
+ }
208
+ if (this.lifecycleManager.isStopped(group, consumer, stream)) {
209
+ return;
210
+ }
211
+ else if (this.throttleManager.isPaused()) {
212
+ setImmediate(consume.bind(this));
213
+ return;
214
+ }
215
+ if (messages.length > 0) {
216
+ // Reset if we were in fallback mode
217
+ this.hasReachedMaxBackoff = false;
218
+ // Process messages - use parallel processing for PostgreSQL
219
+ const features = this.stream.getProviderSpecificFeatures();
220
+ const isPostgres = features.supportsNotifications; // Only PostgreSQL supports notifications currently
221
+ if (isPostgres && messages.length > 1) {
222
+ // Parallel processing for PostgreSQL batches
223
+ this.logger.debug('postgres-stream-parallel-processing-polling', {
224
+ streamName: stream,
225
+ groupName: group,
226
+ messageCount: messages.length
227
+ });
228
+ const processingStart = Date.now();
229
+ const processingPromises = messages.map(async (message) => {
230
+ if (this.lifecycleManager.isStopped(group, consumer, stream)) {
231
+ return;
232
+ }
233
+ return this.consumeOne(stream, group, message.id, message.data, callback);
234
+ });
235
+ // Process all messages in parallel
236
+ await Promise.allSettled(processingPromises);
237
+ this.logger.debug('postgres-stream-parallel-processing-polling-complete', {
238
+ streamName: stream,
239
+ groupName: group,
240
+ messageCount: messages.length,
241
+ processingDuration: Date.now() - processingStart
242
+ });
243
+ }
244
+ else {
245
+ // Sequential processing for other providers or single messages
246
+ for (const message of messages) {
247
+ if (this.lifecycleManager.isStopped(group, consumer, stream)) {
248
+ return;
249
+ }
250
+ await this.consumeOne(stream, group, message.id, message.data, callback);
251
+ }
252
+ }
253
+ // Check for pending messages if supported and enough time has passed
254
+ const now = Date.now();
255
+ if (this.stream.getProviderSpecificFeatures().supportsRetry &&
256
+ now - lastCheckedPendingMessagesAt > this.reclaimDelay) {
257
+ lastCheckedPendingMessagesAt = now;
258
+ const pendingMessages = await this.stream.retryMessages(stream, group, {
259
+ consumerName: consumer,
260
+ minIdleTime: this.reclaimDelay,
261
+ limit: config_1.HMSH_XPENDING_COUNT,
262
+ });
263
+ // Process reclaimed messages (also parallel for PostgreSQL)
264
+ if (isPostgres && pendingMessages.length > 1) {
265
+ const reclaimPromises = pendingMessages.map(async (message) => {
266
+ if (this.lifecycleManager.isStopped(group, consumer, stream)) {
267
+ return;
268
+ }
269
+ return this.consumeOne(stream, group, message.id, message.data, callback);
270
+ });
271
+ await Promise.allSettled(reclaimPromises);
272
+ }
273
+ else {
274
+ for (const message of pendingMessages) {
275
+ if (this.lifecycleManager.isStopped(group, consumer, stream)) {
276
+ return;
277
+ }
278
+ await this.consumeOne(stream, group, message.id, message.data, callback);
279
+ }
280
+ }
281
+ }
282
+ // If we got messages, just continue as normal
283
+ setImmediate(consume.bind(this));
284
+ }
285
+ else {
286
+ // No messages found
287
+ if (!this.hasReachedMaxBackoff) {
288
+ // We were in normal mode and got no messages after maxRetries
289
+ // Switch to fallback mode
290
+ this.hasReachedMaxBackoff = true;
291
+ }
292
+ else {
293
+ // We are already in fallback mode, still no messages
294
+ // Sleep for MAX_STREAM_BACKOFF ms before trying again
295
+ await sleepFor(config_1.MAX_STREAM_BACKOFF);
296
+ }
297
+ // Try again after sleeping
298
+ setImmediate(consume.bind(this));
299
+ }
300
+ }
301
+ catch (error) {
302
+ if (this.lifecycleManager.getShouldConsume() && process.env.NODE_ENV !== 'test') {
303
+ this.logger.error(`router-stream-error`, {
304
+ error,
305
+ stream,
306
+ group,
307
+ consumer,
308
+ });
309
+ this.errorCount++;
310
+ const timeout = Math.min(config_1.HMSH_GRADUATED_INTERVAL_MS * 2 ** this.errorCount, config_1.HMSH_MAX_TIMEOUT_MS);
311
+ setTimeout(consume.bind(this), timeout);
312
+ }
313
+ }
314
+ };
315
+ consume.call(this);
316
+ }
317
+ async consumeOne(stream, group, id, input, callback) {
318
+ this.logger.debug(`stream-read-one`, { group, stream, id });
319
+ let output;
320
+ const telemetry = new telemetry_1.RouterTelemetry(this.appId);
321
+ try {
322
+ telemetry.startStreamSpan(input, this.role);
323
+ output = await this.execStreamLeg(input, stream, id, callback.bind(this));
324
+ telemetry.setStreamErrorFromOutput(output);
325
+ this.errorCount = 0;
326
+ }
327
+ catch (err) {
328
+ this.logger.error(`stream-read-one-error`, { group, stream, id, err });
329
+ telemetry.setStreamErrorFromException(err);
330
+ }
331
+ const messageId = await this.publishResponse(input, output);
332
+ telemetry.setStreamAttributes({ 'app.worker.mid': messageId });
333
+ await this.ackAndDelete(stream, group, id);
334
+ telemetry.endStreamSpan();
335
+ this.logger.debug(`stream-read-one-end`, { group, stream, id });
336
+ }
337
+ async execStreamLeg(input, stream, id, callback) {
338
+ let output;
339
+ try {
340
+ output = await callback(input);
341
+ }
342
+ catch (error) {
343
+ this.logger.error(`stream-call-function-error`, {
344
+ error,
345
+ input: input,
346
+ stack: error.stack,
347
+ message: error.message,
348
+ name: error.name,
349
+ stream,
350
+ id,
351
+ });
352
+ output = this.errorHandler.structureUnhandledError(input, error);
353
+ }
354
+ return output;
355
+ }
356
+ async ackAndDelete(stream, group, id) {
357
+ await this.stream.ackAndDelete(stream, group, [id]);
358
+ }
359
+ // Add batch acknowledgment method for PostgreSQL optimization
360
+ async ackAndDeleteBatch(stream, group, ids) {
361
+ if (ids.length === 0)
362
+ return;
363
+ const features = this.stream.getProviderSpecificFeatures();
364
+ const isPostgres = features.supportsNotifications; // Only PostgreSQL supports notifications
365
+ if (isPostgres && ids.length > 1) {
366
+ // Batch acknowledgment for PostgreSQL
367
+ await this.stream.ackAndDelete(stream, group, ids);
368
+ }
369
+ else {
370
+ // Individual acknowledgments for other providers
371
+ for (const id of ids) {
372
+ await this.stream.ackAndDelete(stream, group, [id]);
373
+ }
374
+ }
375
+ }
376
+ async publishResponse(input, output) {
377
+ if (output && typeof output === 'object') {
378
+ if (output.status === 'error') {
379
+ return await this.errorHandler.handleRetry(input, output, this.publishMessage.bind(this));
380
+ }
381
+ else if (typeof output.metadata !== 'object') {
382
+ output.metadata = { ...input.metadata, guid: (0, utils_1.guid)() };
383
+ }
384
+ else {
385
+ output.metadata.guid = (0, utils_1.guid)();
386
+ }
387
+ output.type = stream_1.StreamDataType.RESPONSE;
388
+ return (await this.publishMessage(null, output));
389
+ }
390
+ }
391
+ isStreamMessage(result) {
392
+ return Array.isArray(result) && Array.isArray(result[0]);
393
+ }
394
+ }
395
+ exports.ConsumptionManager = ConsumptionManager;
@@ -0,0 +1,8 @@
1
+ import { StreamData, StreamDataResponse } from '../../../types/stream';
2
+ export declare class ErrorHandler {
3
+ shouldRetry(input: StreamData, output: StreamDataResponse): [boolean, number];
4
+ structureUnhandledError(input: StreamData, err: Error): StreamDataResponse;
5
+ structureUnacknowledgedError(input: StreamData): StreamDataResponse;
6
+ structureError(input: StreamData, output: StreamDataResponse): StreamDataResponse;
7
+ handleRetry(input: StreamData, output: StreamDataResponse, publishMessage: (topic: string, streamData: StreamData | StreamDataResponse) => Promise<string>): Promise<string>;
8
+ }
@@ -0,0 +1,98 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.ErrorHandler = void 0;
4
+ const utils_1 = require("../../../modules/utils");
5
+ const config_1 = require("../config");
6
+ const stream_1 = require("../../../types/stream");
7
+ class ErrorHandler {
8
+ shouldRetry(input, output) {
9
+ //const isUnhandledEngineError = output.code === 500;
10
+ const policies = input.policies?.retry;
11
+ const errorCode = output.code.toString();
12
+ const policy = policies?.[errorCode];
13
+ const maxRetries = policy?.[0];
14
+ const tryCount = Math.min(input.metadata.try || 0, config_1.HMSH_MAX_RETRIES);
15
+ //only possible values for maxRetries are 1, 2, 3
16
+ //only possible values for tryCount are 0, 1, 2
17
+ if (maxRetries > tryCount) {
18
+ // 10ms, 100ms, or 1000ms delays between system retries
19
+ return [true, Math.pow(10, tryCount + 1)];
20
+ }
21
+ return [false, 0];
22
+ }
23
+ structureUnhandledError(input, err) {
24
+ const error = {};
25
+ if (typeof err.message === 'string') {
26
+ error.message = err.message;
27
+ }
28
+ else {
29
+ error.message = config_1.HMSH_STATUS_UNKNOWN;
30
+ }
31
+ if (typeof err.stack === 'string') {
32
+ error.stack = err.stack;
33
+ }
34
+ if (typeof err.name === 'string') {
35
+ error.name = err.name;
36
+ }
37
+ return {
38
+ status: 'error',
39
+ code: config_1.HMSH_CODE_UNKNOWN,
40
+ metadata: { ...input.metadata, guid: (0, utils_1.guid)() },
41
+ data: error,
42
+ };
43
+ }
44
+ structureUnacknowledgedError(input) {
45
+ const message = 'stream message max delivery count exceeded';
46
+ const code = config_1.HMSH_CODE_UNACKED;
47
+ const data = { message, code };
48
+ const output = {
49
+ metadata: { ...input.metadata, guid: (0, utils_1.guid)() },
50
+ status: stream_1.StreamStatus.ERROR,
51
+ code,
52
+ data,
53
+ };
54
+ //send unacknowleded errors to the engine (it has no topic)
55
+ delete output.metadata.topic;
56
+ return output;
57
+ }
58
+ structureError(input, output) {
59
+ const message = output.data?.message
60
+ ? output.data?.message.toString()
61
+ : config_1.HMSH_STATUS_UNKNOWN;
62
+ const statusCode = output.code || output.data?.code;
63
+ const code = isNaN(statusCode)
64
+ ? config_1.HMSH_CODE_UNKNOWN
65
+ : parseInt(statusCode.toString());
66
+ const stack = output.data?.stack
67
+ ? output.data?.stack.toString()
68
+ : undefined;
69
+ const data = { message, code, stack };
70
+ if (typeof output.data?.error === 'object') {
71
+ data.error = { ...output.data.error };
72
+ }
73
+ return {
74
+ status: stream_1.StreamStatus.ERROR,
75
+ code,
76
+ stack,
77
+ metadata: { ...input.metadata, guid: (0, utils_1.guid)() },
78
+ data,
79
+ };
80
+ }
81
+ async handleRetry(input, output, publishMessage) {
82
+ const [shouldRetry, timeout] = this.shouldRetry(input, output);
83
+ if (shouldRetry) {
84
+ await (0, utils_1.sleepFor)(timeout);
85
+ return await publishMessage(input.metadata.topic, {
86
+ data: input.data,
87
+ //note: retain guid (this is a retry attempt)
88
+ metadata: { ...input.metadata, try: (input.metadata.try || 0) + 1 },
89
+ policies: input.policies,
90
+ });
91
+ }
92
+ else {
93
+ const structuredError = this.structureError(input, output);
94
+ return await publishMessage(null, structuredError);
95
+ }
96
+ }
97
+ }
98
+ exports.ErrorHandler = ErrorHandler;
@@ -0,0 +1,57 @@
1
+ /// <reference types="node" />
2
+ import { ILogger } from '../logger';
3
+ import { StreamService } from '../stream';
4
+ import { RouterConfig, StreamData, StreamDataResponse, StreamRole } from '../../types/stream';
5
+ import { ProviderClient, ProviderTransaction } from '../../types/provider';
6
+ declare class Router<S extends StreamService<ProviderClient, ProviderTransaction>> {
7
+ appId: string;
8
+ guid: string;
9
+ role: StreamRole;
10
+ topic: string | undefined;
11
+ stream: S;
12
+ reclaimDelay: number;
13
+ reclaimCount: number;
14
+ logger: ILogger;
15
+ readonly: boolean;
16
+ errorCount: number;
17
+ counts: {
18
+ [key: string]: number;
19
+ };
20
+ hasReachedMaxBackoff: boolean | undefined;
21
+ currentTimerId: NodeJS.Timeout | null;
22
+ sleepPromiseResolve: (() => void) | null;
23
+ innerPromiseResolve: (() => void) | null;
24
+ isSleeping: boolean;
25
+ sleepTimout: NodeJS.Timeout | null;
26
+ private isUsingNotifications;
27
+ private throttleManager;
28
+ private errorHandler;
29
+ private lifecycleManager;
30
+ private consumptionManager;
31
+ constructor(config: RouterConfig, stream: S, logger: ILogger);
32
+ get throttle(): number;
33
+ get shouldConsume(): boolean;
34
+ set shouldConsume(value: boolean);
35
+ private resetThrottleState;
36
+ createGroup(stream: string, group: string): Promise<void>;
37
+ publishMessage(topic: string, streamData: StreamData | StreamDataResponse, transaction?: ProviderTransaction): Promise<string | ProviderTransaction>;
38
+ customSleep(): Promise<void>;
39
+ consumeMessages(stream: string, group: string, consumer: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
40
+ isStreamMessage(result: any): boolean;
41
+ isPaused(): boolean;
42
+ isStopped(group: string, consumer: string, stream: string): boolean;
43
+ consumeOne(stream: string, group: string, id: string, input: StreamData, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
44
+ execStreamLeg(input: StreamData, stream: string, id: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<StreamDataResponse>;
45
+ ackAndDelete(stream: string, group: string, id: string): Promise<void>;
46
+ publishResponse(input: StreamData, output: StreamDataResponse | void): Promise<string>;
47
+ shouldRetry(input: StreamData, output: StreamDataResponse): [boolean, number];
48
+ structureUnhandledError(input: StreamData, err: Error): StreamDataResponse;
49
+ structureUnacknowledgedError(input: StreamData): StreamDataResponse;
50
+ structureError(input: StreamData, output: StreamDataResponse): StreamDataResponse;
51
+ static stopConsuming(): Promise<void>;
52
+ stopConsuming(): Promise<void>;
53
+ cancelThrottle(): void;
54
+ setThrottle(delayInMillis: number): void;
55
+ static get instances(): Set<Router<any>>;
56
+ }
57
+ export { Router };
@@ -0,0 +1,121 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Router = void 0;
4
+ // Import the new submodules
5
+ const config_1 = require("./config");
6
+ const throttling_1 = require("./throttling");
7
+ const error_handling_1 = require("./error-handling");
8
+ const lifecycle_1 = require("./lifecycle");
9
+ const consumption_1 = require("./consumption");
10
+ class Router {
11
+ constructor(config, stream, logger) {
12
+ // Legacy properties for backward compatibility
13
+ this.errorCount = 0;
14
+ this.counts = {};
15
+ this.currentTimerId = null;
16
+ this.sleepPromiseResolve = null;
17
+ this.innerPromiseResolve = null;
18
+ this.isSleeping = false;
19
+ this.sleepTimout = null;
20
+ this.isUsingNotifications = false;
21
+ // Apply configuration defaults
22
+ const enhancedConfig = config_1.RouterConfigManager.setDefaults(config);
23
+ this.appId = enhancedConfig.appId;
24
+ this.guid = enhancedConfig.guid;
25
+ this.role = enhancedConfig.role;
26
+ this.topic = enhancedConfig.topic;
27
+ this.stream = stream;
28
+ this.reclaimDelay = enhancedConfig.reclaimDelay;
29
+ this.reclaimCount = enhancedConfig.reclaimCount;
30
+ this.logger = logger;
31
+ this.readonly = enhancedConfig.readonly;
32
+ // Initialize submodule managers
33
+ this.throttleManager = new throttling_1.ThrottleManager(enhancedConfig.throttle);
34
+ this.errorHandler = new error_handling_1.ErrorHandler();
35
+ this.lifecycleManager = new lifecycle_1.LifecycleManager(this.readonly, this.topic, this.logger, this.stream);
36
+ this.consumptionManager = new consumption_1.ConsumptionManager(this.stream, this.logger, this.throttleManager, this.errorHandler, this.lifecycleManager, this.reclaimDelay, this.reclaimCount, this.appId, this.role, this);
37
+ this.resetThrottleState();
38
+ }
39
+ // Legacy compatibility methods
40
+ get throttle() {
41
+ return this.throttleManager.getThrottle();
42
+ }
43
+ get shouldConsume() {
44
+ return this.lifecycleManager.getShouldConsume();
45
+ }
46
+ set shouldConsume(value) {
47
+ this.lifecycleManager.setShouldConsume(value);
48
+ }
49
+ resetThrottleState() {
50
+ this.sleepPromiseResolve = null;
51
+ this.innerPromiseResolve = null;
52
+ this.isSleeping = false;
53
+ this.sleepTimout = null;
54
+ }
55
+ // Delegated methods to submodules
56
+ async createGroup(stream, group) {
57
+ return this.consumptionManager.createGroup(stream, group);
58
+ }
59
+ async publishMessage(topic, streamData, transaction) {
60
+ return this.consumptionManager.publishMessage(topic, streamData, transaction);
61
+ }
62
+ async customSleep() {
63
+ return this.throttleManager.customSleep();
64
+ }
65
+ async consumeMessages(stream, group, consumer, callback) {
66
+ return this.consumptionManager.consumeMessages(stream, group, consumer, callback);
67
+ }
68
+ isStreamMessage(result) {
69
+ return this.consumptionManager.isStreamMessage(result);
70
+ }
71
+ isPaused() {
72
+ return this.throttleManager.isPaused();
73
+ }
74
+ isStopped(group, consumer, stream) {
75
+ return this.lifecycleManager.isStopped(group, consumer, stream);
76
+ }
77
+ async consumeOne(stream, group, id, input, callback) {
78
+ return this.consumptionManager.consumeOne(stream, group, id, input, callback);
79
+ }
80
+ async execStreamLeg(input, stream, id, callback) {
81
+ return this.consumptionManager.execStreamLeg(input, stream, id, callback);
82
+ }
83
+ async ackAndDelete(stream, group, id) {
84
+ return this.consumptionManager.ackAndDelete(stream, group, id);
85
+ }
86
+ async publishResponse(input, output) {
87
+ return this.consumptionManager.publishResponse(input, output);
88
+ }
89
+ shouldRetry(input, output) {
90
+ return this.errorHandler.shouldRetry(input, output);
91
+ }
92
+ structureUnhandledError(input, err) {
93
+ return this.errorHandler.structureUnhandledError(input, err);
94
+ }
95
+ structureUnacknowledgedError(input) {
96
+ return this.errorHandler.structureUnacknowledgedError(input);
97
+ }
98
+ structureError(input, output) {
99
+ return this.errorHandler.structureError(input, output);
100
+ }
101
+ // Static methods for instance management
102
+ static async stopConsuming() {
103
+ return lifecycle_1.InstanceRegistry.stopAll();
104
+ }
105
+ async stopConsuming() {
106
+ return this.lifecycleManager.stopConsuming(this);
107
+ }
108
+ cancelThrottle() {
109
+ this.throttleManager.cancelThrottle();
110
+ this.resetThrottleState();
111
+ }
112
+ setThrottle(delayInMillis) {
113
+ config_1.RouterConfigManager.validateThrottle(delayInMillis);
114
+ this.throttleManager.setThrottle(delayInMillis);
115
+ }
116
+ // Static instances property for backward compatibility
117
+ static get instances() {
118
+ return lifecycle_1.InstanceRegistry.getInstances();
119
+ }
120
+ }
121
+ exports.Router = Router;