@volontariapp/post-processors 1.0.0-snap-856af13 → 1.0.0-snap-fb20f49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. package/dist/constants/config.default.d.ts +27 -0
  2. package/dist/constants/config.default.d.ts.map +1 -0
  3. package/dist/constants/config.default.js +29 -0
  4. package/dist/constants/config.default.js.map +1 -0
  5. package/dist/constants/index.d.ts +2 -0
  6. package/dist/constants/index.d.ts.map +1 -0
  7. package/dist/constants/index.js +2 -0
  8. package/dist/constants/index.js.map +1 -0
  9. package/dist/core/helpers/index.d.ts +3 -0
  10. package/dist/core/helpers/index.d.ts.map +1 -0
  11. package/dist/core/helpers/index.js +3 -0
  12. package/dist/core/helpers/index.js.map +1 -0
  13. package/dist/core/{redis-stream.helper.d.ts → helpers/redis-stream.helper.d.ts} +2 -2
  14. package/dist/core/helpers/redis-stream.helper.d.ts.map +1 -0
  15. package/dist/core/helpers/redis-stream.helper.js.map +1 -0
  16. package/dist/core/helpers/retry.helper.d.ts +24 -0
  17. package/dist/core/helpers/retry.helper.d.ts.map +1 -0
  18. package/dist/core/helpers/retry.helper.js +104 -0
  19. package/dist/core/helpers/retry.helper.js.map +1 -0
  20. package/dist/core/index.d.ts +7 -4
  21. package/dist/core/index.d.ts.map +1 -1
  22. package/dist/core/index.js +7 -4
  23. package/dist/core/index.js.map +1 -1
  24. package/dist/core/processors/base.post-processor.d.ts +46 -0
  25. package/dist/core/processors/base.post-processor.d.ts.map +1 -0
  26. package/dist/core/processors/base.post-processor.js +422 -0
  27. package/dist/core/processors/base.post-processor.js.map +1 -0
  28. package/dist/core/{batch.post-processor.d.ts → processors/batch.post-processor.d.ts} +3 -2
  29. package/dist/core/processors/batch.post-processor.d.ts.map +1 -0
  30. package/dist/core/{batch.post-processor.js → processors/batch.post-processor.js} +56 -5
  31. package/dist/core/processors/batch.post-processor.js.map +1 -0
  32. package/dist/core/processors/index.d.ts +4 -0
  33. package/dist/core/processors/index.d.ts.map +1 -0
  34. package/dist/core/processors/index.js +4 -0
  35. package/dist/core/processors/index.js.map +1 -0
  36. package/dist/core/{single.post-processor.d.ts → processors/single.post-processor.d.ts} +3 -1
  37. package/dist/core/processors/single.post-processor.d.ts.map +1 -0
  38. package/dist/core/processors/single.post-processor.js +120 -0
  39. package/dist/core/processors/single.post-processor.js.map +1 -0
  40. package/dist/core/validators/circuit-breaker.d.ts +27 -0
  41. package/dist/core/validators/circuit-breaker.d.ts.map +1 -0
  42. package/dist/core/validators/circuit-breaker.js +83 -0
  43. package/dist/core/validators/circuit-breaker.js.map +1 -0
  44. package/dist/core/validators/index.d.ts +2 -0
  45. package/dist/core/validators/index.d.ts.map +1 -0
  46. package/dist/core/validators/index.js +2 -0
  47. package/dist/core/validators/index.js.map +1 -0
  48. package/dist/core/validators/options-validator.d.ts +5 -0
  49. package/dist/core/validators/options-validator.d.ts.map +1 -0
  50. package/dist/core/validators/options-validator.js +90 -0
  51. package/dist/core/validators/options-validator.js.map +1 -0
  52. package/dist/enums/circuit-breaker-state.enum.d.ts +6 -0
  53. package/dist/enums/circuit-breaker-state.enum.d.ts.map +1 -0
  54. package/dist/enums/circuit-breaker-state.enum.js +7 -0
  55. package/dist/enums/circuit-breaker-state.enum.js.map +1 -0
  56. package/dist/enums/index.d.ts +2 -0
  57. package/dist/enums/index.d.ts.map +1 -0
  58. package/dist/enums/index.js +2 -0
  59. package/dist/enums/index.js.map +1 -0
  60. package/dist/index.d.ts +2 -0
  61. package/dist/index.d.ts.map +1 -1
  62. package/dist/index.js +2 -0
  63. package/dist/index.js.map +1 -1
  64. package/dist/interfaces/index.d.ts +7 -3
  65. package/dist/interfaces/index.d.ts.map +1 -1
  66. package/dist/interfaces/{batch-event-item.interface.d.ts → processors/batch-event-item.interface.d.ts} +1 -1
  67. package/dist/interfaces/processors/batch-event-item.interface.d.ts.map +1 -0
  68. package/dist/interfaces/processors/batch-event-item.interface.js.map +1 -0
  69. package/dist/interfaces/processors/normalized-post-processor-options.interface.d.ts +6 -0
  70. package/dist/interfaces/processors/normalized-post-processor-options.interface.d.ts.map +1 -0
  71. package/dist/interfaces/processors/normalized-post-processor-options.interface.js +2 -0
  72. package/dist/interfaces/processors/normalized-post-processor-options.interface.js.map +1 -0
  73. package/dist/interfaces/processors/pending-message-info.interface.d.ts.map +1 -0
  74. package/dist/interfaces/processors/pending-message-info.interface.js.map +1 -0
  75. package/dist/interfaces/processors/post-processor-options.interface.d.ts +21 -0
  76. package/dist/interfaces/processors/post-processor-options.interface.d.ts.map +1 -0
  77. package/dist/interfaces/processors/post-processor-options.interface.js.map +1 -0
  78. package/dist/interfaces/retry/retry-metadata.interface.d.ts +6 -0
  79. package/dist/interfaces/retry/retry-metadata.interface.d.ts.map +1 -0
  80. package/dist/interfaces/retry/retry-metadata.interface.js +2 -0
  81. package/dist/interfaces/retry/retry-metadata.interface.js.map +1 -0
  82. package/dist/interfaces/retry/retry-options.interface.d.ts +8 -0
  83. package/dist/interfaces/retry/retry-options.interface.d.ts.map +1 -0
  84. package/dist/interfaces/retry/retry-options.interface.js +2 -0
  85. package/dist/interfaces/retry/retry-options.interface.js.map +1 -0
  86. package/dist/interfaces/validators/circuit-breaker-config.interface.d.ts +6 -0
  87. package/dist/interfaces/validators/circuit-breaker-config.interface.d.ts.map +1 -0
  88. package/dist/interfaces/validators/circuit-breaker-config.interface.js +2 -0
  89. package/dist/interfaces/validators/circuit-breaker-config.interface.js.map +1 -0
  90. package/dist/test/data-source.d.ts.map +1 -0
  91. package/dist/test/data-source.js.map +1 -0
  92. package/dist/test/migrations/1776783577425-CreateJobsOutbox.d.ts.map +1 -0
  93. package/dist/test/migrations/1776783577425-CreateJobsOutbox.js.map +1 -0
  94. package/dist/test/migrations/1778328780881-InitialSchemaJobAudit.d.ts.map +1 -0
  95. package/dist/test/migrations/1778328780881-InitialSchemaJobAudit.js.map +1 -0
  96. package/dist/test/specs/batch-post-processor/batch-post-processor.int.spec.js +92 -4
  97. package/dist/test/specs/batch-post-processor/batch-post-processor.int.spec.js.map +1 -1
  98. package/dist/test/specs/batch-post-processor/batch-post-processor.unit.spec.js +174 -7
  99. package/dist/test/specs/batch-post-processor/batch-post-processor.unit.spec.js.map +1 -1
  100. package/dist/test/specs/helpers/redis-stream-helper.int.spec.d.ts +2 -0
  101. package/dist/test/specs/helpers/redis-stream-helper.int.spec.d.ts.map +1 -0
  102. package/dist/test/specs/helpers/redis-stream-helper.int.spec.js +56 -0
  103. package/dist/test/specs/helpers/redis-stream-helper.int.spec.js.map +1 -0
  104. package/dist/test/specs/helpers/redis-stream-helper.unit.spec.d.ts +2 -0
  105. package/dist/test/specs/helpers/redis-stream-helper.unit.spec.d.ts.map +1 -0
  106. package/dist/test/specs/helpers/redis-stream-helper.unit.spec.js +102 -0
  107. package/dist/test/specs/helpers/redis-stream-helper.unit.spec.js.map +1 -0
  108. package/dist/test/specs/helpers/retry-helper.int.spec.d.ts +2 -0
  109. package/dist/test/specs/helpers/retry-helper.int.spec.d.ts.map +1 -0
  110. package/dist/test/specs/helpers/retry-helper.int.spec.js +101 -0
  111. package/dist/test/specs/helpers/retry-helper.int.spec.js.map +1 -0
  112. package/dist/test/specs/helpers/retry-helper.unit.spec.d.ts +2 -0
  113. package/dist/test/specs/helpers/retry-helper.unit.spec.d.ts.map +1 -0
  114. package/dist/test/specs/helpers/retry-helper.unit.spec.js +202 -0
  115. package/dist/test/specs/helpers/retry-helper.unit.spec.js.map +1 -0
  116. package/dist/test/specs/integration/circuit-breaker.integration.spec.d.ts +2 -0
  117. package/dist/test/specs/integration/circuit-breaker.integration.spec.d.ts.map +1 -0
  118. package/dist/test/specs/integration/circuit-breaker.integration.spec.js +112 -0
  119. package/dist/test/specs/integration/circuit-breaker.integration.spec.js.map +1 -0
  120. package/dist/test/specs/single-post-processor/single-post-processor-retry.int.spec.d.ts +2 -0
  121. package/dist/test/specs/single-post-processor/single-post-processor-retry.int.spec.d.ts.map +1 -0
  122. package/dist/test/specs/single-post-processor/single-post-processor-retry.int.spec.js +142 -0
  123. package/dist/test/specs/single-post-processor/single-post-processor-retry.int.spec.js.map +1 -0
  124. package/dist/test/specs/single-post-processor/single-post-processor.int.spec.js +49 -7
  125. package/dist/test/specs/single-post-processor/single-post-processor.int.spec.js.map +1 -1
  126. package/dist/test/specs/single-post-processor/single-post-processor.unit.spec.js +110 -22
  127. package/dist/test/specs/single-post-processor/single-post-processor.unit.spec.js.map +1 -1
  128. package/dist/test/specs/validators/options-validator.unit.spec.d.ts +2 -0
  129. package/dist/test/specs/validators/options-validator.unit.spec.d.ts.map +1 -0
  130. package/dist/test/specs/validators/options-validator.unit.spec.js +290 -0
  131. package/dist/test/specs/validators/options-validator.unit.spec.js.map +1 -0
  132. package/dist/test/utils/classes/test-batch-post-processor.class.d.ts +2 -1
  133. package/dist/test/utils/classes/test-batch-post-processor.class.d.ts.map +1 -1
  134. package/dist/test/utils/classes/test-batch-post-processor.class.js +4 -1
  135. package/dist/test/utils/classes/test-batch-post-processor.class.js.map +1 -1
  136. package/dist/test/utils/classes/test-post-processor.class.d.ts +2 -1
  137. package/dist/test/utils/classes/test-post-processor.class.d.ts.map +1 -1
  138. package/dist/test/utils/classes/test-post-processor.class.js +4 -1
  139. package/dist/test/utils/classes/test-post-processor.class.js.map +1 -1
  140. package/dist/test/utils/helpers/wait.helper.d.ts +2 -0
  141. package/dist/test/utils/helpers/wait.helper.d.ts.map +1 -0
  142. package/dist/test/utils/helpers/wait.helper.js +11 -0
  143. package/dist/test/utils/helpers/wait.helper.js.map +1 -0
  144. package/dist/test/utils/index.d.ts +1 -0
  145. package/dist/test/utils/index.d.ts.map +1 -1
  146. package/dist/test/utils/index.js +1 -0
  147. package/dist/test/utils/index.js.map +1 -1
  148. package/dist/test/utils/mocks/redis-call.mock.d.ts.map +1 -1
  149. package/dist/test/utils/mocks/redis-call.mock.js +9 -0
  150. package/dist/test/utils/mocks/redis-call.mock.js.map +1 -1
  151. package/dist/types/index.d.ts +1 -0
  152. package/dist/types/index.d.ts.map +1 -1
  153. package/dist/types/index.js +1 -1
  154. package/dist/types/index.js.map +1 -1
  155. package/dist/types/parse-event.types.d.ts +13 -0
  156. package/dist/types/parse-event.types.d.ts.map +1 -0
  157. package/dist/types/parse-event.types.js +2 -0
  158. package/dist/types/parse-event.types.js.map +1 -0
  159. package/package.json +1 -1
  160. package/dist/core/base.post-processor.d.ts +0 -27
  161. package/dist/core/base.post-processor.d.ts.map +0 -1
  162. package/dist/core/base.post-processor.js +0 -206
  163. package/dist/core/base.post-processor.js.map +0 -1
  164. package/dist/core/batch.post-processor.d.ts.map +0 -1
  165. package/dist/core/batch.post-processor.js.map +0 -1
  166. package/dist/core/redis-stream.helper.d.ts.map +0 -1
  167. package/dist/core/redis-stream.helper.js.map +0 -1
  168. package/dist/core/single.post-processor.d.ts.map +0 -1
  169. package/dist/core/single.post-processor.js +0 -58
  170. package/dist/core/single.post-processor.js.map +0 -1
  171. package/dist/data-source.d.ts.map +0 -1
  172. package/dist/data-source.js.map +0 -1
  173. package/dist/interfaces/batch-event-item.interface.d.ts.map +0 -1
  174. package/dist/interfaces/batch-event-item.interface.js.map +0 -1
  175. package/dist/interfaces/pending-message-info.interface.d.ts.map +0 -1
  176. package/dist/interfaces/pending-message-info.interface.js.map +0 -1
  177. package/dist/interfaces/post-processor-options.interface.d.ts +0 -11
  178. package/dist/interfaces/post-processor-options.interface.d.ts.map +0 -1
  179. package/dist/interfaces/post-processor-options.interface.js.map +0 -1
  180. package/dist/migrations/1776783577425-CreateJobsOutbox.d.ts.map +0 -1
  181. package/dist/migrations/1776783577425-CreateJobsOutbox.js.map +0 -1
  182. package/dist/migrations/1778328780881-InitialSchemaJobAudit.d.ts.map +0 -1
  183. package/dist/migrations/1778328780881-InitialSchemaJobAudit.js.map +0 -1
  184. /package/dist/core/{redis-stream.helper.js → helpers/redis-stream.helper.js} +0 -0
  185. /package/dist/interfaces/{batch-event-item.interface.js → processors/batch-event-item.interface.js} +0 -0
  186. /package/dist/interfaces/{pending-message-info.interface.d.ts → processors/pending-message-info.interface.d.ts} +0 -0
  187. /package/dist/interfaces/{pending-message-info.interface.js → processors/pending-message-info.interface.js} +0 -0
  188. /package/dist/interfaces/{post-processor-options.interface.js → processors/post-processor-options.interface.js} +0 -0
  189. /package/dist/{data-source.d.ts → test/data-source.d.ts} +0 -0
  190. /package/dist/{data-source.js → test/data-source.js} +0 -0
  191. /package/dist/{migrations → test/migrations}/1776783577425-CreateJobsOutbox.d.ts +0 -0
  192. /package/dist/{migrations → test/migrations}/1776783577425-CreateJobsOutbox.js +0 -0
  193. /package/dist/{migrations → test/migrations}/1778328780881-InitialSchemaJobAudit.d.ts +0 -0
  194. /package/dist/{migrations → test/migrations}/1778328780881-InitialSchemaJobAudit.js +0 -0
@@ -0,0 +1,422 @@
1
+ import { Logger } from '@volontariapp/logger';
2
+ import os from 'node:os';
3
+ import v8 from 'node:v8';
4
+ import { RedisStreamHelper } from '../helpers/redis-stream.helper.js';
5
+ import { RetryHelper } from '../helpers/retry.helper.js';
6
+ import { DEFAULT_POST_PROCESSOR_CONFIG } from '../../constants/index.js';
7
+ import { OptionsValidator } from '../validators/options-validator.js';
8
+ import { CircuitBreaker } from '../validators/circuit-breaker.js';
9
+ export class BasePostProcessor {
10
+ redis;
11
+ logger;
12
+ options;
13
+ retryHelper;
14
+ circuitBreaker;
15
+ isRunning = false;
16
+ readPending = true;
17
+ claimTimeout = null;
18
+ retryLoopTimeout = null;
19
+ dlqSyncTimeout = null;
20
+ loopPromise = null;
21
+ currentBatchSize;
22
+ sigtermHandler = () => {
23
+ void this.handleShutdown('SIGTERM');
24
+ };
25
+ sigintHandler = () => {
26
+ void this.handleShutdown('SIGINT');
27
+ };
28
+ constructor(redis, options) {
29
+ this.redis = redis;
30
+ if (typeof redis.call !== 'function') {
31
+ throw new Error('Invalid Redis instance: must support .call command execution');
32
+ }
33
+ this.logger = new Logger({ context: this.constructor.name });
34
+ const host = os.hostname();
35
+ this.options = {
36
+ ...DEFAULT_POST_PROCESSOR_CONFIG,
37
+ streamName: options.streamName,
38
+ groupName: options.groupName,
39
+ batchSize: options.batchSize ?? DEFAULT_POST_PROCESSOR_CONFIG.batchSize,
40
+ blockMs: options.blockMs ?? DEFAULT_POST_PROCESSOR_CONFIG.blockMs,
41
+ claimIntervalMs: options.claimIntervalMs ?? DEFAULT_POST_PROCESSOR_CONFIG.claimIntervalMs,
42
+ claimMinIdleTimeMs: options.claimMinIdleTimeMs ?? DEFAULT_POST_PROCESSOR_CONFIG.claimMinIdleTimeMs,
43
+ idempotencyTtlSeconds: options.idempotencyTtlSeconds ?? DEFAULT_POST_PROCESSOR_CONFIG.idempotencyTtlSeconds,
44
+ circuitBreaker: options.circuitBreaker ?? DEFAULT_POST_PROCESSOR_CONFIG.circuitBreaker,
45
+ consumerName: options.consumerName ?? `${host}-${this.constructor.name}`,
46
+ retry: RetryHelper.normalizeRetryOptions(options.retry ?? DEFAULT_POST_PROCESSOR_CONFIG.retry),
47
+ dynamicBatching: {
48
+ ...DEFAULT_POST_PROCESSOR_CONFIG.dynamicBatching,
49
+ ...options.dynamicBatching,
50
+ },
51
+ };
52
+ OptionsValidator.validate(this.options);
53
+ this.retryHelper = new RetryHelper(this.options.retry);
54
+ this.circuitBreaker = new CircuitBreaker(this.options.circuitBreaker);
55
+ this.currentBatchSize = this.options.batchSize;
56
+ }
57
+ async start() {
58
+ if (this.isRunning) {
59
+ this.logger.warn('Post-processor is already running');
60
+ return;
61
+ }
62
+ this.isRunning = true;
63
+ this.readPending = true;
64
+ this.logger.info('Starting post-processor', {
65
+ streamName: this.options.streamName,
66
+ groupName: this.options.groupName,
67
+ consumerName: this.options.consumerName,
68
+ });
69
+ try {
70
+ await this.ensureConsumerGroup();
71
+ }
72
+ catch (err) {
73
+ const error = err instanceof Error ? err : new Error(String(err));
74
+ this.logger.error('Failed to initialize consumer group', { error });
75
+ this.isRunning = false;
76
+ throw error;
77
+ }
78
+ this.setupSignalHandlers();
79
+ this.loopPromise = this.runLoop().catch((err) => {
80
+ const error = err instanceof Error ? err : new Error(String(err));
81
+ this.logger.error('Post-processor loop crashed', { error });
82
+ this.isRunning = false;
83
+ });
84
+ this.startClaimLoop();
85
+ this.startRetryLoop();
86
+ this.startDlqSyncLoop();
87
+ }
88
+ getCurrentBatchSize() {
89
+ return this.currentBatchSize;
90
+ }
91
+ async stop() {
92
+ if (!this.isRunning) {
93
+ return;
94
+ }
95
+ this.logger.info('Stopping post-processor...');
96
+ this.isRunning = false;
97
+ this.cleanupSignalHandlers();
98
+ if (this.claimTimeout) {
99
+ clearTimeout(this.claimTimeout);
100
+ this.claimTimeout = null;
101
+ }
102
+ if (this.retryLoopTimeout) {
103
+ clearTimeout(this.retryLoopTimeout);
104
+ this.retryLoopTimeout = null;
105
+ }
106
+ if (this.dlqSyncTimeout) {
107
+ clearTimeout(this.dlqSyncTimeout);
108
+ this.dlqSyncTimeout = null;
109
+ }
110
+ if (this.loopPromise) {
111
+ try {
112
+ await this.loopPromise;
113
+ }
114
+ catch (err) {
115
+ this.logger.error('Failed to stop post-processor', { error: err });
116
+ }
117
+ this.loopPromise = null;
118
+ }
119
+ this.logger.info('Post-processor stopped');
120
+ }
121
+ setupSignalHandlers() {
122
+ process.on('SIGTERM', this.sigtermHandler);
123
+ process.on('SIGINT', this.sigintHandler);
124
+ }
125
+ cleanupSignalHandlers() {
126
+ process.off('SIGTERM', this.sigtermHandler);
127
+ process.off('SIGINT', this.sigintHandler);
128
+ }
129
+ async handleShutdown(signal) {
130
+ this.logger.info(`Received ${signal}, initiating graceful shutdown...`);
131
+ try {
132
+ await this.stop();
133
+ }
134
+ catch (err) {
135
+ this.logger.error('Error during graceful shutdown', { error: err });
136
+ }
137
+ }
138
+ shouldProcess(_eventType) {
139
+ return true;
140
+ }
141
+ async acknowledge(messageId) {
142
+ try {
143
+ await this.redis.call('XACK', this.options.streamName, this.options.groupName, messageId);
144
+ this.logger.info('Acknowledged message', { messageId });
145
+ }
146
+ catch (err) {
147
+ const error = err instanceof Error ? err : new Error(String(err));
148
+ this.logger.error('Failed to acknowledge message', { messageId, error });
149
+ }
150
+ }
151
+ async ensureConsumerGroup() {
152
+ try {
153
+ await this.redis.call('XGROUP', 'CREATE', this.options.streamName, this.options.groupName, '0', 'MKSTREAM');
154
+ this.logger.info('Consumer group created', {
155
+ streamName: this.options.streamName,
156
+ groupName: this.options.groupName,
157
+ });
158
+ }
159
+ catch (err) {
160
+ const error = err instanceof Error ? err : new Error(String(err));
161
+ if (error.message.includes('BUSYGROUP')) {
162
+ this.logger.debug('Consumer group already exists');
163
+ }
164
+ else {
165
+ throw error;
166
+ }
167
+ }
168
+ }
169
+ async runLoop() {
170
+ while (this.isRunning) {
171
+ try {
172
+ if (!this.circuitBreaker.isAllowed()) {
173
+ this.logger.warn('Circuit breaker is OPEN. Suspending message processing.');
174
+ await new Promise((resolve) => setTimeout(resolve, 1000));
175
+ continue;
176
+ }
177
+ await this.processNextCycle();
178
+ }
179
+ catch (err) {
180
+ const error = err instanceof Error ? err : new Error(String(err));
181
+ this.logger.error('Error in post-processor consumption cycle', { error });
182
+ await new Promise((resolve) => setTimeout(resolve, 100));
183
+ }
184
+ }
185
+ }
186
+ async processNextCycle() {
187
+ const idToRead = this.readPending ? '0' : '>';
188
+ const args = this.buildXreadgroupArgs(idToRead);
189
+ if (idToRead === '0') {
190
+ this.readPending = false;
191
+ }
192
+ const rawResult = (await this.redis.call('XREADGROUP', ...args));
193
+ if (!rawResult || rawResult.length === 0) {
194
+ this.logger.warn('No raw entries received', {
195
+ idToRead,
196
+ readPending: this.readPending,
197
+ args,
198
+ });
199
+ if (idToRead === '0')
200
+ this.readPending = false;
201
+ return;
202
+ }
203
+ const rawEntries = rawResult[0][1];
204
+ if (rawEntries.length === 0) {
205
+ this.logger.warn('No raw entries received');
206
+ if (idToRead === '0')
207
+ this.readPending = false;
208
+ return;
209
+ }
210
+ const entries = RedisStreamHelper.parseRawEntries(rawEntries);
211
+ this.logger.info('Fetched entries from stream', {
212
+ count: entries.length,
213
+ readPending: this.readPending,
214
+ });
215
+ const startTime = Date.now();
216
+ try {
217
+ await this.processEntries(entries);
218
+ const latency = Date.now() - startTime;
219
+ this.adjustBatchSize(latency);
220
+ }
221
+ catch (err) {
222
+ const latency = Date.now() - startTime;
223
+ this.adjustBatchSize(latency);
224
+ throw err;
225
+ }
226
+ }
227
+ buildXreadgroupArgs(idToRead) {
228
+ const args = [
229
+ 'GROUP',
230
+ this.options.groupName,
231
+ this.options.consumerName,
232
+ 'COUNT',
233
+ this.currentBatchSize,
234
+ ];
235
+ if (idToRead === '>') {
236
+ args.push('BLOCK', this.options.blockMs);
237
+ }
238
+ args.push('STREAMS', this.options.streamName, idToRead);
239
+ return args;
240
+ }
241
+ adjustBatchSize(latencyMs) {
242
+ if (!this.options.dynamicBatching.enabled) {
243
+ return;
244
+ }
245
+ const config = this.options.dynamicBatching;
246
+ const minSize = config.minBatchSize ?? 1;
247
+ const maxSize = config.maxBatchSize ?? this.options.batchSize;
248
+ const targetLatency = config.targetLatencyMs ?? 1000;
249
+ const heapStats = v8.getHeapStatistics();
250
+ const memoryPressure = heapStats.used_heap_size / heapStats.heap_size_limit;
251
+ if (memoryPressure > 0.85) {
252
+ this.currentBatchSize = minSize;
253
+ this.logger.warn('High memory pressure detected, dropping batch size to minimum', {
254
+ memoryPressure,
255
+ currentBatchSize: this.currentBatchSize,
256
+ });
257
+ return;
258
+ }
259
+ const systemLoad = os.loadavg()[0];
260
+ const cpuCount = os.cpus().length;
261
+ const cpuPressure = systemLoad / cpuCount;
262
+ if (cpuPressure > 1.2) {
263
+ this.currentBatchSize = Math.max(minSize, Math.floor(this.currentBatchSize * 0.7));
264
+ this.logger.warn('High CPU load detected, reducing batch size', {
265
+ cpuPressure,
266
+ currentBatchSize: this.currentBatchSize,
267
+ });
268
+ return;
269
+ }
270
+ const previousSize = this.currentBatchSize;
271
+ if (latencyMs > targetLatency) {
272
+ this.currentBatchSize = Math.max(minSize, Math.floor(this.currentBatchSize * 0.8));
273
+ if (this.currentBatchSize !== previousSize) {
274
+ this.logger.info('Reducing batch size due to high latency', {
275
+ latencyMs,
276
+ targetLatency,
277
+ previousSize,
278
+ currentBatchSize: this.currentBatchSize,
279
+ });
280
+ }
281
+ }
282
+ else {
283
+ this.currentBatchSize = Math.min(maxSize, this.currentBatchSize + 1);
284
+ if (this.currentBatchSize !== previousSize) {
285
+ this.logger.debug('Increasing batch size due to low latency', {
286
+ latencyMs,
287
+ targetLatency,
288
+ previousSize,
289
+ currentBatchSize: this.currentBatchSize,
290
+ });
291
+ }
292
+ }
293
+ }
294
+ startClaimLoop() {
295
+ if (!this.isRunning)
296
+ return;
297
+ this.claimTimeout = setTimeout(() => {
298
+ this.claimPendingMessages()
299
+ .catch((err) => {
300
+ const error = err instanceof Error ? err : new Error(String(err));
301
+ this.logger.error('Failed claiming pending messages', { error });
302
+ })
303
+ .finally(() => {
304
+ this.startClaimLoop();
305
+ });
306
+ }, this.options.claimIntervalMs);
307
+ }
308
+ async claimPendingMessages() {
309
+ if (!this.circuitBreaker.isAllowed()) {
310
+ this.logger.warn('Circuit breaker is OPEN. Skipping claim cycle.');
311
+ return;
312
+ }
313
+ this.logger.debug('Scanning for pending messages to claim');
314
+ const pendingMessages = await RedisStreamHelper.getPendingMessages(this.redis, this.options.streamName, this.options.groupName, this.currentBatchSize);
315
+ const claimable = pendingMessages.filter((msg) => msg.consumerName !== this.options.consumerName &&
316
+ msg.idleTimeMs >= this.options.claimMinIdleTimeMs);
317
+ if (claimable.length === 0)
318
+ return;
319
+ this.logger.info('Claiming idle pending messages', {
320
+ count: claimable.length,
321
+ messageIds: claimable.map((c) => c.messageId),
322
+ });
323
+ const claimedCount = await this.claimMessagesList(claimable.map((c) => c.messageId));
324
+ if (claimedCount > 0) {
325
+ this.readPending = true;
326
+ }
327
+ }
328
+ async claimMessagesList(messageIds) {
329
+ let claimedCount = 0;
330
+ for (const id of messageIds) {
331
+ try {
332
+ await RedisStreamHelper.claimMessage(this.redis, this.options.streamName, this.options.groupName, this.options.consumerName, this.options.claimMinIdleTimeMs, id);
333
+ claimedCount++;
334
+ }
335
+ catch (err) {
336
+ const error = err instanceof Error ? err : new Error(String(err));
337
+ this.logger.error('Failed to claim message', { messageId: id, error });
338
+ }
339
+ }
340
+ return claimedCount;
341
+ }
342
+ startRetryLoop() {
343
+ if (!this.isRunning)
344
+ return;
345
+ this.retryLoopTimeout = setTimeout(() => {
346
+ this.processRetryQueue()
347
+ .catch((err) => {
348
+ const error = err instanceof Error ? err : new Error(String(err));
349
+ this.logger.error('Failed processing retry queue', { error });
350
+ })
351
+ .finally(() => {
352
+ this.startRetryLoop();
353
+ });
354
+ }, this.options.claimIntervalMs);
355
+ }
356
+ async processRetryQueue() {
357
+ if (!this.circuitBreaker.isAllowed()) {
358
+ this.logger.warn('Circuit breaker is OPEN. Skipping retry queue processing.');
359
+ return;
360
+ }
361
+ const readyMessages = await this.retryHelper.getReadyForRetry(this.redis, this.options.groupName);
362
+ if (readyMessages.length === 0)
363
+ return;
364
+ this.logger.info('Found ready-to-retry messages', {
365
+ count: readyMessages.length,
366
+ messageIds: readyMessages.slice(0, 10),
367
+ });
368
+ this.readPending = true;
369
+ }
370
+ startDlqSyncLoop() {
371
+ if (!this.isRunning)
372
+ return;
373
+ this.dlqSyncTimeout = setTimeout(() => {
374
+ this.syncDlqRetention()
375
+ .catch((err) => {
376
+ const error = err instanceof Error ? err : new Error(String(err));
377
+ this.logger.error('Failed syncing DLQ retention', { error });
378
+ })
379
+ .finally(() => {
380
+ this.startDlqSyncLoop();
381
+ });
382
+ }, this.options.claimIntervalMs * 10);
383
+ }
384
+ async syncDlqRetention() {
385
+ if (!this.options.retry.enableDlq)
386
+ return;
387
+ const dlqStreamName = this.retryHelper.getDlqStreamName(this.options.streamName);
388
+ const retentionMs = this.options.idempotencyTtlSeconds * 1000;
389
+ const cutoffTime = Date.now() - retentionMs;
390
+ try {
391
+ const entries = (await this.redis.xrange(dlqStreamName, '-', '+'));
392
+ const idsToRemove = entries
393
+ .filter((entry) => {
394
+ const timestamp = Number(entry[0].split('-')[0]);
395
+ return timestamp < cutoffTime;
396
+ })
397
+ .map((entry) => entry[0]);
398
+ if (idsToRemove.length > 0) {
399
+ await this.redis.xdel(dlqStreamName, ...idsToRemove);
400
+ this.logger.info('Cleaned up DLQ entries', { count: idsToRemove.length });
401
+ }
402
+ }
403
+ catch (err) {
404
+ const error = err instanceof Error ? err : new Error(String(err));
405
+ this.logger.error('Failed to clean DLQ', { error });
406
+ }
407
+ }
408
+ async sendMessageToDlq(messageId, originalPayload, error) {
409
+ if (!this.options.retry.enableDlq)
410
+ return;
411
+ try {
412
+ const dlqStreamName = this.retryHelper.getDlqStreamName(this.options.streamName);
413
+ await this.retryHelper.sendToDlq(this.redis, dlqStreamName, messageId, originalPayload, error);
414
+ this.logger.error('Message sent to DLQ', { messageId, dlqStreamName, error });
415
+ }
416
+ catch (err) {
417
+ const dlqError = err instanceof Error ? err : new Error(String(err));
418
+ this.logger.error('Failed to send message to DLQ', { messageId, error: dlqError });
419
+ }
420
+ }
421
+ }
422
+ //# sourceMappingURL=base.post-processor.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"base.post-processor.js","sourceRoot":"","sources":["../../../src/core/processors/base.post-processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,MAAM,SAAS,CAAC;AAQzB,OAAO,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,6BAA6B,EAAE,MAAM,0BAA0B,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAElE,MAAM,OAAgB,iBAAiB;IAqBhB;IApBF,MAAM,CAAS;IACf,OAAO,CAAiC;IACxC,WAAW,CAAc;IACzB,cAAc,CAAiB;IAC1C,SAAS,GAAG,KAAK,CAAC;IAClB,WAAW,GAAG,IAAI,CAAC;IACnB,YAAY,GAA0B,IAAI,CAAC;IAC3C,gBAAgB,GAA0B,IAAI,CAAC;IAC/C,cAAc,GAA0B,IAAI,CAAC;IAC7C,WAAW,GAAyB,IAAI,CAAC;IACzC,gBAAgB,CAAS;IAEhB,cAAc,GAAG,GAAG,EAAE;QACrC,KAAK,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC,CAAC;IACe,aAAa,GAAG,GAAG,EAAE;QACpC,KAAK,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC,CAAC;IAEF,YACqB,KAAY,EAC/B,OAA6B;QADV,UAAK,GAAL,KAAK,CAAO;QAG/B,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;QAClF,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;QAE3B,IAAI,CAAC,OAAO,GAAG;YACb,GAAG,6BAA6B;YAChC,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,6BAA6B,CAAC,SAAS;YACvE,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,6BAA6B,CAAC,OAAO;YACjE,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,6BAA6B,CAAC,eAAe;YACzF,kBAAkB,EAChB,OAAO,CAAC,kBAAkB,IAAI,6BAA6B,CAAC,kBAAkB;YAChF,qBAAqB,EACnB,OAAO,CAAC,qBAAqB,IAAI,6BAA6B,CAAC,qBAAqB;YACtF,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,6BAA6B,CAAC,cAAc;YACtF,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,GAAG,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE;YACxE,KAAK,EAAE,WAAW,CAAC,qBAAqB,CACtC,OAAO,CAAC,KAAK,IAAI,6BAA6B,CAAC,KAAK,CACrD;YACD,eAAe,EAAE;gBACf,GAAG,6BAA6B,CAAC,eAAe;gBAChD,GAAG,OAAO,CAAC,eAAe;aAC3B;SACF,CAAC;QACF,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACvD,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,CACtC,IAAI,CAAC,OAAO,CAAC,cAAgD,CAC9D,CAAC;QACF,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;IACjD,CAAC;IAID,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;YACtD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;YAC1C,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;YACnC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS;YACjC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY;SACxC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;YACvB,MAAM,KAAK,CAAC;QACd,CAAC;QAED,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAE3B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YACvD,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC5D,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC1B,CAAC;IAKD,mBAAmB;QACjB,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IAKD,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC/C,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QAEvB,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAE7B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAChC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;QAED,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACpC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC/B,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAClC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,WAAW,CAAC;YACzB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACrE,CAAC;YACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC7C,CAAC;IAEO,mBAAmB;QACzB,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QAC3C,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;IAC3C,CAAC;IAEO,qBAAqB;QAC3B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,MAAc;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,MAAM,mCAAmC,CAAC,CAAC;QACxE,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAWS,aAAa,CAAC,UAAkB;QACxC,OAAO,IAAI,CAAC;IACd,CAAC;IAKS,KAAK,CAAC,WAAW,CAAC,SAAiB;QAC3C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC1F,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAC1D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAKO,KAAK,CAAC,mBAAmB;QAC/B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACnB,QAAQ,EACR,QAAQ,EACR,IAAI,CAAC,OAAO,CAAC,UAAU,EACvB,IAAI,CAAC,OAAO,CAAC,SAAS,EACtB,GAAG,EACH,UAAU,CACX,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE;gBACzC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;gBACnC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS;aAClC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;YACrD,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAKO,KAAK,CAAC,OAAO;QACnB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,EAAE,CAAC;oBACrC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;oBAC5E,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;oBAC1D,SAAS;gBACX,CAAC;gBACD,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAChC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAClE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2CAA2C,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC1E,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;IACH,CAAC;IAKO,KAAK,CAAC,gBAAgB;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QAEhD,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;YACrB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QAC3B,CAAC;QAED,MAAM,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACtC,YAAY,EACZ,GAAG,IAAI,CACR,CAAkC,CAAC;QAEpC,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;gBAC1C,QAAQ;gBACR,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,IAAI;aACL,CAAC,CAAC;YACH,IAAI,QAAQ,KAAK,GAAG;gBAAE,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YAC/C,OAAO;QACT,CAAC;QAED,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;YAC5C,IAAI,QAAQ,KAAK,GAAG;gBAAE,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YAC/C,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QAC9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,EAAE;YAC9C,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,WAAW,EAAE,IAAI,CAAC,WAAW;SAC9B,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YACnC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACvC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAChC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACvC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAC9B,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAKO,mBAAmB,CAAC,QAAgB;QAC1C,MAAM,IAAI,GAAwB;YAChC,OAAO;YACP,IAAI,CAAC,OAAO,CAAC,SAAS;YACtB,IAAI,CAAC,OAAO,CAAC,YAAY;YACzB,OAAO;YACP,IAAI,CAAC,gBAAgB;SACtB,CAAC;QAEF,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;YACrB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,eAAe,CAAC,SAAiB;QACvC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;YAC1C,OAAO;QACT,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC;QAC5C,MAAM,OAAO,GAAG,MAAM,CAAC,YAAY,IAAI,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,CAAC,YAAY,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;QAC9D,MAAM,aAAa,GAAG,MAAM,CAAC,eAAe,IAAI,IAAI,CAAC;QAGrD,MAAM,SAAS,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC;QACzC,MAAM,cAAc,GAAG,SAAS,CAAC,cAAc,GAAG,SAAS,CAAC,eAAe,CAAC;QAE5E,IAAI,cAAc,GAAG,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC;YAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+DAA+D,EAAE;gBAChF,cAAc;gBACd,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;aACxC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAGD,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;QAClC,MAAM,WAAW,GAAG,UAAU,GAAG,QAAQ,CAAC;QAE1C,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;YACtB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,GAAG,GAAG,CAAC,CAAC,CAAC;YACnF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6CAA6C,EAAE;gBAC9D,WAAW;gBACX,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;aACxC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAGD,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC;QAC3C,IAAI,SAAS,GAAG,aAAa,EAAE,CAAC;YAC9B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,GAAG,GAAG,CAAC,CAAC,CAAC;YACnF,IAAI,IAAI,CAAC,gBAAgB,KAAK,YAAY,EAAE,CAAC;gBAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,EAAE;oBAC1D,SAAS;oBACT,aAAa;oBACb,YAAY;oBACZ,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;iBACxC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;YACrE,IAAI,IAAI,CAAC,gBAAgB,KAAK,YAAY,EAAE,CAAC;gBAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE;oBAC5D,SAAS;oBACT,aAAa;oBACb,YAAY;oBACZ,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;iBACxC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAKO,cAAc;QACpB,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAE5B,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;YAClC,IAAI,CAAC,oBAAoB,EAAE;iBACxB,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;gBACtB,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAClE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACnE,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,CAAC,CAAC,CAAC;QACP,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IACnC,CAAC;IAKO,KAAK,CAAC,oBAAoB;QAChC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,EAAE,CAAC;YACrC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;YACnE,OAAO;QACT,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAE5D,MAAM,eAAe,GAAG,MAAM,iBAAiB,CAAC,kBAAkB,CAChE,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CAAC,UAAU,EACvB,IAAI,CAAC,OAAO,CAAC,SAAS,EACtB,IAAI,CAAC,gBAAgB,CACtB,CAAC;QAEF,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CACtC,CAAC,GAAG,EAAE,EAAE,CACN,GAAG,CAAC,YAAY,KAAK,IAAI,CAAC,OAAO,CAAC,YAAY;YAC9C,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,kBAAkB,CACpD,CAAC;QAEF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEnC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE;YACjD,KAAK,EAAE,SAAS,CAAC,MAAM;YACvB,UAAU,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;SAC9C,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;QACrF,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC;IAKO,KAAK,CAAC,iBAAiB,CAAC,UAAoB;QAClD,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,iBAAiB,CAAC,YAAY,CAClC,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CAAC,UAAU,EACvB,IAAI,CAAC,OAAO,CAAC,SAAS,EACtB,IAAI,CAAC,OAAO,CAAC,YAAY,EACzB,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAC/B,EAAE,CACH,CAAC;gBACF,YAAY,EAAE,CAAC;YACjB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAClE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACzE,CAAC;QACH,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IAMO,cAAc;QACpB,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAE5B,IAAI,CAAC,gBAAgB,GAAG,UAAU,CAAC,GAAG,EAAE;YACtC,IAAI,CAAC,iBAAiB,EAAE;iBACrB,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;gBACtB,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAClE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAChE,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,CAAC,CAAC,CAAC;QACP,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IACnC,CAAC;IAKO,KAAK,CAAC,iBAAiB;QAC7B,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,EAAE,CAAC;YACrC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC;YAC9E,OAAO;QACT,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAC3D,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CAAC,SAAS,CACvB,CAAC;QAEF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE;YAChD,KAAK,EAAE,aAAa,CAAC,MAAM;YAC3B,UAAU,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;SACvC,CAAC,CAAC;QAGH,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAMO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAE5B,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC,GAAG,EAAE;YACpC,IAAI,CAAC,gBAAgB,EAAE;iBACpB,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;gBACtB,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAClE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/D,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,CAAC,CAAC,CAAC;QACP,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC;IACxC,CAAC;IAKO,KAAK,CAAC,gBAAgB;QAC5B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS;YAAE,OAAO;QAE1C,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACjF,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAC9D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;QAE5C,IAAI,CAAC;YAEH,MAAM,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,GAAG,EAAE,GAAG,CAAC,CAAyB,CAAC;YAE3F,MAAM,WAAW,GAAG,OAAO;iBACxB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;gBAChB,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjD,OAAO,SAAS,GAAG,UAAU,CAAC;YAChC,CAAC,CAAC;iBACD,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAE5B,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,EAAE,GAAG,WAAW,CAAC,CAAC;gBACrD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAMS,KAAK,CAAC,gBAAgB,CAC9B,SAAiB,EACjB,eAA4B,EAC5B,KAAa;QAEb,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS;YAAE,OAAO;QAE1C,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YACjF,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAC9B,IAAI,CAAC,KAAK,EACV,aAAa,EACb,SAAS,EACT,eAAe,EACf,KAAK,CACN,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC;QAChF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YACrE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrF,CAAC;IACH,CAAC;CACF"}
@@ -1,13 +1,14 @@
1
1
  import { BasePostProcessor } from './base.post-processor.js';
2
2
  import type { EventMessagingType } from '@volontariapp/messaging';
3
- import type { RedisStreamEntry } from '../types/index.js';
4
- import type { BatchEventItem } from '../interfaces/index.js';
3
+ import type { RedisStreamEntry } from '../../types/index.js';
4
+ import type { BatchEventItem } from '../../interfaces/index.js';
5
5
  export declare abstract class BatchPostProcessor<TKey extends EventMessagingType = EventMessagingType> extends BasePostProcessor {
6
6
  protected abstract processEvents(events: BatchEventItem<TKey>[]): Promise<void>;
7
7
  protected processEntries(entries: RedisStreamEntry[]): Promise<void>;
8
8
  private filterAndLockEntry;
9
9
  private parseAndAccumulateEntry;
10
10
  private executeBatchProcessing;
11
+ private handleBatchProcessingFailure;
11
12
  private releaseIdempotencyLocks;
12
13
  }
13
14
  //# sourceMappingURL=batch.post-processor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"batch.post-processor.d.ts","sourceRoot":"","sources":["../../../src/core/processors/batch.post-processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,KAAK,EAAE,kBAAkB,EAA8B,MAAM,yBAAyB,CAAC;AAC9F,OAAO,KAAK,EAAE,gBAAgB,EAA+B,MAAM,sBAAsB,CAAC;AAC1F,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAGhE,8BAAsB,kBAAkB,CACtC,IAAI,SAAS,kBAAkB,GAAG,kBAAkB,CACpD,SAAQ,iBAAiB;IACzB,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;cAKtD,cAAc,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;YAiBrE,kBAAkB;YA0ClB,uBAAuB;YAuCvB,sBAAsB;YA+BtB,4BAA4B;YAgE5B,uBAAuB;CAUtC"}
@@ -1,5 +1,5 @@
1
1
  import { BasePostProcessor } from './base.post-processor.js';
2
- import { RedisStreamHelper } from './redis-stream.helper.js';
2
+ import { RedisStreamHelper } from '../helpers/redis-stream.helper.js';
3
3
  export class BatchPostProcessor extends BasePostProcessor {
4
4
  async processEntries(entries) {
5
5
  const items = [];
@@ -42,6 +42,15 @@ export class BatchPostProcessor extends BasePostProcessor {
42
42
  async parseAndAccumulateEntry(id, rawEvent, useIdempotency, items, acquiredMessageIds) {
43
43
  try {
44
44
  const event = JSON.parse(rawEvent);
45
+ if (typeof event !== 'object') {
46
+ throw new Error('Event payload must be a non-null object');
47
+ }
48
+ if (typeof event.id !== 'string' || event.id.trim() === '') {
49
+ throw new Error('Event payload is missing a valid string id');
50
+ }
51
+ if (typeof event.type !== 'string' || event.type.trim() === '') {
52
+ throw new Error('Event payload is missing a valid string type');
53
+ }
45
54
  items.push({ event, messageId: id });
46
55
  if (useIdempotency)
47
56
  acquiredMessageIds.push(id);
@@ -49,7 +58,7 @@ export class BatchPostProcessor extends BasePostProcessor {
49
58
  }
50
59
  catch (err) {
51
60
  const error = err instanceof Error ? err : new Error(String(err));
52
- this.logger.error('Failed to parse event payload, acknowledging and skipping', {
61
+ this.logger.error('Failed to parse or validate event payload, acknowledging and skipping', {
53
62
  messageId: id,
54
63
  error,
55
64
  });
@@ -60,11 +69,13 @@ export class BatchPostProcessor extends BasePostProcessor {
60
69
  return false;
61
70
  }
62
71
  }
63
- async executeBatchProcessing(items, acquiredMessageIds, useIdempotency) {
72
+ async executeBatchProcessing(items, _acquiredMessageIds, useIdempotency) {
64
73
  try {
65
74
  this.logger.info('Processing batch of events', { count: items.length });
66
75
  await this.processEvents(items);
76
+ this.circuitBreaker.recordSuccess();
67
77
  for (const item of items) {
78
+ await this.retryHelper.clearRetryData(this.redis, this.options.groupName, item.messageId);
68
79
  await this.acknowledge(item.messageId);
69
80
  }
70
81
  this.logger.info('Successfully processed batch of events', { count: items.length });
@@ -72,9 +83,49 @@ export class BatchPostProcessor extends BasePostProcessor {
72
83
  catch (err) {
73
84
  const error = err instanceof Error ? err : new Error(String(err));
74
85
  this.logger.error('Failed to process batch of events from stream', { error });
75
- if (useIdempotency) {
76
- await this.releaseIdempotencyLocks(acquiredMessageIds);
86
+ this.circuitBreaker.recordFailure();
87
+ await this.handleBatchProcessingFailure(items, error, useIdempotency);
88
+ }
89
+ }
90
+ async handleBatchProcessingFailure(items, error, useIdempotency) {
91
+ for (const item of items) {
92
+ try {
93
+ const attemptCount = await this.retryHelper.recordRetry(this.redis, this.options.groupName, item.messageId, error, this.options.idempotencyTtlSeconds);
94
+ if (this.retryHelper.shouldRetry(attemptCount)) {
95
+ await this.retryHelper.enqueueForRetry(this.redis, this.options.groupName, item.messageId, attemptCount);
96
+ const delayMs = this.retryHelper.calculateDelay(attemptCount);
97
+ this.logger.warn('Batch item scheduled for retry', {
98
+ messageId: item.messageId,
99
+ attemptCount,
100
+ delayMs,
101
+ });
102
+ }
103
+ else {
104
+ const payloadFields = {
105
+ success: true,
106
+ type: item.event.type,
107
+ id: item.event.id,
108
+ payload: JSON.stringify(item.event),
109
+ };
110
+ await this.sendMessageToDlq(item.messageId, payloadFields, error.message);
111
+ this.logger.error('Batch item max retries exceeded, sent to DLQ', {
112
+ messageId: item.messageId,
113
+ attemptCount,
114
+ maxRetries: this.options.retry.maxRetries,
115
+ });
116
+ await this.acknowledge(item.messageId);
117
+ }
77
118
  }
119
+ catch (handleErr) {
120
+ const handleError = handleErr instanceof Error ? handleErr : new Error(String(handleErr));
121
+ this.logger.error('Failed to handle batch item retry', {
122
+ messageId: item.messageId,
123
+ error: handleError,
124
+ });
125
+ }
126
+ }
127
+ if (useIdempotency) {
128
+ await this.releaseIdempotencyLocks(items.map((item) => item.messageId));
78
129
  }
79
130
  }
80
131
  async releaseIdempotencyLocks(messageIds) {
@@ -0,0 +1 @@
1
+ {"version":3,"file":"batch.post-processor.js","sourceRoot":"","sources":["../../../src/core/processors/batch.post-processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAI7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAC;AAEtE,MAAM,OAAgB,kBAEpB,SAAQ,iBAAiB;IAMN,KAAK,CAAC,cAAc,CAAC,OAA2B;QACjE,MAAM,KAAK,GAA2B,EAAE,CAAC;QACzC,MAAM,kBAAkB,GAAa,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC;QAE/C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,kBAAkB,CAAC,CAAC;QACvE,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAE/B,MAAM,IAAI,CAAC,sBAAsB,CAAC,KAAK,EAAE,kBAAkB,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;IACxE,CAAC;IAKO,KAAK,CAAC,kBAAkB,CAC9B,KAAuB,EACvB,GAAW,EACX,KAA6B,EAC7B,kBAA4B;QAE5B,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;QAE7B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAClB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kEAAkE,EAAE;gBACnF,SAAS,EAAE,EAAE;aACd,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAC3B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC;YAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+CAA+C,EAAE;gBACjE,SAAS,EAAE,EAAE;gBACb,IAAI,EAAE,MAAM,CAAC,IAAI;aAClB,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAC3B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IACE,GAAG,GAAG,CAAC;YACP,CAAC,CAAC,MAAM,iBAAiB,CAAC,sBAAsB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC,EAC9F,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uEAAuE,EAAE;gBACxF,SAAS,EAAE,EAAE;aACd,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAC3B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC,uBAAuB,CAAC,EAAE,EAAE,MAAM,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,EAAE,KAAK,EAAE,kBAAkB,CAAC,CAAC;IAC5F,CAAC;IAKO,KAAK,CAAC,uBAAuB,CACnC,EAAU,EACV,QAAgB,EAChB,cAAuB,EACvB,KAA6B,EAC7B,kBAA4B;QAE5B,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAqD,CAAC;YACvF,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;YAC7D,CAAC;YACD,IAAI,OAAO,KAAK,CAAC,EAAE,KAAK,QAAQ,IAAI,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBAC3D,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAChE,CAAC;YACD,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBAC/D,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAClE,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;YACrC,IAAI,cAAc;gBAAE,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAChD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uEAAuE,EAAE;gBACzF,SAAS,EAAE,EAAE;gBACb,KAAK;aACN,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAC3B,IAAI,cAAc,EAAE,CAAC;gBACnB,MAAM,iBAAiB,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACxF,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAKO,KAAK,CAAC,sBAAsB,CAClC,KAA6B,EAC7B,mBAA6B,EAC7B,cAAuB;QAEvB,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YACxE,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,CAAC;YAGpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC1F,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACzC,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QACtF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+CAA+C,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAE9E,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,CAAC;YAGpC,MAAM,IAAI,CAAC,4BAA4B,CAAC,KAAK,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IAMO,KAAK,CAAC,4BAA4B,CACxC,KAA6B,EAC7B,KAAY,EACZ,cAAuB;QAEvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CACrD,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CAAC,SAAS,EACtB,IAAI,CAAC,SAAS,EACd,KAAK,EACL,IAAI,CAAC,OAAO,CAAC,qBAAqB,CACnC,CAAC;gBAEF,IAAI,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,YAAY,CAAC,EAAE,CAAC;oBAC/C,MAAM,IAAI,CAAC,WAAW,CAAC,eAAe,CACpC,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CAAC,SAAS,EACtB,IAAI,CAAC,SAAS,EACd,YAAY,CACb,CAAC;oBAEF,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;oBAC9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE;wBACjD,SAAS,EAAE,IAAI,CAAC,SAAS;wBACzB,YAAY;wBACZ,OAAO;qBACR,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,MAAM,aAAa,GAAgB;wBACjC,OAAO,EAAE,IAAI;wBACb,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;wBACrB,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE;wBACjB,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC;qBACpC,CAAC;oBAEF,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;oBAE1E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8CAA8C,EAAE;wBAChE,SAAS,EAAE,IAAI,CAAC,SAAS;wBACzB,YAAY;wBACZ,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU;qBAC1C,CAAC,CAAC;oBAEH,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBACnB,MAAM,WAAW,GAAG,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;gBAC1F,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE;oBACrD,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,KAAK,EAAE,WAAW;iBACnB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAKO,KAAK,CAAC,uBAAuB,CAAC,UAAoB;QACxD,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,iBAAiB,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACxF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAClE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACpF,CAAC;QACH,CAAC;IACH,CAAC;CACF"}
@@ -0,0 +1,4 @@
1
+ export * from './base.post-processor.js';
2
+ export * from './single.post-processor.js';
3
+ export * from './batch.post-processor.js';
4
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/core/processors/index.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAC;AACzC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,2BAA2B,CAAC"}
@@ -0,0 +1,4 @@
1
+ export * from './base.post-processor.js';
2
+ export * from './single.post-processor.js';
3
+ export * from './batch.post-processor.js';
4
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/processors/index.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAC;AACzC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,2BAA2B,CAAC"}
@@ -1,12 +1,14 @@
1
1
  import { BasePostProcessor } from './base.post-processor.js';
2
2
  import type { EventMessagingType, EventRegistry, StreamEvent, EventChangedPayload } from '@volontariapp/messaging';
3
- import type { RedisStreamEntry } from '../types/index.js';
3
+ import type { RedisStreamEntry } from '../../types/index.js';
4
4
  type ExtractPayload<T> = T extends EventChangedPayload<infer P> ? P : T;
5
5
  export declare abstract class SinglePostProcessor<TKey extends EventMessagingType = EventMessagingType> extends BasePostProcessor {
6
6
  protected abstract processEvent(event: StreamEvent<ExtractPayload<EventRegistry[TKey]>>, messageId: string): Promise<void>;
7
7
  protected processEntries(entries: RedisStreamEntry[]): Promise<void>;
8
8
  private processSingleEntry;
9
9
  private executeEventProcessing;
10
+ private handleProcessingFailure;
11
+ private parsePayloadFields;
10
12
  }
11
13
  export {};
12
14
  //# sourceMappingURL=single.post-processor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"single.post-processor.d.ts","sourceRoot":"","sources":["../../../src/core/processors/single.post-processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,KAAK,EACV,kBAAkB,EAClB,aAAa,EACb,WAAW,EACX,mBAAmB,EACpB,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAe,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAG1E,KAAK,cAAc,CAAC,CAAC,IAAI,CAAC,SAAS,mBAAmB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAExE,8BAAsB,mBAAmB,CACvC,IAAI,SAAS,kBAAkB,GAAG,kBAAkB,CACpD,SAAQ,iBAAiB;IAIzB,SAAS,CAAC,QAAQ,CAAC,YAAY,CAC7B,KAAK,EAAE,WAAW,CAAC,cAAc,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EACvD,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC;cAKS,cAAc,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;YASrE,kBAAkB;YAsClB,sBAAsB;YA2CtB,uBAAuB;IA4DrC,OAAO,CAAC,kBAAkB;CAkB3B"}