@agentlensai/server 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. package/dist/cloud/auth/api-key-middleware.d.ts +66 -0
  2. package/dist/cloud/auth/api-key-middleware.d.ts.map +1 -0
  3. package/dist/cloud/auth/api-key-middleware.js +147 -0
  4. package/dist/cloud/auth/api-key-middleware.js.map +1 -0
  5. package/dist/cloud/auth/api-keys.d.ts +90 -0
  6. package/dist/cloud/auth/api-keys.d.ts.map +1 -0
  7. package/dist/cloud/auth/api-keys.js +162 -0
  8. package/dist/cloud/auth/api-keys.js.map +1 -0
  9. package/dist/cloud/auth/audit-log.d.ts +66 -0
  10. package/dist/cloud/auth/audit-log.d.ts.map +1 -0
  11. package/dist/cloud/auth/audit-log.js +92 -0
  12. package/dist/cloud/auth/audit-log.js.map +1 -0
  13. package/dist/cloud/auth/auth-service.d.ts +77 -0
  14. package/dist/cloud/auth/auth-service.d.ts.map +1 -0
  15. package/dist/cloud/auth/auth-service.js +229 -0
  16. package/dist/cloud/auth/auth-service.js.map +1 -0
  17. package/dist/cloud/auth/brute-force.d.ts +36 -0
  18. package/dist/cloud/auth/brute-force.d.ts.map +1 -0
  19. package/dist/cloud/auth/brute-force.js +67 -0
  20. package/dist/cloud/auth/brute-force.js.map +1 -0
  21. package/dist/cloud/auth/index.d.ts +11 -0
  22. package/dist/cloud/auth/index.d.ts.map +1 -0
  23. package/dist/cloud/auth/index.js +11 -0
  24. package/dist/cloud/auth/index.js.map +1 -0
  25. package/dist/cloud/auth/jwt.d.ts +34 -0
  26. package/dist/cloud/auth/jwt.d.ts.map +1 -0
  27. package/dist/cloud/auth/jwt.js +68 -0
  28. package/dist/cloud/auth/jwt.js.map +1 -0
  29. package/dist/cloud/auth/oauth.d.ts +37 -0
  30. package/dist/cloud/auth/oauth.d.ts.map +1 -0
  31. package/dist/cloud/auth/oauth.js +120 -0
  32. package/dist/cloud/auth/oauth.js.map +1 -0
  33. package/dist/cloud/auth/passwords.d.ts +25 -0
  34. package/dist/cloud/auth/passwords.d.ts.map +1 -0
  35. package/dist/cloud/auth/passwords.js +50 -0
  36. package/dist/cloud/auth/passwords.js.map +1 -0
  37. package/dist/cloud/auth/rbac.d.ts +51 -0
  38. package/dist/cloud/auth/rbac.d.ts.map +1 -0
  39. package/dist/cloud/auth/rbac.js +89 -0
  40. package/dist/cloud/auth/rbac.js.map +1 -0
  41. package/dist/cloud/auth/tokens.d.ts +18 -0
  42. package/dist/cloud/auth/tokens.d.ts.map +1 -0
  43. package/dist/cloud/auth/tokens.js +29 -0
  44. package/dist/cloud/auth/tokens.js.map +1 -0
  45. package/dist/cloud/billing/billing-service.d.ts +44 -0
  46. package/dist/cloud/billing/billing-service.d.ts.map +1 -0
  47. package/dist/cloud/billing/billing-service.js +153 -0
  48. package/dist/cloud/billing/billing-service.js.map +1 -0
  49. package/dist/cloud/billing/index.d.ts +11 -0
  50. package/dist/cloud/billing/index.d.ts.map +1 -0
  51. package/dist/cloud/billing/index.js +11 -0
  52. package/dist/cloud/billing/index.js.map +1 -0
  53. package/dist/cloud/billing/invoice-service.d.ts +57 -0
  54. package/dist/cloud/billing/invoice-service.d.ts.map +1 -0
  55. package/dist/cloud/billing/invoice-service.js +123 -0
  56. package/dist/cloud/billing/invoice-service.js.map +1 -0
  57. package/dist/cloud/billing/plan-management.d.ts +46 -0
  58. package/dist/cloud/billing/plan-management.d.ts.map +1 -0
  59. package/dist/cloud/billing/plan-management.js +157 -0
  60. package/dist/cloud/billing/plan-management.js.map +1 -0
  61. package/dist/cloud/billing/quota-enforcement.d.ts +53 -0
  62. package/dist/cloud/billing/quota-enforcement.d.ts.map +1 -0
  63. package/dist/cloud/billing/quota-enforcement.js +143 -0
  64. package/dist/cloud/billing/quota-enforcement.js.map +1 -0
  65. package/dist/cloud/billing/stripe-client.d.ts +142 -0
  66. package/dist/cloud/billing/stripe-client.d.ts.map +1 -0
  67. package/dist/cloud/billing/stripe-client.js +169 -0
  68. package/dist/cloud/billing/stripe-client.js.map +1 -0
  69. package/dist/cloud/billing/trial-service.d.ts +47 -0
  70. package/dist/cloud/billing/trial-service.d.ts.map +1 -0
  71. package/dist/cloud/billing/trial-service.js +104 -0
  72. package/dist/cloud/billing/trial-service.js.map +1 -0
  73. package/dist/cloud/billing/usage-metering.d.ts +83 -0
  74. package/dist/cloud/billing/usage-metering.d.ts.map +1 -0
  75. package/dist/cloud/billing/usage-metering.js +174 -0
  76. package/dist/cloud/billing/usage-metering.js.map +1 -0
  77. package/dist/cloud/ingestion/backpressure.d.ts +107 -0
  78. package/dist/cloud/ingestion/backpressure.d.ts.map +1 -0
  79. package/dist/cloud/ingestion/backpressure.js +134 -0
  80. package/dist/cloud/ingestion/backpressure.js.map +1 -0
  81. package/dist/cloud/ingestion/batch-writer.d.ts +115 -0
  82. package/dist/cloud/ingestion/batch-writer.d.ts.map +1 -0
  83. package/dist/cloud/ingestion/batch-writer.js +319 -0
  84. package/dist/cloud/ingestion/batch-writer.js.map +1 -0
  85. package/dist/cloud/ingestion/dlq-manager.d.ts +116 -0
  86. package/dist/cloud/ingestion/dlq-manager.d.ts.map +1 -0
  87. package/dist/cloud/ingestion/dlq-manager.js +244 -0
  88. package/dist/cloud/ingestion/dlq-manager.js.map +1 -0
  89. package/dist/cloud/ingestion/event-queue.d.ts +105 -0
  90. package/dist/cloud/ingestion/event-queue.d.ts.map +1 -0
  91. package/dist/cloud/ingestion/event-queue.js +185 -0
  92. package/dist/cloud/ingestion/event-queue.js.map +1 -0
  93. package/dist/cloud/ingestion/gateway.d.ts +68 -0
  94. package/dist/cloud/ingestion/gateway.d.ts.map +1 -0
  95. package/dist/cloud/ingestion/gateway.js +198 -0
  96. package/dist/cloud/ingestion/gateway.js.map +1 -0
  97. package/dist/cloud/ingestion/index.d.ts +7 -0
  98. package/dist/cloud/ingestion/index.d.ts.map +1 -0
  99. package/dist/cloud/ingestion/index.js +7 -0
  100. package/dist/cloud/ingestion/index.js.map +1 -0
  101. package/dist/cloud/ingestion/rate-limiter.d.ts +73 -0
  102. package/dist/cloud/ingestion/rate-limiter.d.ts.map +1 -0
  103. package/dist/cloud/ingestion/rate-limiter.js +153 -0
  104. package/dist/cloud/ingestion/rate-limiter.js.map +1 -0
  105. package/dist/cloud/migrate.d.ts +45 -0
  106. package/dist/cloud/migrate.d.ts.map +1 -0
  107. package/dist/cloud/migrate.js +147 -0
  108. package/dist/cloud/migrate.js.map +1 -0
  109. package/dist/cloud/migration/export-import.d.ts +56 -0
  110. package/dist/cloud/migration/export-import.d.ts.map +1 -0
  111. package/dist/cloud/migration/export-import.js +289 -0
  112. package/dist/cloud/migration/export-import.js.map +1 -0
  113. package/dist/cloud/migration/index.d.ts +5 -0
  114. package/dist/cloud/migration/index.d.ts.map +1 -0
  115. package/dist/cloud/migration/index.js +5 -0
  116. package/dist/cloud/migration/index.js.map +1 -0
  117. package/dist/cloud/org-service.d.ts +68 -0
  118. package/dist/cloud/org-service.d.ts.map +1 -0
  119. package/dist/cloud/org-service.js +169 -0
  120. package/dist/cloud/org-service.js.map +1 -0
  121. package/dist/cloud/partition-maintenance.d.ts +29 -0
  122. package/dist/cloud/partition-maintenance.d.ts.map +1 -0
  123. package/dist/cloud/partition-maintenance.js +96 -0
  124. package/dist/cloud/partition-maintenance.js.map +1 -0
  125. package/dist/cloud/retention/index.d.ts +7 -0
  126. package/dist/cloud/retention/index.d.ts.map +1 -0
  127. package/dist/cloud/retention/index.js +7 -0
  128. package/dist/cloud/retention/index.js.map +1 -0
  129. package/dist/cloud/retention/partition-management.d.ts +61 -0
  130. package/dist/cloud/retention/partition-management.d.ts.map +1 -0
  131. package/dist/cloud/retention/partition-management.js +167 -0
  132. package/dist/cloud/retention/partition-management.js.map +1 -0
  133. package/dist/cloud/retention/retention-job.d.ts +70 -0
  134. package/dist/cloud/retention/retention-job.d.ts.map +1 -0
  135. package/dist/cloud/retention/retention-job.js +160 -0
  136. package/dist/cloud/retention/retention-job.js.map +1 -0
  137. package/dist/cloud/retention/retention-policy.d.ts +27 -0
  138. package/dist/cloud/retention/retention-policy.d.ts.map +1 -0
  139. package/dist/cloud/retention/retention-policy.js +36 -0
  140. package/dist/cloud/retention/retention-policy.js.map +1 -0
  141. package/dist/cloud/routes/api-key-routes.d.ts +38 -0
  142. package/dist/cloud/routes/api-key-routes.d.ts.map +1 -0
  143. package/dist/cloud/routes/api-key-routes.js +84 -0
  144. package/dist/cloud/routes/api-key-routes.js.map +1 -0
  145. package/dist/cloud/routes/audit-routes.d.ts +36 -0
  146. package/dist/cloud/routes/audit-routes.d.ts.map +1 -0
  147. package/dist/cloud/routes/audit-routes.js +47 -0
  148. package/dist/cloud/routes/audit-routes.js.map +1 -0
  149. package/dist/cloud/routes/billing-routes.d.ts +51 -0
  150. package/dist/cloud/routes/billing-routes.d.ts.map +1 -0
  151. package/dist/cloud/routes/billing-routes.js +114 -0
  152. package/dist/cloud/routes/billing-routes.js.map +1 -0
  153. package/dist/cloud/routes/onboarding-routes.d.ts +34 -0
  154. package/dist/cloud/routes/onboarding-routes.d.ts.map +1 -0
  155. package/dist/cloud/routes/onboarding-routes.js +58 -0
  156. package/dist/cloud/routes/onboarding-routes.js.map +1 -0
  157. package/dist/cloud/routes/org-routes.d.ts +80 -0
  158. package/dist/cloud/routes/org-routes.d.ts.map +1 -0
  159. package/dist/cloud/routes/org-routes.js +153 -0
  160. package/dist/cloud/routes/org-routes.js.map +1 -0
  161. package/dist/cloud/routes/usage-routes.d.ts +18 -0
  162. package/dist/cloud/routes/usage-routes.d.ts.map +1 -0
  163. package/dist/cloud/routes/usage-routes.js +66 -0
  164. package/dist/cloud/routes/usage-routes.js.map +1 -0
  165. package/dist/cloud/storage/adapter.d.ts +102 -0
  166. package/dist/cloud/storage/adapter.d.ts.map +1 -0
  167. package/dist/cloud/storage/adapter.js +21 -0
  168. package/dist/cloud/storage/adapter.js.map +1 -0
  169. package/dist/cloud/storage/index.d.ts +8 -0
  170. package/dist/cloud/storage/index.d.ts.map +1 -0
  171. package/dist/cloud/storage/index.js +7 -0
  172. package/dist/cloud/storage/index.js.map +1 -0
  173. package/dist/cloud/storage/postgres-adapter.d.ts +34 -0
  174. package/dist/cloud/storage/postgres-adapter.d.ts.map +1 -0
  175. package/dist/cloud/storage/postgres-adapter.js +544 -0
  176. package/dist/cloud/storage/postgres-adapter.js.map +1 -0
  177. package/dist/cloud/storage/sqlite-adapter.d.ts +29 -0
  178. package/dist/cloud/storage/sqlite-adapter.d.ts.map +1 -0
  179. package/dist/cloud/storage/sqlite-adapter.js +176 -0
  180. package/dist/cloud/storage/sqlite-adapter.js.map +1 -0
  181. package/dist/cloud/tenant-pool.d.ts +49 -0
  182. package/dist/cloud/tenant-pool.d.ts.map +1 -0
  183. package/dist/cloud/tenant-pool.js +61 -0
  184. package/dist/cloud/tenant-pool.js.map +1 -0
  185. package/package.json +1 -1
@@ -0,0 +1,319 @@
1
+ /**
2
+ * Batch Writer — Queue Worker (S-3.3)
3
+ *
4
+ * Consumer group workers: XREADGROUP up to 50 messages from the
5
+ * `event_ingestion` stream, enrich (cost calculation, hash chain),
6
+ * batch INSERT into Postgres, XACK on success. After 3 failures → DLQ.
7
+ */
8
+ import { createHash } from 'crypto';
9
+ import { STREAM_NAME, DLQ_STREAM_NAME, CONSUMER_GROUP } from './event-queue.js';
10
+ // ═══════════════════════════════════════════
11
+ // Cost Calculation
12
+ // ═══════════════════════════════════════════
13
+ /** Known model pricing (per 1K tokens) */
14
+ const MODEL_COSTS = {
15
+ 'gpt-4': { input: 0.03, output: 0.06 },
16
+ 'gpt-4-turbo': { input: 0.01, output: 0.03 },
17
+ 'gpt-4o': { input: 0.005, output: 0.015 },
18
+ 'gpt-4o-mini': { input: 0.00015, output: 0.0006 },
19
+ 'gpt-3.5-turbo': { input: 0.0005, output: 0.0015 },
20
+ 'claude-3-opus': { input: 0.015, output: 0.075 },
21
+ 'claude-3-sonnet': { input: 0.003, output: 0.015 },
22
+ 'claude-3-haiku': { input: 0.00025, output: 0.00125 },
23
+ 'claude-3.5-sonnet': { input: 0.003, output: 0.015 },
24
+ 'claude-4-opus': { input: 0.015, output: 0.075 },
25
+ 'claude-4-sonnet': { input: 0.003, output: 0.015 },
26
+ };
27
+ /**
28
+ * Calculate cost for an LLM call event.
29
+ * Returns cost in USD or null if not calculable.
30
+ */
31
+ export function calculateCost(event) {
32
+ if (event.type !== 'llm_call')
33
+ return null;
34
+ const data = event.data;
35
+ const model = data.model ?? '';
36
+ const inputTokens = data.input_tokens ?? data.prompt_tokens ?? 0;
37
+ const outputTokens = data.output_tokens ?? data.completion_tokens ?? 0;
38
+ // Find matching model (prefix match for versioned names)
39
+ let pricing;
40
+ for (const [key, cost] of Object.entries(MODEL_COSTS)) {
41
+ if (model === key || model.startsWith(key + '-')) {
42
+ pricing = cost;
43
+ break;
44
+ }
45
+ }
46
+ if (!pricing)
47
+ return null;
48
+ if (inputTokens === 0 && outputTokens === 0)
49
+ return null;
50
+ return (inputTokens / 1000) * pricing.input + (outputTokens / 1000) * pricing.output;
51
+ }
52
+ // ═══════════════════════════════════════════
53
+ // Hash Chain
54
+ // ═══════════════════════════════════════════
55
+ /**
56
+ * Compute hash for event in the hash chain.
57
+ * hash = SHA-256(previousHash + event.id + event.type + event.timestamp)
58
+ */
59
+ export function computeHash(event, previousHash) {
60
+ const content = `${previousHash}:${event.id}:${event.type}:${event.timestamp}`;
61
+ return createHash('sha256').update(content).digest('hex');
62
+ }
63
+ // ═══════════════════════════════════════════
64
+ // Batch Writer
65
+ // ═══════════════════════════════════════════
66
+ export class BatchWriter {
67
+ redis;
68
+ pool;
69
+ config;
70
+ running = false;
71
+ stats = { processed: 0, failed: 0, dlqd: 0 };
72
+ /** Retry counts by stream message ID */
73
+ retryCounts = new Map();
74
+ /** Last hash per org for hash chain continuity */
75
+ lastHashByOrg = new Map();
76
+ constructor(redis, pool, config) {
77
+ this.redis = redis;
78
+ this.pool = pool;
79
+ this.config = {
80
+ consumerName: config.consumerName,
81
+ batchSize: config.batchSize ?? 50,
82
+ blockMs: config.blockMs ?? 5000,
83
+ maxRetries: config.maxRetries ?? 3,
84
+ };
85
+ }
86
+ /** Start the worker loop */
87
+ async start() {
88
+ this.running = true;
89
+ while (this.running) {
90
+ await this.processBatch();
91
+ }
92
+ }
93
+ /** Stop the worker loop */
94
+ stop() {
95
+ this.running = false;
96
+ }
97
+ /** Get current stats */
98
+ getStats() {
99
+ return { ...this.stats };
100
+ }
101
+ /**
102
+ * Process a single batch: XREADGROUP → enrich → INSERT → XACK.
103
+ * Exposed for testing.
104
+ */
105
+ async processBatch() {
106
+ const messages = await this.readMessages();
107
+ if (messages.length === 0)
108
+ return 0;
109
+ // Enrich all messages
110
+ const enriched = this.enrichMessages(messages);
111
+ // Group by org_id for batch insert
112
+ const byOrg = new Map();
113
+ for (const msg of enriched) {
114
+ const orgEvents = byOrg.get(msg.event.org_id) ?? [];
115
+ orgEvents.push(msg);
116
+ byOrg.set(msg.event.org_id, orgEvents);
117
+ }
118
+ const ackedIds = [];
119
+ const failedMsgs = [];
120
+ for (const [orgId, orgMessages] of byOrg) {
121
+ try {
122
+ await this.batchInsert(orgId, orgMessages.map((m) => m.event));
123
+ ackedIds.push(...orgMessages.map((m) => m.streamId));
124
+ this.stats.processed += orgMessages.length;
125
+ }
126
+ catch {
127
+ failedMsgs.push(...orgMessages);
128
+ }
129
+ }
130
+ // ACK successful messages
131
+ if (ackedIds.length > 0) {
132
+ await this.ackMessages(ackedIds);
133
+ }
134
+ // Handle failures (retry or DLQ)
135
+ for (const msg of failedMsgs) {
136
+ const retries = (this.retryCounts.get(msg.streamId) ?? 0) + 1;
137
+ this.retryCounts.set(msg.streamId, retries);
138
+ if (retries >= this.config.maxRetries) {
139
+ await this.moveToDlq(msg);
140
+ await this.ackMessages([msg.streamId]);
141
+ this.retryCounts.delete(msg.streamId);
142
+ this.stats.dlqd++;
143
+ }
144
+ else {
145
+ this.stats.failed++;
146
+ }
147
+ }
148
+ return messages.length;
149
+ }
150
+ // ── Private methods ──
151
+ async readMessages() {
152
+ const result = await this.redis.xreadgroup('GROUP', CONSUMER_GROUP, this.config.consumerName, 'COUNT', this.config.batchSize, 'BLOCK', this.config.blockMs, 'STREAMS', STREAM_NAME, '>');
153
+ if (!result || result.length === 0)
154
+ return [];
155
+ const messages = [];
156
+ for (const [, entries] of result) {
157
+ for (const [streamId, fields] of entries) {
158
+ // fields is [key, value, key, value, ...]
159
+ const payloadIdx = fields.indexOf('payload');
160
+ if (payloadIdx === -1 || payloadIdx + 1 >= fields.length)
161
+ continue;
162
+ try {
163
+ const event = JSON.parse(fields[payloadIdx + 1]);
164
+ messages.push({ streamId, event });
165
+ }
166
+ catch {
167
+ // Unparseable message → DLQ immediately
168
+ await this.moveToDlq({ streamId, event: { id: 'unknown', type: 'unknown', timestamp: '', session_id: '', data: {}, org_id: '', api_key_id: '', received_at: '', request_id: '' } });
169
+ await this.ackMessages([streamId]);
170
+ }
171
+ }
172
+ }
173
+ return messages;
174
+ }
175
+ enrichMessages(messages) {
176
+ for (const msg of messages) {
177
+ // Cost calculation
178
+ const cost = calculateCost(msg.event);
179
+ if (cost !== null) {
180
+ msg.event.data = { ...msg.event.data, estimated_cost_usd: cost };
181
+ }
182
+ // Hash chain — persist both prev_hash and hash for verifiability
183
+ const prevHash = this.lastHashByOrg.get(msg.event.org_id) ?? '0';
184
+ const hash = computeHash(msg.event, prevHash);
185
+ msg.event.data = { ...msg.event.data, prev_hash: prevHash, hash };
186
+ this.lastHashByOrg.set(msg.event.org_id, hash);
187
+ }
188
+ return messages;
189
+ }
190
+ async batchInsert(orgId, events) {
191
+ if (events.length === 0)
192
+ return;
193
+ const client = await this.pool.connect();
194
+ try {
195
+ await client.query('BEGIN');
196
+ await client.query('SET LOCAL app.current_org = $1', [orgId]);
197
+ // Build batch INSERT
198
+ const values = [];
199
+ const placeholders = [];
200
+ let paramIdx = 1;
201
+ for (const event of events) {
202
+ placeholders.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
203
+ values.push(event.id, orgId, event.session_id, event.data.agent_id ?? 'unknown', event.type, event.data.severity ?? 'info', event.timestamp, JSON.stringify(event.data), event.data.prev_hash ?? null, event.data.hash ?? null);
204
+ }
205
+ await client.query(`INSERT INTO events (id, org_id, session_id, agent_id, event_type, severity, timestamp, payload, prev_hash, hash)
206
+ VALUES ${placeholders.join(', ')}
207
+ ON CONFLICT (id, timestamp) DO NOTHING`, values);
208
+ // Update usage records (atomic increment)
209
+ const hour = new Date(events[0].timestamp);
210
+ hour.setMinutes(0, 0, 0);
211
+ await client.query(`INSERT INTO usage_records (org_id, hour, event_count, api_key_id)
212
+ VALUES ($1, $2, $3, $4)
213
+ ON CONFLICT (org_id, hour, api_key_id)
214
+ DO UPDATE SET event_count = usage_records.event_count + $3`, [orgId, hour.toISOString(), events.length, events[0].api_key_id || '00000000-0000-0000-0000-000000000000']);
215
+ await client.query('COMMIT');
216
+ }
217
+ catch (err) {
218
+ await client.query('ROLLBACK');
219
+ throw err;
220
+ }
221
+ finally {
222
+ client.release();
223
+ }
224
+ }
225
+ async ackMessages(ids) {
226
+ if (ids.length === 0)
227
+ return;
228
+ await this.redis.xack(STREAM_NAME, CONSUMER_GROUP, ...ids);
229
+ }
230
+ async moveToDlq(msg) {
231
+ const payload = JSON.stringify({
232
+ ...msg.event,
233
+ _dlq_reason: 'max_retries_exceeded',
234
+ _dlq_stream_id: msg.streamId,
235
+ _dlq_timestamp: new Date().toISOString(),
236
+ });
237
+ await this.redis.xadd(DLQ_STREAM_NAME, '*', 'payload', payload);
238
+ }
239
+ }
240
+ /**
241
+ * In-memory batch writer for testing without Redis/Postgres.
242
+ */
243
+ export class InMemoryBatchWriter {
244
+ deps;
245
+ stats = { processed: 0, failed: 0, dlqd: 0 };
246
+ retryCounts = new Map();
247
+ lastHashByOrg = new Map();
248
+ maxRetries;
249
+ batchSize;
250
+ constructor(deps, config) {
251
+ this.deps = deps;
252
+ this.maxRetries = config?.maxRetries ?? 3;
253
+ this.batchSize = config?.batchSize ?? 50;
254
+ }
255
+ getStats() {
256
+ return { ...this.stats };
257
+ }
258
+ /**
259
+ * Process one batch from pendingEvents.
260
+ */
261
+ async processBatch() {
262
+ const batch = this.deps.pendingEvents.splice(0, this.batchSize);
263
+ if (batch.length === 0)
264
+ return 0;
265
+ // Enrich
266
+ for (const msg of batch) {
267
+ const cost = calculateCost(msg.event);
268
+ if (cost !== null) {
269
+ msg.event.data = { ...msg.event.data, estimated_cost_usd: cost };
270
+ }
271
+ const prevHash = this.lastHashByOrg.get(msg.event.org_id) ?? '0';
272
+ const hash = computeHash(msg.event, prevHash);
273
+ msg.event.data = { ...msg.event.data, hash };
274
+ this.lastHashByOrg.set(msg.event.org_id, hash);
275
+ }
276
+ // Group by org
277
+ const byOrg = new Map();
278
+ for (const msg of batch) {
279
+ const list = byOrg.get(msg.event.org_id) ?? [];
280
+ list.push(msg);
281
+ byOrg.set(msg.event.org_id, list);
282
+ }
283
+ const failedMsgs = [];
284
+ for (const [orgId, orgMsgs] of byOrg) {
285
+ if (this.deps.failingOrgs?.has(orgId)) {
286
+ failedMsgs.push(...orgMsgs);
287
+ continue;
288
+ }
289
+ // "Insert"
290
+ const existing = this.deps.insertedEvents.get(orgId) ?? [];
291
+ existing.push(...orgMsgs.map((m) => m.event));
292
+ this.deps.insertedEvents.set(orgId, existing);
293
+ // Usage
294
+ this.deps.usageIncrements.push({ orgId, count: orgMsgs.length });
295
+ // ACK
296
+ this.deps.ackedIds.push(...orgMsgs.map((m) => m.streamId));
297
+ this.stats.processed += orgMsgs.length;
298
+ }
299
+ // Handle failures
300
+ for (const msg of failedMsgs) {
301
+ const retries = (this.retryCounts.get(msg.streamId) ?? 0) + 1;
302
+ this.retryCounts.set(msg.streamId, retries);
303
+ if (retries >= this.maxRetries) {
304
+ this.deps.dlqEvents.push(msg.event);
305
+ this.deps.ackedIds.push(msg.streamId);
306
+ this.retryCounts.delete(msg.streamId);
307
+ this.stats.dlqd++;
308
+ // Re-add to pending won't happen; it's DLQ'd
309
+ }
310
+ else {
311
+ this.stats.failed++;
312
+ // Re-add to pending for retry
313
+ this.deps.pendingEvents.push(msg);
314
+ }
315
+ }
316
+ return batch.length;
317
+ }
318
+ }
319
+ //# sourceMappingURL=batch-writer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"batch-writer.js","sourceRoot":"","sources":["../../../src/cloud/ingestion/batch-writer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAwChF,8CAA8C;AAC9C,mBAAmB;AACnB,8CAA8C;AAE9C,0CAA0C;AAC1C,MAAM,WAAW,GAAsD;IACrE,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IACtC,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAC5C,QAAQ,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IACzC,aAAa,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE;IACjD,eAAe,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE;IAClD,eAAe,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IAChD,iBAAiB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IAClD,gBAAgB,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;IACrD,mBAAmB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IACpD,eAAe,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IAChD,iBAAiB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;CACnD,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAkB;IAC9C,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU;QAAE,OAAO,IAAI,CAAC;IAE3C,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACxB,MAAM,KAAK,GAAI,IAAI,CAAC,KAAgB,IAAI,EAAE,CAAC;IAC3C,MAAM,WAAW,GAAI,IAAI,CAAC,YAAuB,IAAK,IAAI,CAAC,aAAwB,IAAI,CAAC,CAAC;IACzF,MAAM,YAAY,GAAI,IAAI,CAAC,aAAwB,IAAK,IAAI,CAAC,iBAA4B,IAAI,CAAC,CAAC;IAE/F,yDAAyD;IACzD,IAAI,OAAsD,CAAC;IAC3D,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;QACtD,IAAI,KAAK,KAAK,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,CAAC,EAAE,CAAC;YACjD,OAAO,GAAG,IAAI,CAAC;YACf,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,IAAI,WAAW,KAAK,CAAC,IAAI,YAAY,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEzD,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,GAAG,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;AACvF,CAAC;AAED,8CAA8C;AAC9C,aAAa;AACb,8CAA8C;AAE9C;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,KAAkB,EAAE,YAAoB;IAClE,MAAM,OAAO,GAAG,GAAG,YAAY,IAAI,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;IAC/E,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5D,CAAC;AAED,8CAA8C;AAC9C,eAAe;AACf,8CAA8C;AAE9C,MAAM,OAAO,WAAW;IAUZ;IACA;IAVF,MAAM,CAA8B;IACpC,OAAO,GAAG,KAAK,CAAC;IAChB,KAAK,GAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAClE,wCAAwC;IAChC,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,kDAAkD;IAC1C,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAElD,YACU,KAA0B,EAC1B,IAAU,EAClB,MAAyB;QAFjB,UAAK,GAAL,KAAK,CAAqB;QAC1B,SAAI,GAAJ,IAAI,CAAM;QAGlB,IAAI,CAAC,MAAM,GAAG;YACZ,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,EAAE;YACjC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;YAC/B,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,CAAC;SACnC,CAAC;IACJ,CAAC;IAED,4BAA4B;IAC5B,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,2BAA2B;IAC3B,IAAI;QACF,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;IACvB,CAAC;IAED,wBAAwB;IACxB,QAAQ;QACN,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY;QAChB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC3C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEpC,sBAAsB;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAE/C,mCAAmC;QACnC,MAAM,KAAK,GAAG,IAAI,GAAG,EAA2D,CAAC;QACjF,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACzC,CAAC;QAED,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAoB,EAAE,CAAC;QAEvC,KAAK,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,IAAI,KAAK,EAAE,CAAC;YACzC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC/D,QAAQ,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACrD,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,WAAW,CAAC,MAAM,CAAC;YAC7C,CAAC;YAAC,MAAM,CAAC;gBACP,UAAU,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QAED,0BAA0B;QAC1B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC;QAED,iCAAiC;QACjC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAC9D,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAE5C,IAAI,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACvC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YACpB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtB,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC,MAAM,CAAC;IACzB,CAAC;IAED,wBAAwB;IAEhB,KAAK,CAAC,YAAY;QACxB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CACxC,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,EACjD,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,EAC9B,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAC5B,SAAS,EAAE,WAAW,EAAE,GAAG,CAC5B,CAAC;QAEF,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAE9C,MAAM,QAAQ,GAAoB,EAAE,CAAC;QACrC,KAAK,MAAM,CAAC,EAAE,OAAO,CAAC,IAAI,MAAM,EAAE,CAAC;YACjC,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACzC,0CAA0C;gBAC1C,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;gBAC7C,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,UAAU,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM;oBAAE,SAAS;gBACnE,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,CAAgB,CAAC;oBAChE,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;gBACrC,CAAC;gBAAC,MAAM,CAAC;oBACP,wCAAwC;oBACxC,MAAM,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;oBACpL,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,cAAc,CAAC,QAAyB;QAC9C,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,mBAAmB;YACnB,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACtC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBAClB,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;YACnE,CAAC;YAED,iEAAiE;YACjE,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC;YACjE,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAC9C,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YAClE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,KAAa,EAAE,MAAqB;QAC5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEhC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5B,MAAM,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;YAE9D,qBAAqB;YACrB,MAAM,MAAM,GAAc,EAAE,CAAC;YAC7B,MAAM,YAAY,GAAa,EAAE,CAAC;YAClC,IAAI,QAAQ,GAAG,CAAC,CAAC;YAEjB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,YAAY,CAAC,IAAI,CACf,KAAK,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,GAAG,CACnK,CAAC;gBACF,MAAM,CAAC,IAAI,CACT,KAAK,CAAC,EAAE,EACR,KAAK,EACL,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,SAAS,EAChC,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,MAAM,EAC7B,KAAK,CAAC,SAAS,EACf,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,EAC1B,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,EAC5B,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CACxB,CAAC;YACJ,CAAC;YAED,MAAM,MAAM,CAAC,KAAK,CAChB;kBACU,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;gDACO,EACxC,MAAM,CACP,CAAC;YAEF,0CAA0C;YAC1C,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAC3C,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAEzB,MAAM,MAAM,CAAC,KAAK,CAChB;;;oEAG4D,EAC5D,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,sCAAsC,CAAC,CAC3G,CAAC;YAEF,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAC/B,MAAM,GAAG,CAAC;QACZ,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,GAAa;QACrC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC7B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,EAAE,GAAG,GAAG,CAAC,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAkB;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;YAC7B,GAAG,GAAG,CAAC,KAAK;YACZ,WAAW,EAAE,sBAAsB;YACnC,cAAc,EAAE,GAAG,CAAC,QAAQ;YAC5B,cAAc,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACzC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAClE,CAAC;CACF;AAqBD;;GAEG;AACH,MAAM,OAAO,mBAAmB;IAQpB;IAPF,KAAK,GAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAC1D,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,UAAU,CAAS;IACnB,SAAS,CAAS;IAE1B,YACU,IAA6B,EACrC,MAAoD;QAD5C,SAAI,GAAJ,IAAI,CAAyB;QAGrC,IAAI,CAAC,UAAU,GAAG,MAAM,EAAE,UAAU,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,MAAM,EAAE,SAAS,IAAI,EAAE,CAAC;IAC3C,CAAC;IAED,QAAQ;QACN,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAChE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEjC,SAAS;QACT,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACtC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBAClB,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;YACnE,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC;YACjE,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAC9C,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;YAC7C,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACjD,CAAC;QAED,eAAe;QACf,MAAM,KAAK,GAAG,IAAI,GAAG,EAA2D,CAAC;QACjF,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC/C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACf,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACpC,CAAC;QAED,MAAM,UAAU,GAAoD,EAAE,CAAC;QAEvE,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,KAAK,EAAE,CAAC;YACrC,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtC,UAAU,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;gBAC5B,SAAS;YACX,CAAC;YAED,WAAW;YACX,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAC3D,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9C,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAE9C,QAAQ;YACR,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAEjE,MAAM;YACN,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3D,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;QACzC,CAAC;QAED,kBAAkB;QAClB,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAC9D,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAE5C,IAAI,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC/B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACpC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBAClB,6CAA6C;YAC/C,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;gBACpB,8BAA8B;gBAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC,MAAM,CAAC;IACtB,CAAC;CACF"}
@@ -0,0 +1,116 @@
1
+ /**
2
+ * DLQ Management & Monitoring (S-3.6)
3
+ *
4
+ * Manages the dead-letter queue stream:
5
+ * - Inspect DLQ entries (list with metadata)
6
+ * - Replay events back to the main ingestion stream
7
+ * - Expire old entries (> 7 days)
8
+ * - Report DLQ depth in health checks
9
+ */
10
+ import type { QueuedEvent, RedisClient } from './event-queue.js';
11
+ export interface DlqEntry {
12
+ /** Redis stream message ID */
13
+ streamId: string;
14
+ /** Original event payload */
15
+ event: QueuedEvent;
16
+ /** Why it was sent to DLQ */
17
+ dlqReason: string;
18
+ /** Original stream ID before DLQ */
19
+ originalStreamId: string;
20
+ /** When it was moved to DLQ */
21
+ dlqTimestamp: string;
22
+ }
23
+ export interface DlqStats {
24
+ depth: number;
25
+ oldestEntryAge?: number;
26
+ }
27
+ export interface DlqHealthInfo {
28
+ dlqDepth: number;
29
+ dlqHealthy: boolean;
30
+ dlqWarning?: string;
31
+ }
32
+ /** Extended Redis client for DLQ operations */
33
+ export interface DlqRedisClient extends RedisClient {
34
+ xrange(key: string, start: string, end: string, ...args: unknown[]): Promise<Array<[string, string[]]>>;
35
+ xdel(key: string, ...ids: string[]): Promise<number>;
36
+ xadd(...args: unknown[]): Promise<string>;
37
+ xlen(key: string): Promise<number>;
38
+ }
39
+ export declare class DlqManager {
40
+ private redis;
41
+ constructor(redis: DlqRedisClient);
42
+ /**
43
+ * Get DLQ depth (number of entries).
44
+ */
45
+ getDepth(): Promise<number>;
46
+ /**
47
+ * List DLQ entries, optionally with pagination.
48
+ * @param limit Max entries to return (default 50)
49
+ * @param startId Stream ID to start from (default '-' = beginning)
50
+ */
51
+ listEntries(limit?: number, startId?: string): Promise<DlqEntry[]>;
52
+ /**
53
+ * Get a single DLQ entry by stream ID.
54
+ */
55
+ getEntry(streamId: string): Promise<DlqEntry | null>;
56
+ /**
57
+ * Replay a single DLQ entry back to the main ingestion stream.
58
+ * Removes the entry from DLQ after successful re-queue.
59
+ */
60
+ replayEntry(streamId: string): Promise<{
61
+ success: boolean;
62
+ newStreamId?: string;
63
+ error?: string;
64
+ }>;
65
+ /**
66
+ * Replay multiple DLQ entries. Returns count of successfully replayed.
67
+ */
68
+ replayBatch(streamIds: string[]): Promise<{
69
+ replayed: number;
70
+ failed: number;
71
+ errors: string[];
72
+ }>;
73
+ /**
74
+ * Expire DLQ entries older than 7 days.
75
+ * Returns count of expired entries.
76
+ */
77
+ expireOldEntries(): Promise<number>;
78
+ /**
79
+ * Get DLQ health info for inclusion in health check response.
80
+ */
81
+ healthInfo(): Promise<DlqHealthInfo>;
82
+ private parseEntry;
83
+ }
84
+ export interface InMemoryDlqEntry {
85
+ streamId: string;
86
+ payload: Record<string, unknown>;
87
+ addedAt: number;
88
+ }
89
+ export declare class InMemoryDlqManager {
90
+ private entries;
91
+ private replayedToMain;
92
+ private counter;
93
+ /** Add an entry to the in-memory DLQ */
94
+ addEntry(event: QueuedEvent, reason: string, originalStreamId: string): string;
95
+ getDepth(): Promise<number>;
96
+ listEntries(limit?: number): Promise<DlqEntry[]>;
97
+ getEntry(streamId: string): Promise<DlqEntry | null>;
98
+ replayEntry(streamId: string): Promise<{
99
+ success: boolean;
100
+ newStreamId?: string;
101
+ error?: string;
102
+ }>;
103
+ replayBatch(streamIds: string[]): Promise<{
104
+ replayed: number;
105
+ failed: number;
106
+ errors: string[];
107
+ }>;
108
+ expireOldEntries(expiryMs?: number): Promise<number>;
109
+ healthInfo(): Promise<DlqHealthInfo>;
110
+ /** Test helper: get replayed events */
111
+ getReplayed(): Array<Record<string, unknown>>;
112
+ /** Test helper: clear all */
113
+ clear(): void;
114
+ private toEntry;
115
+ }
116
+ //# sourceMappingURL=dlq-manager.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"dlq-manager.d.ts","sourceRoot":"","sources":["../../../src/cloud/ingestion/dlq-manager.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAc,WAAW,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAO7E,MAAM,WAAW,QAAQ;IACvB,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,6BAA6B;IAC7B,KAAK,EAAE,WAAW,CAAC;IACnB,6BAA6B;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,oCAAoC;IACpC,gBAAgB,EAAE,MAAM,CAAC;IACzB,+BAA+B;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AASD,+CAA+C;AAC/C,MAAM,WAAW,cAAe,SAAQ,WAAW;IACjD,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;IACxG,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACrD,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACpC;AAED,qBAAa,UAAU;IACT,OAAO,CAAC,KAAK;gBAAL,KAAK,EAAE,cAAc;IAEzC;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;IAQjC;;;;OAIG;IACG,WAAW,CAAC,KAAK,SAAK,EAAE,OAAO,SAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAKjE;;OAEG;IACG,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAM1D;;;OAGG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAwBxG;;OAEG;IACG,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAkBvG;;;OAGG;IACG,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;IAczC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,aAAa,CAAC;IAe1C,OAAO,CAAC,UAAU;CA6BnB;AAMD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,cAAc,CAAsC;IAC5D,OAAO,CAAC,OAAO,CAAK;IAEpB,wCAAwC;IACxC,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,GAAG,MAAM;IAexE,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;IAI3B,WAAW,CAAC,KAAK,SAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAI5C,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAKpD,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAgBlG,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAYjG,gBAAgB,CAAC,QAAQ,SAAgB,GAAG,OAAO,CAAC,MAAM,CAAC;IAO3D,UAAU,IAAI,OAAO,CAAC,aAAa,CAAC;IAS1C,uCAAuC;IACvC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAI7C,6BAA6B;IAC7B,KAAK,IAAI,IAAI;IAMb,OAAO,CAAC,OAAO;CAchB"}