@agentlensai/server 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cloud/auth/api-key-middleware.d.ts +66 -0
- package/dist/cloud/auth/api-key-middleware.d.ts.map +1 -0
- package/dist/cloud/auth/api-key-middleware.js +147 -0
- package/dist/cloud/auth/api-key-middleware.js.map +1 -0
- package/dist/cloud/auth/api-keys.d.ts +90 -0
- package/dist/cloud/auth/api-keys.d.ts.map +1 -0
- package/dist/cloud/auth/api-keys.js +162 -0
- package/dist/cloud/auth/api-keys.js.map +1 -0
- package/dist/cloud/auth/audit-log.d.ts +66 -0
- package/dist/cloud/auth/audit-log.d.ts.map +1 -0
- package/dist/cloud/auth/audit-log.js +92 -0
- package/dist/cloud/auth/audit-log.js.map +1 -0
- package/dist/cloud/auth/auth-service.d.ts +77 -0
- package/dist/cloud/auth/auth-service.d.ts.map +1 -0
- package/dist/cloud/auth/auth-service.js +229 -0
- package/dist/cloud/auth/auth-service.js.map +1 -0
- package/dist/cloud/auth/brute-force.d.ts +36 -0
- package/dist/cloud/auth/brute-force.d.ts.map +1 -0
- package/dist/cloud/auth/brute-force.js +67 -0
- package/dist/cloud/auth/brute-force.js.map +1 -0
- package/dist/cloud/auth/index.d.ts +11 -0
- package/dist/cloud/auth/index.d.ts.map +1 -0
- package/dist/cloud/auth/index.js +11 -0
- package/dist/cloud/auth/index.js.map +1 -0
- package/dist/cloud/auth/jwt.d.ts +34 -0
- package/dist/cloud/auth/jwt.d.ts.map +1 -0
- package/dist/cloud/auth/jwt.js +68 -0
- package/dist/cloud/auth/jwt.js.map +1 -0
- package/dist/cloud/auth/oauth.d.ts +37 -0
- package/dist/cloud/auth/oauth.d.ts.map +1 -0
- package/dist/cloud/auth/oauth.js +120 -0
- package/dist/cloud/auth/oauth.js.map +1 -0
- package/dist/cloud/auth/passwords.d.ts +25 -0
- package/dist/cloud/auth/passwords.d.ts.map +1 -0
- package/dist/cloud/auth/passwords.js +50 -0
- package/dist/cloud/auth/passwords.js.map +1 -0
- package/dist/cloud/auth/rbac.d.ts +51 -0
- package/dist/cloud/auth/rbac.d.ts.map +1 -0
- package/dist/cloud/auth/rbac.js +89 -0
- package/dist/cloud/auth/rbac.js.map +1 -0
- package/dist/cloud/auth/tokens.d.ts +18 -0
- package/dist/cloud/auth/tokens.d.ts.map +1 -0
- package/dist/cloud/auth/tokens.js +29 -0
- package/dist/cloud/auth/tokens.js.map +1 -0
- package/dist/cloud/billing/billing-service.d.ts +44 -0
- package/dist/cloud/billing/billing-service.d.ts.map +1 -0
- package/dist/cloud/billing/billing-service.js +153 -0
- package/dist/cloud/billing/billing-service.js.map +1 -0
- package/dist/cloud/billing/index.d.ts +11 -0
- package/dist/cloud/billing/index.d.ts.map +1 -0
- package/dist/cloud/billing/index.js +11 -0
- package/dist/cloud/billing/index.js.map +1 -0
- package/dist/cloud/billing/invoice-service.d.ts +57 -0
- package/dist/cloud/billing/invoice-service.d.ts.map +1 -0
- package/dist/cloud/billing/invoice-service.js +123 -0
- package/dist/cloud/billing/invoice-service.js.map +1 -0
- package/dist/cloud/billing/plan-management.d.ts +46 -0
- package/dist/cloud/billing/plan-management.d.ts.map +1 -0
- package/dist/cloud/billing/plan-management.js +157 -0
- package/dist/cloud/billing/plan-management.js.map +1 -0
- package/dist/cloud/billing/quota-enforcement.d.ts +53 -0
- package/dist/cloud/billing/quota-enforcement.d.ts.map +1 -0
- package/dist/cloud/billing/quota-enforcement.js +143 -0
- package/dist/cloud/billing/quota-enforcement.js.map +1 -0
- package/dist/cloud/billing/stripe-client.d.ts +142 -0
- package/dist/cloud/billing/stripe-client.d.ts.map +1 -0
- package/dist/cloud/billing/stripe-client.js +169 -0
- package/dist/cloud/billing/stripe-client.js.map +1 -0
- package/dist/cloud/billing/trial-service.d.ts +47 -0
- package/dist/cloud/billing/trial-service.d.ts.map +1 -0
- package/dist/cloud/billing/trial-service.js +104 -0
- package/dist/cloud/billing/trial-service.js.map +1 -0
- package/dist/cloud/billing/usage-metering.d.ts +83 -0
- package/dist/cloud/billing/usage-metering.d.ts.map +1 -0
- package/dist/cloud/billing/usage-metering.js +174 -0
- package/dist/cloud/billing/usage-metering.js.map +1 -0
- package/dist/cloud/ingestion/backpressure.d.ts +107 -0
- package/dist/cloud/ingestion/backpressure.d.ts.map +1 -0
- package/dist/cloud/ingestion/backpressure.js +134 -0
- package/dist/cloud/ingestion/backpressure.js.map +1 -0
- package/dist/cloud/ingestion/batch-writer.d.ts +115 -0
- package/dist/cloud/ingestion/batch-writer.d.ts.map +1 -0
- package/dist/cloud/ingestion/batch-writer.js +319 -0
- package/dist/cloud/ingestion/batch-writer.js.map +1 -0
- package/dist/cloud/ingestion/dlq-manager.d.ts +116 -0
- package/dist/cloud/ingestion/dlq-manager.d.ts.map +1 -0
- package/dist/cloud/ingestion/dlq-manager.js +244 -0
- package/dist/cloud/ingestion/dlq-manager.js.map +1 -0
- package/dist/cloud/ingestion/event-queue.d.ts +105 -0
- package/dist/cloud/ingestion/event-queue.d.ts.map +1 -0
- package/dist/cloud/ingestion/event-queue.js +185 -0
- package/dist/cloud/ingestion/event-queue.js.map +1 -0
- package/dist/cloud/ingestion/gateway.d.ts +68 -0
- package/dist/cloud/ingestion/gateway.d.ts.map +1 -0
- package/dist/cloud/ingestion/gateway.js +198 -0
- package/dist/cloud/ingestion/gateway.js.map +1 -0
- package/dist/cloud/ingestion/index.d.ts +7 -0
- package/dist/cloud/ingestion/index.d.ts.map +1 -0
- package/dist/cloud/ingestion/index.js +7 -0
- package/dist/cloud/ingestion/index.js.map +1 -0
- package/dist/cloud/ingestion/rate-limiter.d.ts +73 -0
- package/dist/cloud/ingestion/rate-limiter.d.ts.map +1 -0
- package/dist/cloud/ingestion/rate-limiter.js +153 -0
- package/dist/cloud/ingestion/rate-limiter.js.map +1 -0
- package/dist/cloud/migrate.d.ts +45 -0
- package/dist/cloud/migrate.d.ts.map +1 -0
- package/dist/cloud/migrate.js +147 -0
- package/dist/cloud/migrate.js.map +1 -0
- package/dist/cloud/migration/export-import.d.ts +56 -0
- package/dist/cloud/migration/export-import.d.ts.map +1 -0
- package/dist/cloud/migration/export-import.js +289 -0
- package/dist/cloud/migration/export-import.js.map +1 -0
- package/dist/cloud/migration/index.d.ts +5 -0
- package/dist/cloud/migration/index.d.ts.map +1 -0
- package/dist/cloud/migration/index.js +5 -0
- package/dist/cloud/migration/index.js.map +1 -0
- package/dist/cloud/org-service.d.ts +68 -0
- package/dist/cloud/org-service.d.ts.map +1 -0
- package/dist/cloud/org-service.js +169 -0
- package/dist/cloud/org-service.js.map +1 -0
- package/dist/cloud/partition-maintenance.d.ts +29 -0
- package/dist/cloud/partition-maintenance.d.ts.map +1 -0
- package/dist/cloud/partition-maintenance.js +96 -0
- package/dist/cloud/partition-maintenance.js.map +1 -0
- package/dist/cloud/retention/index.d.ts +7 -0
- package/dist/cloud/retention/index.d.ts.map +1 -0
- package/dist/cloud/retention/index.js +7 -0
- package/dist/cloud/retention/index.js.map +1 -0
- package/dist/cloud/retention/partition-management.d.ts +61 -0
- package/dist/cloud/retention/partition-management.d.ts.map +1 -0
- package/dist/cloud/retention/partition-management.js +167 -0
- package/dist/cloud/retention/partition-management.js.map +1 -0
- package/dist/cloud/retention/retention-job.d.ts +70 -0
- package/dist/cloud/retention/retention-job.d.ts.map +1 -0
- package/dist/cloud/retention/retention-job.js +160 -0
- package/dist/cloud/retention/retention-job.js.map +1 -0
- package/dist/cloud/retention/retention-policy.d.ts +27 -0
- package/dist/cloud/retention/retention-policy.d.ts.map +1 -0
- package/dist/cloud/retention/retention-policy.js +36 -0
- package/dist/cloud/retention/retention-policy.js.map +1 -0
- package/dist/cloud/routes/api-key-routes.d.ts +38 -0
- package/dist/cloud/routes/api-key-routes.d.ts.map +1 -0
- package/dist/cloud/routes/api-key-routes.js +84 -0
- package/dist/cloud/routes/api-key-routes.js.map +1 -0
- package/dist/cloud/routes/audit-routes.d.ts +36 -0
- package/dist/cloud/routes/audit-routes.d.ts.map +1 -0
- package/dist/cloud/routes/audit-routes.js +47 -0
- package/dist/cloud/routes/audit-routes.js.map +1 -0
- package/dist/cloud/routes/billing-routes.d.ts +51 -0
- package/dist/cloud/routes/billing-routes.d.ts.map +1 -0
- package/dist/cloud/routes/billing-routes.js +114 -0
- package/dist/cloud/routes/billing-routes.js.map +1 -0
- package/dist/cloud/routes/onboarding-routes.d.ts +34 -0
- package/dist/cloud/routes/onboarding-routes.d.ts.map +1 -0
- package/dist/cloud/routes/onboarding-routes.js +58 -0
- package/dist/cloud/routes/onboarding-routes.js.map +1 -0
- package/dist/cloud/routes/org-routes.d.ts +80 -0
- package/dist/cloud/routes/org-routes.d.ts.map +1 -0
- package/dist/cloud/routes/org-routes.js +153 -0
- package/dist/cloud/routes/org-routes.js.map +1 -0
- package/dist/cloud/routes/usage-routes.d.ts +18 -0
- package/dist/cloud/routes/usage-routes.d.ts.map +1 -0
- package/dist/cloud/routes/usage-routes.js +66 -0
- package/dist/cloud/routes/usage-routes.js.map +1 -0
- package/dist/cloud/storage/adapter.d.ts +102 -0
- package/dist/cloud/storage/adapter.d.ts.map +1 -0
- package/dist/cloud/storage/adapter.js +21 -0
- package/dist/cloud/storage/adapter.js.map +1 -0
- package/dist/cloud/storage/index.d.ts +8 -0
- package/dist/cloud/storage/index.d.ts.map +1 -0
- package/dist/cloud/storage/index.js +7 -0
- package/dist/cloud/storage/index.js.map +1 -0
- package/dist/cloud/storage/postgres-adapter.d.ts +34 -0
- package/dist/cloud/storage/postgres-adapter.d.ts.map +1 -0
- package/dist/cloud/storage/postgres-adapter.js +544 -0
- package/dist/cloud/storage/postgres-adapter.js.map +1 -0
- package/dist/cloud/storage/sqlite-adapter.d.ts +29 -0
- package/dist/cloud/storage/sqlite-adapter.d.ts.map +1 -0
- package/dist/cloud/storage/sqlite-adapter.js +176 -0
- package/dist/cloud/storage/sqlite-adapter.js.map +1 -0
- package/dist/cloud/tenant-pool.d.ts +49 -0
- package/dist/cloud/tenant-pool.d.ts.map +1 -0
- package/dist/cloud/tenant-pool.js +61 -0
- package/dist/cloud/tenant-pool.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Batch Writer — Queue Worker (S-3.3)
|
|
3
|
+
*
|
|
4
|
+
* Consumer group workers: XREADGROUP up to 50 messages from the
|
|
5
|
+
* `event_ingestion` stream, enrich (cost calculation, hash chain),
|
|
6
|
+
* batch INSERT into Postgres, XACK on success. After 3 failures → DLQ.
|
|
7
|
+
*/
|
|
8
|
+
import { createHash } from 'crypto';
|
|
9
|
+
import { STREAM_NAME, DLQ_STREAM_NAME, CONSUMER_GROUP } from './event-queue.js';
|
|
10
|
+
// ═══════════════════════════════════════════
|
|
11
|
+
// Cost Calculation
|
|
12
|
+
// ═══════════════════════════════════════════
|
|
13
|
+
/** Known model pricing (per 1K tokens) */
|
|
14
|
+
const MODEL_COSTS = {
|
|
15
|
+
'gpt-4': { input: 0.03, output: 0.06 },
|
|
16
|
+
'gpt-4-turbo': { input: 0.01, output: 0.03 },
|
|
17
|
+
'gpt-4o': { input: 0.005, output: 0.015 },
|
|
18
|
+
'gpt-4o-mini': { input: 0.00015, output: 0.0006 },
|
|
19
|
+
'gpt-3.5-turbo': { input: 0.0005, output: 0.0015 },
|
|
20
|
+
'claude-3-opus': { input: 0.015, output: 0.075 },
|
|
21
|
+
'claude-3-sonnet': { input: 0.003, output: 0.015 },
|
|
22
|
+
'claude-3-haiku': { input: 0.00025, output: 0.00125 },
|
|
23
|
+
'claude-3.5-sonnet': { input: 0.003, output: 0.015 },
|
|
24
|
+
'claude-4-opus': { input: 0.015, output: 0.075 },
|
|
25
|
+
'claude-4-sonnet': { input: 0.003, output: 0.015 },
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* Calculate cost for an LLM call event.
|
|
29
|
+
* Returns cost in USD or null if not calculable.
|
|
30
|
+
*/
|
|
31
|
+
export function calculateCost(event) {
|
|
32
|
+
if (event.type !== 'llm_call')
|
|
33
|
+
return null;
|
|
34
|
+
const data = event.data;
|
|
35
|
+
const model = data.model ?? '';
|
|
36
|
+
const inputTokens = data.input_tokens ?? data.prompt_tokens ?? 0;
|
|
37
|
+
const outputTokens = data.output_tokens ?? data.completion_tokens ?? 0;
|
|
38
|
+
// Find matching model (prefix match for versioned names)
|
|
39
|
+
let pricing;
|
|
40
|
+
for (const [key, cost] of Object.entries(MODEL_COSTS)) {
|
|
41
|
+
if (model === key || model.startsWith(key + '-')) {
|
|
42
|
+
pricing = cost;
|
|
43
|
+
break;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
if (!pricing)
|
|
47
|
+
return null;
|
|
48
|
+
if (inputTokens === 0 && outputTokens === 0)
|
|
49
|
+
return null;
|
|
50
|
+
return (inputTokens / 1000) * pricing.input + (outputTokens / 1000) * pricing.output;
|
|
51
|
+
}
|
|
52
|
+
// ═══════════════════════════════════════════
|
|
53
|
+
// Hash Chain
|
|
54
|
+
// ═══════════════════════════════════════════
|
|
55
|
+
/**
|
|
56
|
+
* Compute hash for event in the hash chain.
|
|
57
|
+
* hash = SHA-256(previousHash + event.id + event.type + event.timestamp)
|
|
58
|
+
*/
|
|
59
|
+
export function computeHash(event, previousHash) {
|
|
60
|
+
const content = `${previousHash}:${event.id}:${event.type}:${event.timestamp}`;
|
|
61
|
+
return createHash('sha256').update(content).digest('hex');
|
|
62
|
+
}
|
|
63
|
+
// ═══════════════════════════════════════════
|
|
64
|
+
// Batch Writer
|
|
65
|
+
// ═══════════════════════════════════════════
|
|
66
|
+
export class BatchWriter {
|
|
67
|
+
redis;
|
|
68
|
+
pool;
|
|
69
|
+
config;
|
|
70
|
+
running = false;
|
|
71
|
+
stats = { processed: 0, failed: 0, dlqd: 0 };
|
|
72
|
+
/** Retry counts by stream message ID */
|
|
73
|
+
retryCounts = new Map();
|
|
74
|
+
/** Last hash per org for hash chain continuity */
|
|
75
|
+
lastHashByOrg = new Map();
|
|
76
|
+
constructor(redis, pool, config) {
|
|
77
|
+
this.redis = redis;
|
|
78
|
+
this.pool = pool;
|
|
79
|
+
this.config = {
|
|
80
|
+
consumerName: config.consumerName,
|
|
81
|
+
batchSize: config.batchSize ?? 50,
|
|
82
|
+
blockMs: config.blockMs ?? 5000,
|
|
83
|
+
maxRetries: config.maxRetries ?? 3,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
/** Start the worker loop */
|
|
87
|
+
async start() {
|
|
88
|
+
this.running = true;
|
|
89
|
+
while (this.running) {
|
|
90
|
+
await this.processBatch();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
/** Stop the worker loop */
|
|
94
|
+
stop() {
|
|
95
|
+
this.running = false;
|
|
96
|
+
}
|
|
97
|
+
/** Get current stats */
|
|
98
|
+
getStats() {
|
|
99
|
+
return { ...this.stats };
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Process a single batch: XREADGROUP → enrich → INSERT → XACK.
|
|
103
|
+
* Exposed for testing.
|
|
104
|
+
*/
|
|
105
|
+
async processBatch() {
|
|
106
|
+
const messages = await this.readMessages();
|
|
107
|
+
if (messages.length === 0)
|
|
108
|
+
return 0;
|
|
109
|
+
// Enrich all messages
|
|
110
|
+
const enriched = this.enrichMessages(messages);
|
|
111
|
+
// Group by org_id for batch insert
|
|
112
|
+
const byOrg = new Map();
|
|
113
|
+
for (const msg of enriched) {
|
|
114
|
+
const orgEvents = byOrg.get(msg.event.org_id) ?? [];
|
|
115
|
+
orgEvents.push(msg);
|
|
116
|
+
byOrg.set(msg.event.org_id, orgEvents);
|
|
117
|
+
}
|
|
118
|
+
const ackedIds = [];
|
|
119
|
+
const failedMsgs = [];
|
|
120
|
+
for (const [orgId, orgMessages] of byOrg) {
|
|
121
|
+
try {
|
|
122
|
+
await this.batchInsert(orgId, orgMessages.map((m) => m.event));
|
|
123
|
+
ackedIds.push(...orgMessages.map((m) => m.streamId));
|
|
124
|
+
this.stats.processed += orgMessages.length;
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
failedMsgs.push(...orgMessages);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
// ACK successful messages
|
|
131
|
+
if (ackedIds.length > 0) {
|
|
132
|
+
await this.ackMessages(ackedIds);
|
|
133
|
+
}
|
|
134
|
+
// Handle failures (retry or DLQ)
|
|
135
|
+
for (const msg of failedMsgs) {
|
|
136
|
+
const retries = (this.retryCounts.get(msg.streamId) ?? 0) + 1;
|
|
137
|
+
this.retryCounts.set(msg.streamId, retries);
|
|
138
|
+
if (retries >= this.config.maxRetries) {
|
|
139
|
+
await this.moveToDlq(msg);
|
|
140
|
+
await this.ackMessages([msg.streamId]);
|
|
141
|
+
this.retryCounts.delete(msg.streamId);
|
|
142
|
+
this.stats.dlqd++;
|
|
143
|
+
}
|
|
144
|
+
else {
|
|
145
|
+
this.stats.failed++;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return messages.length;
|
|
149
|
+
}
|
|
150
|
+
// ── Private methods ──
|
|
151
|
+
async readMessages() {
|
|
152
|
+
const result = await this.redis.xreadgroup('GROUP', CONSUMER_GROUP, this.config.consumerName, 'COUNT', this.config.batchSize, 'BLOCK', this.config.blockMs, 'STREAMS', STREAM_NAME, '>');
|
|
153
|
+
if (!result || result.length === 0)
|
|
154
|
+
return [];
|
|
155
|
+
const messages = [];
|
|
156
|
+
for (const [, entries] of result) {
|
|
157
|
+
for (const [streamId, fields] of entries) {
|
|
158
|
+
// fields is [key, value, key, value, ...]
|
|
159
|
+
const payloadIdx = fields.indexOf('payload');
|
|
160
|
+
if (payloadIdx === -1 || payloadIdx + 1 >= fields.length)
|
|
161
|
+
continue;
|
|
162
|
+
try {
|
|
163
|
+
const event = JSON.parse(fields[payloadIdx + 1]);
|
|
164
|
+
messages.push({ streamId, event });
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
// Unparseable message → DLQ immediately
|
|
168
|
+
await this.moveToDlq({ streamId, event: { id: 'unknown', type: 'unknown', timestamp: '', session_id: '', data: {}, org_id: '', api_key_id: '', received_at: '', request_id: '' } });
|
|
169
|
+
await this.ackMessages([streamId]);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return messages;
|
|
174
|
+
}
|
|
175
|
+
enrichMessages(messages) {
|
|
176
|
+
for (const msg of messages) {
|
|
177
|
+
// Cost calculation
|
|
178
|
+
const cost = calculateCost(msg.event);
|
|
179
|
+
if (cost !== null) {
|
|
180
|
+
msg.event.data = { ...msg.event.data, estimated_cost_usd: cost };
|
|
181
|
+
}
|
|
182
|
+
// Hash chain — persist both prev_hash and hash for verifiability
|
|
183
|
+
const prevHash = this.lastHashByOrg.get(msg.event.org_id) ?? '0';
|
|
184
|
+
const hash = computeHash(msg.event, prevHash);
|
|
185
|
+
msg.event.data = { ...msg.event.data, prev_hash: prevHash, hash };
|
|
186
|
+
this.lastHashByOrg.set(msg.event.org_id, hash);
|
|
187
|
+
}
|
|
188
|
+
return messages;
|
|
189
|
+
}
|
|
190
|
+
async batchInsert(orgId, events) {
|
|
191
|
+
if (events.length === 0)
|
|
192
|
+
return;
|
|
193
|
+
const client = await this.pool.connect();
|
|
194
|
+
try {
|
|
195
|
+
await client.query('BEGIN');
|
|
196
|
+
await client.query('SET LOCAL app.current_org = $1', [orgId]);
|
|
197
|
+
// Build batch INSERT
|
|
198
|
+
const values = [];
|
|
199
|
+
const placeholders = [];
|
|
200
|
+
let paramIdx = 1;
|
|
201
|
+
for (const event of events) {
|
|
202
|
+
placeholders.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
|
|
203
|
+
values.push(event.id, orgId, event.session_id, event.data.agent_id ?? 'unknown', event.type, event.data.severity ?? 'info', event.timestamp, JSON.stringify(event.data), event.data.prev_hash ?? null, event.data.hash ?? null);
|
|
204
|
+
}
|
|
205
|
+
await client.query(`INSERT INTO events (id, org_id, session_id, agent_id, event_type, severity, timestamp, payload, prev_hash, hash)
|
|
206
|
+
VALUES ${placeholders.join(', ')}
|
|
207
|
+
ON CONFLICT (id, timestamp) DO NOTHING`, values);
|
|
208
|
+
// Update usage records (atomic increment)
|
|
209
|
+
const hour = new Date(events[0].timestamp);
|
|
210
|
+
hour.setMinutes(0, 0, 0);
|
|
211
|
+
await client.query(`INSERT INTO usage_records (org_id, hour, event_count, api_key_id)
|
|
212
|
+
VALUES ($1, $2, $3, $4)
|
|
213
|
+
ON CONFLICT (org_id, hour, api_key_id)
|
|
214
|
+
DO UPDATE SET event_count = usage_records.event_count + $3`, [orgId, hour.toISOString(), events.length, events[0].api_key_id || '00000000-0000-0000-0000-000000000000']);
|
|
215
|
+
await client.query('COMMIT');
|
|
216
|
+
}
|
|
217
|
+
catch (err) {
|
|
218
|
+
await client.query('ROLLBACK');
|
|
219
|
+
throw err;
|
|
220
|
+
}
|
|
221
|
+
finally {
|
|
222
|
+
client.release();
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
async ackMessages(ids) {
|
|
226
|
+
if (ids.length === 0)
|
|
227
|
+
return;
|
|
228
|
+
await this.redis.xack(STREAM_NAME, CONSUMER_GROUP, ...ids);
|
|
229
|
+
}
|
|
230
|
+
async moveToDlq(msg) {
|
|
231
|
+
const payload = JSON.stringify({
|
|
232
|
+
...msg.event,
|
|
233
|
+
_dlq_reason: 'max_retries_exceeded',
|
|
234
|
+
_dlq_stream_id: msg.streamId,
|
|
235
|
+
_dlq_timestamp: new Date().toISOString(),
|
|
236
|
+
});
|
|
237
|
+
await this.redis.xadd(DLQ_STREAM_NAME, '*', 'payload', payload);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* In-memory batch writer for testing without Redis/Postgres.
|
|
242
|
+
*/
|
|
243
|
+
export class InMemoryBatchWriter {
|
|
244
|
+
deps;
|
|
245
|
+
stats = { processed: 0, failed: 0, dlqd: 0 };
|
|
246
|
+
retryCounts = new Map();
|
|
247
|
+
lastHashByOrg = new Map();
|
|
248
|
+
maxRetries;
|
|
249
|
+
batchSize;
|
|
250
|
+
constructor(deps, config) {
|
|
251
|
+
this.deps = deps;
|
|
252
|
+
this.maxRetries = config?.maxRetries ?? 3;
|
|
253
|
+
this.batchSize = config?.batchSize ?? 50;
|
|
254
|
+
}
|
|
255
|
+
getStats() {
|
|
256
|
+
return { ...this.stats };
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* Process one batch from pendingEvents.
|
|
260
|
+
*/
|
|
261
|
+
async processBatch() {
|
|
262
|
+
const batch = this.deps.pendingEvents.splice(0, this.batchSize);
|
|
263
|
+
if (batch.length === 0)
|
|
264
|
+
return 0;
|
|
265
|
+
// Enrich
|
|
266
|
+
for (const msg of batch) {
|
|
267
|
+
const cost = calculateCost(msg.event);
|
|
268
|
+
if (cost !== null) {
|
|
269
|
+
msg.event.data = { ...msg.event.data, estimated_cost_usd: cost };
|
|
270
|
+
}
|
|
271
|
+
const prevHash = this.lastHashByOrg.get(msg.event.org_id) ?? '0';
|
|
272
|
+
const hash = computeHash(msg.event, prevHash);
|
|
273
|
+
msg.event.data = { ...msg.event.data, hash };
|
|
274
|
+
this.lastHashByOrg.set(msg.event.org_id, hash);
|
|
275
|
+
}
|
|
276
|
+
// Group by org
|
|
277
|
+
const byOrg = new Map();
|
|
278
|
+
for (const msg of batch) {
|
|
279
|
+
const list = byOrg.get(msg.event.org_id) ?? [];
|
|
280
|
+
list.push(msg);
|
|
281
|
+
byOrg.set(msg.event.org_id, list);
|
|
282
|
+
}
|
|
283
|
+
const failedMsgs = [];
|
|
284
|
+
for (const [orgId, orgMsgs] of byOrg) {
|
|
285
|
+
if (this.deps.failingOrgs?.has(orgId)) {
|
|
286
|
+
failedMsgs.push(...orgMsgs);
|
|
287
|
+
continue;
|
|
288
|
+
}
|
|
289
|
+
// "Insert"
|
|
290
|
+
const existing = this.deps.insertedEvents.get(orgId) ?? [];
|
|
291
|
+
existing.push(...orgMsgs.map((m) => m.event));
|
|
292
|
+
this.deps.insertedEvents.set(orgId, existing);
|
|
293
|
+
// Usage
|
|
294
|
+
this.deps.usageIncrements.push({ orgId, count: orgMsgs.length });
|
|
295
|
+
// ACK
|
|
296
|
+
this.deps.ackedIds.push(...orgMsgs.map((m) => m.streamId));
|
|
297
|
+
this.stats.processed += orgMsgs.length;
|
|
298
|
+
}
|
|
299
|
+
// Handle failures
|
|
300
|
+
for (const msg of failedMsgs) {
|
|
301
|
+
const retries = (this.retryCounts.get(msg.streamId) ?? 0) + 1;
|
|
302
|
+
this.retryCounts.set(msg.streamId, retries);
|
|
303
|
+
if (retries >= this.maxRetries) {
|
|
304
|
+
this.deps.dlqEvents.push(msg.event);
|
|
305
|
+
this.deps.ackedIds.push(msg.streamId);
|
|
306
|
+
this.retryCounts.delete(msg.streamId);
|
|
307
|
+
this.stats.dlqd++;
|
|
308
|
+
// Re-add to pending won't happen; it's DLQ'd
|
|
309
|
+
}
|
|
310
|
+
else {
|
|
311
|
+
this.stats.failed++;
|
|
312
|
+
// Re-add to pending for retry
|
|
313
|
+
this.deps.pendingEvents.push(msg);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
return batch.length;
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
//# sourceMappingURL=batch-writer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batch-writer.js","sourceRoot":"","sources":["../../../src/cloud/ingestion/batch-writer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAwChF,8CAA8C;AAC9C,mBAAmB;AACnB,8CAA8C;AAE9C,0CAA0C;AAC1C,MAAM,WAAW,GAAsD;IACrE,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IACtC,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAC5C,QAAQ,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IACzC,aAAa,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE;IACjD,eAAe,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE;IAClD,eAAe,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IAChD,iBAAiB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IAClD,gBAAgB,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;IACrD,mBAAmB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IACpD,eAAe,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IAChD,iBAAiB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;CACnD,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAkB;IAC9C,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU;QAAE,OAAO,IAAI,CAAC;IAE3C,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACxB,MAAM,KAAK,GAAI,IAAI,CAAC,KAAgB,IAAI,EAAE,CAAC;IAC3C,MAAM,WAAW,GAAI,IAAI,CAAC,YAAuB,IAAK,IAAI,CAAC,aAAwB,IAAI,CAAC,CAAC;IACzF,MAAM,YAAY,GAAI,IAAI,CAAC,aAAwB,IAAK,IAAI,CAAC,iBAA4B,IAAI,CAAC,CAAC;IAE/F,yDAAyD;IACzD,IAAI,OAAsD,CAAC;IAC3D,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;QACtD,IAAI,KAAK,KAAK,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,CAAC,EAAE,CAAC;YACjD,OAAO,GAAG,IAAI,CAAC;YACf,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,IAAI,WAAW,KAAK,CAAC,IAAI,YAAY,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEzD,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,GAAG,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;AACvF,CAAC;AAED,8CAA8C;AAC9C,aAAa;AACb,8CAA8C;AAE9C;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,KAAkB,EAAE,YAAoB;IAClE,MAAM,OAAO,GAAG,GAAG,YAAY,IAAI,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;IAC/E,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5D,CAAC;AAED,8CAA8C;AAC9C,eAAe;AACf,8CAA8C;AAE9C,MAAM,OAAO,WAAW;IAUZ;IACA;IAVF,MAAM,CAA8B;IACpC,OAAO,GAAG,KAAK,CAAC;IAChB,KAAK,GAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAClE,wCAAwC;IAChC,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,kDAAkD;IAC1C,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAElD,YACU,KAA0B,EAC1B,IAAU,EAClB,MAAyB;QAFjB,UAAK,GAAL,KAAK,CAAqB;QAC1B,SAAI,GAAJ,IAAI,CAAM;QAGlB,IAAI,CAAC,MAAM,GAAG;YACZ,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,EAAE;YACjC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;YAC/B,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,CAAC;SACnC,CAAC;IACJ,CAAC;IAED,4BAA4B;IAC5B,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,2BAA2B;IAC3B,IAAI;QACF,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;IACvB,CAAC;IAED,wBAAwB;IACxB,QAAQ;QACN,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY;QAChB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC3C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEpC,sBAAsB;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAE/C,mCAAmC;QACnC,MAAM,KAAK,GAAG,IAAI,GAAG,EAA2D,CAAC;QACjF,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACzC,CAAC;QAED,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAoB,EAAE,CAAC;QAEvC,KAAK,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,IAAI,KAAK,EAAE,CAAC;YACzC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC/D,QAAQ,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACrD,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,WAAW,CAAC,MAAM,CAAC;YAC7C,CAAC;YAAC,MAAM,CAAC;gBACP,UAAU,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QAED,0BAA0B;QAC1B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC;QAED,iCAAiC;QACjC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAC9D,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAE5C,IAAI,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACvC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YACpB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtB,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC,MAAM,CAAC;IACzB,CAAC;IAED,wBAAwB;IAEhB,KAAK,CAAC,YAAY;QACxB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CACxC,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,EACjD,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,EAC9B,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAC5B,SAAS,EAAE,WAAW,EAAE,GAAG,CAC5B,CAAC;QAEF,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAE9C,MAAM,QAAQ,GAAoB,EAAE,CAAC;QACrC,KAAK,MAAM,CAAC,EAAE,OAAO,CAAC,IAAI,MAAM,EAAE,CAAC;YACjC,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACzC,0CAA0C;gBAC1C,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;gBAC7C,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,UAAU,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM;oBAAE,SAAS;gBACnE,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,CAAgB,CAAC;oBAChE,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;gBACrC,CAAC;gBAAC,MAAM,CAAC;oBACP,wCAAwC;oBACxC,MAAM,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;oBACpL,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,cAAc,CAAC,QAAyB;QAC9C,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,mBAAmB;YACnB,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACtC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBAClB,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;YACnE,CAAC;YAED,iEAAiE;YACjE,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC;YACjE,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAC9C,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YAClE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,KAAa,EAAE,MAAqB;QAC5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEhC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5B,MAAM,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;YAE9D,qBAAqB;YACrB,MAAM,MAAM,GAAc,EAAE,CAAC;YAC7B,MAAM,YAAY,GAAa,EAAE,CAAC;YAClC,IAAI,QAAQ,GAAG,CAAC,CAAC;YAEjB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,YAAY,CAAC,IAAI,CACf,KAAK,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,GAAG,CACnK,CAAC;gBACF,MAAM,CAAC,IAAI,CACT,KAAK,CAAC,EAAE,EACR,KAAK,EACL,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,SAAS,EAChC,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,MAAM,EAC7B,KAAK,CAAC,SAAS,EACf,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,EAC1B,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,EAC5B,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CACxB,CAAC;YACJ,CAAC;YAED,MAAM,MAAM,CAAC,KAAK,CAChB;kBACU,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;gDACO,EACxC,MAAM,CACP,CAAC;YAEF,0CAA0C;YAC1C,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAC3C,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAEzB,MAAM,MAAM,CAAC,KAAK,CAChB;;;oEAG4D,EAC5D,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,sCAAsC,CAAC,CAC3G,CAAC;YAEF,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAC/B,MAAM,GAAG,CAAC;QACZ,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,GAAa;QACrC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC7B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,EAAE,GAAG,GAAG,CAAC,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAkB;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;YAC7B,GAAG,GAAG,CAAC,KAAK;YACZ,WAAW,EAAE,sBAAsB;YACnC,cAAc,EAAE,GAAG,CAAC,QAAQ;YAC5B,cAAc,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACzC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAClE,CAAC;CACF;AAqBD;;GAEG;AACH,MAAM,OAAO,mBAAmB;IAQpB;IAPF,KAAK,GAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAC1D,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,UAAU,CAAS;IACnB,SAAS,CAAS;IAE1B,YACU,IAA6B,EACrC,MAAoD;QAD5C,SAAI,GAAJ,IAAI,CAAyB;QAGrC,IAAI,CAAC,UAAU,GAAG,MAAM,EAAE,UAAU,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,MAAM,EAAE,SAAS,IAAI,EAAE,CAAC;IAC3C,CAAC;IAED,QAAQ;QACN,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAChE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEjC,SAAS;QACT,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACtC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBAClB,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;YACnE,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC;YACjE,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAC9C,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;YAC7C,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACjD,CAAC;QAED,eAAe;QACf,MAAM,KAAK,GAAG,IAAI,GAAG,EAA2D,CAAC;QACjF,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC/C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACf,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACpC,CAAC;QAED,MAAM,UAAU,GAAoD,EAAE,CAAC;QAEvE,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,KAAK,EAAE,CAAC;YACrC,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtC,UAAU,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;gBAC5B,SAAS;YACX,CAAC;YAED,WAAW;YACX,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAC3D,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9C,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAE9C,QAAQ;YACR,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAEjE,MAAM;YACN,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3D,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;QACzC,CAAC;QAED,kBAAkB;QAClB,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAC9D,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAE5C,IAAI,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC/B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACpC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBAClB,6CAA6C;YAC/C,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;gBACpB,8BAA8B;gBAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC,MAAM,CAAC;IACtB,CAAC;CACF"}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DLQ Management & Monitoring (S-3.6)
|
|
3
|
+
*
|
|
4
|
+
* Manages the dead-letter queue stream:
|
|
5
|
+
* - Inspect DLQ entries (list with metadata)
|
|
6
|
+
* - Replay events back to the main ingestion stream
|
|
7
|
+
* - Expire old entries (> 7 days)
|
|
8
|
+
* - Report DLQ depth in health checks
|
|
9
|
+
*/
|
|
10
|
+
import type { QueuedEvent, RedisClient } from './event-queue.js';
|
|
11
|
+
export interface DlqEntry {
|
|
12
|
+
/** Redis stream message ID */
|
|
13
|
+
streamId: string;
|
|
14
|
+
/** Original event payload */
|
|
15
|
+
event: QueuedEvent;
|
|
16
|
+
/** Why it was sent to DLQ */
|
|
17
|
+
dlqReason: string;
|
|
18
|
+
/** Original stream ID before DLQ */
|
|
19
|
+
originalStreamId: string;
|
|
20
|
+
/** When it was moved to DLQ */
|
|
21
|
+
dlqTimestamp: string;
|
|
22
|
+
}
|
|
23
|
+
export interface DlqStats {
|
|
24
|
+
depth: number;
|
|
25
|
+
oldestEntryAge?: number;
|
|
26
|
+
}
|
|
27
|
+
export interface DlqHealthInfo {
|
|
28
|
+
dlqDepth: number;
|
|
29
|
+
dlqHealthy: boolean;
|
|
30
|
+
dlqWarning?: string;
|
|
31
|
+
}
|
|
32
|
+
/** Extended Redis client for DLQ operations */
|
|
33
|
+
export interface DlqRedisClient extends RedisClient {
|
|
34
|
+
xrange(key: string, start: string, end: string, ...args: unknown[]): Promise<Array<[string, string[]]>>;
|
|
35
|
+
xdel(key: string, ...ids: string[]): Promise<number>;
|
|
36
|
+
xadd(...args: unknown[]): Promise<string>;
|
|
37
|
+
xlen(key: string): Promise<number>;
|
|
38
|
+
}
|
|
39
|
+
export declare class DlqManager {
|
|
40
|
+
private redis;
|
|
41
|
+
constructor(redis: DlqRedisClient);
|
|
42
|
+
/**
|
|
43
|
+
* Get DLQ depth (number of entries).
|
|
44
|
+
*/
|
|
45
|
+
getDepth(): Promise<number>;
|
|
46
|
+
/**
|
|
47
|
+
* List DLQ entries, optionally with pagination.
|
|
48
|
+
* @param limit Max entries to return (default 50)
|
|
49
|
+
* @param startId Stream ID to start from (default '-' = beginning)
|
|
50
|
+
*/
|
|
51
|
+
listEntries(limit?: number, startId?: string): Promise<DlqEntry[]>;
|
|
52
|
+
/**
|
|
53
|
+
* Get a single DLQ entry by stream ID.
|
|
54
|
+
*/
|
|
55
|
+
getEntry(streamId: string): Promise<DlqEntry | null>;
|
|
56
|
+
/**
|
|
57
|
+
* Replay a single DLQ entry back to the main ingestion stream.
|
|
58
|
+
* Removes the entry from DLQ after successful re-queue.
|
|
59
|
+
*/
|
|
60
|
+
replayEntry(streamId: string): Promise<{
|
|
61
|
+
success: boolean;
|
|
62
|
+
newStreamId?: string;
|
|
63
|
+
error?: string;
|
|
64
|
+
}>;
|
|
65
|
+
/**
|
|
66
|
+
* Replay multiple DLQ entries. Returns count of successfully replayed.
|
|
67
|
+
*/
|
|
68
|
+
replayBatch(streamIds: string[]): Promise<{
|
|
69
|
+
replayed: number;
|
|
70
|
+
failed: number;
|
|
71
|
+
errors: string[];
|
|
72
|
+
}>;
|
|
73
|
+
/**
|
|
74
|
+
* Expire DLQ entries older than 7 days.
|
|
75
|
+
* Returns count of expired entries.
|
|
76
|
+
*/
|
|
77
|
+
expireOldEntries(): Promise<number>;
|
|
78
|
+
/**
|
|
79
|
+
* Get DLQ health info for inclusion in health check response.
|
|
80
|
+
*/
|
|
81
|
+
healthInfo(): Promise<DlqHealthInfo>;
|
|
82
|
+
private parseEntry;
|
|
83
|
+
}
|
|
84
|
+
export interface InMemoryDlqEntry {
|
|
85
|
+
streamId: string;
|
|
86
|
+
payload: Record<string, unknown>;
|
|
87
|
+
addedAt: number;
|
|
88
|
+
}
|
|
89
|
+
export declare class InMemoryDlqManager {
|
|
90
|
+
private entries;
|
|
91
|
+
private replayedToMain;
|
|
92
|
+
private counter;
|
|
93
|
+
/** Add an entry to the in-memory DLQ */
|
|
94
|
+
addEntry(event: QueuedEvent, reason: string, originalStreamId: string): string;
|
|
95
|
+
getDepth(): Promise<number>;
|
|
96
|
+
listEntries(limit?: number): Promise<DlqEntry[]>;
|
|
97
|
+
getEntry(streamId: string): Promise<DlqEntry | null>;
|
|
98
|
+
replayEntry(streamId: string): Promise<{
|
|
99
|
+
success: boolean;
|
|
100
|
+
newStreamId?: string;
|
|
101
|
+
error?: string;
|
|
102
|
+
}>;
|
|
103
|
+
replayBatch(streamIds: string[]): Promise<{
|
|
104
|
+
replayed: number;
|
|
105
|
+
failed: number;
|
|
106
|
+
errors: string[];
|
|
107
|
+
}>;
|
|
108
|
+
expireOldEntries(expiryMs?: number): Promise<number>;
|
|
109
|
+
healthInfo(): Promise<DlqHealthInfo>;
|
|
110
|
+
/** Test helper: get replayed events */
|
|
111
|
+
getReplayed(): Array<Record<string, unknown>>;
|
|
112
|
+
/** Test helper: clear all */
|
|
113
|
+
clear(): void;
|
|
114
|
+
private toEntry;
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=dlq-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dlq-manager.d.ts","sourceRoot":"","sources":["../../../src/cloud/ingestion/dlq-manager.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAc,WAAW,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAO7E,MAAM,WAAW,QAAQ;IACvB,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,6BAA6B;IAC7B,KAAK,EAAE,WAAW,CAAC;IACnB,6BAA6B;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,oCAAoC;IACpC,gBAAgB,EAAE,MAAM,CAAC;IACzB,+BAA+B;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AASD,+CAA+C;AAC/C,MAAM,WAAW,cAAe,SAAQ,WAAW;IACjD,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;IACxG,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACrD,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACpC;AAED,qBAAa,UAAU;IACT,OAAO,CAAC,KAAK;gBAAL,KAAK,EAAE,cAAc;IAEzC;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;IAQjC;;;;OAIG;IACG,WAAW,CAAC,KAAK,SAAK,EAAE,OAAO,SAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAKjE;;OAEG;IACG,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAM1D;;;OAGG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAwBxG;;OAEG;IACG,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAkBvG;;;OAGG;IACG,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;IAczC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,aAAa,CAAC;IAe1C,OAAO,CAAC,UAAU;CA6BnB;AAMD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,cAAc,CAAsC;IAC5D,OAAO,CAAC,OAAO,CAAK;IAEpB,wCAAwC;IACxC,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,GAAG,MAAM;IAexE,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;IAI3B,WAAW,CAAC,KAAK,SAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAI5C,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAKpD,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAgBlG,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAYjG,gBAAgB,CAAC,QAAQ,SAAgB,GAAG,OAAO,CAAC,MAAM,CAAC;IAO3D,UAAU,IAAI,OAAO,CAAC,aAAa,CAAC;IAS1C,uCAAuC;IACvC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAI7C,6BAA6B;IAC7B,KAAK,IAAI,IAAI;IAMb,OAAO,CAAC,OAAO;CAchB"}
|