@shalwin04/x404r-sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +391 -0
- package/dist/ai/index.d.ts +21 -0
- package/dist/ai/index.d.ts.map +1 -0
- package/dist/ai/index.js +344 -0
- package/dist/ai/index.js.map +1 -0
- package/dist/backend/cloud.d.ts +91 -0
- package/dist/backend/cloud.d.ts.map +1 -0
- package/dist/backend/cloud.js +257 -0
- package/dist/backend/cloud.js.map +1 -0
- package/dist/backend/embedded.d.ts +79 -0
- package/dist/backend/embedded.d.ts.map +1 -0
- package/dist/backend/embedded.js +307 -0
- package/dist/backend/embedded.js.map +1 -0
- package/dist/backend/index.d.ts +11 -0
- package/dist/backend/index.d.ts.map +1 -0
- package/dist/backend/index.js +8 -0
- package/dist/backend/index.js.map +1 -0
- package/dist/backend/interface.d.ts +88 -0
- package/dist/backend/interface.d.ts.map +1 -0
- package/dist/backend/interface.js +11 -0
- package/dist/backend/interface.js.map +1 -0
- package/dist/chaos.d.ts +117 -0
- package/dist/chaos.d.ts.map +1 -0
- package/dist/chaos.js +215 -0
- package/dist/chaos.js.map +1 -0
- package/dist/client.d.ts +242 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +401 -0
- package/dist/client.js.map +1 -0
- package/dist/context.d.ts +18 -0
- package/dist/context.d.ts.map +1 -0
- package/dist/context.js +65 -0
- package/dist/context.js.map +1 -0
- package/dist/durable.d.ts +90 -0
- package/dist/durable.d.ts.map +1 -0
- package/dist/durable.js +143 -0
- package/dist/durable.js.map +1 -0
- package/dist/index.d.ts +74 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +76 -0
- package/dist/index.js.map +1 -0
- package/dist/metrics.d.ts +279 -0
- package/dist/metrics.d.ts.map +1 -0
- package/dist/metrics.js +693 -0
- package/dist/metrics.js.map +1 -0
- package/dist/types.d.ts +240 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +16 -0
- package/dist/types.js.map +1 -0
- package/dist/worker.d.ts +74 -0
- package/dist/worker.d.ts.map +1 -0
- package/dist/worker.js +269 -0
- package/dist/worker.js.map +1 -0
- package/dist/workflow.d.ts +48 -0
- package/dist/workflow.d.ts.map +1 -0
- package/dist/workflow.js +190 -0
- package/dist/workflow.js.map +1 -0
- package/package.json +93 -0
package/dist/metrics.js
ADDED
|
@@ -0,0 +1,693 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* x404-r Metrics & Observability
|
|
3
|
+
*
|
|
4
|
+
* Real-world production metrics for AI agent monitoring
|
|
5
|
+
* Supports hybrid mode: in-memory tracking + database persistence
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Histogram for tracking distributions
|
|
9
|
+
*/
|
|
10
|
+
export class Histogram {
|
|
11
|
+
values = [];
|
|
12
|
+
maxSize;
|
|
13
|
+
constructor(maxSize = 1000) {
|
|
14
|
+
this.maxSize = maxSize;
|
|
15
|
+
}
|
|
16
|
+
record(value) {
|
|
17
|
+
this.values.push(value);
|
|
18
|
+
if (this.values.length > this.maxSize) {
|
|
19
|
+
this.values.shift();
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
percentile(p) {
|
|
23
|
+
if (this.values.length === 0)
|
|
24
|
+
return 0;
|
|
25
|
+
const sorted = [...this.values].sort((a, b) => a - b);
|
|
26
|
+
const index = Math.ceil((p / 100) * sorted.length) - 1;
|
|
27
|
+
return sorted[Math.max(0, index)];
|
|
28
|
+
}
|
|
29
|
+
average() {
|
|
30
|
+
if (this.values.length === 0)
|
|
31
|
+
return 0;
|
|
32
|
+
return this.values.reduce((a, b) => a + b, 0) / this.values.length;
|
|
33
|
+
}
|
|
34
|
+
count() {
|
|
35
|
+
return this.values.length;
|
|
36
|
+
}
|
|
37
|
+
reset() {
|
|
38
|
+
this.values = [];
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Rate calculator for throughput metrics
|
|
43
|
+
*/
|
|
44
|
+
export class RateCalculator {
|
|
45
|
+
events = [];
|
|
46
|
+
windowMs;
|
|
47
|
+
constructor(windowMs = 60000) {
|
|
48
|
+
this.windowMs = windowMs;
|
|
49
|
+
}
|
|
50
|
+
record() {
|
|
51
|
+
this.events.push(Date.now());
|
|
52
|
+
this.cleanup();
|
|
53
|
+
}
|
|
54
|
+
rate() {
|
|
55
|
+
this.cleanup();
|
|
56
|
+
return this.events.length;
|
|
57
|
+
}
|
|
58
|
+
cleanup() {
|
|
59
|
+
const cutoff = Date.now() - this.windowMs;
|
|
60
|
+
this.events = this.events.filter(t => t > cutoff);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Main metrics collector
|
|
65
|
+
* Supports in-memory tracking with optional database persistence
|
|
66
|
+
*/
|
|
67
|
+
export class MetricsCollector {
|
|
68
|
+
counters = new Map();
|
|
69
|
+
gauges = new Map();
|
|
70
|
+
histograms = new Map();
|
|
71
|
+
rates = new Map();
|
|
72
|
+
events = [];
|
|
73
|
+
maxEvents;
|
|
74
|
+
listeners = [];
|
|
75
|
+
// Timing state
|
|
76
|
+
taskStartTimes = new Map();
|
|
77
|
+
lastFailureTime = 0;
|
|
78
|
+
failureCount = 0;
|
|
79
|
+
recoveryTimes = [];
|
|
80
|
+
// Recovery benchmark tracking
|
|
81
|
+
recoveryEvents = [];
|
|
82
|
+
taskTokenUsage = new Map();
|
|
83
|
+
taskTimeUsage = new Map();
|
|
84
|
+
// Database persistence
|
|
85
|
+
dbConfig = null;
|
|
86
|
+
flushInterval = null;
|
|
87
|
+
lastFlushTime = 0;
|
|
88
|
+
flushInProgress = false;
|
|
89
|
+
constructor(maxEvents = 10000) {
|
|
90
|
+
this.maxEvents = maxEvents;
|
|
91
|
+
// Initialize default histograms
|
|
92
|
+
this.histograms.set('task.latency', new Histogram());
|
|
93
|
+
this.histograms.set('ai.latency', new Histogram());
|
|
94
|
+
this.histograms.set('checkpoint.latency', new Histogram());
|
|
95
|
+
this.histograms.set('recovery.time', new Histogram());
|
|
96
|
+
// Initialize default rates
|
|
97
|
+
this.rates.set('tasks.completed', new RateCalculator());
|
|
98
|
+
this.rates.set('tasks.failed', new RateCalculator());
|
|
99
|
+
this.rates.set('ai.generations', new RateCalculator());
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Subscribe to metric events
|
|
103
|
+
*/
|
|
104
|
+
subscribe(listener) {
|
|
105
|
+
this.listeners.push(listener);
|
|
106
|
+
return () => {
|
|
107
|
+
const index = this.listeners.indexOf(listener);
|
|
108
|
+
if (index > -1)
|
|
109
|
+
this.listeners.splice(index, 1);
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Emit a metric event
|
|
114
|
+
*/
|
|
115
|
+
emit(event) {
|
|
116
|
+
this.events.push(event);
|
|
117
|
+
if (this.events.length > this.maxEvents) {
|
|
118
|
+
this.events.shift();
|
|
119
|
+
}
|
|
120
|
+
for (const listener of this.listeners) {
|
|
121
|
+
listener(event);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
// ============ Counter Operations ============
|
|
125
|
+
increment(name, value = 1, tags = {}) {
|
|
126
|
+
const current = this.counters.get(name) || 0;
|
|
127
|
+
this.counters.set(name, current + value);
|
|
128
|
+
this.emit({
|
|
129
|
+
name,
|
|
130
|
+
value: current + value,
|
|
131
|
+
timestamp: new Date(),
|
|
132
|
+
tags,
|
|
133
|
+
type: 'counter',
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
// ============ Gauge Operations ============
|
|
137
|
+
gauge(name, value, tags = {}) {
|
|
138
|
+
this.gauges.set(name, value);
|
|
139
|
+
this.emit({
|
|
140
|
+
name,
|
|
141
|
+
value,
|
|
142
|
+
timestamp: new Date(),
|
|
143
|
+
tags,
|
|
144
|
+
type: 'gauge',
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
// ============ Histogram Operations ============
|
|
148
|
+
recordHistogram(name, value, tags = {}) {
|
|
149
|
+
let histogram = this.histograms.get(name);
|
|
150
|
+
if (!histogram) {
|
|
151
|
+
histogram = new Histogram();
|
|
152
|
+
this.histograms.set(name, histogram);
|
|
153
|
+
}
|
|
154
|
+
histogram.record(value);
|
|
155
|
+
this.emit({
|
|
156
|
+
name,
|
|
157
|
+
value,
|
|
158
|
+
timestamp: new Date(),
|
|
159
|
+
tags,
|
|
160
|
+
type: 'histogram',
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
// ============ Timer Operations ============
|
|
164
|
+
startTimer(name) {
|
|
165
|
+
const start = performance.now();
|
|
166
|
+
return () => {
|
|
167
|
+
const duration = performance.now() - start;
|
|
168
|
+
this.recordHistogram(name, duration);
|
|
169
|
+
return duration;
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
// ============ Task Lifecycle ============
|
|
173
|
+
taskStarted(taskId, taskType, workflowId) {
|
|
174
|
+
this.taskStartTimes.set(taskId, Date.now());
|
|
175
|
+
this.increment('tasks.started', 1, { taskType, workflowId });
|
|
176
|
+
this.gauge('tasks.running', (this.gauges.get('tasks.running') || 0) + 1);
|
|
177
|
+
}
|
|
178
|
+
taskCompleted(taskId, taskType, workflowId) {
|
|
179
|
+
const startTime = this.taskStartTimes.get(taskId);
|
|
180
|
+
if (startTime) {
|
|
181
|
+
const duration = Date.now() - startTime;
|
|
182
|
+
this.recordHistogram('task.latency', duration, { taskType });
|
|
183
|
+
this.taskStartTimes.delete(taskId);
|
|
184
|
+
}
|
|
185
|
+
this.increment('tasks.completed', 1, { taskType, workflowId });
|
|
186
|
+
this.rates.get('tasks.completed')?.record();
|
|
187
|
+
this.gauge('tasks.running', Math.max(0, (this.gauges.get('tasks.running') || 0) - 1));
|
|
188
|
+
}
|
|
189
|
+
taskFailed(taskId, taskType, workflowId, error) {
|
|
190
|
+
const startTime = this.taskStartTimes.get(taskId);
|
|
191
|
+
if (startTime) {
|
|
192
|
+
this.taskStartTimes.delete(taskId);
|
|
193
|
+
}
|
|
194
|
+
this.increment('tasks.failed', 1, { taskType, workflowId, error: error.slice(0, 50) });
|
|
195
|
+
this.rates.get('tasks.failed')?.record();
|
|
196
|
+
this.gauge('tasks.running', Math.max(0, (this.gauges.get('tasks.running') || 0) - 1));
|
|
197
|
+
// Track failure timing for MTBF
|
|
198
|
+
const now = Date.now();
|
|
199
|
+
if (this.lastFailureTime > 0) {
|
|
200
|
+
const timeBetweenFailures = now - this.lastFailureTime;
|
|
201
|
+
this.recordHistogram('failure.interval', timeBetweenFailures);
|
|
202
|
+
}
|
|
203
|
+
this.lastFailureTime = now;
|
|
204
|
+
this.failureCount++;
|
|
205
|
+
}
|
|
206
|
+
taskRecovered(taskId, taskType, recoveryTimeMs) {
|
|
207
|
+
this.increment('tasks.recovered', 1, { taskType });
|
|
208
|
+
this.recordHistogram('recovery.time', recoveryTimeMs);
|
|
209
|
+
this.recoveryTimes.push(recoveryTimeMs);
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Record a crash recovery event with full context
|
|
213
|
+
* This is used to calculate the "what if" benchmark
|
|
214
|
+
*/
|
|
215
|
+
recordCrashRecovery(event) {
|
|
216
|
+
this.recoveryEvents.push({
|
|
217
|
+
taskId: event.taskId,
|
|
218
|
+
tokensBeforeCrash: event.tokensUsedBeforeCrash,
|
|
219
|
+
tokensAfterRecovery: event.tokensUsedAfterRecovery,
|
|
220
|
+
timeBeforeCrash: event.timeSpentBeforeCrashMs,
|
|
221
|
+
checkpointAgeMs: event.checkpointAgeMs,
|
|
222
|
+
recoveredAt: Date.now(),
|
|
223
|
+
});
|
|
224
|
+
// Update counters
|
|
225
|
+
this.increment('recovery.tokens_saved', event.tokensUsedBeforeCrash);
|
|
226
|
+
this.increment('recovery.tokens_used', event.tokensUsedAfterRecovery);
|
|
227
|
+
this.recordHistogram('recovery.checkpoint_age', event.checkpointAgeMs);
|
|
228
|
+
// Track recovery time
|
|
229
|
+
this.taskRecovered(event.taskId, 'recovered', event.recoveryTimeMs);
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Track token usage for a task (used for benchmark calculation)
|
|
233
|
+
*/
|
|
234
|
+
trackTaskTokens(taskId, tokens) {
|
|
235
|
+
const current = this.taskTokenUsage.get(taskId) || 0;
|
|
236
|
+
this.taskTokenUsage.set(taskId, current + tokens);
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Track time usage for a task (used for benchmark calculation)
|
|
240
|
+
*/
|
|
241
|
+
trackTaskTime(taskId, timeMs) {
|
|
242
|
+
const current = this.taskTimeUsage.get(taskId) || 0;
|
|
243
|
+
this.taskTimeUsage.set(taskId, current + timeMs);
|
|
244
|
+
}
|
|
245
|
+
// ============ Checkpoint Lifecycle ============
|
|
246
|
+
checkpointCreated(taskId, stepNumber, dataSize) {
|
|
247
|
+
this.increment('checkpoints.created', 1);
|
|
248
|
+
this.recordHistogram('checkpoint.size', dataSize);
|
|
249
|
+
}
|
|
250
|
+
checkpointRestored(taskId, stepNumber) {
|
|
251
|
+
this.increment('checkpoints.restored', 1);
|
|
252
|
+
}
|
|
253
|
+
// ============ AI Operations ============
|
|
254
|
+
aiGenerationStarted(model) {
|
|
255
|
+
const stopTimer = this.startTimer('ai.latency');
|
|
256
|
+
return () => {
|
|
257
|
+
stopTimer();
|
|
258
|
+
this.increment('ai.generations', 1, { model });
|
|
259
|
+
this.rates.get('ai.generations')?.record();
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
aiTokensUsed(model, input, output) {
|
|
263
|
+
this.increment('ai.tokens.input', input, { model });
|
|
264
|
+
this.increment('ai.tokens.output', output, { model });
|
|
265
|
+
this.increment('ai.tokens.total', input + output, { model });
|
|
266
|
+
// Track model breakdown
|
|
267
|
+
const modelKey = `ai.model.${model}`;
|
|
268
|
+
this.increment(modelKey, 1);
|
|
269
|
+
}
|
|
270
|
+
aiCostIncurred(model, costUsd) {
|
|
271
|
+
this.increment('ai.cost.total', costUsd * 10000, { model }); // Store in 0.0001 USD units
|
|
272
|
+
}
|
|
273
|
+
// ============ Memory Operations ============
|
|
274
|
+
memoryStored(taskType) {
|
|
275
|
+
this.increment('memory.stored', 1, { taskType });
|
|
276
|
+
}
|
|
277
|
+
memoryQueried(taskType, resultsCount, relevanceScore) {
|
|
278
|
+
this.increment('memory.queries', 1, { taskType });
|
|
279
|
+
this.recordHistogram('memory.results', resultsCount);
|
|
280
|
+
this.recordHistogram('memory.relevance', relevanceScore * 100);
|
|
281
|
+
}
|
|
282
|
+
// ============ Aggregation ============
|
|
283
|
+
getSummary() {
|
|
284
|
+
const tasksCompleted = this.counters.get('tasks.completed') || 0;
|
|
285
|
+
const tasksFailed = this.counters.get('tasks.failed') || 0;
|
|
286
|
+
const tasksTotal = tasksCompleted + tasksFailed + (this.counters.get('tasks.started') || 0);
|
|
287
|
+
const taskLatency = this.histograms.get('task.latency');
|
|
288
|
+
const aiLatency = this.histograms.get('ai.latency');
|
|
289
|
+
const checkpointLatency = this.histograms.get('checkpoint.latency');
|
|
290
|
+
const recoveryTime = this.histograms.get('recovery.time');
|
|
291
|
+
const tokensInput = this.counters.get('ai.tokens.input') || 0;
|
|
292
|
+
const tokensOutput = this.counters.get('ai.tokens.output') || 0;
|
|
293
|
+
const totalCost = (this.counters.get('ai.cost.total') || 0) / 10000;
|
|
294
|
+
const checkpointsCreated = this.counters.get('checkpoints.created') || 0;
|
|
295
|
+
const checkpointsRestored = this.counters.get('checkpoints.restored') || 0;
|
|
296
|
+
const memoryStored = this.counters.get('memory.stored') || 0;
|
|
297
|
+
const memoryRelevance = this.histograms.get('memory.relevance');
|
|
298
|
+
// Calculate model breakdown
|
|
299
|
+
const modelBreakdown = {};
|
|
300
|
+
for (const [key, value] of this.counters.entries()) {
|
|
301
|
+
if (key.startsWith('ai.model.')) {
|
|
302
|
+
const model = key.replace('ai.model.', '');
|
|
303
|
+
modelBreakdown[model] = value;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
const completedRate = this.rates.get('tasks.completed');
|
|
307
|
+
const throughput = completedRate?.rate() || 0;
|
|
308
|
+
return {
|
|
309
|
+
execution: {
|
|
310
|
+
tasksTotal,
|
|
311
|
+
tasksCompleted,
|
|
312
|
+
tasksFailed,
|
|
313
|
+
tasksPending: this.gauges.get('tasks.pending') || 0,
|
|
314
|
+
tasksRunning: this.gauges.get('tasks.running') || 0,
|
|
315
|
+
successRate: tasksTotal > 0 ? (tasksCompleted / tasksTotal) * 100 : 100,
|
|
316
|
+
throughput,
|
|
317
|
+
queueDepth: this.gauges.get('queue.depth') || 0,
|
|
318
|
+
activeWorkers: this.gauges.get('workers.active') || 0,
|
|
319
|
+
},
|
|
320
|
+
cost: {
|
|
321
|
+
totalCostUsd: totalCost,
|
|
322
|
+
costPerTask: tasksCompleted > 0 ? totalCost / tasksCompleted : 0,
|
|
323
|
+
costPerWorkflow: totalCost, // Simplified
|
|
324
|
+
tokensInput,
|
|
325
|
+
tokensOutput,
|
|
326
|
+
tokensTotal: tokensInput + tokensOutput,
|
|
327
|
+
savingsFromCheckpoints: checkpointsRestored * 0.001, // Estimated savings
|
|
328
|
+
projectedMonthlyCost: totalCost * 30 * 24, // Rough projection
|
|
329
|
+
},
|
|
330
|
+
performance: {
|
|
331
|
+
latencyP50Ms: taskLatency?.percentile(50) || 0,
|
|
332
|
+
latencyP95Ms: taskLatency?.percentile(95) || 0,
|
|
333
|
+
latencyP99Ms: taskLatency?.percentile(99) || 0,
|
|
334
|
+
latencyAvgMs: taskLatency?.average() || 0,
|
|
335
|
+
throughputPerMinute: throughput,
|
|
336
|
+
checkpointLatencyMs: checkpointLatency?.average() || 0,
|
|
337
|
+
aiLatencyMs: aiLatency?.average() || 0,
|
|
338
|
+
},
|
|
339
|
+
reliability: {
|
|
340
|
+
uptimePercent: 99.9, // Would need actual uptime tracking
|
|
341
|
+
mtbfMinutes: this.failureCount > 1
|
|
342
|
+
? (Date.now() - this.lastFailureTime) / 60000 / this.failureCount
|
|
343
|
+
: 999,
|
|
344
|
+
mttrSeconds: recoveryTime?.average() ? recoveryTime.average() / 1000 : 0,
|
|
345
|
+
crashRecoveries: this.counters.get('tasks.recovered') || 0,
|
|
346
|
+
checkpointHitRate: checkpointsCreated > 0
|
|
347
|
+
? (checkpointsRestored / checkpointsCreated) * 100
|
|
348
|
+
: 0,
|
|
349
|
+
failedRecoveries: 0,
|
|
350
|
+
},
|
|
351
|
+
ai: {
|
|
352
|
+
totalGenerations: this.counters.get('ai.generations') || 0,
|
|
353
|
+
modelBreakdown,
|
|
354
|
+
avgTokensPerGeneration: (this.counters.get('ai.generations') || 0) > 0
|
|
355
|
+
? (tokensInput + tokensOutput) / (this.counters.get('ai.generations') || 1)
|
|
356
|
+
: 0,
|
|
357
|
+
contextUtilization: 0.35, // Would need actual tracking
|
|
358
|
+
cacheHitRate: 0,
|
|
359
|
+
streamingRatio: 0,
|
|
360
|
+
},
|
|
361
|
+
memory: {
|
|
362
|
+
vectorCount: memoryStored,
|
|
363
|
+
avgRetrievalRelevance: memoryRelevance?.average() || 0,
|
|
364
|
+
memoryHitRate: 0,
|
|
365
|
+
learningImprovement: 0,
|
|
366
|
+
},
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
/**
|
|
370
|
+
* Get recent events for streaming
|
|
371
|
+
*/
|
|
372
|
+
getRecentEvents(limit = 100) {
|
|
373
|
+
return this.events.slice(-limit);
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* Get the recovery benchmark - the value x404-r provides
|
|
377
|
+
* This shows "with x404-r" vs "without x404-r" comparison
|
|
378
|
+
*/
|
|
379
|
+
getRecoveryBenchmark() {
|
|
380
|
+
const tokensInput = this.counters.get('ai.tokens.input') || 0;
|
|
381
|
+
const tokensOutput = this.counters.get('ai.tokens.output') || 0;
|
|
382
|
+
const totalTokens = tokensInput + tokensOutput;
|
|
383
|
+
const totalCost = (this.counters.get('ai.cost.total') || 0) / 10000;
|
|
384
|
+
const tasksCompleted = this.counters.get('tasks.completed') || 0;
|
|
385
|
+
const crashes = this.counters.get('tasks.recovered') || 0;
|
|
386
|
+
// Calculate tokens and time saved from recovery events
|
|
387
|
+
let tokensSavedFromRecovery = 0;
|
|
388
|
+
let timeSavedFromRecovery = 0;
|
|
389
|
+
let totalCheckpointAge = 0;
|
|
390
|
+
for (const event of this.recoveryEvents) {
|
|
391
|
+
// Without x404-r, we'd have to re-run all tokens before crash
|
|
392
|
+
tokensSavedFromRecovery += event.tokensBeforeCrash;
|
|
393
|
+
timeSavedFromRecovery += event.timeBeforeCrash;
|
|
394
|
+
totalCheckpointAge += event.checkpointAgeMs;
|
|
395
|
+
}
|
|
396
|
+
// Cost calculation (using approximate pricing)
|
|
397
|
+
// Claude: ~$3/1M input, ~$15/1M output (Sonnet pricing)
|
|
398
|
+
const avgCostPerToken = 0.000009; // ~$9 per 1M tokens blended
|
|
399
|
+
const costSaved = tokensSavedFromRecovery * avgCostPerToken;
|
|
400
|
+
// What would have happened without x404-r
|
|
401
|
+
const tokensWithoutX404r = totalTokens + tokensSavedFromRecovery;
|
|
402
|
+
const costWithoutX404r = totalCost + costSaved;
|
|
403
|
+
// Time estimation: assume average task latency from histogram
|
|
404
|
+
const taskLatency = this.histograms.get('task.latency');
|
|
405
|
+
const avgTaskTimeMs = taskLatency?.average() || 5000;
|
|
406
|
+
const totalTimeMs = tasksCompleted * avgTaskTimeMs;
|
|
407
|
+
const timeWithoutX404r = totalTimeMs + timeSavedFromRecovery;
|
|
408
|
+
// Recovery quality
|
|
409
|
+
const recoveryTime = this.histograms.get('recovery.time');
|
|
410
|
+
const checkpointAgeHist = this.histograms.get('recovery.checkpoint_age');
|
|
411
|
+
const failedRecoveries = this.counters.get('recovery.failed') || 0;
|
|
412
|
+
const successfulRecoveries = crashes;
|
|
413
|
+
const totalRecoveryAttempts = successfulRecoveries + failedRecoveries;
|
|
414
|
+
return {
|
|
415
|
+
actual: {
|
|
416
|
+
crashes,
|
|
417
|
+
tokensUsed: totalTokens,
|
|
418
|
+
costUsd: totalCost,
|
|
419
|
+
timeMs: totalTimeMs,
|
|
420
|
+
tasksCompleted,
|
|
421
|
+
},
|
|
422
|
+
withoutX404r: {
|
|
423
|
+
tokensRequired: tokensWithoutX404r,
|
|
424
|
+
costUsd: costWithoutX404r,
|
|
425
|
+
timeMs: timeWithoutX404r,
|
|
426
|
+
tasksRestarted: crashes, // Each crash = full restart
|
|
427
|
+
},
|
|
428
|
+
savings: {
|
|
429
|
+
tokensSaved: tokensSavedFromRecovery,
|
|
430
|
+
costSavedUsd: costSaved,
|
|
431
|
+
timeSavedMs: timeSavedFromRecovery,
|
|
432
|
+
percentTokensSaved: tokensWithoutX404r > 0
|
|
433
|
+
? (tokensSavedFromRecovery / tokensWithoutX404r) * 100
|
|
434
|
+
: 0,
|
|
435
|
+
percentCostSaved: costWithoutX404r > 0
|
|
436
|
+
? (costSaved / costWithoutX404r) * 100
|
|
437
|
+
: 0,
|
|
438
|
+
percentTimeSaved: timeWithoutX404r > 0
|
|
439
|
+
? (timeSavedFromRecovery / timeWithoutX404r) * 100
|
|
440
|
+
: 0,
|
|
441
|
+
},
|
|
442
|
+
quality: {
|
|
443
|
+
recoverySuccessRate: totalRecoveryAttempts > 0
|
|
444
|
+
? (successfulRecoveries / totalRecoveryAttempts) * 100
|
|
445
|
+
: 100,
|
|
446
|
+
avgCheckpointAge: checkpointAgeHist?.average() || 0,
|
|
447
|
+
avgRecoveryTime: recoveryTime?.average() || 0,
|
|
448
|
+
},
|
|
449
|
+
};
|
|
450
|
+
}
|
|
451
|
+
/**
|
|
452
|
+
* Export metrics in Prometheus format
|
|
453
|
+
*/
|
|
454
|
+
toPrometheus() {
|
|
455
|
+
const lines = [];
|
|
456
|
+
for (const [name, value] of this.counters.entries()) {
|
|
457
|
+
const metricName = name.replace(/\./g, '_');
|
|
458
|
+
lines.push(`# TYPE ${metricName} counter`);
|
|
459
|
+
lines.push(`${metricName} ${value}`);
|
|
460
|
+
}
|
|
461
|
+
for (const [name, value] of this.gauges.entries()) {
|
|
462
|
+
const metricName = name.replace(/\./g, '_');
|
|
463
|
+
lines.push(`# TYPE ${metricName} gauge`);
|
|
464
|
+
lines.push(`${metricName} ${value}`);
|
|
465
|
+
}
|
|
466
|
+
for (const [name, histogram] of this.histograms.entries()) {
|
|
467
|
+
const metricName = name.replace(/\./g, '_');
|
|
468
|
+
lines.push(`# TYPE ${metricName} histogram`);
|
|
469
|
+
lines.push(`${metricName}_p50 ${histogram.percentile(50)}`);
|
|
470
|
+
lines.push(`${metricName}_p95 ${histogram.percentile(95)}`);
|
|
471
|
+
lines.push(`${metricName}_p99 ${histogram.percentile(99)}`);
|
|
472
|
+
lines.push(`${metricName}_avg ${histogram.average()}`);
|
|
473
|
+
lines.push(`${metricName}_count ${histogram.count()}`);
|
|
474
|
+
}
|
|
475
|
+
return lines.join('\n');
|
|
476
|
+
}
|
|
477
|
+
/**
|
|
478
|
+
* Reset all metrics
|
|
479
|
+
*/
|
|
480
|
+
reset() {
|
|
481
|
+
this.counters.clear();
|
|
482
|
+
this.gauges.clear();
|
|
483
|
+
this.events = [];
|
|
484
|
+
for (const histogram of this.histograms.values()) {
|
|
485
|
+
histogram.reset();
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
// ============ Database Persistence ============
|
|
489
|
+
/**
|
|
490
|
+
* Configure database connection for metrics persistence
|
|
491
|
+
* Call this to enable automatic flushing to CockroachDB
|
|
492
|
+
*/
|
|
493
|
+
setDatabase(config) {
|
|
494
|
+
this.dbConfig = config;
|
|
495
|
+
// Start automatic flush interval
|
|
496
|
+
if (config.enabled !== false) {
|
|
497
|
+
const intervalMs = config.flushIntervalMs || 30000;
|
|
498
|
+
this.startAutoFlush(intervalMs);
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
/**
|
|
502
|
+
* Start automatic periodic flushing
|
|
503
|
+
*/
|
|
504
|
+
startAutoFlush(intervalMs) {
|
|
505
|
+
if (this.flushInterval) {
|
|
506
|
+
clearInterval(this.flushInterval);
|
|
507
|
+
}
|
|
508
|
+
this.flushInterval = setInterval(async () => {
|
|
509
|
+
try {
|
|
510
|
+
await this.flush('periodic');
|
|
511
|
+
}
|
|
512
|
+
catch (error) {
|
|
513
|
+
console.error('[x404-r Metrics] Auto-flush failed:', error);
|
|
514
|
+
}
|
|
515
|
+
}, intervalMs);
|
|
516
|
+
// Don't prevent process exit
|
|
517
|
+
this.flushInterval.unref();
|
|
518
|
+
}
|
|
519
|
+
/**
|
|
520
|
+
* Stop automatic flushing
|
|
521
|
+
*/
|
|
522
|
+
stopAutoFlush() {
|
|
523
|
+
if (this.flushInterval) {
|
|
524
|
+
clearInterval(this.flushInterval);
|
|
525
|
+
this.flushInterval = null;
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
/**
|
|
529
|
+
* Flush current metrics to the database
|
|
530
|
+
* @param snapshotType - Type of snapshot: 'periodic', 'flush', 'shutdown'
|
|
531
|
+
*/
|
|
532
|
+
async flush(snapshotType = 'flush') {
|
|
533
|
+
if (!this.dbConfig) {
|
|
534
|
+
return; // No database configured, skip
|
|
535
|
+
}
|
|
536
|
+
if (this.flushInProgress) {
|
|
537
|
+
return; // Prevent concurrent flushes
|
|
538
|
+
}
|
|
539
|
+
this.flushInProgress = true;
|
|
540
|
+
const now = Date.now();
|
|
541
|
+
const periodSeconds = Math.round((now - (this.lastFlushTime || now)) / 1000);
|
|
542
|
+
try {
|
|
543
|
+
const summary = this.getSummary();
|
|
544
|
+
await this.dbConfig.pool.query(`INSERT INTO metrics_snapshots (
|
|
545
|
+
tenant_id, snapshot_type, period_seconds,
|
|
546
|
+
tasks_total, tasks_completed, tasks_failed, tasks_pending, tasks_running,
|
|
547
|
+
success_rate, throughput_per_min,
|
|
548
|
+
total_cost_usd, tokens_input, tokens_output,
|
|
549
|
+
latency_p50_ms, latency_p95_ms, latency_p99_ms, latency_avg_ms,
|
|
550
|
+
ai_latency_ms, checkpoint_latency_ms,
|
|
551
|
+
crash_recoveries, checkpoint_hit_rate,
|
|
552
|
+
ai_generations, model_breakdown,
|
|
553
|
+
memory_vectors_stored, avg_retrieval_relevance,
|
|
554
|
+
full_snapshot
|
|
555
|
+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)`, [
|
|
556
|
+
this.dbConfig.tenantId,
|
|
557
|
+
snapshotType,
|
|
558
|
+
periodSeconds || 30,
|
|
559
|
+
summary.execution.tasksTotal,
|
|
560
|
+
summary.execution.tasksCompleted,
|
|
561
|
+
summary.execution.tasksFailed,
|
|
562
|
+
summary.execution.tasksPending,
|
|
563
|
+
summary.execution.tasksRunning,
|
|
564
|
+
summary.execution.successRate,
|
|
565
|
+
summary.execution.throughput,
|
|
566
|
+
summary.cost.totalCostUsd,
|
|
567
|
+
summary.cost.tokensInput,
|
|
568
|
+
summary.cost.tokensOutput,
|
|
569
|
+
summary.performance.latencyP50Ms,
|
|
570
|
+
summary.performance.latencyP95Ms,
|
|
571
|
+
summary.performance.latencyP99Ms,
|
|
572
|
+
summary.performance.latencyAvgMs,
|
|
573
|
+
summary.performance.aiLatencyMs,
|
|
574
|
+
summary.performance.checkpointLatencyMs,
|
|
575
|
+
summary.reliability.crashRecoveries,
|
|
576
|
+
summary.reliability.checkpointHitRate,
|
|
577
|
+
summary.ai.totalGenerations,
|
|
578
|
+
JSON.stringify(summary.ai.modelBreakdown),
|
|
579
|
+
summary.memory.vectorCount,
|
|
580
|
+
Math.min(summary.memory.avgRetrievalRelevance / 100, 1), // Normalize to 0-1 range
|
|
581
|
+
JSON.stringify(summary),
|
|
582
|
+
]);
|
|
583
|
+
this.lastFlushTime = now;
|
|
584
|
+
}
|
|
585
|
+
finally {
|
|
586
|
+
this.flushInProgress = false;
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
/**
|
|
590
|
+
* Query historical metrics from database
|
|
591
|
+
*/
|
|
592
|
+
async getHistory(options = {}) {
|
|
593
|
+
if (!this.dbConfig) {
|
|
594
|
+
return [];
|
|
595
|
+
}
|
|
596
|
+
const hours = options.hours || 24;
|
|
597
|
+
const limit = options.limit || 100;
|
|
598
|
+
const result = await this.dbConfig.pool.query(`SELECT full_snapshot FROM metrics_snapshots
|
|
599
|
+
WHERE tenant_id = $1
|
|
600
|
+
AND created_at > now() - INTERVAL '${hours} hours'
|
|
601
|
+
ORDER BY created_at DESC
|
|
602
|
+
LIMIT $2`, [this.dbConfig.tenantId, limit]);
|
|
603
|
+
return result.rows.map(r => r.full_snapshot);
|
|
604
|
+
}
|
|
605
|
+
/**
|
|
606
|
+
* Get aggregated hourly metrics
|
|
607
|
+
*/
|
|
608
|
+
async getHourlyMetrics(hours = 24) {
|
|
609
|
+
if (!this.dbConfig) {
|
|
610
|
+
return [];
|
|
611
|
+
}
|
|
612
|
+
const result = await this.dbConfig.pool.query(`SELECT
|
|
613
|
+
date_trunc('hour', created_at) as hour,
|
|
614
|
+
AVG(tasks_completed)::INT as avg_tasks_completed,
|
|
615
|
+
AVG(tasks_failed)::INT as avg_tasks_failed,
|
|
616
|
+
AVG(success_rate) as avg_success_rate,
|
|
617
|
+
SUM(total_cost_usd) as total_cost,
|
|
618
|
+
AVG(latency_p50_ms) as avg_latency_p50
|
|
619
|
+
FROM metrics_snapshots
|
|
620
|
+
WHERE tenant_id = $1
|
|
621
|
+
AND created_at > now() - INTERVAL '${hours} hours'
|
|
622
|
+
GROUP BY date_trunc('hour', created_at)
|
|
623
|
+
ORDER BY hour DESC`, [this.dbConfig.tenantId]);
|
|
624
|
+
return result.rows.map(r => ({
|
|
625
|
+
hour: r.hour,
|
|
626
|
+
avgTasksCompleted: r.avg_tasks_completed,
|
|
627
|
+
avgTasksFailed: r.avg_tasks_failed,
|
|
628
|
+
avgSuccessRate: Number(r.avg_success_rate),
|
|
629
|
+
totalCost: Number(r.total_cost),
|
|
630
|
+
avgLatencyP50: Number(r.avg_latency_p50),
|
|
631
|
+
}));
|
|
632
|
+
}
|
|
633
|
+
/**
|
|
634
|
+
* Graceful shutdown - flush final metrics
|
|
635
|
+
*/
|
|
636
|
+
async shutdown() {
|
|
637
|
+
this.stopAutoFlush();
|
|
638
|
+
await this.flush('shutdown');
|
|
639
|
+
}
|
|
640
|
+
/**
|
|
641
|
+
* Check if database persistence is configured
|
|
642
|
+
*/
|
|
643
|
+
isDatabaseEnabled() {
|
|
644
|
+
return this.dbConfig !== null && this.dbConfig.enabled !== false;
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
// Global metrics instance
|
|
648
|
+
export const metrics = new MetricsCollector();
|
|
649
|
+
/**
|
|
650
|
+
* Decorator for automatic metrics collection
|
|
651
|
+
*/
|
|
652
|
+
export function observe(name) {
|
|
653
|
+
return function (target, propertyKey, descriptor) {
|
|
654
|
+
const original = descriptor.value;
|
|
655
|
+
descriptor.value = async function (...args) {
|
|
656
|
+
const stopTimer = metrics.startTimer(`${name}.duration`);
|
|
657
|
+
metrics.increment(`${name}.calls`);
|
|
658
|
+
try {
|
|
659
|
+
const result = await original.apply(this, args);
|
|
660
|
+
metrics.increment(`${name}.success`);
|
|
661
|
+
return result;
|
|
662
|
+
}
|
|
663
|
+
catch (error) {
|
|
664
|
+
metrics.increment(`${name}.errors`);
|
|
665
|
+
throw error;
|
|
666
|
+
}
|
|
667
|
+
finally {
|
|
668
|
+
stopTimer();
|
|
669
|
+
}
|
|
670
|
+
};
|
|
671
|
+
return descriptor;
|
|
672
|
+
};
|
|
673
|
+
}
|
|
674
|
+
/**
|
|
675
|
+
* Helper to wrap async functions with metrics
|
|
676
|
+
*/
|
|
677
|
+
export function withMetrics(name, fn, tags = {}) {
|
|
678
|
+
const stopTimer = metrics.startTimer(`${name}.duration`);
|
|
679
|
+
metrics.increment(`${name}.calls`, 1, tags);
|
|
680
|
+
return fn()
|
|
681
|
+
.then((result) => {
|
|
682
|
+
metrics.increment(`${name}.success`, 1, tags);
|
|
683
|
+
return result;
|
|
684
|
+
})
|
|
685
|
+
.catch((error) => {
|
|
686
|
+
metrics.increment(`${name}.errors`, 1, tags);
|
|
687
|
+
throw error;
|
|
688
|
+
})
|
|
689
|
+
.finally(() => {
|
|
690
|
+
stopTimer();
|
|
691
|
+
});
|
|
692
|
+
}
|
|
693
|
+
//# sourceMappingURL=metrics.js.map
|