@orka-js/devtools 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/index.cjs +857 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +300 -0
- package/dist/index.d.ts +300 -0
- package/dist/index.js +811 -0
- package/dist/index.js.map +1 -0
- package/package.json +62 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,811 @@
|
|
|
1
|
+
// src/collector.ts
|
|
2
|
+
import { generateId } from "@orka-js/core";
|
|
3
|
+
var TraceCollector = class {
|
|
4
|
+
sessions = /* @__PURE__ */ new Map();
|
|
5
|
+
activeSessionId;
|
|
6
|
+
runStack = /* @__PURE__ */ new Map();
|
|
7
|
+
maxTraces;
|
|
8
|
+
retentionMs;
|
|
9
|
+
listeners = /* @__PURE__ */ new Set();
|
|
10
|
+
constructor(config = {}) {
|
|
11
|
+
this.maxTraces = config.maxTraces ?? 1e3;
|
|
12
|
+
this.retentionMs = config.retentionMs ?? 24 * 60 * 60 * 1e3;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Start a new trace session
|
|
16
|
+
*/
|
|
17
|
+
startSession(name) {
|
|
18
|
+
const sessionId = generateId();
|
|
19
|
+
const session = {
|
|
20
|
+
id: sessionId,
|
|
21
|
+
name: name ?? `Session ${this.sessions.size + 1}`,
|
|
22
|
+
startTime: Date.now(),
|
|
23
|
+
runs: []
|
|
24
|
+
};
|
|
25
|
+
this.sessions.set(sessionId, session);
|
|
26
|
+
this.activeSessionId = sessionId;
|
|
27
|
+
this.runStack.set(sessionId, []);
|
|
28
|
+
this.emit({
|
|
29
|
+
type: "session:start",
|
|
30
|
+
timestamp: Date.now(),
|
|
31
|
+
sessionId
|
|
32
|
+
});
|
|
33
|
+
this.cleanup();
|
|
34
|
+
return sessionId;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* End the current session
|
|
38
|
+
*/
|
|
39
|
+
endSession(sessionId) {
|
|
40
|
+
const id = sessionId ?? this.activeSessionId;
|
|
41
|
+
if (!id) return;
|
|
42
|
+
const session = this.sessions.get(id);
|
|
43
|
+
if (session) {
|
|
44
|
+
session.endTime = Date.now();
|
|
45
|
+
this.emit({
|
|
46
|
+
type: "session:end",
|
|
47
|
+
timestamp: Date.now(),
|
|
48
|
+
sessionId: id
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
if (this.activeSessionId === id) {
|
|
52
|
+
this.activeSessionId = void 0;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Start a new trace run
|
|
57
|
+
*/
|
|
58
|
+
startRun(type, name, input, metadata) {
|
|
59
|
+
const sessionId = this.activeSessionId ?? this.startSession();
|
|
60
|
+
const session = this.sessions.get(sessionId);
|
|
61
|
+
const stack = this.runStack.get(sessionId);
|
|
62
|
+
const run = {
|
|
63
|
+
id: generateId(),
|
|
64
|
+
parentId: stack.length > 0 ? stack[stack.length - 1].id : void 0,
|
|
65
|
+
type,
|
|
66
|
+
name,
|
|
67
|
+
startTime: Date.now(),
|
|
68
|
+
status: "running",
|
|
69
|
+
input,
|
|
70
|
+
metadata,
|
|
71
|
+
children: []
|
|
72
|
+
};
|
|
73
|
+
if (stack.length > 0) {
|
|
74
|
+
stack[stack.length - 1].children.push(run);
|
|
75
|
+
} else {
|
|
76
|
+
session.runs.push(run);
|
|
77
|
+
}
|
|
78
|
+
stack.push(run);
|
|
79
|
+
this.emit({
|
|
80
|
+
type: "run:start",
|
|
81
|
+
timestamp: Date.now(),
|
|
82
|
+
sessionId,
|
|
83
|
+
run
|
|
84
|
+
});
|
|
85
|
+
return run.id;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* End a trace run
|
|
89
|
+
*/
|
|
90
|
+
endRun(runId, output, metadata) {
|
|
91
|
+
const sessionId = this.activeSessionId;
|
|
92
|
+
if (!sessionId) return;
|
|
93
|
+
const stack = this.runStack.get(sessionId);
|
|
94
|
+
if (!stack) return;
|
|
95
|
+
const runIndex = stack.findIndex((r) => r.id === runId);
|
|
96
|
+
if (runIndex === -1) return;
|
|
97
|
+
const run = stack[runIndex];
|
|
98
|
+
run.endTime = Date.now();
|
|
99
|
+
run.latencyMs = run.endTime - run.startTime;
|
|
100
|
+
run.status = "success";
|
|
101
|
+
run.output = output;
|
|
102
|
+
if (metadata) {
|
|
103
|
+
run.metadata = { ...run.metadata, ...metadata };
|
|
104
|
+
}
|
|
105
|
+
stack.splice(runIndex, 1);
|
|
106
|
+
this.emit({
|
|
107
|
+
type: "run:end",
|
|
108
|
+
timestamp: Date.now(),
|
|
109
|
+
sessionId,
|
|
110
|
+
run
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Mark a run as errored
|
|
115
|
+
*/
|
|
116
|
+
errorRun(runId, error) {
|
|
117
|
+
const sessionId = this.activeSessionId;
|
|
118
|
+
if (!sessionId) return;
|
|
119
|
+
const stack = this.runStack.get(sessionId);
|
|
120
|
+
if (!stack) return;
|
|
121
|
+
const runIndex = stack.findIndex((r) => r.id === runId);
|
|
122
|
+
if (runIndex === -1) return;
|
|
123
|
+
const run = stack[runIndex];
|
|
124
|
+
run.endTime = Date.now();
|
|
125
|
+
run.latencyMs = run.endTime - run.startTime;
|
|
126
|
+
run.status = "error";
|
|
127
|
+
run.error = error instanceof Error ? error.message : error;
|
|
128
|
+
stack.splice(runIndex, 1);
|
|
129
|
+
this.emit({
|
|
130
|
+
type: "run:error",
|
|
131
|
+
timestamp: Date.now(),
|
|
132
|
+
sessionId,
|
|
133
|
+
run,
|
|
134
|
+
error: run.error
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Get all sessions
|
|
139
|
+
*/
|
|
140
|
+
getSessions() {
|
|
141
|
+
return Array.from(this.sessions.values());
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Get a specific session
|
|
145
|
+
*/
|
|
146
|
+
getSession(sessionId) {
|
|
147
|
+
return this.sessions.get(sessionId);
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Get metrics for a session or all sessions
|
|
151
|
+
*/
|
|
152
|
+
getMetrics(sessionId) {
|
|
153
|
+
const sessions = sessionId ? [this.sessions.get(sessionId)].filter(Boolean) : Array.from(this.sessions.values());
|
|
154
|
+
const metrics = {
|
|
155
|
+
totalRuns: 0,
|
|
156
|
+
totalLatencyMs: 0,
|
|
157
|
+
avgLatencyMs: 0,
|
|
158
|
+
totalTokens: 0,
|
|
159
|
+
totalCost: 0,
|
|
160
|
+
errorRate: 0,
|
|
161
|
+
runsByType: {},
|
|
162
|
+
tokensByModel: {},
|
|
163
|
+
costByModel: {}
|
|
164
|
+
};
|
|
165
|
+
let errorCount = 0;
|
|
166
|
+
const processRun = (run) => {
|
|
167
|
+
metrics.totalRuns++;
|
|
168
|
+
metrics.totalLatencyMs += run.latencyMs ?? 0;
|
|
169
|
+
if (run.status === "error") errorCount++;
|
|
170
|
+
metrics.runsByType[run.type] = (metrics.runsByType[run.type] ?? 0) + 1;
|
|
171
|
+
if (run.metadata) {
|
|
172
|
+
const { totalTokens, cost, model } = run.metadata;
|
|
173
|
+
if (totalTokens) {
|
|
174
|
+
metrics.totalTokens += totalTokens;
|
|
175
|
+
if (model) {
|
|
176
|
+
metrics.tokensByModel[model] = (metrics.tokensByModel[model] ?? 0) + totalTokens;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
if (cost) {
|
|
180
|
+
metrics.totalCost += cost;
|
|
181
|
+
if (model) {
|
|
182
|
+
metrics.costByModel[model] = (metrics.costByModel[model] ?? 0) + cost;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
for (const child of run.children) {
|
|
187
|
+
processRun(child);
|
|
188
|
+
}
|
|
189
|
+
};
|
|
190
|
+
for (const session of sessions) {
|
|
191
|
+
for (const run of session.runs) {
|
|
192
|
+
processRun(run);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
metrics.avgLatencyMs = metrics.totalRuns > 0 ? metrics.totalLatencyMs / metrics.totalRuns : 0;
|
|
196
|
+
metrics.errorRate = metrics.totalRuns > 0 ? errorCount / metrics.totalRuns : 0;
|
|
197
|
+
return metrics;
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Find a run by ID
|
|
201
|
+
*/
|
|
202
|
+
findRun(runId, sessionId) {
|
|
203
|
+
const sessions = sessionId ? [this.sessions.get(sessionId)].filter(Boolean) : Array.from(this.sessions.values());
|
|
204
|
+
const findInRuns = (runs) => {
|
|
205
|
+
for (const run of runs) {
|
|
206
|
+
if (run.id === runId) return run;
|
|
207
|
+
const found = findInRuns(run.children);
|
|
208
|
+
if (found) return found;
|
|
209
|
+
}
|
|
210
|
+
return void 0;
|
|
211
|
+
};
|
|
212
|
+
for (const session of sessions) {
|
|
213
|
+
const found = findInRuns(session.runs);
|
|
214
|
+
if (found) return found;
|
|
215
|
+
}
|
|
216
|
+
return void 0;
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Subscribe to trace events
|
|
220
|
+
*/
|
|
221
|
+
subscribe(listener) {
|
|
222
|
+
this.listeners.add(listener);
|
|
223
|
+
return () => this.listeners.delete(listener);
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Emit an event to all listeners
|
|
227
|
+
*/
|
|
228
|
+
emit(event) {
|
|
229
|
+
for (const listener of this.listeners) {
|
|
230
|
+
try {
|
|
231
|
+
listener(event);
|
|
232
|
+
} catch {
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Cleanup old sessions
|
|
238
|
+
*/
|
|
239
|
+
cleanup() {
|
|
240
|
+
const now = Date.now();
|
|
241
|
+
const cutoff = now - this.retentionMs;
|
|
242
|
+
for (const [id, session] of this.sessions) {
|
|
243
|
+
if (session.endTime && session.endTime < cutoff) {
|
|
244
|
+
this.sessions.delete(id);
|
|
245
|
+
this.runStack.delete(id);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
if (this.sessions.size > this.maxTraces) {
|
|
249
|
+
const sorted = Array.from(this.sessions.entries()).sort((a, b) => a[1].startTime - b[1].startTime);
|
|
250
|
+
const toDelete = sorted.slice(0, this.sessions.size - this.maxTraces);
|
|
251
|
+
for (const [id] of toDelete) {
|
|
252
|
+
this.sessions.delete(id);
|
|
253
|
+
this.runStack.delete(id);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
258
|
+
* Clear all traces
|
|
259
|
+
*/
|
|
260
|
+
clear() {
|
|
261
|
+
this.sessions.clear();
|
|
262
|
+
this.runStack.clear();
|
|
263
|
+
this.activeSessionId = void 0;
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Export traces as JSON
|
|
267
|
+
*/
|
|
268
|
+
export() {
|
|
269
|
+
return JSON.stringify({
|
|
270
|
+
sessions: Array.from(this.sessions.values()),
|
|
271
|
+
exportedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
272
|
+
}, null, 2);
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* Import traces from JSON
|
|
276
|
+
*/
|
|
277
|
+
import(json) {
|
|
278
|
+
const data = JSON.parse(json);
|
|
279
|
+
for (const session of data.sessions) {
|
|
280
|
+
this.sessions.set(session.id, session);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
};
|
|
284
|
+
var globalCollector;
|
|
285
|
+
function getCollector(config) {
|
|
286
|
+
if (!globalCollector) {
|
|
287
|
+
globalCollector = new TraceCollector(config);
|
|
288
|
+
}
|
|
289
|
+
return globalCollector;
|
|
290
|
+
}
|
|
291
|
+
function resetCollector() {
|
|
292
|
+
globalCollector = void 0;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
// src/server.ts
|
|
296
|
+
var DevToolsServer = class {
|
|
297
|
+
collector;
|
|
298
|
+
config;
|
|
299
|
+
server;
|
|
300
|
+
clients = /* @__PURE__ */ new Set();
|
|
301
|
+
constructor(collector, config = {}) {
|
|
302
|
+
this.collector = collector;
|
|
303
|
+
this.config = {
|
|
304
|
+
port: config.port ?? 3001,
|
|
305
|
+
host: config.host ?? "localhost",
|
|
306
|
+
cors: config.cors ?? true
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Start the DevTools server
|
|
311
|
+
*/
|
|
312
|
+
async start() {
|
|
313
|
+
let expressModule;
|
|
314
|
+
let httpModule;
|
|
315
|
+
try {
|
|
316
|
+
expressModule = await import("express");
|
|
317
|
+
httpModule = await import("http");
|
|
318
|
+
} catch {
|
|
319
|
+
throw new Error(
|
|
320
|
+
"Express is required for DevTools server. Install it with: npm install express"
|
|
321
|
+
);
|
|
322
|
+
}
|
|
323
|
+
const app = expressModule.default();
|
|
324
|
+
if (this.config.cors) {
|
|
325
|
+
app.use((_req, res, next) => {
|
|
326
|
+
res.header("Access-Control-Allow-Origin", "*");
|
|
327
|
+
res.header("Access-Control-Allow-Headers", "Content-Type");
|
|
328
|
+
res.header("Access-Control-Allow-Methods", "GET, POST, DELETE");
|
|
329
|
+
next();
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
app.use(expressModule.json());
|
|
333
|
+
this.setupRoutes(app);
|
|
334
|
+
this.server = httpModule.createServer(app);
|
|
335
|
+
await new Promise((resolve) => {
|
|
336
|
+
this.server.listen(this.config.port, this.config.host, () => {
|
|
337
|
+
console.log(`
|
|
338
|
+
\u{1F50D} OrkaJS DevTools running at http://${this.config.host}:${this.config.port}
|
|
339
|
+
`);
|
|
340
|
+
resolve();
|
|
341
|
+
});
|
|
342
|
+
});
|
|
343
|
+
this.collector.subscribe((event) => {
|
|
344
|
+
this.broadcastEvent(event);
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Stop the server
|
|
349
|
+
*/
|
|
350
|
+
async stop() {
|
|
351
|
+
if (this.server) {
|
|
352
|
+
await new Promise((resolve, reject) => {
|
|
353
|
+
this.server.close((err) => {
|
|
354
|
+
if (err) reject(err);
|
|
355
|
+
else resolve();
|
|
356
|
+
});
|
|
357
|
+
});
|
|
358
|
+
this.server = void 0;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Setup API routes
|
|
363
|
+
*/
|
|
364
|
+
setupRoutes(app) {
|
|
365
|
+
app.get("/api/health", (_req, res) => {
|
|
366
|
+
res.json({ status: "ok", timestamp: Date.now() });
|
|
367
|
+
});
|
|
368
|
+
app.get("/api/sessions", (_req, res) => {
|
|
369
|
+
const sessions = this.collector.getSessions();
|
|
370
|
+
res.json(sessions);
|
|
371
|
+
});
|
|
372
|
+
app.get("/api/sessions/:id", (req, res) => {
|
|
373
|
+
const session = this.collector.getSession(req.params.id);
|
|
374
|
+
if (!session) {
|
|
375
|
+
res.status(404).json({ error: "Session not found" });
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
res.json(session);
|
|
379
|
+
});
|
|
380
|
+
app.get("/api/metrics", (req, res) => {
|
|
381
|
+
const sessionId = req.query.sessionId;
|
|
382
|
+
const metrics = this.collector.getMetrics(sessionId);
|
|
383
|
+
res.json(metrics);
|
|
384
|
+
});
|
|
385
|
+
app.get("/api/runs/:id", (req, res) => {
|
|
386
|
+
const sessionId = req.query.sessionId;
|
|
387
|
+
const run = this.collector.findRun(req.params.id, sessionId);
|
|
388
|
+
if (!run) {
|
|
389
|
+
res.status(404).json({ error: "Run not found" });
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
res.json(run);
|
|
393
|
+
});
|
|
394
|
+
app.delete("/api/sessions", (_req, res) => {
|
|
395
|
+
this.collector.clear();
|
|
396
|
+
res.json({ success: true });
|
|
397
|
+
});
|
|
398
|
+
app.get("/api/export", (_req, res) => {
|
|
399
|
+
const data = this.collector.export();
|
|
400
|
+
res.setHeader("Content-Type", "application/json");
|
|
401
|
+
res.setHeader("Content-Disposition", "attachment; filename=orka-traces.json");
|
|
402
|
+
res.send(data);
|
|
403
|
+
});
|
|
404
|
+
app.post("/api/import", (req, res) => {
|
|
405
|
+
try {
|
|
406
|
+
this.collector.import(JSON.stringify(req.body));
|
|
407
|
+
res.json({ success: true });
|
|
408
|
+
} catch {
|
|
409
|
+
res.status(400).json({ error: "Invalid trace data" });
|
|
410
|
+
}
|
|
411
|
+
});
|
|
412
|
+
app.get("/api/events", (req, res) => {
|
|
413
|
+
res.setHeader("Content-Type", "text/event-stream");
|
|
414
|
+
res.setHeader("Cache-Control", "no-cache");
|
|
415
|
+
res.setHeader("Connection", "keep-alive");
|
|
416
|
+
this.clients.add(res);
|
|
417
|
+
req.on("close", () => {
|
|
418
|
+
this.clients.delete(res);
|
|
419
|
+
});
|
|
420
|
+
});
|
|
421
|
+
app.get("/", (_req, res) => {
|
|
422
|
+
res.send(this.getDashboardHTML());
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* Broadcast event to all SSE clients
|
|
427
|
+
*/
|
|
428
|
+
broadcastEvent(event) {
|
|
429
|
+
const data = JSON.stringify(event);
|
|
430
|
+
for (const client of this.clients) {
|
|
431
|
+
client.write(`data: ${data}
|
|
432
|
+
|
|
433
|
+
`);
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
/**
|
|
437
|
+
* Get embedded dashboard HTML
|
|
438
|
+
*/
|
|
439
|
+
getDashboardHTML() {
|
|
440
|
+
return `<!DOCTYPE html>
|
|
441
|
+
<html lang="en">
|
|
442
|
+
<head>
|
|
443
|
+
<meta charset="UTF-8">
|
|
444
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
445
|
+
<title>OrkaJS DevTools</title>
|
|
446
|
+
<script src="https://cdn.tailwindcss.com"></script>
|
|
447
|
+
<style>
|
|
448
|
+
.tree-line { border-left: 2px solid #e2e8f0; }
|
|
449
|
+
.dark .tree-line { border-left-color: #334155; }
|
|
450
|
+
</style>
|
|
451
|
+
</head>
|
|
452
|
+
<body class="bg-slate-50 dark:bg-slate-900 text-slate-900 dark:text-white min-h-screen">
|
|
453
|
+
<div id="app" class="max-w-7xl mx-auto p-6">
|
|
454
|
+
<header class="flex items-center justify-between mb-8">
|
|
455
|
+
<div class="flex items-center gap-3">
|
|
456
|
+
<div class="w-10 h-10 bg-gradient-to-br from-purple-500 to-pink-500 rounded-lg flex items-center justify-center">
|
|
457
|
+
<svg class="w-6 h-6 text-white" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
|
458
|
+
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9.75 17L9 20l-1 1h8l-1-1-.75-3M3 13h18M5 17h14a2 2 0 002-2V5a2 2 0 00-2-2H5a2 2 0 00-2 2v10a2 2 0 002 2z"/>
|
|
459
|
+
</svg>
|
|
460
|
+
</div>
|
|
461
|
+
<div>
|
|
462
|
+
<h1 class="text-2xl font-bold">OrkaJS DevTools</h1>
|
|
463
|
+
<p class="text-sm text-slate-500">Real-time LLM observability</p>
|
|
464
|
+
</div>
|
|
465
|
+
</div>
|
|
466
|
+
<div class="flex items-center gap-4">
|
|
467
|
+
<span id="status" class="flex items-center gap-2 text-sm">
|
|
468
|
+
<span class="w-2 h-2 bg-green-500 rounded-full animate-pulse"></span>
|
|
469
|
+
Connected
|
|
470
|
+
</span>
|
|
471
|
+
<button onclick="clearTraces()" class="px-3 py-1.5 text-sm bg-red-500/10 text-red-500 rounded-lg hover:bg-red-500/20">
|
|
472
|
+
Clear
|
|
473
|
+
</button>
|
|
474
|
+
<button onclick="exportTraces()" class="px-3 py-1.5 text-sm bg-purple-500/10 text-purple-500 rounded-lg hover:bg-purple-500/20">
|
|
475
|
+
Export
|
|
476
|
+
</button>
|
|
477
|
+
</div>
|
|
478
|
+
</header>
|
|
479
|
+
|
|
480
|
+
<!-- Metrics -->
|
|
481
|
+
<div id="metrics" class="grid grid-cols-4 gap-4 mb-8">
|
|
482
|
+
<div class="bg-white dark:bg-slate-800 rounded-xl p-4 shadow-sm">
|
|
483
|
+
<p class="text-sm text-slate-500 mb-1">Total Runs</p>
|
|
484
|
+
<p id="metric-runs" class="text-2xl font-bold">0</p>
|
|
485
|
+
</div>
|
|
486
|
+
<div class="bg-white dark:bg-slate-800 rounded-xl p-4 shadow-sm">
|
|
487
|
+
<p class="text-sm text-slate-500 mb-1">Avg Latency</p>
|
|
488
|
+
<p id="metric-latency" class="text-2xl font-bold">0ms</p>
|
|
489
|
+
</div>
|
|
490
|
+
<div class="bg-white dark:bg-slate-800 rounded-xl p-4 shadow-sm">
|
|
491
|
+
<p class="text-sm text-slate-500 mb-1">Total Tokens</p>
|
|
492
|
+
<p id="metric-tokens" class="text-2xl font-bold">0</p>
|
|
493
|
+
</div>
|
|
494
|
+
<div class="bg-white dark:bg-slate-800 rounded-xl p-4 shadow-sm">
|
|
495
|
+
<p class="text-sm text-slate-500 mb-1">Error Rate</p>
|
|
496
|
+
<p id="metric-errors" class="text-2xl font-bold">0%</p>
|
|
497
|
+
</div>
|
|
498
|
+
</div>
|
|
499
|
+
|
|
500
|
+
<!-- Sessions & Traces -->
|
|
501
|
+
<div class="grid grid-cols-3 gap-6">
|
|
502
|
+
<div class="col-span-1">
|
|
503
|
+
<h2 class="text-lg font-semibold mb-4">Sessions</h2>
|
|
504
|
+
<div id="sessions" class="space-y-2"></div>
|
|
505
|
+
</div>
|
|
506
|
+
<div class="col-span-2">
|
|
507
|
+
<h2 class="text-lg font-semibold mb-4">Trace Viewer</h2>
|
|
508
|
+
<div id="traces" class="bg-white dark:bg-slate-800 rounded-xl p-4 shadow-sm min-h-[400px]">
|
|
509
|
+
<p class="text-slate-500 text-center py-8">Select a session to view traces</p>
|
|
510
|
+
</div>
|
|
511
|
+
</div>
|
|
512
|
+
</div>
|
|
513
|
+
</div>
|
|
514
|
+
|
|
515
|
+
<script>
|
|
516
|
+
let selectedSession = null;
|
|
517
|
+
|
|
518
|
+
// SSE connection
|
|
519
|
+
const events = new EventSource('/api/events');
|
|
520
|
+
events.onmessage = (e) => {
|
|
521
|
+
const event = JSON.parse(e.data);
|
|
522
|
+
console.log('Event:', event);
|
|
523
|
+
refreshData();
|
|
524
|
+
};
|
|
525
|
+
events.onerror = () => {
|
|
526
|
+
document.getElementById('status').innerHTML = '<span class="w-2 h-2 bg-red-500 rounded-full"></span> Disconnected';
|
|
527
|
+
};
|
|
528
|
+
|
|
529
|
+
async function refreshData() {
|
|
530
|
+
// Fetch metrics
|
|
531
|
+
const metrics = await fetch('/api/metrics').then(r => r.json());
|
|
532
|
+
document.getElementById('metric-runs').textContent = metrics.totalRuns;
|
|
533
|
+
document.getElementById('metric-latency').textContent = Math.round(metrics.avgLatencyMs) + 'ms';
|
|
534
|
+
document.getElementById('metric-tokens').textContent = metrics.totalTokens.toLocaleString();
|
|
535
|
+
document.getElementById('metric-errors').textContent = (metrics.errorRate * 100).toFixed(1) + '%';
|
|
536
|
+
|
|
537
|
+
// Fetch sessions
|
|
538
|
+
const sessions = await fetch('/api/sessions').then(r => r.json());
|
|
539
|
+
renderSessions(sessions);
|
|
540
|
+
|
|
541
|
+
if (selectedSession) {
|
|
542
|
+
const session = await fetch('/api/sessions/' + selectedSession).then(r => r.json());
|
|
543
|
+
renderTraces(session.runs);
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
function renderSessions(sessions) {
|
|
548
|
+
const container = document.getElementById('sessions');
|
|
549
|
+
container.innerHTML = sessions.map(s => \`
|
|
550
|
+
<div onclick="selectSession('\${s.id}')" class="p-3 rounded-lg cursor-pointer \${selectedSession === s.id ? 'bg-purple-500/20 border border-purple-500' : 'bg-white dark:bg-slate-800 hover:bg-slate-100 dark:hover:bg-slate-700'}">
|
|
551
|
+
<p class="font-medium">\${s.name || 'Session'}</p>
|
|
552
|
+
<p class="text-xs text-slate-500">\${s.runs.length} runs \u2022 \${new Date(s.startTime).toLocaleTimeString()}</p>
|
|
553
|
+
</div>
|
|
554
|
+
\`).join('');
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
function selectSession(id) {
|
|
558
|
+
selectedSession = id;
|
|
559
|
+
refreshData();
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
function renderTraces(runs, depth = 0) {
|
|
563
|
+
if (!runs || runs.length === 0) {
|
|
564
|
+
document.getElementById('traces').innerHTML = '<p class="text-slate-500 text-center py-8">No traces in this session</p>';
|
|
565
|
+
return;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
const html = runs.map(run => renderRun(run, depth)).join('');
|
|
569
|
+
document.getElementById('traces').innerHTML = html;
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
function renderRun(run, depth) {
|
|
573
|
+
const statusColor = run.status === 'success' ? 'bg-green-500' : run.status === 'error' ? 'bg-red-500' : 'bg-yellow-500';
|
|
574
|
+
const typeColors = {
|
|
575
|
+
llm: 'text-purple-500',
|
|
576
|
+
agent: 'text-blue-500',
|
|
577
|
+
tool: 'text-orange-500',
|
|
578
|
+
retrieval: 'text-green-500',
|
|
579
|
+
chain: 'text-pink-500',
|
|
580
|
+
};
|
|
581
|
+
|
|
582
|
+
return \`
|
|
583
|
+
<div class="mb-2" style="margin-left: \${depth * 20}px">
|
|
584
|
+
<div class="flex items-center gap-2 p-2 rounded-lg hover:bg-slate-100 dark:hover:bg-slate-700">
|
|
585
|
+
<span class="w-2 h-2 rounded-full \${statusColor}"></span>
|
|
586
|
+
<span class="text-xs font-medium \${typeColors[run.type] || 'text-slate-500'}">\${run.type.toUpperCase()}</span>
|
|
587
|
+
<span class="font-medium">\${run.name}</span>
|
|
588
|
+
<span class="text-xs text-slate-500 ml-auto">\${run.latencyMs ? run.latencyMs + 'ms' : 'running...'}</span>
|
|
589
|
+
\${run.metadata?.totalTokens ? '<span class="text-xs text-slate-400">' + run.metadata.totalTokens + ' tokens</span>' : ''}
|
|
590
|
+
</div>
|
|
591
|
+
\${run.children.map(c => renderRun(c, depth + 1)).join('')}
|
|
592
|
+
</div>
|
|
593
|
+
\`;
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
async function clearTraces() {
|
|
597
|
+
await fetch('/api/sessions', { method: 'DELETE' });
|
|
598
|
+
selectedSession = null;
|
|
599
|
+
refreshData();
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
function exportTraces() {
|
|
603
|
+
window.open('/api/export', '_blank');
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
// Initial load
|
|
607
|
+
refreshData();
|
|
608
|
+
</script>
|
|
609
|
+
</body>
|
|
610
|
+
</html>`;
|
|
611
|
+
}
|
|
612
|
+
};
|
|
613
|
+
|
|
614
|
+
// src/tracer-hook.ts
|
|
615
|
+
function mapEventType(type) {
|
|
616
|
+
const typeMap = {
|
|
617
|
+
"llm": "llm",
|
|
618
|
+
"llm_call": "llm",
|
|
619
|
+
"embedding": "embedding",
|
|
620
|
+
"retrieval": "retrieval",
|
|
621
|
+
"tool": "tool",
|
|
622
|
+
"tool_call": "tool",
|
|
623
|
+
"agent": "agent",
|
|
624
|
+
"agent_step": "agent",
|
|
625
|
+
"chain": "chain",
|
|
626
|
+
"workflow": "workflow",
|
|
627
|
+
"graph": "graph",
|
|
628
|
+
"node": "node"
|
|
629
|
+
};
|
|
630
|
+
return typeMap[type.toLowerCase()] ?? "custom";
|
|
631
|
+
}
|
|
632
|
+
function createDevToolsHook() {
|
|
633
|
+
const collector = getCollector();
|
|
634
|
+
const traceToSession = /* @__PURE__ */ new Map();
|
|
635
|
+
const eventToRun = /* @__PURE__ */ new Map();
|
|
636
|
+
return {
|
|
637
|
+
onTraceStart(trace2) {
|
|
638
|
+
const sessionId = collector.startSession(trace2.name);
|
|
639
|
+
traceToSession.set(trace2.id, sessionId);
|
|
640
|
+
},
|
|
641
|
+
onTraceEnd(trace2) {
|
|
642
|
+
const sessionId = traceToSession.get(trace2.id);
|
|
643
|
+
if (sessionId) {
|
|
644
|
+
collector.endSession(sessionId);
|
|
645
|
+
traceToSession.delete(trace2.id);
|
|
646
|
+
}
|
|
647
|
+
},
|
|
648
|
+
onEvent(event) {
|
|
649
|
+
const runType = mapEventType(event.type);
|
|
650
|
+
const metadata = {
|
|
651
|
+
...event.metadata
|
|
652
|
+
};
|
|
653
|
+
if (event.usage) {
|
|
654
|
+
metadata.promptTokens = event.usage.promptTokens;
|
|
655
|
+
metadata.completionTokens = event.usage.completionTokens;
|
|
656
|
+
metadata.totalTokens = event.usage.totalTokens;
|
|
657
|
+
}
|
|
658
|
+
if (event.startTime && event.endTime) {
|
|
659
|
+
const runId = collector.startRun(runType, event.name, event.input, metadata);
|
|
660
|
+
collector.endRun(runId, event.output, {
|
|
661
|
+
...metadata
|
|
662
|
+
// Override latency calculation since we have actual times
|
|
663
|
+
});
|
|
664
|
+
eventToRun.set(event.id, runId);
|
|
665
|
+
} else if (event.startTime && !event.endTime) {
|
|
666
|
+
const runId = collector.startRun(runType, event.name, event.input, metadata);
|
|
667
|
+
eventToRun.set(event.id, runId);
|
|
668
|
+
} else {
|
|
669
|
+
const runId = eventToRun.get(event.id);
|
|
670
|
+
if (runId) {
|
|
671
|
+
collector.endRun(runId, event.output, metadata);
|
|
672
|
+
eventToRun.delete(event.id);
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
},
|
|
676
|
+
onError(error, context) {
|
|
677
|
+
const sessions = collector.getSessions();
|
|
678
|
+
if (sessions.length > 0) {
|
|
679
|
+
const lastSession = sessions[sessions.length - 1];
|
|
680
|
+
if (lastSession.runs.length > 0) {
|
|
681
|
+
const findRunningRun = (runs) => {
|
|
682
|
+
for (const run of runs) {
|
|
683
|
+
if (run.status === "running") return run.id;
|
|
684
|
+
const childRun = findRunningRun(run.children);
|
|
685
|
+
if (childRun) return childRun;
|
|
686
|
+
}
|
|
687
|
+
return void 0;
|
|
688
|
+
};
|
|
689
|
+
const runningRunId = findRunningRun(lastSession.runs);
|
|
690
|
+
if (runningRunId) {
|
|
691
|
+
collector.errorRun(runningRunId, error);
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
if (context) {
|
|
696
|
+
console.error("[DevTools] Error context:", context);
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
function createTracerWithDevTools(options = {}) {
|
|
702
|
+
const hook = createDevToolsHook();
|
|
703
|
+
return {
|
|
704
|
+
hook,
|
|
705
|
+
config: {
|
|
706
|
+
...options,
|
|
707
|
+
hooks: [hook]
|
|
708
|
+
}
|
|
709
|
+
};
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
// src/index.ts
|
|
713
|
+
async function devtools(config = {}) {
|
|
714
|
+
const collector = getCollector(config);
|
|
715
|
+
const server = new DevToolsServer(collector, config);
|
|
716
|
+
await server.start();
|
|
717
|
+
if (config.open !== false) {
|
|
718
|
+
const url = `http://${config.host ?? "localhost"}:${config.port ?? 3001}`;
|
|
719
|
+
try {
|
|
720
|
+
const { exec } = await import("child_process");
|
|
721
|
+
const command = process.platform === "darwin" ? "open" : process.platform === "win32" ? "start" : "xdg-open";
|
|
722
|
+
exec(`${command} ${url}`);
|
|
723
|
+
} catch {
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
return {
|
|
727
|
+
collector,
|
|
728
|
+
server,
|
|
729
|
+
stop: () => server.stop()
|
|
730
|
+
};
|
|
731
|
+
}
|
|
732
|
+
function withTrace(fn, options = {}) {
|
|
733
|
+
const collector = options.collector ?? getCollector();
|
|
734
|
+
const name = options.name ?? fn.name ?? "anonymous";
|
|
735
|
+
const type = options.type ?? "custom";
|
|
736
|
+
return (async (...args) => {
|
|
737
|
+
const runId = collector.startRun(type, name, args);
|
|
738
|
+
try {
|
|
739
|
+
const result = await fn(...args);
|
|
740
|
+
collector.endRun(runId, result);
|
|
741
|
+
return result;
|
|
742
|
+
} catch (error) {
|
|
743
|
+
collector.errorRun(runId, error);
|
|
744
|
+
throw error;
|
|
745
|
+
}
|
|
746
|
+
});
|
|
747
|
+
}
|
|
748
|
+
function Trace(options = {}) {
|
|
749
|
+
return function(_target, propertyKey, descriptor) {
|
|
750
|
+
const originalMethod = descriptor.value;
|
|
751
|
+
const name = options.name ?? propertyKey;
|
|
752
|
+
const type = options.type ?? "custom";
|
|
753
|
+
descriptor.value = async function(...args) {
|
|
754
|
+
const collector = getCollector();
|
|
755
|
+
const runId = collector.startRun(type, name, args);
|
|
756
|
+
try {
|
|
757
|
+
const result = await originalMethod.apply(this, args);
|
|
758
|
+
collector.endRun(runId, result);
|
|
759
|
+
return result;
|
|
760
|
+
} catch (error) {
|
|
761
|
+
collector.errorRun(runId, error);
|
|
762
|
+
throw error;
|
|
763
|
+
}
|
|
764
|
+
};
|
|
765
|
+
return descriptor;
|
|
766
|
+
};
|
|
767
|
+
}
|
|
768
|
+
var trace = {
|
|
769
|
+
start(type, name, input, metadata) {
|
|
770
|
+
return getCollector().startRun(type, name, input, metadata);
|
|
771
|
+
},
|
|
772
|
+
end(runId, output, metadata) {
|
|
773
|
+
getCollector().endRun(runId, output, metadata);
|
|
774
|
+
},
|
|
775
|
+
error(runId, error) {
|
|
776
|
+
getCollector().errorRun(runId, error);
|
|
777
|
+
},
|
|
778
|
+
session(name) {
|
|
779
|
+
return getCollector().startSession(name);
|
|
780
|
+
},
|
|
781
|
+
endSession(sessionId) {
|
|
782
|
+
getCollector().endSession(sessionId);
|
|
783
|
+
},
|
|
784
|
+
/**
|
|
785
|
+
* Wrap an async function with tracing
|
|
786
|
+
*/
|
|
787
|
+
async wrap(type, name, fn, metadata) {
|
|
788
|
+
const runId = getCollector().startRun(type, name, void 0, metadata);
|
|
789
|
+
try {
|
|
790
|
+
const result = await fn();
|
|
791
|
+
getCollector().endRun(runId, result);
|
|
792
|
+
return result;
|
|
793
|
+
} catch (error) {
|
|
794
|
+
getCollector().errorRun(runId, error);
|
|
795
|
+
throw error;
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
};
|
|
799
|
+
export {
|
|
800
|
+
DevToolsServer,
|
|
801
|
+
Trace,
|
|
802
|
+
TraceCollector,
|
|
803
|
+
createDevToolsHook,
|
|
804
|
+
createTracerWithDevTools,
|
|
805
|
+
devtools,
|
|
806
|
+
getCollector,
|
|
807
|
+
resetCollector,
|
|
808
|
+
trace,
|
|
809
|
+
withTrace
|
|
810
|
+
};
|
|
811
|
+
//# sourceMappingURL=index.js.map
|