@townco/debugger 0.1.28 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -4
- package/src/App.tsx +6 -0
- package/src/analysis/analyzer.ts +272 -0
- package/src/analysis/embeddings.ts +97 -0
- package/src/analysis/schema.ts +91 -0
- package/src/analysis/types.ts +157 -0
- package/src/analysis-db.ts +238 -0
- package/src/comparison-db.test.ts +28 -5
- package/src/comparison-db.ts +57 -9
- package/src/components/AnalyzeAllButton.tsx +81 -0
- package/src/components/DebuggerHeader.tsx +12 -0
- package/src/components/SessionAnalysisButton.tsx +109 -0
- package/src/components/SessionAnalysisDialog.tsx +240 -0
- package/src/components/UnifiedTimeline.tsx +3 -3
- package/src/components/ui/dialog.tsx +120 -0
- package/src/db.ts +3 -2
- package/src/lib/metrics.ts +131 -11
- package/src/pages/ComparisonView.tsx +618 -177
- package/src/pages/FindSessions.tsx +247 -0
- package/src/pages/SessionList.tsx +76 -10
- package/src/pages/SessionView.tsx +33 -1
- package/src/pages/TownHall.tsx +345 -187
- package/src/schemas.ts +27 -8
- package/src/server.ts +423 -3
- package/src/types.ts +11 -2
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
import type { DetailedToolCall, SessionAnalysis } from "../analysis/types";
|
|
2
|
+
import { formatCost, formatDuration, formatTokens } from "../lib/metrics";
|
|
3
|
+
import { Dialog, DialogContent, DialogHeader, DialogTitle } from "./ui/dialog";
|
|
4
|
+
|
|
5
|
+
interface Props {
|
|
6
|
+
open: boolean;
|
|
7
|
+
onClose: () => void;
|
|
8
|
+
analysis: SessionAnalysis;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function formatDate(isoString: string): string {
|
|
12
|
+
return new Date(isoString).toLocaleString();
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function formatToolTime(ns?: number): string {
|
|
16
|
+
if (!ns) return "";
|
|
17
|
+
return new Date(ns / 1_000_000).toLocaleTimeString();
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function Section({
|
|
21
|
+
title,
|
|
22
|
+
children,
|
|
23
|
+
}: {
|
|
24
|
+
title: string;
|
|
25
|
+
children: React.ReactNode;
|
|
26
|
+
}) {
|
|
27
|
+
return (
|
|
28
|
+
<div className="space-y-3">
|
|
29
|
+
<h3 className="text-sm font-semibold text-foreground">{title}</h3>
|
|
30
|
+
<div className="space-y-2">{children}</div>
|
|
31
|
+
</div>
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function Field({ label, value }: { label: string; value: string }) {
|
|
36
|
+
return (
|
|
37
|
+
<div className="space-y-1">
|
|
38
|
+
<div className="text-xs font-medium text-muted-foreground">{label}</div>
|
|
39
|
+
<div className="text-sm text-foreground whitespace-pre-wrap break-words">
|
|
40
|
+
{value}
|
|
41
|
+
</div>
|
|
42
|
+
</div>
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function Badge({ label, value }: { label: string; value: string }) {
|
|
47
|
+
return (
|
|
48
|
+
<div className="space-y-1">
|
|
49
|
+
<div className="text-xs font-medium text-muted-foreground">{label}</div>
|
|
50
|
+
<div className="inline-block px-3 py-1 bg-primary/10 text-primary rounded-full text-sm font-medium">
|
|
51
|
+
{value}
|
|
52
|
+
</div>
|
|
53
|
+
</div>
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function Metric({ label, value }: { label: string; value: number }) {
|
|
58
|
+
return (
|
|
59
|
+
<div className="space-y-1">
|
|
60
|
+
<div className="text-xs font-medium text-muted-foreground">{label}</div>
|
|
61
|
+
<div className="text-2xl font-bold text-foreground">{value}</div>
|
|
62
|
+
</div>
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function ToolCallDetails({ toolCalls }: { toolCalls: DetailedToolCall[] }) {
|
|
67
|
+
if (!toolCalls || toolCalls.length === 0) {
|
|
68
|
+
return <div className="text-xs text-muted-foreground">No tool calls</div>;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
return (
|
|
72
|
+
<div className="space-y-2">
|
|
73
|
+
{toolCalls.map((call, idx) => (
|
|
74
|
+
<details
|
|
75
|
+
key={`${call.name}-${call.startTimeUnixNano ?? idx}`}
|
|
76
|
+
className="rounded-md border px-3 py-2 bg-muted/50"
|
|
77
|
+
>
|
|
78
|
+
<summary className="text-xs font-medium cursor-pointer flex items-center justify-between">
|
|
79
|
+
<span>
|
|
80
|
+
{call.name}{" "}
|
|
81
|
+
{call.startTimeUnixNano ? (
|
|
82
|
+
<span className="text-muted-foreground">
|
|
83
|
+
@ {formatToolTime(call.startTimeUnixNano)}
|
|
84
|
+
</span>
|
|
85
|
+
) : null}
|
|
86
|
+
</span>
|
|
87
|
+
<span className="text-muted-foreground text-[11px]">view</span>
|
|
88
|
+
</summary>
|
|
89
|
+
<div className="mt-2 text-[11px] space-y-1 break-words">
|
|
90
|
+
<div>
|
|
91
|
+
<span className="font-semibold">Args:</span>{" "}
|
|
92
|
+
<pre className="break-words whitespace-pre-wrap bg-muted rounded p-2 mt-1 overflow-x-auto max-h-40">
|
|
93
|
+
{JSON.stringify(call.input, null, 2)}
|
|
94
|
+
</pre>
|
|
95
|
+
</div>
|
|
96
|
+
<div>
|
|
97
|
+
<span className="font-semibold">Result:</span>{" "}
|
|
98
|
+
<pre className="break-words whitespace-pre-wrap bg-muted rounded p-2 mt-1 overflow-x-auto max-h-40">
|
|
99
|
+
{JSON.stringify(call.output, null, 2)}
|
|
100
|
+
</pre>
|
|
101
|
+
</div>
|
|
102
|
+
</div>
|
|
103
|
+
</details>
|
|
104
|
+
))}
|
|
105
|
+
</div>
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export function SessionAnalysisDialog({ open, onClose, analysis }: Props) {
|
|
110
|
+
return (
|
|
111
|
+
<Dialog open={open} onOpenChange={onClose}>
|
|
112
|
+
<DialogContent className="max-w-3xl max-h-[85vh] overflow-y-auto">
|
|
113
|
+
<DialogHeader>
|
|
114
|
+
<DialogTitle>Session Analysis</DialogTitle>
|
|
115
|
+
</DialogHeader>
|
|
116
|
+
|
|
117
|
+
<div className="space-y-6">
|
|
118
|
+
{/* Task Section */}
|
|
119
|
+
<Section title="Task">
|
|
120
|
+
<Field label="User Query" value={analysis.task.user_query} />
|
|
121
|
+
<Field label="Summary" value={analysis.task.task_summary} />
|
|
122
|
+
<Badge label="Intent" value={analysis.task.intent_type} />
|
|
123
|
+
</Section>
|
|
124
|
+
|
|
125
|
+
{/* Trajectory Section */}
|
|
126
|
+
<Section title="Trajectory">
|
|
127
|
+
<Field
|
|
128
|
+
label="High Level Plan"
|
|
129
|
+
value={analysis.trajectory.high_level_plan}
|
|
130
|
+
/>
|
|
131
|
+
<div className="grid grid-cols-2 gap-4 pt-2">
|
|
132
|
+
<Metric label="Steps" value={analysis.trajectory.num_steps} />
|
|
133
|
+
<Metric
|
|
134
|
+
label="Tool Calls"
|
|
135
|
+
value={analysis.trajectory.num_tool_calls}
|
|
136
|
+
/>
|
|
137
|
+
</div>
|
|
138
|
+
{analysis.trajectory.tools_used.length > 0 && (
|
|
139
|
+
<div className="space-y-2 pt-2">
|
|
140
|
+
<div className="text-xs font-medium text-muted-foreground">
|
|
141
|
+
Tools Used
|
|
142
|
+
</div>
|
|
143
|
+
<div className="flex flex-wrap gap-2">
|
|
144
|
+
{analysis.trajectory.tools_used.map((tool) => (
|
|
145
|
+
<span
|
|
146
|
+
key={tool}
|
|
147
|
+
className="px-2 py-1 bg-secondary text-secondary-foreground rounded text-xs font-medium"
|
|
148
|
+
>
|
|
149
|
+
{tool}
|
|
150
|
+
</span>
|
|
151
|
+
))}
|
|
152
|
+
</div>
|
|
153
|
+
</div>
|
|
154
|
+
)}
|
|
155
|
+
{/* Detailed Tool Calls */}
|
|
156
|
+
{analysis.trajectory.tool_calls &&
|
|
157
|
+
analysis.trajectory.tool_calls.length > 0 && (
|
|
158
|
+
<div className="space-y-2 pt-2">
|
|
159
|
+
<div className="text-xs font-medium text-muted-foreground">
|
|
160
|
+
Tool Call Details
|
|
161
|
+
</div>
|
|
162
|
+
<ToolCallDetails toolCalls={analysis.trajectory.tool_calls} />
|
|
163
|
+
</div>
|
|
164
|
+
)}
|
|
165
|
+
</Section>
|
|
166
|
+
|
|
167
|
+
{/* Outcome Section */}
|
|
168
|
+
<Section title="Outcome">
|
|
169
|
+
<div className="flex gap-4">
|
|
170
|
+
<Badge label="Status" value={analysis.outcome.status} />
|
|
171
|
+
<Badge label="Answer Type" value={analysis.outcome.answer_type} />
|
|
172
|
+
</div>
|
|
173
|
+
<Field label="Assessment" value={analysis.outcome.assessment} />
|
|
174
|
+
</Section>
|
|
175
|
+
|
|
176
|
+
{/* Metrics Section */}
|
|
177
|
+
{analysis.metrics && (
|
|
178
|
+
<Section title="Metrics">
|
|
179
|
+
<div className="grid grid-cols-5 gap-4">
|
|
180
|
+
<div className="space-y-1">
|
|
181
|
+
<div className="text-xs font-medium text-muted-foreground">
|
|
182
|
+
Duration
|
|
183
|
+
</div>
|
|
184
|
+
<div className="text-lg font-semibold">
|
|
185
|
+
{formatDuration(analysis.metrics.durationMs)}
|
|
186
|
+
</div>
|
|
187
|
+
</div>
|
|
188
|
+
<div className="space-y-1">
|
|
189
|
+
<div className="text-xs font-medium text-muted-foreground">
|
|
190
|
+
Input Tokens
|
|
191
|
+
</div>
|
|
192
|
+
<div className="text-lg font-semibold">
|
|
193
|
+
{formatTokens(analysis.metrics.inputTokens)}
|
|
194
|
+
</div>
|
|
195
|
+
</div>
|
|
196
|
+
<div className="space-y-1">
|
|
197
|
+
<div className="text-xs font-medium text-muted-foreground">
|
|
198
|
+
Output Tokens
|
|
199
|
+
</div>
|
|
200
|
+
<div className="text-lg font-semibold">
|
|
201
|
+
{formatTokens(analysis.metrics.outputTokens)}
|
|
202
|
+
</div>
|
|
203
|
+
</div>
|
|
204
|
+
<div className="space-y-1">
|
|
205
|
+
<div className="text-xs font-medium text-muted-foreground">
|
|
206
|
+
Total Tokens
|
|
207
|
+
</div>
|
|
208
|
+
<div className="text-lg font-semibold">
|
|
209
|
+
{formatTokens(analysis.metrics.totalTokens)}
|
|
210
|
+
</div>
|
|
211
|
+
</div>
|
|
212
|
+
<div className="space-y-1">
|
|
213
|
+
<div className="text-xs font-medium text-muted-foreground">
|
|
214
|
+
Estimated Cost
|
|
215
|
+
</div>
|
|
216
|
+
<div className="text-lg font-semibold text-green-600 dark:text-green-400">
|
|
217
|
+
{formatCost(analysis.metrics.estimatedCost)}
|
|
218
|
+
</div>
|
|
219
|
+
</div>
|
|
220
|
+
</div>
|
|
221
|
+
</Section>
|
|
222
|
+
)}
|
|
223
|
+
|
|
224
|
+
{/* Metadata Section */}
|
|
225
|
+
<Section title="Metadata">
|
|
226
|
+
<div className="grid grid-cols-2 gap-4">
|
|
227
|
+
<Field label="Started" value={formatDate(analysis.started_at)} />
|
|
228
|
+
<Field label="Ended" value={formatDate(analysis.ended_at)} />
|
|
229
|
+
</div>
|
|
230
|
+
<Field label="Agent" value={analysis.agent_name} />
|
|
231
|
+
<Field
|
|
232
|
+
label="Session ID"
|
|
233
|
+
value={analysis.session_id.slice(0, 16)}
|
|
234
|
+
/>
|
|
235
|
+
</Section>
|
|
236
|
+
</div>
|
|
237
|
+
</DialogContent>
|
|
238
|
+
</Dialog>
|
|
239
|
+
);
|
|
240
|
+
}
|
|
@@ -231,7 +231,7 @@ export function UnifiedTimeline({
|
|
|
231
231
|
{/* Timeline container */}
|
|
232
232
|
<div
|
|
233
233
|
ref={timelineRef}
|
|
234
|
-
className="border border-border rounded-lg overflow-hidden bg-card flex"
|
|
234
|
+
className="border border-border rounded-lg overflow-hidden bg-card flex max-h-[calc(100vh-250px)]"
|
|
235
235
|
onWheel={handleWheel}
|
|
236
236
|
>
|
|
237
237
|
{/* Fixed left column - processes labels */}
|
|
@@ -245,7 +245,7 @@ export function UnifiedTimeline({
|
|
|
245
245
|
</div>
|
|
246
246
|
|
|
247
247
|
{/* Span labels */}
|
|
248
|
-
<div className="flex-1 bg-white dark:bg-gray-900">
|
|
248
|
+
<div className="flex-1 bg-white dark:bg-gray-900 overflow-y-auto">
|
|
249
249
|
{flatSpans.map(({ node, depth }) => {
|
|
250
250
|
const span = spans.find((s) => s.span_id === node.span_id);
|
|
251
251
|
// This should never be null now due to filtering
|
|
@@ -310,7 +310,7 @@ export function UnifiedTimeline({
|
|
|
310
310
|
|
|
311
311
|
{/* Scrollable right side - message bubbles + timeline */}
|
|
312
312
|
<div
|
|
313
|
-
className="flex-1 overflow-x-auto overflow-y-
|
|
313
|
+
className="flex-1 overflow-x-auto overflow-y-auto"
|
|
314
314
|
style={{ background: "#404040" }}
|
|
315
315
|
>
|
|
316
316
|
<div
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import * as DialogPrimitive from "@radix-ui/react-dialog";
|
|
2
|
+
import { X } from "lucide-react";
|
|
3
|
+
import * as React from "react";
|
|
4
|
+
|
|
5
|
+
import { cn } from "@/lib/utils";
|
|
6
|
+
|
|
7
|
+
const Dialog = DialogPrimitive.Root;
|
|
8
|
+
|
|
9
|
+
const DialogTrigger = DialogPrimitive.Trigger;
|
|
10
|
+
|
|
11
|
+
const DialogPortal = DialogPrimitive.Portal;
|
|
12
|
+
|
|
13
|
+
const DialogClose = DialogPrimitive.Close;
|
|
14
|
+
|
|
15
|
+
const DialogOverlay = React.forwardRef<
|
|
16
|
+
React.ElementRef<typeof DialogPrimitive.Overlay>,
|
|
17
|
+
React.ComponentPropsWithoutRef<typeof DialogPrimitive.Overlay>
|
|
18
|
+
>(({ className, ...props }, ref) => (
|
|
19
|
+
<DialogPrimitive.Overlay
|
|
20
|
+
ref={ref}
|
|
21
|
+
className={cn(
|
|
22
|
+
"fixed inset-0 z-50 bg-black/80 data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0",
|
|
23
|
+
className,
|
|
24
|
+
)}
|
|
25
|
+
{...props}
|
|
26
|
+
/>
|
|
27
|
+
));
|
|
28
|
+
DialogOverlay.displayName = DialogPrimitive.Overlay.displayName;
|
|
29
|
+
|
|
30
|
+
const DialogContent = React.forwardRef<
|
|
31
|
+
React.ElementRef<typeof DialogPrimitive.Content>,
|
|
32
|
+
React.ComponentPropsWithoutRef<typeof DialogPrimitive.Content>
|
|
33
|
+
>(({ className, children, ...props }, ref) => (
|
|
34
|
+
<DialogPortal>
|
|
35
|
+
<DialogOverlay />
|
|
36
|
+
<DialogPrimitive.Content
|
|
37
|
+
ref={ref}
|
|
38
|
+
className={cn(
|
|
39
|
+
"fixed left-[50%] top-[50%] z-50 grid w-full max-w-lg translate-x-[-50%] translate-y-[-50%] gap-4 border bg-background p-6 shadow-lg duration-200 data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[state=closed]:slide-out-to-left-1/2 data-[state=closed]:slide-out-to-top-[48%] data-[state=open]:slide-in-from-left-1/2 data-[state=open]:slide-in-from-top-[48%] sm:rounded-lg",
|
|
40
|
+
className,
|
|
41
|
+
)}
|
|
42
|
+
{...props}
|
|
43
|
+
>
|
|
44
|
+
{children}
|
|
45
|
+
<DialogPrimitive.Close className="absolute right-4 top-4 rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground">
|
|
46
|
+
<X className="size-4" />
|
|
47
|
+
<span className="sr-only">Close</span>
|
|
48
|
+
</DialogPrimitive.Close>
|
|
49
|
+
</DialogPrimitive.Content>
|
|
50
|
+
</DialogPortal>
|
|
51
|
+
));
|
|
52
|
+
DialogContent.displayName = DialogPrimitive.Content.displayName;
|
|
53
|
+
|
|
54
|
+
const DialogHeader = ({
|
|
55
|
+
className,
|
|
56
|
+
...props
|
|
57
|
+
}: React.HTMLAttributes<HTMLDivElement>) => (
|
|
58
|
+
<div
|
|
59
|
+
className={cn(
|
|
60
|
+
"flex flex-col space-y-1.5 text-center sm:text-left",
|
|
61
|
+
className,
|
|
62
|
+
)}
|
|
63
|
+
{...props}
|
|
64
|
+
/>
|
|
65
|
+
);
|
|
66
|
+
DialogHeader.displayName = "DialogHeader";
|
|
67
|
+
|
|
68
|
+
const DialogFooter = ({
|
|
69
|
+
className,
|
|
70
|
+
...props
|
|
71
|
+
}: React.HTMLAttributes<HTMLDivElement>) => (
|
|
72
|
+
<div
|
|
73
|
+
className={cn(
|
|
74
|
+
"flex flex-col-reverse sm:flex-row sm:justify-end sm:space-x-2",
|
|
75
|
+
className,
|
|
76
|
+
)}
|
|
77
|
+
{...props}
|
|
78
|
+
/>
|
|
79
|
+
);
|
|
80
|
+
DialogFooter.displayName = "DialogFooter";
|
|
81
|
+
|
|
82
|
+
const DialogTitle = React.forwardRef<
|
|
83
|
+
React.ElementRef<typeof DialogPrimitive.Title>,
|
|
84
|
+
React.ComponentPropsWithoutRef<typeof DialogPrimitive.Title>
|
|
85
|
+
>(({ className, ...props }, ref) => (
|
|
86
|
+
<DialogPrimitive.Title
|
|
87
|
+
ref={ref}
|
|
88
|
+
className={cn(
|
|
89
|
+
"text-lg font-semibold leading-none tracking-tight",
|
|
90
|
+
className,
|
|
91
|
+
)}
|
|
92
|
+
{...props}
|
|
93
|
+
/>
|
|
94
|
+
));
|
|
95
|
+
DialogTitle.displayName = DialogPrimitive.Title.displayName;
|
|
96
|
+
|
|
97
|
+
const DialogDescription = React.forwardRef<
|
|
98
|
+
React.ElementRef<typeof DialogPrimitive.Description>,
|
|
99
|
+
React.ComponentPropsWithoutRef<typeof DialogPrimitive.Description>
|
|
100
|
+
>(({ className, ...props }, ref) => (
|
|
101
|
+
<DialogPrimitive.Description
|
|
102
|
+
ref={ref}
|
|
103
|
+
className={cn("text-sm text-muted-foreground", className)}
|
|
104
|
+
{...props}
|
|
105
|
+
/>
|
|
106
|
+
));
|
|
107
|
+
DialogDescription.displayName = DialogPrimitive.Description.displayName;
|
|
108
|
+
|
|
109
|
+
export {
|
|
110
|
+
Dialog,
|
|
111
|
+
DialogPortal,
|
|
112
|
+
DialogOverlay,
|
|
113
|
+
DialogClose,
|
|
114
|
+
DialogTrigger,
|
|
115
|
+
DialogContent,
|
|
116
|
+
DialogHeader,
|
|
117
|
+
DialogFooter,
|
|
118
|
+
DialogTitle,
|
|
119
|
+
DialogDescription,
|
|
120
|
+
};
|
package/src/db.ts
CHANGED
|
@@ -89,15 +89,16 @@ export class DebuggerDb {
|
|
|
89
89
|
getSpansBySessionAttribute(sessionId: string): Span[] {
|
|
90
90
|
// Use JSON extract to find spans where agent.session_id matches
|
|
91
91
|
return this.db
|
|
92
|
-
.query<Span, [string]>(
|
|
92
|
+
.query<Span, [string, string]>(
|
|
93
93
|
`
|
|
94
94
|
SELECT *
|
|
95
95
|
FROM spans
|
|
96
96
|
WHERE json_extract(attributes, '$."agent.session_id"') = ?
|
|
97
|
+
OR json_extract(attributes, '$."session.id"') = ?
|
|
97
98
|
ORDER BY start_time_unix_nano ASC
|
|
98
99
|
`,
|
|
99
100
|
)
|
|
100
|
-
.all(sessionId);
|
|
101
|
+
.all(sessionId, sessionId);
|
|
101
102
|
}
|
|
102
103
|
|
|
103
104
|
/**
|
package/src/lib/metrics.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { SessionMetrics, Span, Trace } from "../types";
|
|
1
|
+
import type { SessionMetrics, Span, ToolCall, Trace } from "../types";
|
|
2
2
|
import { getModelPricing } from "./pricing";
|
|
3
3
|
|
|
4
4
|
/**
|
|
@@ -28,7 +28,7 @@ export function extractMetricsFromSpans(
|
|
|
28
28
|
): Omit<SessionMetrics, "durationMs"> {
|
|
29
29
|
let inputTokens = 0;
|
|
30
30
|
let outputTokens = 0;
|
|
31
|
-
|
|
31
|
+
const toolCalls: ToolCall[] = [];
|
|
32
32
|
|
|
33
33
|
for (const span of spans) {
|
|
34
34
|
const attrs = span.attributes ? JSON.parse(span.attributes) : {};
|
|
@@ -41,14 +41,114 @@ export function extractMetricsFromSpans(
|
|
|
41
41
|
outputTokens += attrs["gen_ai.usage.output_tokens"];
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
-
// Count tool calls
|
|
45
|
-
|
|
46
|
-
|
|
44
|
+
// Count tool calls from dedicated tool spans
|
|
45
|
+
const isToolSpan =
|
|
46
|
+
span.name.includes("tool_call") || span.name.startsWith("tool:");
|
|
47
|
+
if (isToolSpan) {
|
|
48
|
+
const name =
|
|
49
|
+
attrs["tool.name"] ||
|
|
50
|
+
attrs["tool.type"] ||
|
|
51
|
+
attrs["gen_ai.tool_call.name"] ||
|
|
52
|
+
span.name;
|
|
53
|
+
|
|
54
|
+
const rawInput =
|
|
55
|
+
attrs["tool.input"] || attrs["gen_ai.tool_call.input"] || null;
|
|
56
|
+
const rawOutput = attrs["tool.output"] || null;
|
|
57
|
+
|
|
58
|
+
const parseMaybeJson = (val: unknown) => {
|
|
59
|
+
if (typeof val !== "string") return val;
|
|
60
|
+
try {
|
|
61
|
+
return JSON.parse(val);
|
|
62
|
+
} catch {
|
|
63
|
+
return val;
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
toolCalls.push({
|
|
68
|
+
name: typeof name === "string" ? name : String(name),
|
|
69
|
+
input: parseMaybeJson(rawInput),
|
|
70
|
+
output: parseMaybeJson(rawOutput),
|
|
71
|
+
startTimeUnixNano: span.start_time_unix_nano,
|
|
72
|
+
endTimeUnixNano: span.end_time_unix_nano,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// Also inspect model outputs for declared tool calls (e.g. Anthropic tool_use)
|
|
77
|
+
const messagesRaw =
|
|
78
|
+
attrs["gen_ai.output.messages"] ?? attrs["gen_ai.output.tool_calls"];
|
|
79
|
+
if (messagesRaw) {
|
|
80
|
+
try {
|
|
81
|
+
const messages =
|
|
82
|
+
typeof messagesRaw === "string"
|
|
83
|
+
? JSON.parse(messagesRaw)
|
|
84
|
+
: messagesRaw;
|
|
85
|
+
if (Array.isArray(messages)) {
|
|
86
|
+
for (const msg of messages) {
|
|
87
|
+
if (msg && typeof msg === "object") {
|
|
88
|
+
const toolCallsArray =
|
|
89
|
+
(msg as Record<string, unknown>).tool_calls ?? [];
|
|
90
|
+
if (Array.isArray(toolCallsArray)) {
|
|
91
|
+
for (const call of toolCallsArray) {
|
|
92
|
+
if (call && typeof call === "object") {
|
|
93
|
+
toolCalls.push({
|
|
94
|
+
name:
|
|
95
|
+
(call as Record<string, unknown>).name?.toString() ||
|
|
96
|
+
"tool_call",
|
|
97
|
+
input: (call as Record<string, unknown>).args,
|
|
98
|
+
output: null,
|
|
99
|
+
startTimeUnixNano: span.start_time_unix_nano,
|
|
100
|
+
endTimeUnixNano: span.end_time_unix_nano,
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const content = (msg as Record<string, unknown>).content;
|
|
107
|
+
if (Array.isArray(content)) {
|
|
108
|
+
for (const c of content) {
|
|
109
|
+
if (
|
|
110
|
+
c &&
|
|
111
|
+
typeof c === "object" &&
|
|
112
|
+
(c as Record<string, unknown>).type === "tool_use"
|
|
113
|
+
) {
|
|
114
|
+
toolCalls.push({
|
|
115
|
+
name:
|
|
116
|
+
((c as Record<string, unknown>).name as
|
|
117
|
+
| string
|
|
118
|
+
| undefined) || "tool_use",
|
|
119
|
+
input: (c as Record<string, unknown>).input,
|
|
120
|
+
output: null,
|
|
121
|
+
startTimeUnixNano: span.start_time_unix_nano,
|
|
122
|
+
endTimeUnixNano: span.end_time_unix_nano,
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
} catch {
|
|
131
|
+
// ignore parse errors
|
|
132
|
+
}
|
|
47
133
|
}
|
|
48
134
|
}
|
|
49
135
|
|
|
50
136
|
const totalTokens = inputTokens + outputTokens;
|
|
51
137
|
const estimatedCost = calculateCost(model, inputTokens, outputTokens);
|
|
138
|
+
// Dedupe tool calls using name + start time to avoid double counting when captured in multiple places
|
|
139
|
+
// Prefer entries with actual output over entries with null output
|
|
140
|
+
const deduped = new Map<string, ToolCall>();
|
|
141
|
+
for (const call of toolCalls) {
|
|
142
|
+
const key = `${call.name}-${call.startTimeUnixNano ?? ""}`;
|
|
143
|
+
const existing = deduped.get(key);
|
|
144
|
+
if (!existing) {
|
|
145
|
+
deduped.set(key, call);
|
|
146
|
+
} else if (existing.output == null && call.output != null) {
|
|
147
|
+
// Replace null-output entry with one that has actual output
|
|
148
|
+
deduped.set(key, call);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
const toolCallCount = deduped.size;
|
|
52
152
|
|
|
53
153
|
return {
|
|
54
154
|
inputTokens,
|
|
@@ -56,6 +156,7 @@ export function extractMetricsFromSpans(
|
|
|
56
156
|
totalTokens,
|
|
57
157
|
estimatedCost,
|
|
58
158
|
toolCallCount,
|
|
159
|
+
toolCalls: Array.from(deduped.values()),
|
|
59
160
|
};
|
|
60
161
|
}
|
|
61
162
|
|
|
@@ -67,8 +168,10 @@ export function extractSessionMetrics(
|
|
|
67
168
|
spans: Span[],
|
|
68
169
|
model: string,
|
|
69
170
|
): SessionMetrics {
|
|
70
|
-
// Calculate total duration from traces
|
|
71
|
-
|
|
171
|
+
// Calculate total duration from traces first
|
|
172
|
+
// Note: Using Infinity instead of Number.MAX_SAFE_INTEGER because nanosecond
|
|
173
|
+
// timestamps exceed MAX_SAFE_INTEGER and JS number comparison doesn't work correctly
|
|
174
|
+
let minStartTime = Infinity;
|
|
72
175
|
let maxEndTime = 0;
|
|
73
176
|
|
|
74
177
|
for (const trace of traces) {
|
|
@@ -80,10 +183,27 @@ export function extractSessionMetrics(
|
|
|
80
183
|
}
|
|
81
184
|
}
|
|
82
185
|
|
|
83
|
-
|
|
84
|
-
minStartTime <
|
|
85
|
-
|
|
86
|
-
|
|
186
|
+
let durationMs =
|
|
187
|
+
minStartTime < Infinity ? (maxEndTime - minStartTime) / 1_000_000 : 0;
|
|
188
|
+
|
|
189
|
+
// If traces didn't give us duration, calculate from spans as fallback
|
|
190
|
+
if (durationMs === 0 && spans.length > 0) {
|
|
191
|
+
let spanMinStart = Infinity;
|
|
192
|
+
let spanMaxEnd = 0;
|
|
193
|
+
|
|
194
|
+
for (const span of spans) {
|
|
195
|
+
if (span.start_time_unix_nano < spanMinStart) {
|
|
196
|
+
spanMinStart = span.start_time_unix_nano;
|
|
197
|
+
}
|
|
198
|
+
if (span.end_time_unix_nano > spanMaxEnd) {
|
|
199
|
+
spanMaxEnd = span.end_time_unix_nano;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
if (spanMinStart < Infinity) {
|
|
204
|
+
durationMs = (spanMaxEnd - spanMinStart) / 1_000_000;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
87
207
|
|
|
88
208
|
// Extract token metrics from spans
|
|
89
209
|
const tokenMetrics = extractMetricsFromSpans(spans, model);
|