@townco/debugger 0.1.32 → 0.1.34

package/src/pages/FindSessions.tsx

@@ -114,7 +114,9 @@ export function FindSessions() {
             </CardHeader>
             <CardContent className="space-y-4">
               <div className="space-y-2">
-                <label className="text-sm font-medium">Select Session</label>
+                <label htmlFor="session-select" className="text-sm font-medium">
+                  Select Session
+                </label>
                 <Select
                   value={selectedSessionId}
                   onValueChange={setSelectedSessionId}

package/src/pages/TownHall.tsx

@@ -19,13 +19,6 @@ import type {
   Session,
 } from "../types";
 
-function formatDuration(startNano: number, endNano: number): string {
-  const ms = (endNano - startNano) / 1_000_000;
-  if (ms < 1000) return `${ms.toFixed(0)}ms`;
-  if (ms < 60000) return `${(ms / 1000).toFixed(1)}s`;
-  return `${(ms / 60000).toFixed(1)}m`;
-}
-
 function formatRelativeTime(nanoseconds: number): string {
   const ms = Date.now() - nanoseconds / 1_000_000;
   const seconds = Math.floor(ms / 1000);
@@ -47,9 +40,9 @@ interface SessionWithMessage extends Session {
 export function TownHall() {
   const [sessions, setSessions] = useState<SessionWithMessage[]>([]);
   const [comparisonRuns, setComparisonRuns] = useState<ComparisonRun[]>([]);
-  const [comparisonSessionIds, setComparisonSessionIds] = useState<Set<string>>(
-    new Set(),
-  );
+  const [_comparisonSessionIds, setComparisonSessionIds] = useState<
+    Set<string>
+  >(new Set());
   const [loading, setLoading] = useState(true);
   const [error, setError] = useState<string | null>(null);
 
@@ -64,6 +57,7 @@ export function TownHall() {
   const [variantModel, setVariantModel] = useState<string | null>(null);
   const [variantSystemPrompt, setVariantSystemPrompt] = useState<string>("");
   const [variantTools, setVariantTools] = useState<string[]>([]);
+  const [hypothesis, setHypothesis] = useState<string>("");
 
   // Running comparison state
   const [runningComparison, setRunningComparison] = useState<string | null>(
@@ -175,13 +169,14 @@ export function TownHall() {
           variantSystemPrompt !== "" &&
           variantSystemPrompt !== agentConfig?.systemPrompt
         );
-      case "tools":
+      case "tools": {
         const currentToolNames =
           agentConfig?.tools?.map((t) => t.name).sort() || [];
         const variantToolNames = [...variantTools].sort();
         return (
           JSON.stringify(currentToolNames) !== JSON.stringify(variantToolNames)
         );
+      }
       default:
         return false;
     }
@@ -224,6 +219,8 @@ export function TownHall() {
         // Include tools if selected
         ...(selectedDimensions.has("tools") &&
           variantTools.length > 0 && { variantTools }),
+        // Include hypothesis if provided
+        ...(hypothesis && { hypothesis }),
       };
 
       // Save config
@@ -477,6 +474,25 @@ export function TownHall() {
                       )}
                   </div>
                 )}
+
+                {/* Hypothesis input - shown when any dimension is selected */}
+                {selectedDimensions.size > 0 && (
+                  <div className="space-y-1 pt-2 border-t">
+                    <Label className="text-xs text-muted-foreground">
+                      Hypothesis (optional)
+                    </Label>
+                    <Textarea
+                      value={hypothesis}
+                      onChange={(e) => setHypothesis(e.target.value)}
+                      className="min-h-[60px] text-xs"
+                      placeholder="I expect this change to... (e.g., 'reduce token usage by caching file reads' or 'improve answer accuracy by adding web search')"
+                    />
+                    <p className="text-xs text-muted-foreground">
+                      Describe what you expect the change to achieve. This helps
+                      the analysis evaluate if your hypothesis was correct.
+                    </p>
+                  </div>
+                )}
               </CardContent>
             </Card>
             {/* Status indicator - inline below card */}
@@ -527,9 +543,9 @@ export function TownHall() {
                         variant="outline"
                         size="sm"
                         className="h-7 px-2 text-xs"
-                        onClick={() =>
-                          (window.location.href = `/sessions/${session.session_id}`)
-                        }
+                        onClick={() => {
+                          window.location.href = `/sessions/${session.session_id}`;
+                        }}
                       >
                         View
                       </Button>

package/src/server.ts

@@ -12,7 +12,6 @@ import type {
   ComparisonConfig,
   ConversationTrace,
   SessionMetrics,
-  Span,
 } from "./types";
 
 export const DEFAULT_DEBUGGER_PORT = 4000;
@@ -113,6 +112,7 @@ export function startDebuggerServer(
   // Start debugger UI server
   const server = serve({
     port,
+    idleTimeout: 120, // 2 minutes for long-running LLM analysis requests
     routes: {
       "/api/config": {
         GET() {
@@ -140,8 +140,12 @@ export function startDebuggerServer(
           const url = new URL(req.url);
           const limit = Number.parseInt(
             url.searchParams.get("limit") || "1000",
+            10,
+          );
+          const offset = Number.parseInt(
+            url.searchParams.get("offset") || "0",
+            10,
           );
-          const offset = Number.parseInt(url.searchParams.get("offset") || "0");
           const sessions = db.listSessions(limit, offset);
           return Response.json(sessions);
         },
@@ -150,8 +154,14 @@ export function startDebuggerServer(
       "/api/traces": {
         GET(req) {
           const url = new URL(req.url);
-          const limit = Number.parseInt(url.searchParams.get("limit") || "50");
-          const offset = Number.parseInt(url.searchParams.get("offset") || "0");
+          const limit = Number.parseInt(
+            url.searchParams.get("limit") || "50",
+            10,
+          );
+          const offset = Number.parseInt(
+            url.searchParams.get("offset") || "0",
+            10,
+          );
           const sessionId = url.searchParams.get("sessionId") || undefined;
           const traces = db.listTraces(limit, offset, sessionId);
           return Response.json(traces);
@@ -309,8 +319,14 @@ export function startDebuggerServer(
       "/api/comparison-runs": {
         GET(req) {
           const url = new URL(req.url);
-          const limit = Number.parseInt(url.searchParams.get("limit") || "50");
-          const offset = Number.parseInt(url.searchParams.get("offset") || "0");
+          const limit = Number.parseInt(
+            url.searchParams.get("limit") || "50",
+            10,
+          );
+          const offset = Number.parseInt(
+            url.searchParams.get("offset") || "0",
+            10,
+          );
           const sourceSessionId = url.searchParams.get("sourceSessionId");
 
           if (sourceSessionId) {
@@ -481,7 +497,7 @@ export function startDebuggerServer(
             });
 
             return Response.json({ success: true });
-          } catch (error) {
+          } catch (_error) {
             return Response.json(
               { error: "Failed to update comparison run" },
               { status: 500 },
@@ -799,9 +815,11 @@ export function startDebuggerServer(
             // List all analyses
             const limit = Number.parseInt(
               url.searchParams.get("limit") || "50",
+              10,
             );
             const offset = Number.parseInt(
               url.searchParams.get("offset") || "0",
+              10,
             );
 
             const analyses = analysisDb.listAnalyses(limit, offset);
@@ -828,6 +846,7 @@ export function startDebuggerServer(
             const url = new URL(req.url);
             const limit = Number.parseInt(
               url.searchParams.get("limit") || "10",
+              10,
             );
 
             // Get embedding for this session
@@ -862,6 +881,157 @@ export function startDebuggerServer(
         },
       },
 
+      // Comparison analysis endpoints
+      "/api/analyze-comparison/:runId": {
+        async POST(req) {
+          const runId = req.params.runId;
+
+          try {
+            // Import analyzer dynamically
+            const { analyzeComparison } = await import(
+              "./analysis/comparison-analyzer.js"
+            );
+
+            // Get the comparison run
+            const run = comparisonDb.getRun(runId);
+            if (!run) {
+              return Response.json(
+                { error: "Comparison run not found" },
+                { status: 404 },
+              );
+            }
+
+            // Get the comparison config
+            const config = comparisonDb.getConfig(run.configId);
+            if (!config) {
+              return Response.json(
+                { error: "Comparison config not found" },
+                { status: 404 },
+              );
+            }
+
+            // Verify all sessions exist
+            if (!run.controlSessionId || !run.variantSessionId) {
+              return Response.json(
+                { error: "Comparison run is incomplete - missing session IDs" },
+                { status: 400 },
+              );
+            }
+
+            // Fetch all three sessions from agent server
+            const [originalRes, controlRes, variantRes] = await Promise.all([
+              fetch(`${agentServerUrl}/sessions/${run.sourceSessionId}`),
+              fetch(`${agentServerUrl}/sessions/${run.controlSessionId}`),
+              fetch(`${agentServerUrl}/sessions/${run.variantSessionId}`),
+            ]);
+
+            if (!originalRes.ok || !controlRes.ok || !variantRes.ok) {
+              return Response.json(
+                { error: "Failed to fetch one or more sessions" },
+                { status: 500 },
+              );
+            }
+
+            const [originalSession, controlSession, variantSession] =
+              await Promise.all([
+                originalRes.json(),
+                controlRes.json(),
+                variantRes.json(),
+              ]);
+
+            // Get agent config for original tools and system prompt
+            const agentConfig = await fetchAgentConfig();
+
+            // Get metrics for each session
+            const getMetrics = (sessionId: string) => {
+              const spans = db.getSpansBySessionAttribute(sessionId);
+              const traces = db.listTraces(100, 0, sessionId);
+              return extractSessionMetrics(
+                traces,
+                spans,
+                agentConfig?.model || "unknown",
+              );
+            };
+
+            const originalMetrics = getMetrics(run.sourceSessionId);
+            const controlMetrics = getMetrics(run.controlSessionId);
+            const variantMetrics = getMetrics(run.variantSessionId);
+
+            // Run the comparison analysis
+            const analysis = await analyzeComparison({
+              runId,
+              hypothesis: config.hypothesis || "",
+              config,
+              originalSession,
+              controlSession,
+              variantSession,
+              originalMetrics,
+              controlMetrics,
+              variantMetrics,
+              originalSystemPrompt: agentConfig?.systemPrompt || undefined,
+              originalTools: agentConfig?.tools?.map((t) => t.name) || [],
+            });
+
+            // Save to database
+            comparisonDb.saveComparisonAnalysis(runId, analysis);
+
+            return Response.json(analysis);
+          } catch (error) {
+            console.error("Comparison analysis error:", error);
+            return Response.json(
+              {
+                error:
+                  error instanceof Error
+                    ? error.message
+                    : "Comparison analysis failed",
+              },
+              { status: 500 },
+            );
+          }
+        },
+      },
+
+      "/api/comparison-analysis/:runId": {
+        async GET(req) {
+          try {
+            const runId = req.params.runId;
+            const analysis = comparisonDb.getComparisonAnalysis(runId);
+
+            if (!analysis) {
+              return Response.json(
+                { error: "Comparison analysis not found" },
+                { status: 404 },
+              );
+            }
+
+            return Response.json(analysis);
+          } catch (error) {
+            console.error("Error fetching comparison analysis:", error);
+            return Response.json(
+              {
+                error:
+                  error instanceof Error
+                    ? error.message
+                    : "Failed to fetch comparison analysis",
+              },
+              { status: 500 },
+            );
+          }
+        },
+      },
+
+      "/api/comparison-analysis/:runId/exists": {
+        async GET(req) {
+          try {
+            const runId = req.params.runId;
+            const exists = comparisonDb.hasComparisonAnalysis(runId);
+            return Response.json({ exists });
+          } catch (_error) {
+            return Response.json({ exists: false });
+          }
+        },
+      },
+
       // Serve index.html for all unmatched routes (SPA routing)
       "/*": index,
     },

package/src/types.ts

@@ -54,6 +54,8 @@ export interface AgentMessage {
   timestamp: number; // end_time_unix_nano of the chat span
   type: "chat" | "tool_call";
   toolName?: string; // Only for tool_call type
+  toolInput?: unknown; // Only for tool_call type - the input/args to the tool
+  toolOutput?: unknown; // Only for tool_call type - the result from the tool
 }
 
 export interface TraceDetail extends TraceDetailRaw {
@@ -90,6 +92,7 @@ export interface ComparisonConfig {
   variantModel?: string | undefined;
   variantSystemPrompt?: string | undefined;
   variantTools?: string[] | undefined; // JSON array of tool names
+  hypothesis?: string | undefined; // User's expected outcome: "I expect this change to..."
   createdAt: string;
   updatedAt: string;
 }
@@ -101,6 +104,7 @@ export interface ComparisonConfigRow {
   variant_model: string | null;
   variant_system_prompt: string | null;
   variant_tools: string | null; // JSON string
+  hypothesis: string | null;
   created_at: string;
   updated_at: string;
 }

package/styles/globals.css

@@ -0,0 +1,120 @@
+@import "tailwindcss";
+@import "tw-animate-css";
+
+@custom-variant dark (&:is(.dark *));
+
+@theme inline {
+  --color-background: var(--background);
+  --color-foreground: var(--foreground);
+  --color-sidebar-ring: var(--sidebar-ring);
+  --color-sidebar-border: var(--sidebar-border);
+  --color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
+  --color-sidebar-accent: var(--sidebar-accent);
+  --color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
+  --color-sidebar-primary: var(--sidebar-primary);
+  --color-sidebar-foreground: var(--sidebar-foreground);
+  --color-sidebar: var(--sidebar);
+  --color-chart-5: var(--chart-5);
+  --color-chart-4: var(--chart-4);
+  --color-chart-3: var(--chart-3);
+  --color-chart-2: var(--chart-2);
+  --color-chart-1: var(--chart-1);
+  --color-ring: var(--ring);
+  --color-input: var(--input);
+  --color-border: var(--border);
+  --color-destructive: var(--destructive);
+  --color-accent-foreground: var(--accent-foreground);
+  --color-accent: var(--accent);
+  --color-muted-foreground: var(--muted-foreground);
+  --color-muted: var(--muted);
+  --color-secondary-foreground: var(--secondary-foreground);
+  --color-secondary: var(--secondary);
+  --color-primary-foreground: var(--primary-foreground);
+  --color-primary: var(--primary);
+  --color-popover-foreground: var(--popover-foreground);
+  --color-popover: var(--popover);
+  --color-card-foreground: var(--card-foreground);
+  --color-card: var(--card);
+  --radius-sm: calc(var(--radius) - 4px);
+  --radius-md: calc(var(--radius) - 2px);
+  --radius-lg: var(--radius);
+  --radius-xl: calc(var(--radius) + 4px);
+}
+
+:root {
+  --radius: 0.625rem;
+  --background: oklch(1 0 0);
+  --foreground: oklch(0.145 0 0);
+  --card: oklch(1 0 0);
+  --card-foreground: oklch(0.145 0 0);
+  --popover: oklch(1 0 0);
+  --popover-foreground: oklch(0.145 0 0);
+  --primary: oklch(0.205 0 0);
+  --primary-foreground: oklch(0.985 0 0);
+  --secondary: oklch(0.97 0 0);
+  --secondary-foreground: oklch(0.205 0 0);
+  --muted: oklch(0.97 0 0);
+  --muted-foreground: oklch(0.556 0 0);
+  --accent: oklch(0.97 0 0);
+  --accent-foreground: oklch(0.205 0 0);
+  --destructive: oklch(0.577 0.245 27.325);
+  --border: oklch(0.922 0 0);
+  --input: oklch(0.922 0 0);
+  --ring: oklch(0.708 0 0);
+  --chart-1: oklch(0.646 0.222 41.116);
+  --chart-2: oklch(0.6 0.118 184.704);
+  --chart-3: oklch(0.398 0.07 227.392);
+  --chart-4: oklch(0.828 0.189 84.429);
+  --chart-5: oklch(0.769 0.188 70.08);
+  --sidebar: oklch(0.985 0 0);
+  --sidebar-foreground: oklch(0.145 0 0);
+  --sidebar-primary: oklch(0.205 0 0);
+  --sidebar-primary-foreground: oklch(0.985 0 0);
+  --sidebar-accent: oklch(0.97 0 0);
+  --sidebar-accent-foreground: oklch(0.205 0 0);
+  --sidebar-border: oklch(0.922 0 0);
+  --sidebar-ring: oklch(0.708 0 0);
+}
+
+.dark {
+  --background: oklch(0.145 0 0);
+  --foreground: oklch(0.985 0 0);
+  --card: oklch(0.205 0 0);
+  --card-foreground: oklch(0.985 0 0);
+  --popover: oklch(0.205 0 0);
+  --popover-foreground: oklch(0.985 0 0);
+  --primary: oklch(0.922 0 0);
+  --primary-foreground: oklch(0.205 0 0);
+  --secondary: oklch(0.269 0 0);
+  --secondary-foreground: oklch(0.985 0 0);
+  --muted: oklch(0.269 0 0);
+  --muted-foreground: oklch(0.708 0 0);
+  --accent: oklch(0.269 0 0);
+  --accent-foreground: oklch(0.985 0 0);
+  --destructive: oklch(0.704 0.191 22.216);
+  --border: oklch(1 0 0 / 10%);
+  --input: oklch(1 0 0 / 15%);
+  --ring: oklch(0.556 0 0);
+  --chart-1: oklch(0.488 0.243 264.376);
+  --chart-2: oklch(0.696 0.17 162.48);
+  --chart-3: oklch(0.769 0.188 70.08);
+  --chart-4: oklch(0.627 0.265 303.9);
+  --chart-5: oklch(0.645 0.246 16.439);
+  --sidebar: oklch(0.205 0 0);
+  --sidebar-foreground: oklch(0.985 0 0);
+  --sidebar-primary: oklch(0.488 0.243 264.376);
+  --sidebar-primary-foreground: oklch(0.985 0 0);
+  --sidebar-accent: oklch(0.269 0 0);
+  --sidebar-accent-foreground: oklch(0.985 0 0);
+  --sidebar-border: oklch(1 0 0 / 10%);
+  --sidebar-ring: oklch(0.556 0 0);
+}
+
+@layer base {
+  * {
+    @apply border-border outline-ring/50;
+  }
+  body {
+    @apply bg-background text-foreground;
+  }
+}

package/tsconfig.json

@@ -3,10 +3,10 @@
   "compilerOptions": {
     "composite": false,
     "declaration": false,
-    "baseUrl": ".",
     "outDir": "dist",
     "paths": {
-      "@/*": ["./src/*"]
+      "@/*": ["./src/*"],
+      "*": ["./*"]
     }
   },
   "include": ["src/**/*"],