@townco/debugger 0.1.28 → 0.1.30

package/src/schemas.ts

@@ -1,15 +1,34 @@
 import { z } from "zod";
 
-export const VariantToolsSchema = z.array(z.string());
+const numberWithDefault = z
+  .number()
+  .optional()
+  .transform((val) => val ?? 0);
 
-export const SessionMetricsSchema = z.object({
-  durationMs: z.number(),
-  inputTokens: z.number(),
-  outputTokens: z.number(),
-  totalTokens: z.number(),
-  estimatedCost: z.number(),
-  toolCallCount: z.number(),
+const ToolCallSchema = z.object({
+  name: z.string(),
+  input: z.unknown(),
+  output: z.unknown(),
+  startTimeUnixNano: z.number().optional(),
+  endTimeUnixNano: z.number().optional(),
 });
 
+export const VariantToolsSchema = z.array(z.string());
+
+export const SessionMetricsSchema = z
+  .object({
+    durationMs: numberWithDefault,
+    inputTokens: numberWithDefault,
+    outputTokens: numberWithDefault,
+    totalTokens: numberWithDefault,
+    estimatedCost: z.number().catch(0),
+    toolCallCount: numberWithDefault,
+    toolCalls: z.array(ToolCallSchema).optional().default([]),
+  })
+  .transform((metrics) => ({
+    ...metrics,
+    toolCalls: metrics.toolCalls ?? [],
+  }));
+
 export type VariantTools = z.infer<typeof VariantToolsSchema>;
 export type SessionMetrics = z.infer<typeof SessionMetricsSchema>;

package/src/server.ts

@@ -1,6 +1,7 @@
 import { resetDb } from "@townco/otlp-server/db";
 import { createOtlpServer } from "@townco/otlp-server/http";
 import { serve } from "bun";
+import { AnalysisDb } from "./analysis-db";
 import { ComparisonDb } from "./comparison-db";
 import { DebuggerDb } from "./db";
 import index from "./index.html";
@@ -10,6 +11,7 @@ import type {
   AgentConfig,
   ComparisonConfig,
   ConversationTrace,
+  SessionMetrics,
   Span,
 } from "./types";
 
@@ -56,6 +58,9 @@ export function startDebuggerServer(
   const comparisonDbPath = dbPath.replace(/\.db$/, "-comparison.db");
   const comparisonDb = new ComparisonDb(comparisonDbPath);
 
+  // Create analysis database - uses main debugger database
+  const analysisDb = new AnalysisDb(dbPath);
+
   // Helper to fetch agent config from agent server
   async function fetchAgentConfig(): Promise<AgentConfig | null> {
     try {
@@ -133,7 +138,9 @@ export function startDebuggerServer(
       "/api/sessions": {
         GET(req) {
           const url = new URL(req.url);
-          const limit = Number.parseInt(url.searchParams.get("limit") || "50");
+          const limit = Number.parseInt(
+            url.searchParams.get("limit") || "1000",
+          );
           const offset = Number.parseInt(url.searchParams.get("offset") || "0");
           const sessions = db.listSessions(limit, offset);
           return Response.json(sessions);
@@ -258,7 +265,7 @@ export function startDebuggerServer(
             const body = await req.json();
             const config: ComparisonConfig = {
               id: body.id || crypto.randomUUID(),
-              dimension: body.dimension,
+              dimensions: body.dimensions || [],
               controlModel: body.controlModel,
               variantModel: body.variantModel,
               variantSystemPrompt: body.variantSystemPrompt,
@@ -269,6 +276,7 @@ export function startDebuggerServer(
             comparisonDb.saveConfig(config);
             return Response.json({ id: config.id });
           } catch (error) {
+            console.error("Error saving comparison config:", error);
             return Response.json(
               { error: "Invalid request body" },
               { status: 400 },
@@ -277,6 +285,20 @@ export function startDebuggerServer(
         },
       },
 
+      "/api/comparison-config/:configId": {
+        GET(req) {
+          const configId = req.params.configId;
+          const config = comparisonDb.getConfig(configId);
+          if (!config) {
+            return Response.json(
+              { error: "Comparison config not found" },
+              { status: 404 },
+            );
+          }
+          return Response.json(config);
+        },
+      },
+
       "/api/comparison-session-ids": {
         GET() {
           const sessionIds = comparisonDb.getComparisonSessionIds();
@@ -311,7 +333,53 @@ export function startDebuggerServer(
               { status: 404 },
             );
           }
-          return Response.json(run);
+
+          const config = comparisonDb.getConfig(run.configId);
+          const controlModel =
+            config?.controlModel ??
+            config?.variantModel ??
+            "claude-sonnet-4-5-20250929";
+          const variantModel =
+            config?.variantModel ??
+            config?.controlModel ??
+            "claude-sonnet-4-5-20250929";
+
+          const maybeRefreshMetrics = (
+            sessionId: string | null,
+            cached: SessionMetrics | null,
+            model: string,
+          ): SessionMetrics | null => {
+            if (!sessionId) return cached;
+            const needsRefresh =
+              !cached ||
+              cached.totalTokens === 0 ||
+              cached.toolCallCount === 0 ||
+              !cached.toolCalls ||
+              cached.toolCalls.length === 0;
+            if (!needsRefresh) return cached;
+
+            const spans = db.getSpansBySessionAttribute(sessionId);
+            if (spans.length === 0) return cached;
+            const traces = db.listTraces(100, 0, sessionId);
+            return extractSessionMetrics(traces, spans, model);
+          };
+
+          const controlMetrics = maybeRefreshMetrics(
+            run.controlSessionId,
+            run.controlMetrics,
+            controlModel,
+          );
+          const variantMetrics = maybeRefreshMetrics(
+            run.variantSessionId,
+            run.variantMetrics,
+            variantModel,
+          );
+
+          return Response.json({
+            ...run,
+            controlMetrics,
+            variantMetrics,
+          });
         },
       },
 
@@ -449,6 +517,351 @@ export function startDebuggerServer(
         },
       },
 
+      "/api/analyze-session/:sessionId": {
+        async POST(req) {
+          const sessionId = req.params.sessionId;
+
+          try {
+            // Import analyzer dynamically to avoid loading at startup
+            const { analyzeSession } = await import("./analysis/analyzer.js");
+
+            // Fetch session from agent server via ACP HTTP API
+            const sessionResponse = await fetch(
+              `${agentServerUrl}/sessions/${sessionId}`,
+            );
+
+            if (!sessionResponse.ok) {
+              if (sessionResponse.status === 404) {
+                return Response.json(
+                  { error: "Session not found" },
+                  { status: 404 },
+                );
+              }
+              throw new Error(
+                `Failed to fetch session: ${sessionResponse.statusText}`,
+              );
+            }
+
+            const sessionData = await sessionResponse.json();
+
+            // Fetch agent config to get model for cost calculation
+            const agentConfig = await fetchAgentConfig();
+            const model = agentConfig?.model || "unknown";
+
+            // Fetch metrics from OTLP spans
+            const allSpans = db.getSpansBySessionAttribute(sessionId);
+            const traces = db.listTraces(100, 0, sessionId);
+            const sessionMetrics = extractSessionMetrics(
+              traces,
+              allSpans,
+              model,
+            );
+
+            // Convert to AnalysisMetrics format
+            const metrics = {
+              inputTokens: sessionMetrics.inputTokens,
+              outputTokens: sessionMetrics.outputTokens,
+              totalTokens: sessionMetrics.totalTokens,
+              estimatedCost: sessionMetrics.estimatedCost,
+              durationMs: sessionMetrics.durationMs,
+            };
+
+            // Convert tool calls to DetailedToolCall format
+            const toolCalls = (sessionMetrics.toolCalls || []).map((tc) => ({
+              name: tc.name,
+              input: tc.input,
+              output: tc.output,
+              startTimeUnixNano: tc.startTimeUnixNano,
+              endTimeUnixNano: tc.endTimeUnixNano,
+            }));
+
+            // Analyze with LLM
+            const analysis = await analyzeSession({
+              session: sessionData,
+              metrics,
+              toolCalls,
+            });
+
+            // Persist to database
+            analysisDb.saveAnalysis(analysis);
+
+            // Generate and save embedding
+            try {
+              const { embedAnalysis } = await import(
+                "./analysis/embeddings.js"
+              );
+              const embedding = await embedAnalysis(analysis);
+              await analysisDb.saveEmbedding(analysis.session_id, embedding);
+            } catch (error) {
+              console.error(
+                `Failed to generate embedding for ${sessionId}:`,
+                error,
+              );
+              // Continue - don't fail entire analysis
+            }
+
+            return Response.json(analysis);
+          } catch (error) {
+            console.error("Session analysis error:", error);
+            return Response.json(
+              {
+                error:
+                  error instanceof Error ? error.message : "Analysis failed",
+              },
+              { status: 500 },
+            );
+          }
+        },
+      },
+
+      "/api/analyze-all-sessions": {
+        async POST(req) {
+          try {
+            const body = await req.json();
+            const { sessionIds } = body as { sessionIds: string[] };
+
+            if (!Array.isArray(sessionIds)) {
+              return Response.json(
+                { error: "sessionIds must be an array" },
+                { status: 400 },
+              );
+            }
+
+            // Import analyzer dynamically
+            const { analyzeSession } = await import("./analysis/analyzer.js");
+
+            // Fetch agent config once for all sessions
+            const agentConfig = await fetchAgentConfig();
+            const model = agentConfig?.model || "unknown";
+
+            // Process in batches of 25
+            const BATCH_SIZE = 25;
+            const results: Array<{
+              session_id: string;
+              success: boolean;
+              error?: string;
+            }> = [];
+
+            const totalBatches = Math.ceil(sessionIds.length / BATCH_SIZE);
+            console.log(
+              `✨ Starting batch analysis of ${sessionIds.length} sessions (${totalBatches} batches)...`,
+            );
+
+            for (let i = 0; i < sessionIds.length; i += BATCH_SIZE) {
+              const batch = sessionIds.slice(i, i + BATCH_SIZE);
+              const batchNum = Math.floor(i / BATCH_SIZE) + 1;
+
+              console.log(
+                `📊 Processing batch ${batchNum}/${totalBatches} (${batch.length} sessions)...`,
+              );
+
+              // Run batch in parallel
+              const batchResults = await Promise.allSettled(
+                batch.map(async (sessionId) => {
+                  // Fetch session data
+                  const sessionResponse = await fetch(
+                    `${agentServerUrl}/sessions/${sessionId}`,
+                  );
+
+                  if (!sessionResponse.ok) {
+                    throw new Error(`Failed to fetch session ${sessionId}`);
+                  }
+
+                  const sessionData = await sessionResponse.json();
+
+                  // Fetch metrics from OTLP spans
+                  const allSpans = db.getSpansBySessionAttribute(sessionId);
+                  const traces = db.listTraces(100, 0, sessionId);
+                  const sessionMetrics = extractSessionMetrics(
+                    traces,
+                    allSpans,
+                    model,
+                  );
+
+                  // Convert to AnalysisMetrics format
+                  const metrics = {
+                    inputTokens: sessionMetrics.inputTokens,
+                    outputTokens: sessionMetrics.outputTokens,
+                    totalTokens: sessionMetrics.totalTokens,
+                    estimatedCost: sessionMetrics.estimatedCost,
+                    durationMs: sessionMetrics.durationMs,
+                  };
+
+                  // Convert tool calls to DetailedToolCall format
+                  const toolCalls = (sessionMetrics.toolCalls || []).map(
+                    (tc) => ({
+                      name: tc.name,
+                      input: tc.input,
+                      output: tc.output,
+                      startTimeUnixNano: tc.startTimeUnixNano,
+                      endTimeUnixNano: tc.endTimeUnixNano,
+                    }),
+                  );
+
+                  // Analyze
+                  const analysis = await analyzeSession({
+                    session: sessionData,
+                    metrics,
+                    toolCalls,
+                  });
+
+                  // Persist
+                  analysisDb.saveAnalysis(analysis);
+
+                  // Generate and save embedding
+                  try {
+                    const { embedAnalysis } = await import(
+                      "./analysis/embeddings.js"
+                    );
+                    const embedding = await embedAnalysis(analysis);
+                    await analysisDb.saveEmbedding(sessionId, embedding);
+                  } catch (error) {
+                    console.error(
+                      `Failed to generate embedding for ${sessionId}:`,
+                      error,
+                    );
+                    // Continue - batch processing continues
+                  }
+
+                  return { session_id: sessionId, success: true };
+                }),
+              );
+
+              // Collect results
+              for (let j = 0; j < batchResults.length; j++) {
+                const result = batchResults[j];
+                const sessionId = batch[j];
+                if (!sessionId) continue;
+
+                if (result && result.status === "fulfilled") {
+                  results.push(result.value);
+                } else if (result && result.status === "rejected") {
+                  results.push({
+                    session_id: sessionId,
+                    success: false,
+                    error:
+                      result.reason instanceof Error
+                        ? result.reason.message
+                        : String(result.reason || "Unknown error"),
+                  });
+                }
+              }
+
+              const batchSuccesses = batchResults.filter(
+                (r) => r.status === "fulfilled",
+              ).length;
+              const batchErrors = batchResults.filter(
+                (r) => r.status === "rejected",
+              ).length;
+              console.log(
+                `✅ Batch ${batchNum}/${totalBatches} complete: ${batchSuccesses} successful, ${batchErrors} failed`,
+              );
+            }
+
+            const totalSuccesses = results.filter((r) => r.success).length;
+            const totalErrors = results.filter((r) => !r.success).length;
+            console.log(
+              `🎉 Batch analysis complete: ${totalSuccesses} successful, ${totalErrors} failed`,
+            );
+
+            return Response.json({ results });
+          } catch (error) {
+            console.error("Batch analysis error:", error);
+            return Response.json(
+              {
+                error:
+                  error instanceof Error ? error.message : "Analysis failed",
+              },
+              { status: 500 },
+            );
+          }
+        },
+      },
+
+      "/api/session-analyses": {
+        async GET(req) {
+          try {
+            const url = new URL(req.url);
+            const sessionId = url.searchParams.get("sessionId");
+
+            if (sessionId) {
+              // Get single analysis
+              const analysis = analysisDb.getAnalysis(sessionId);
+              if (!analysis) {
+                return Response.json(
+                  { error: "Analysis not found" },
+                  { status: 404 },
+                );
+              }
+              return Response.json(analysis);
+            }
+
+            // List all analyses
+            const limit = Number.parseInt(
+              url.searchParams.get("limit") || "50",
+            );
+            const offset = Number.parseInt(
+              url.searchParams.get("offset") || "0",
+            );
+
+            const analyses = analysisDb.listAnalyses(limit, offset);
+            return Response.json({ analyses });
+          } catch (error) {
+            console.error("Error retrieving analyses:", error);
+            return Response.json(
+              {
+                error:
+                  error instanceof Error
+                    ? error.message
+                    : "Failed to retrieve analyses",
+              },
+              { status: 500 },
+            );
+          }
+        },
+      },
+
+      "/api/session-analyses/:sessionId/similar": {
+        async GET(req) {
+          try {
+            const sessionId = req.params.sessionId;
+            const url = new URL(req.url);
+            const limit = Number.parseInt(
+              url.searchParams.get("limit") || "10",
+            );
+
+            // Get embedding for this session
+            const embedding = await analysisDb.getEmbedding(sessionId);
+            if (!embedding) {
+              return Response.json(
+                { error: "No embedding found for this session" },
+                { status: 404 },
+              );
+            }
+
+            // Search for similar sessions
+            const similar = (
+              await analysisDb.searchSimilarSessions(embedding, limit + 1)
+            )
+              .filter((s) => s.session_id !== sessionId)
+              .slice(0, limit);
+
+            return Response.json({ similar });
+          } catch (error) {
+            console.error("Error finding similar sessions:", error);
+            return Response.json(
+              {
+                error:
+                  error instanceof Error
+                    ? error.message
+                    : "Failed to find similar sessions",
+              },
+              { status: 500 },
+            );
+          }
+        },
+      },
+
       // Serve index.html for all unmatched routes (SPA routing)
       "/*": index,
     },
@@ -464,5 +877,12 @@ export function startDebuggerServer(
     otlpServer.stop();
   };
 
+  console.log(`🔍 Debugger UI: http://${server.hostname}:${server.port}`);
+  console.log(
+    `📊 OTLP endpoint: http://${otlpServer.hostname}:${otlpServer.port}`,
+  );
+  console.log(`📁 Database: ${dbPath}`);
+  console.log(`🤖 Agent server: ${agentServerUrl}`);
+
   return { server, otlpServer, stop };
 }

package/src/types.ts

@@ -85,7 +85,7 @@ export type ComparisonDimension = "model" | "system_prompt" | "tools";
 
 export interface ComparisonConfig {
   id: string;
-  dimension: ComparisonDimension;
+  dimensions: ComparisonDimension[]; // Now supports multiple dimensions
   controlModel?: string | undefined; // Original model for comparison
   variantModel?: string | undefined;
   variantSystemPrompt?: string | undefined;
@@ -96,7 +96,7 @@ export interface ComparisonConfig {
 
 export interface ComparisonConfigRow {
   id: string;
-  dimension: string;
+  dimensions: string; // JSON array of dimensions
   control_model: string | null;
   variant_model: string | null;
   variant_system_prompt: string | null;
@@ -112,6 +112,15 @@ export interface SessionMetrics {
   totalTokens: number;
   estimatedCost: number;
   toolCallCount: number;
+  toolCalls?: ToolCall[];
+}
+
+export interface ToolCall {
+  name: string;
+  input: unknown;
+  output: unknown;
+  startTimeUnixNano?: number | undefined;
+  endTimeUnixNano?: number | undefined;
 }
 
 export interface ComparisonRun {