@townco/debugger 0.1.23 → 0.1.25

package/src/server.ts

@@ -1,9 +1,17 @@
+import { resetDb } from "@townco/otlp-server/db";
 import { createOtlpServer } from "@townco/otlp-server/http";
 import { serve } from "bun";
+import { ComparisonDb } from "./comparison-db";
 import { DebuggerDb } from "./db";
 import index from "./index.html";
+import { extractSessionMetrics } from "./lib/metrics";
 import { extractTurnMessages } from "./lib/turnExtractor";
-import type { ConversationTrace } from "./types";
+import type {
+  AgentConfig,
+  ComparisonConfig,
+  ConversationTrace,
+  Span,
+} from "./types";
 
 export const DEFAULT_DEBUGGER_PORT = 4000;
 export const DEFAULT_OTLP_PORT = 4318;
@@ -13,6 +21,7 @@ export interface DebuggerServerOptions {
   otlpPort?: number;
   dbPath: string;
   agentName?: string;
+  agentServerUrl?: string;
 }
 
 export interface DebuggerServerResult {
@@ -29,6 +38,7 @@ export function startDebuggerServer(
     otlpPort = DEFAULT_OTLP_PORT,
     dbPath,
     agentName = "Agent",
+    agentServerUrl = "http://localhost:3100",
   } = options;
 
   // Start OTLP server (initializes database internally)
@@ -41,6 +51,59 @@ export function startDebuggerServer(
   // Create debugger database connection for reading
   const db = new DebuggerDb(dbPath);
 
+  // Create comparison database for Town Hall feature
+  const comparisonDbPath = dbPath.replace(/\.db$/, "-comparison.db");
+  const comparisonDb = new ComparisonDb(comparisonDbPath);
+
+  // Helper to fetch agent config from agent server
+  async function fetchAgentConfig(): Promise<AgentConfig | null> {
+    try {
+      // Call agent's initialize RPC to get config
+      const response = await fetch(`${agentServerUrl}/rpc`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          jsonrpc: "2.0",
+          id: "debugger-config",
+          method: "initialize",
+          params: {
+            protocolVersion: 1, // ACP protocol version as number
+            clientCapabilities: {},
+          },
+        }),
+      });
+
+      if (!response.ok) {
+        console.error("Failed to fetch agent config:", response.statusText);
+        return null;
+      }
+
+      const data = await response.json();
+
+      // Check for JSON-RPC error
+      if (data.error) {
+        console.error("Agent RPC error:", data.error);
+        return null;
+      }
+
+      const result = data.result;
+      if (!result) {
+        console.error("No result in agent response");
+        return null;
+      }
+
+      // Extract config from initialize response
+      return {
+        model: result._meta?.model || "unknown",
+        systemPrompt: result._meta?.systemPrompt || null,
+        tools: result._meta?.tools || [],
+      };
+    } catch (error) {
+      console.error("Error fetching agent config:", error);
+      return null;
+    }
+  }
+
   // Start debugger UI server
   const server = serve({
     port,
@@ -51,6 +114,21 @@ export function startDebuggerServer(
         },
       },
 
+      "/api/reset-database": {
+        POST() {
+          try {
+            resetDb();
+            return new Response("Database reset successfully", { status: 200 });
+          } catch (error) {
+            console.error("Error resetting database:", error);
+            return new Response(
+              `Failed to reset database: ${error instanceof Error ? error.message : String(error)}`,
+              { status: 500 },
+            );
+          }
+        },
+      },
+
       "/api/sessions": {
         GET(req) {
           const url = new URL(req.url);
@@ -96,22 +174,277 @@ export function startDebuggerServer(
             );
           }
 
-          // Get all traces for the session (already sorted ASC)
-          const traces = db.listTraces(50, 0, sessionId);
+          // Query traces by session attribute to avoid race conditions
+          const traceIds = db.getTraceIdsBySessionAttribute(sessionId);
 
           // Extract messages for each trace
-          const conversation: ConversationTrace[] = traces.map((trace) => {
+          const conversation: ConversationTrace[] = traceIds.map(
+            (traceInfo) => {
+              const data = db.getTraceById(traceInfo.trace_id);
+              const messages = extractTurnMessages(data.spans, data.logs);
+              return {
+                trace_id: traceInfo.trace_id,
+                start_time_unix_nano: traceInfo.start_time_unix_nano,
+                userInput: messages.userInput,
+                llmOutput: messages.llmOutput,
+                agentMessages: messages.agentMessages,
+              };
+            },
+          );
+
+          return Response.json(conversation);
+        },
+      },
+
+      // Town Hall API endpoints
+
+      "/api/agent-config": {
+        async GET() {
+          const config = await fetchAgentConfig();
+          if (!config) {
+            return Response.json(
+              { error: "Failed to fetch agent config" },
+              { status: 503 },
+            );
+          }
+          return Response.json(config);
+        },
+      },
+
+      "/api/available-models": {
+        GET() {
+          // List of supported models for comparison
+          const models = [
+            // Anthropic models
+            "claude-sonnet-4-5-20250929",
+            "claude-3-5-haiku-20241022",
+            "claude-opus-4-5-20251101",
+            // Google Gemini models
+            "gemini-2.0-flash",
+            "gemini-1.5-pro",
+            "gemini-1.5-flash",
+          ];
+          return Response.json({ models });
+        },
+      },
+
+      "/api/session-first-message/:sessionId": {
+        GET(req) {
+          const sessionId = req.params.sessionId;
+
+          // Query logs directly by session attribute to avoid race conditions
+          // with trace.session_id association during concurrent sessions
+          const message = db.getFirstUserMessageBySession(sessionId);
+
+          if (!message) {
+            return Response.json(
+              { error: "Session not found or has no user message" },
+              { status: 404 },
+            );
+          }
+
+          return Response.json({ message });
+        },
+      },
+
+      "/api/comparison-config": {
+        GET() {
+          const config = comparisonDb.getLatestConfig();
+          return Response.json(config);
+        },
+        async POST(req) {
+          try {
+            const body = await req.json();
+            const config: ComparisonConfig = {
+              id: body.id || crypto.randomUUID(),
+              dimension: body.dimension,
+              controlModel: body.controlModel,
+              variantModel: body.variantModel,
+              variantSystemPrompt: body.variantSystemPrompt,
+              variantTools: body.variantTools,
+              createdAt: body.createdAt || new Date().toISOString(),
+              updatedAt: new Date().toISOString(),
+            };
+            comparisonDb.saveConfig(config);
+            return Response.json({ id: config.id });
+          } catch (error) {
+            return Response.json(
+              { error: "Invalid request body" },
+              { status: 400 },
+            );
+          }
+        },
+      },
+
+      "/api/comparison-session-ids": {
+        GET() {
+          const sessionIds = comparisonDb.getComparisonSessionIds();
+          return Response.json({ sessionIds });
+        },
+      },
+
+      "/api/comparison-runs": {
+        GET(req) {
+          const url = new URL(req.url);
+          const limit = Number.parseInt(url.searchParams.get("limit") || "50");
+          const offset = Number.parseInt(url.searchParams.get("offset") || "0");
+          const sourceSessionId = url.searchParams.get("sourceSessionId");
+
+          if (sourceSessionId) {
+            const runs = comparisonDb.listRunsBySourceSession(sourceSessionId);
+            return Response.json(runs);
+          }
+
+          const runs = comparisonDb.listRuns(limit, offset);
+          return Response.json(runs);
+        },
+      },
+
+      "/api/comparison-run/:runId": {
+        GET(req) {
+          const runId = req.params.runId;
+          const run = comparisonDb.getRun(runId);
+          if (!run) {
+            return Response.json(
+              { error: "Comparison run not found" },
+              { status: 404 },
+            );
+          }
+          return Response.json(run);
+        },
+      },
+
+      "/api/run-comparison": {
+        async POST(req) {
+          try {
+            const body = await req.json();
+            const { sessionId, configId } = body;
+
+            if (!sessionId || !configId) {
+              return Response.json(
+                { error: "sessionId and configId are required" },
+                { status: 400 },
+              );
+            }
+
+            // Get the comparison config
+            const config = comparisonDb.getConfig(configId);
+            if (!config) {
+              return Response.json(
+                { error: "Comparison config not found" },
+                { status: 404 },
+              );
+            }
+
+            // Get the first user message from the source session
+            const traces = db.listTraces(1, 0, sessionId);
+            if (traces.length === 0) {
+              return Response.json(
+                { error: "Source session not found" },
+                { status: 404 },
+              );
+            }
+
+            const trace = traces[0];
+            if (!trace) {
+              return Response.json(
+                { error: "Source session not found" },
+                { status: 404 },
+              );
+            }
+
             const data = db.getTraceById(trace.trace_id);
             const messages = extractTurnMessages(data.spans, data.logs);
-            return {
-              trace_id: trace.trace_id,
-              start_time_unix_nano: trace.start_time_unix_nano,
-              userInput: messages.userInput,
-              llmOutput: messages.llmOutput,
-            };
-          });
 
-          return Response.json(conversation);
+            if (!messages.userInput) {
+              return Response.json(
+                { error: "No user message found in source session" },
+                { status: 400 },
+              );
+            }
+
+            // Create the comparison run
+            const run = comparisonDb.createRun(
+              configId,
+              sessionId,
+              messages.userInput,
+            );
+
+            // Return the run info - actual execution will be handled by the frontend
+            // which will create two ACP sessions and run them in parallel
+            return Response.json({
+              runId: run.id,
+              firstUserMessage: run.firstUserMessage,
+              config,
+            });
+          } catch (error) {
+            console.error("Error starting comparison:", error);
+            return Response.json(
+              { error: "Failed to start comparison" },
+              { status: 500 },
+            );
+          }
+        },
+      },
+
+      "/api/comparison-run/:runId/update": {
+        async POST(req) {
+          try {
+            const runId = req.params.runId;
+            const body = await req.json();
+            const {
+              status,
+              controlSessionId,
+              variantSessionId,
+              controlMetrics,
+              variantMetrics,
+              controlResponse,
+              variantResponse,
+            } = body;
+
+            comparisonDb.updateRunStatus(runId, status, {
+              controlSessionId,
+              variantSessionId,
+              controlMetrics,
+              variantMetrics,
+              controlResponse,
+              variantResponse,
+            });
+
+            return Response.json({ success: true });
+          } catch (error) {
+            return Response.json(
+              { error: "Failed to update comparison run" },
+              { status: 500 },
+            );
+          }
+        },
+      },
+
+      "/api/session-metrics/:sessionId": {
+        async GET(req) {
+          const sessionId = req.params.sessionId;
+          const url = new URL(req.url);
+          const model = url.searchParams.get("model") || "unknown";
+
+          // Query spans by their agent.session_id attribute directly
+          // This is more reliable than trace-based lookup because concurrent
+          // sessions can cause race conditions in trace association
+          const allSpans = db.getSpansBySessionAttribute(sessionId);
+
+          if (allSpans.length === 0) {
+            return Response.json(
+              { error: "Session not found or has no traces" },
+              { status: 404 },
+            );
+          }
+
+          // Get traces for duration calculation (use empty array if not found)
+          const traces = db.listTraces(100, 0, sessionId);
+
+          // Extract metrics
+          const metrics = extractSessionMetrics(traces, allSpans, model);
+          return Response.json(metrics);
         },
       },
 

package/src/types.ts

@@ -48,10 +48,19 @@ export interface TraceDetailRaw {
   logs: Log[];
 }
 
+export interface AgentMessage {
+  content: string;
+  spanId: string;
+  timestamp: number; // end_time_unix_nano of the chat span
+  type: "chat" | "tool_call";
+  toolName?: string; // Only for tool_call type
+}
+
 export interface TraceDetail extends TraceDetailRaw {
   messages: {
     userInput: string | null;
     llmOutput: string | null;
+    agentMessages: AgentMessage[];
   };
 }
 
@@ -60,6 +69,7 @@ export interface ConversationTrace {
   start_time_unix_nano: number;
   userInput: string | null;
   llmOutput: string | null;
+  agentMessages: AgentMessage[];
 }
 
 export interface Session {
@@ -68,3 +78,80 @@ export interface Session {
   first_trace_time: number;
   last_trace_time: number;
 }
+
+// Town Hall comparison types
+
+export type ComparisonDimension = "model" | "system_prompt" | "tools";
+
+export interface ComparisonConfig {
+  id: string;
+  dimension: ComparisonDimension;
+  controlModel?: string | undefined; // Original model for comparison
+  variantModel?: string | undefined;
+  variantSystemPrompt?: string | undefined;
+  variantTools?: string[] | undefined; // JSON array of tool names
+  createdAt: string;
+  updatedAt: string;
+}
+
+export interface ComparisonConfigRow {
+  id: string;
+  dimension: string;
+  control_model: string | null;
+  variant_model: string | null;
+  variant_system_prompt: string | null;
+  variant_tools: string | null; // JSON string
+  created_at: string;
+  updated_at: string;
+}
+
+export interface SessionMetrics {
+  durationMs: number;
+  inputTokens: number;
+  outputTokens: number;
+  totalTokens: number;
+  estimatedCost: number;
+  toolCallCount: number;
+}
+
+export interface ComparisonRun {
+  id: string;
+  configId: string;
+  sourceSessionId: string;
+  firstUserMessage: string;
+  startMessageIndex: number;
+  turnCount: number;
+  controlSessionId: string | null;
+  variantSessionId: string | null;
+  status: "pending" | "running" | "completed" | "failed";
+  startedAt: string;
+  completedAt: string | null;
+  controlMetrics: SessionMetrics | null;
+  variantMetrics: SessionMetrics | null;
+  controlResponse: string | null;
+  variantResponse: string | null;
+}
+
+export interface ComparisonRunRow {
+  id: string;
+  config_id: string;
+  source_session_id: string;
+  first_user_message: string;
+  start_message_index: number;
+  turn_count: number;
+  control_session_id: string | null;
+  variant_session_id: string | null;
+  status: string;
+  started_at: string;
+  completed_at: string | null;
+  control_metrics: string | null; // JSON string
+  variant_metrics: string | null; // JSON string
+  control_response: string | null;
+  variant_response: string | null;
+}
+
+export interface AgentConfig {
+  model: string;
+  systemPrompt: string | null;
+  tools: Array<{ name: string; description?: string }>;
+}

package/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "@townco/tsconfig",
+  "compilerOptions": {
+    "composite": false,
+    "declaration": false,
+    "baseUrl": ".",
+    "outDir": "dist",
+    "paths": {
+      "@/*": ["./src/*"]
+    }
+  },
+  "include": ["src/**/*"],
+  "exclude": ["dist", "node_modules"]
+}