@townco/debugger 0.1.31 → 0.1.33

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@townco/debugger",
-  "version": "0.1.31",
+  "version": "0.1.33",
   "type": "module",
   "engines": {
     "bun": ">=1.3.0"
   },
   "files": [
     "src",
+    "styles",
     "tsconfig.json"
   ],
   "scripts": {
@@ -18,27 +19,26 @@
     "@anthropic-ai/sdk": "^0.70.0",
     "@lancedb/lancedb": "^0.22.3",
     "@radix-ui/react-dialog": "^1.1.15",
-    "@radix-ui/react-label": "^2.1.7",
+    "@radix-ui/react-label": "^2.1.8",
     "@radix-ui/react-select": "^2.2.6",
-    "@radix-ui/react-slot": "^1.2.3",
-    "@radix-ui/react-tabs": "^1.1.0",
-    "@townco/otlp-server": "0.1.31",
-    "@townco/ui": "0.1.76",
-    "bun-plugin-tailwind": "^0.1.2",
+    "@radix-ui/react-slot": "^1.2.4",
+    "@radix-ui/react-tabs": "^1.1.13",
+    "@townco/otlp-server": "0.1.33",
+    "@townco/ui": "^0.1.77",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
-    "lucide-react": "^0.545.0",
+    "lucide-react": "^0.556.0",
     "openai": "^4.77.3",
-    "react": "19.2.1",
-    "react-dom": "19.2.1",
-    "tailwind-merge": "^3.3.1",
+    "react-dom": "^19.2.1",
+    "tailwind-merge": "^3.4.0",
     "zod": "^4.1.13"
   },
   "devDependencies": {
-    "@townco/tsconfig": "0.1.73",
+    "@townco/tsconfig": "0.1.75",
     "@types/bun": "latest",
-    "@types/react": "^19",
-    "@types/react-dom": "^19",
+    "@types/react": "^19.2.7",
+    "@types/react-dom": "^19.2.3",
+    "bun-plugin-tailwind": "^0.1.2",
     "tailwindcss": "^4.1.11",
     "tw-animate-css": "^1.4.0",
     "typescript": "^5.9.3"

package/src/App.tsx

@@ -57,6 +57,7 @@ class ErrorBoundary extends Component<
             {this.state.error?.message}
           </pre>
           <button
+            type="button"
             onClick={() => window.location.reload()}
             style={{
               marginTop: "1rem",

package/src/analysis/analyzer.ts

@@ -7,7 +7,6 @@ import { LLMAnalysisOutputSchema, SessionAnalysisSchema } from "./schema";
 import type {
   AnalysisMetrics,
   DetailedToolCall,
-  LLMAnalysisOutput,
   PreComputedFields,
   SessionAnalysis,
 } from "./types";
@@ -118,7 +117,7 @@ function formatConversationTranscript(session: StoredSession): string {
 
     for (const block of msg.content) {
       if (block.type === "text") {
-        transcript += block.text + "\n";
+        transcript += `${block.text}\n`;
       } else if (block.type === "tool_call") {
         transcript += `[Tool: ${block.title}`;
         if (block.status === "completed") {

package/src/analysis/comparison-analyzer.ts

@@ -0,0 +1,528 @@
+/**
+ * Comparison analyzer - analyzes comparison runs using Claude to produce
+ * Reproducibility Reports and Change Impact Reports.
+ */
+
+import Anthropic from "@anthropic-ai/sdk";
+import type { ComparisonConfig, SessionMetrics } from "../types";
+import {
+  LLMComparisonOutputSchema,
+  SessionComparisonAnalysisSchema,
+} from "./comparison-schema";
+import type {
+  ConfigSummary,
+  LLMComparisonOutput,
+  SessionComparisonAnalysis,
+} from "./comparison-types";
+
+// Session types (same as analyzer.ts)
+type StoredSession = {
+  sessionId: string;
+  messages: SessionMessage[];
+  metadata: {
+    createdAt: string;
+    updatedAt: string;
+    agentName: string;
+  };
+  context: unknown[];
+};
+
+type SessionMessage = {
+  role: "user" | "assistant";
+  content: ContentBlock[];
+  timestamp: string;
+};
+
+type ContentBlock =
+  | { type: "text"; text: string }
+  | { type: "image"; [key: string]: unknown }
+  | {
+      type: "tool_call";
+      id: string;
+      title: string;
+      status: "pending" | "in_progress" | "completed" | "failed";
+      error?: string;
+    };
+
+const anthropic = new Anthropic({
+  apiKey: process.env.ANTHROPIC_API_KEY,
+});
+
+const ANALYSIS_MODEL = "claude-sonnet-4-5-20250929";
+
+const COMPARISON_SYSTEM_PROMPT = `You are an expert AI agent behavior analyst helping software engineers improve their agents.
+
+You will analyze 3 versions of the same agent task:
+- ORIGINAL: The historical session the engineer is trying to understand/improve
+- CONTROL: A fresh replay with the same configuration (tests reproducibility)
+- VARIANT: A replay with modified configuration (tests the engineer's changes)
+
+Your job is to produce TWO reports:
+
+1. REPRODUCIBILITY REPORT (Original vs Control)
+   - Assess whether the baseline behavior is stable
+   - Identify any concerning divergences that suggest non-determinism
+   - Help the engineer understand if they can trust A/B comparisons
+
+2. CHANGE IMPACT REPORT (Control vs Variant)
+   - Evaluate whether the engineer's changes achieved their stated hypothesis
+   - Identify intended effects (did the change work?)
+   - Identify unintended effects (regressions, side effects)
+   - Provide specific, actionable recommendations
+
+CRITICAL ANALYSIS PRINCIPLES:
+
+1. SPECIFICITY: Reference exact tool names, quote transcripts, cite evidence
+   - BAD: "The variant used fewer tool calls"
+   - GOOD: "Variant made 3 [TOOL:read_file] calls vs control's 7. It skipped redundant reads of config.json (called 3x in control) by caching the result."
+
+2. EVIDENCE-BASED: Always cite your evidence using these formats:
+   - [TOOL:name] for tool names
+   - [ARG:key=value] for tool arguments
+   - [MSG:"quote..."] for quoting assistant messages
+   - [OUTPUT:"excerpt..."] for tool outputs
+
+3. ACTIONABLE RECOMMENDATIONS:
+   - BAD: "Consider making the prompt more specific"
+   - GOOD: "Add to system prompt: 'When reading configuration files, cache the contents for the session to avoid redundant reads.'"
+
+4. HYPOTHESIS-FOCUSED: The user stated a hypothesis about what their change would do. Directly evaluate whether that hypothesis was achieved.
+
+5. METRICS INTERPRETATION: Don't just report numbers - explain what they mean for the user.
+   - BAD: "Token usage increased by 20%"
+   - GOOD: "Token usage increased by 20% (from 5,000 to 6,000), adding ~$0.03 per session, but the improved answer quality likely justifies this cost."
+
+You must respond with valid JSON matching this schema:
+{
+  "reproducibility": {
+    "verdict": "STABLE" | "UNSTABLE" | "PARTIALLY_STABLE",
+    "confidence": "HIGH" | "MEDIUM" | "LOW",
+    "summary": "2-3 sentences summarizing reproducibility findings",
+    "behavioral_differences": [
+      {
+        "category": "TOOL_USAGE" | "RESPONSE_CONTENT" | "REASONING_PATH" | "ERROR_HANDLING" | "PERFORMANCE",
+        "observation": "What was different",
+        "evidence": "Specific quotes/citations",
+        "significance": "CRITICAL" | "NOTABLE" | "MINOR"
+      }
+    ],
+    "metric_comparison": {
+      "duration_delta_pct": number,
+      "token_delta_pct": number,
+      "cost_delta_pct": number,
+      "tool_call_delta": number,
+      "interpretation": "What these metrics mean"
+    },
+    "recommendations": [
+      {
+        "priority": "HIGH" | "MEDIUM" | "LOW",
+        "action": "Specific action to take",
+        "rationale": "Why this helps"
+      }
+    ]
+  },
+  "change_impact": {
+    "verdict": "IMPROVED" | "DEGRADED" | "NEUTRAL" | "MIXED",
+    "confidence": "HIGH" | "MEDIUM" | "LOW",
+    "summary": "2-3 sentences summarizing change impact",
+    "hypothesis_assessment": "Did the changes achieve the user's hypothesis?",
+    "intended_effects": [
+      {
+        "expected_change": "What was supposed to happen",
+        "observed_outcome": "What actually happened",
+        "evidence": "Specific quotes/citations",
+        "assessment": "ACHIEVED" | "PARTIALLY_ACHIEVED" | "NOT_ACHIEVED" | "OPPOSITE_EFFECT"
+      }
+    ],
+    "unintended_effects": [
+      {
+        "observation": "What unexpected thing happened",
+        "evidence": "Specific quotes/citations",
+        "impact": "POSITIVE" | "NEGATIVE" | "NEUTRAL",
+        "severity": "CRITICAL" | "NOTABLE" | "MINOR"
+      }
+    ],
+    "metric_comparison": {
+      "duration_delta_pct": number,
+      "token_delta_pct": number,
+      "cost_delta_pct": number,
+      "tool_call_delta": number,
+      "interpretation": "What these metrics mean"
+    },
+    "tool_usage_changes": [
+      {
+        "tool_name": "name",
+        "control_calls": number,
+        "variant_calls": number,
+        "pattern_change": "How usage changed"
+      }
+    ],
+    "recommendations": [
+      {
+        "priority": "HIGH" | "MEDIUM" | "LOW",
+        "action": "Specific action to take",
+        "rationale": "Why this helps",
+        "expected_impact": "What improvement to expect"
+      }
+    ]
+  },
+  "next_experiments": [
+    {
+      "hypothesis": "What you want to test",
+      "suggested_change": {
+        "dimension": "model" | "system_prompt" | "tools",
+        "description": "What to change",
+        "example": "Concrete example if prompt change"
+      },
+      "expected_outcome": "What you expect to happen"
+    }
+  ]
+}
+
+Respond with ONLY the JSON object, no additional text.`;
+
+/**
+ * Format a single transcript with intelligent truncation
+ */
+function formatTranscript(
+  session: StoredSession,
+  label: string,
+  maxChars: number = 15000,
+): string {
+  let transcript = `### ${label}\n\n`;
+  let currentLength = transcript.length;
+
+  for (const msg of session.messages) {
+    const roleHeader = `## ${msg.role.toUpperCase()}\n`;
+
+    if (currentLength + roleHeader.length > maxChars) {
+      transcript += "\n[...transcript truncated...]\n";
+      break;
+    }
+
+    transcript += roleHeader;
+    currentLength += roleHeader.length;
+
+    for (const block of msg.content) {
+      if (block.type === "text") {
+        let text = block.text;
+
+        // Truncate very long text blocks
+        if (text.length > 1500) {
+          text = `${text.slice(0, 800)}\n[...truncated...]\n${text.slice(-400)}`;
+        }
+
+        if (currentLength + text.length > maxChars) {
+          transcript += text.slice(0, maxChars - currentLength - 50);
+          transcript += "\n[...transcript truncated...]\n";
+          return transcript;
+        }
+
+        transcript += `${text}\n`;
+        currentLength += text.length + 1;
+      } else if (block.type === "tool_call") {
+        const toolInfo = `[TOOL:${block.title} - ${block.status}${block.error ? `: ${block.error}` : ""}]\n`;
+
+        if (currentLength + toolInfo.length > maxChars) {
+          transcript += "\n[...transcript truncated...]\n";
+          return transcript;
+        }
+
+        transcript += toolInfo;
+        currentLength += toolInfo.length;
+      }
+    }
+
+    transcript += "\n";
+    currentLength += 1;
+  }
+
+  return transcript;
+}
+
+/**
+ * Format config diff for the prompt
+ */
+function formatConfigDiff(
+  config: ComparisonConfig,
+  originalSystemPrompt?: string,
+  originalTools?: string[],
+): string {
+  const parts: string[] = [];
+
+  if (config.dimensions.includes("model") && config.variantModel) {
+    parts.push(`MODEL CHANGE:
+- Control: ${config.controlModel || "default"}
+- Variant: ${config.variantModel}`);
+  }
+
+  if (
+    config.dimensions.includes("system_prompt") &&
+    config.variantSystemPrompt
+  ) {
+    // Truncate very long prompts for display
+    const originalTruncated = originalSystemPrompt
+      ? originalSystemPrompt.length > 2000
+        ? `${originalSystemPrompt.slice(0, 1500)}\n[...truncated...]`
+        : originalSystemPrompt
+      : "[not available]";
+
+    const variantTruncated =
+      config.variantSystemPrompt.length > 2000
+        ? `${config.variantSystemPrompt.slice(0, 1500)}\n[...truncated...]`
+        : config.variantSystemPrompt;
+
+    parts.push(`SYSTEM PROMPT CHANGE:
+
+Control (Original):
+"""
+${originalTruncated}
+"""
+
+Variant (Modified):
+"""
+${variantTruncated}
+"""`);
+  }
+
+  if (config.dimensions.includes("tools") && config.variantTools) {
+    const controlTools = originalTools || [];
+    const variantTools = config.variantTools;
+    const added = variantTools.filter((t) => !controlTools.includes(t));
+    const removed = controlTools.filter((t) => !variantTools.includes(t));
+
+    parts.push(`TOOLS CHANGE:
+- Added: ${added.length > 0 ? added.join(", ") : "none"}
+- Removed: ${removed.length > 0 ? removed.join(", ") : "none"}`);
+  }
+
+  return parts.length > 0 ? parts.join("\n\n") : "No config changes specified";
+}
+
+/**
+ * Format metrics comparison table
+ */
+function formatMetricsTable(
+  original: SessionMetrics | null,
+  control: SessionMetrics | null,
+  variant: SessionMetrics | null,
+): string {
+  const fmt = (val: number | undefined, suffix = "") =>
+    val !== undefined ? `${val.toLocaleString()}${suffix}` : "N/A";
+  const fmtCost = (val: number | undefined) =>
+    val !== undefined ? `$${val.toFixed(4)}` : "N/A";
+  const fmtDur = (val: number | undefined) =>
+    val !== undefined ? `${(val / 1000).toFixed(1)}s` : "N/A";
+
+  return `| Metric | Original | Control | Variant |
+|--------|----------|---------|---------|
+| Duration | ${fmtDur(original?.durationMs)} | ${fmtDur(control?.durationMs)} | ${fmtDur(variant?.durationMs)} |
+| Input Tokens | ${fmt(original?.inputTokens)} | ${fmt(control?.inputTokens)} | ${fmt(variant?.inputTokens)} |
+| Output Tokens | ${fmt(original?.outputTokens)} | ${fmt(control?.outputTokens)} | ${fmt(variant?.outputTokens)} |
+| Total Tokens | ${fmt(original?.totalTokens)} | ${fmt(control?.totalTokens)} | ${fmt(variant?.totalTokens)} |
+| Est. Cost | ${fmtCost(original?.estimatedCost)} | ${fmtCost(control?.estimatedCost)} | ${fmtCost(variant?.estimatedCost)} |
+| Tool Calls | ${fmt(original?.toolCallCount)} | ${fmt(control?.toolCallCount)} | ${fmt(variant?.toolCallCount)} |`;
+}
+
+/**
+ * Build the comparison prompt
+ */
+function buildComparisonPrompt(
+  hypothesis: string,
+  configDiff: string,
+  metricsTable: string,
+  originalTranscript: string,
+  controlTranscript: string,
+  variantTranscript: string,
+): string {
+  return `# COMPARISON ANALYSIS REQUEST
+
+## USER'S HYPOTHESIS
+${hypothesis || "No hypothesis provided - infer the expected change from the config diff."}
+
+## CONFIG DIFF
+${configDiff}
+
+## METRICS COMPARISON
+${metricsTable}
+
+## TRANSCRIPTS
+
+${originalTranscript}
+
+${controlTranscript}
+
+${variantTranscript}
+
+---
+
+Analyze these three sessions and produce:
+1. A REPRODUCIBILITY REPORT comparing Original vs Control
+2. A CHANGE IMPACT REPORT comparing Control vs Variant
+
+Focus on specific, evidence-based observations with actionable recommendations.`;
+}
+
+/**
+ * Extract JSON from potential markdown code blocks
+ */
+function extractJSON(text: string): string {
+  const jsonMatch = text.match(/```(?:json)?\n([\s\S]*?)\n```/);
+  if (jsonMatch?.[1]) {
+    return jsonMatch[1];
+  }
+  return text.trim();
+}
+
+/**
+ * Build config summary from comparison config
+ */
+function buildConfigSummary(
+  config: ComparisonConfig,
+  originalTools?: string[],
+): ConfigSummary {
+  const summary: ConfigSummary = {
+    system_prompt_changed: config.dimensions.includes("system_prompt"),
+    tools_added: [],
+    tools_removed: [],
+  };
+
+  if (config.dimensions.includes("model") && config.variantModel) {
+    summary.model_change = {
+      from: config.controlModel || "default",
+      to: config.variantModel,
+    };
+  }
+
+  if (config.dimensions.includes("tools") && config.variantTools) {
+    const controlTools = originalTools || [];
+    summary.tools_added = config.variantTools.filter(
+      (t) => !controlTools.includes(t),
+    );
+    summary.tools_removed = controlTools.filter(
+      (t) => !config.variantTools?.includes(t),
+    );
+  }
+
+  return summary;
+}
+
+/**
+ * Options for comparison analysis
+ */
+export interface AnalyzeComparisonOptions {
+  runId: string;
+  hypothesis: string;
+  config: ComparisonConfig;
+  originalSession: StoredSession;
+  controlSession: StoredSession;
+  variantSession: StoredSession;
+  originalMetrics: SessionMetrics | null;
+  controlMetrics: SessionMetrics | null;
+  variantMetrics: SessionMetrics | null;
+  originalSystemPrompt?: string | undefined;
+  originalTools?: string[] | undefined;
+}
+
+/**
+ * Analyze a comparison run using Claude
+ */
+export async function analyzeComparison(
+  options: AnalyzeComparisonOptions,
+): Promise<SessionComparisonAnalysis> {
+  const {
+    runId,
+    hypothesis,
+    config,
+    originalSession,
+    controlSession,
+    variantSession,
+    originalMetrics,
+    controlMetrics,
+    variantMetrics,
+    originalSystemPrompt,
+    originalTools,
+  } = options;
+
+  // 1. Format all components
+  const configDiff = formatConfigDiff(
+    config,
+    originalSystemPrompt,
+    originalTools,
+  );
+  const metricsTable = formatMetricsTable(
+    originalMetrics,
+    controlMetrics,
+    variantMetrics,
+  );
+  const originalTranscript = formatTranscript(
+    originalSession,
+    "ORIGINAL SESSION",
+  );
+  const controlTranscript = formatTranscript(controlSession, "CONTROL SESSION");
+  const variantTranscript = formatTranscript(variantSession, "VARIANT SESSION");
+
+  // 2. Build the prompt
+  const prompt = buildComparisonPrompt(
+    hypothesis,
+    configDiff,
+    metricsTable,
+    originalTranscript,
+    controlTranscript,
+    variantTranscript,
+  );
+
+  // 3. Call Claude
+  const response = await anthropic.messages.create({
+    model: ANALYSIS_MODEL,
+    max_tokens: 8192,
+    temperature: 0,
+    system: COMPARISON_SYSTEM_PROMPT,
+    messages: [{ role: "user", content: prompt }],
+  });
+
+  // 4. Parse and validate response
+  const content = response.content[0];
+  if (!content) {
+    throw new Error("No content in response");
+  }
+  if (content.type !== "text") {
+    throw new Error("Unexpected response format");
+  }
+
+  const jsonText = extractJSON(content.text);
+
+  // Debug: log the raw response if parsing fails
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(jsonText);
+  } catch (parseError) {
+    console.error("Failed to parse LLM response as JSON:");
+    console.error("Raw response:", content.text.slice(0, 1000));
+    console.error("Extracted JSON text:", jsonText.slice(0, 1000));
+    throw new Error(
+      `Invalid JSON in LLM response: ${parseError instanceof Error ? parseError.message : "Unknown parse error"}`,
+    );
+  }
+
+  const llmOutput: LLMComparisonOutput =
+    LLMComparisonOutputSchema.parse(parsed);
+
+  // 5. Build full analysis object
+  const analysis: SessionComparisonAnalysis = {
+    comparison_run_id: runId,
+    created_at: new Date().toISOString(),
+    original_session_id: originalSession.sessionId,
+    control_session_id: controlSession.sessionId,
+    variant_session_id: variantSession.sessionId,
+    hypothesis,
+    dimensions_compared: config.dimensions,
+    config_summary: buildConfigSummary(config, originalTools),
+    reproducibility: llmOutput.reproducibility,
+    change_impact: llmOutput.change_impact,
+    next_experiments: llmOutput.next_experiments,
+  };
+
+  // 6. Validate final schema
+  return SessionComparisonAnalysisSchema.parse(analysis);
+}