@townco/debugger 0.1.22 → 0.1.24

package/src/pages/ComparisonView.tsx

@@ -0,0 +1,685 @@
+import { useCallback, useEffect, useState } from "react";
+import { Button } from "@/components/ui/button";
+import {
+  Card,
+  CardContent,
+  CardDescription,
+  CardHeader,
+  CardTitle,
+} from "@/components/ui/card";
+import { DebuggerLayout } from "../components/DebuggerLayout";
+import { formatCost, formatDuration, formatTokens } from "../lib/metrics";
+import type { ComparisonConfig, ComparisonRun, SessionMetrics } from "../types";
+
+interface ComparisonViewProps {
+  runId: string;
+}
+
+interface ChatMessage {
+  role: "user" | "assistant";
+  content: string;
+}
+
+interface SessionState {
+  sessionId: string | null;
+  messages: ChatMessage[];
+  isStreaming: boolean;
+  metrics: SessionMetrics | null;
+  error: string | null;
+}
+
+const AGENT_SERVER_URL =
+  typeof window !== "undefined"
+    ? window.location.origin.replace(":4000", ":3100")
+    : "http://localhost:3100";
+
+export function ComparisonView({ runId }: ComparisonViewProps) {
+  const [run, setRun] = useState<ComparisonRun | null>(null);
+  const [config, setConfig] = useState<ComparisonConfig | null>(null);
+  const [loading, setLoading] = useState(true);
+  const [error, setError] = useState<string | null>(null);
+
+  // Session states
+  const [controlState, setControlState] = useState<SessionState>({
+    sessionId: null,
+    messages: [],
+    isStreaming: false,
+    metrics: null,
+    error: null,
+  });
+  const [variantState, setVariantState] = useState<SessionState>({
+    sessionId: null,
+    messages: [],
+    isStreaming: false,
+    metrics: null,
+    error: null,
+  });
+
+  const [isRunning, setIsRunning] = useState(false);
+  const [hasRun, setHasRun] = useState(false);
+
+  // Fetch comparison run details and restore saved messages
+  useEffect(() => {
+    Promise.all([
+      fetch(`/api/comparison-run/${runId}`).then((res) => res.json()),
+    ])
+      .then(([runData]) => {
+        setRun(runData);
+
+        // Restore saved messages if the run has been completed
+        if (runData.status === "completed" || runData.status === "running") {
+          setHasRun(true);
+
+          // Restore control messages
+          if (runData.controlResponse) {
+            setControlState({
+              sessionId: runData.controlSessionId,
+              messages: [
+                { role: "user", content: runData.firstUserMessage },
+                { role: "assistant", content: runData.controlResponse },
+              ],
+              isStreaming: false,
+              metrics: runData.controlMetrics,
+              error: null,
+            });
+          }
+
+          // Restore variant messages
+          if (runData.variantResponse) {
+            setVariantState({
+              sessionId: runData.variantSessionId,
+              messages: [
+                { role: "user", content: runData.firstUserMessage },
+                { role: "assistant", content: runData.variantResponse },
+              ],
+              isStreaming: false,
+              metrics: runData.variantMetrics,
+              error: null,
+            });
+          }
+        }
+
+        // Fetch the config
+        return fetch(`/api/comparison-config`).then((res) => res.json());
+      })
+      .then((configData) => {
+        setConfig(configData);
+        setLoading(false);
+      })
+      .catch((err) => {
+        setError(err.message);
+        setLoading(false);
+      });
+  }, [runId]);
+
+  // Create a new session with the agent server
+  const createSession = async (
+    configOverrides?: Record<string, unknown>,
+  ): Promise<string> => {
+    const initRes = await fetch(`${AGENT_SERVER_URL}/rpc`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        jsonrpc: "2.0",
+        id: `init-${Date.now()}`,
+        method: "initialize",
+        params: {
+          protocolVersion: 1,
+          clientCapabilities: {},
+        },
+      }),
+    });
+    await initRes.json();
+
+    const sessionRes = await fetch(`${AGENT_SERVER_URL}/rpc`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        jsonrpc: "2.0",
+        id: `session-${Date.now()}`,
+        method: "session/new",
+        params: {
+          cwd: "/",
+          mcpServers: [],
+          _meta: configOverrides ? { configOverrides } : undefined,
+        },
+      }),
+    });
+    const sessionData = await sessionRes.json();
+    return sessionData.result.sessionId;
+  };
+
+  // Send a message and collect the response via SSE
+  const sendMessageAndCollect = async (
+    sessionId: string,
+    message: string,
+    onUpdate: (content: string) => void,
+  ): Promise<void> => {
+    let accumulatedContent = "";
+    let abortController: AbortController | null = new AbortController();
+
+    // Start SSE connection (don't await - runs in background)
+    const ssePromise = fetch(`${AGENT_SERVER_URL}/events`, {
+      headers: {
+        "X-Session-ID": sessionId,
+      },
+      signal: abortController.signal,
+    })
+      .then(async (response) => {
+        const reader = response.body?.getReader();
+        if (!reader) return;
+
+        const decoder = new TextDecoder();
+        let buffer = "";
+
+        try {
+          while (true) {
+            const { done, value } = await reader.read();
+            if (done) break;
+
+            buffer += decoder.decode(value, { stream: true });
+
+            // Process complete SSE events
+            const lines = buffer.split("\n");
+            buffer = lines.pop() || ""; // Keep incomplete line in buffer
+
+            for (const line of lines) {
+              if (line.startsWith("data: ")) {
+                try {
+                  const data = JSON.parse(line.slice(6));
+                  // Handle session/update notifications
+                  if (
+                    data.method === "session/update" &&
+                    data.params?.update?.sessionUpdate === "agent_message_chunk"
+                  ) {
+                    const content = data.params.update.content;
+                    if (content?.type === "text" && content.text) {
+                      accumulatedContent += content.text;
+                      onUpdate(accumulatedContent);
+                    }
+                  }
+                } catch {
+                  // Ignore parse errors
+                }
+              }
+            }
+          }
+        } catch (err) {
+          // AbortError is expected when we cancel the stream
+          if (err instanceof Error && err.name !== "AbortError") {
+            throw err;
+          }
+        }
+      })
+      .catch((err) => {
+        // AbortError is expected when we cancel the stream
+        if (err instanceof Error && err.name !== "AbortError") {
+          console.error("SSE error:", err);
+        }
+      });
+
+    // Small delay to ensure SSE connection is established
+    await new Promise((r) => setTimeout(r, 100));
+
+    // Send the prompt and wait for it to complete
+    // When the prompt RPC returns, the agent has finished responding
+    await fetch(`${AGENT_SERVER_URL}/rpc`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        jsonrpc: "2.0",
+        id: `prompt-${Date.now()}`,
+        method: "session/prompt",
+        params: {
+          sessionId,
+          prompt: [{ type: "text", text: message }],
+        },
+      }),
+    });
+
+    // Give a small delay for any final SSE chunks to arrive
+    await new Promise((r) => setTimeout(r, 200));
+
+    // Abort the SSE connection since we're done
+    abortController.abort();
+    abortController = null;
+  };
+
+  // Run the comparison
+  const runComparison = useCallback(async () => {
+    if (!run || !config) return;
+
+    setIsRunning(true);
+    setHasRun(true);
+
+    const firstMessage = run.firstUserMessage;
+
+    // Reset states
+    setControlState({
+      sessionId: null,
+      messages: [{ role: "user", content: firstMessage }],
+      isStreaming: true,
+      metrics: null,
+      error: null,
+    });
+    setVariantState({
+      sessionId: null,
+      messages: [{ role: "user", content: firstMessage }],
+      isStreaming: true,
+      metrics: null,
+      error: null,
+    });
+
+    try {
+      // Build config overrides based on dimension
+      const variantOverrides: Record<string, unknown> = {};
+      if (config.dimension === "model" && config.variantModel) {
+        variantOverrides.model = config.variantModel;
+      }
+      if (config.dimension === "system_prompt" && config.variantSystemPrompt) {
+        variantOverrides.systemPrompt = config.variantSystemPrompt;
+      }
+      if (config.dimension === "tools" && config.variantTools) {
+        variantOverrides.tools = config.variantTools;
+      }
+
+      // Create sessions in parallel
+      const [controlSessionId, variantSessionId] = await Promise.all([
+        createSession(), // Control - no overrides
+        createSession(variantOverrides), // Variant - with overrides
+      ]);
+
+      setControlState((prev) => ({ ...prev, sessionId: controlSessionId }));
+      setVariantState((prev) => ({ ...prev, sessionId: variantSessionId }));
+
+      // Update run with session IDs
+      await fetch(`/api/comparison-run/${runId}/update`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          status: "running",
+          controlSessionId,
+          variantSessionId,
+        }),
+      });
+
+      // Run both sessions in parallel
+      const startTime = Date.now();
+
+      // Track final responses and metrics
+      let finalControlResponse = "";
+      let finalVariantResponse = "";
+      let finalControlMetrics: SessionMetrics | null = null;
+      let finalVariantMetrics: SessionMetrics | null = null;
+
+      await Promise.all([
+        // Control session
+        sendMessageAndCollect(controlSessionId, firstMessage, (content) => {
+          finalControlResponse = content;
+          setControlState((prev) => ({
+            ...prev,
+            messages: [
+              { role: "user", content: firstMessage },
+              { role: "assistant", content },
+            ],
+          }));
+        })
+          .then(async () => {
+            const duration = Date.now() - startTime;
+            // Wait for telemetry data to be written to the database
+            await new Promise((r) => setTimeout(r, 2000));
+            // Fetch metrics - use control model for cost calculation
+            const controlModel =
+              config.controlModel || "claude-sonnet-4-5-20250929";
+            try {
+              const metricsRes = await fetch(
+                `/api/session-metrics/${controlSessionId}?model=${encodeURIComponent(controlModel)}`,
+              );
+              const metrics = await metricsRes.json();
+              metrics.durationMs = duration;
+              finalControlMetrics = metrics;
+              setControlState((prev) => ({
+                ...prev,
+                isStreaming: false,
+                metrics,
+              }));
+            } catch {
+              finalControlMetrics = {
+                durationMs: duration,
+                inputTokens: 0,
+                outputTokens: 0,
+                totalTokens: 0,
+                estimatedCost: 0,
+                toolCallCount: 0,
+              };
+              setControlState((prev) => ({
+                ...prev,
+                isStreaming: false,
+                metrics: finalControlMetrics,
+              }));
+            }
+          })
+          .catch((err) => {
+            setControlState((prev) => ({
+              ...prev,
+              isStreaming: false,
+              error: err.message,
+            }));
+          }),
+
+        // Variant session
+        sendMessageAndCollect(variantSessionId, firstMessage, (content) => {
+          finalVariantResponse = content;
+          setVariantState((prev) => ({
+            ...prev,
+            messages: [
+              { role: "user", content: firstMessage },
+              { role: "assistant", content },
+            ],
+          }));
+        })
+          .then(async () => {
+            const duration = Date.now() - startTime;
+            // Wait for telemetry data to be written to the database
+            await new Promise((r) => setTimeout(r, 2000));
+            // Fetch metrics - use variant model for cost calculation
+            const variantModel =
+              config.variantModel ||
+              config.controlModel ||
+              "claude-sonnet-4-5-20250929";
+            try {
+              const metricsRes = await fetch(
+                `/api/session-metrics/${variantSessionId}?model=${encodeURIComponent(variantModel)}`,
+              );
+              const metrics = await metricsRes.json();
+              metrics.durationMs = duration;
+              finalVariantMetrics = metrics;
+              setVariantState((prev) => ({
+                ...prev,
+                isStreaming: false,
+                metrics,
+              }));
+            } catch {
+              finalVariantMetrics = {
+                durationMs: duration,
+                inputTokens: 0,
+                outputTokens: 0,
+                totalTokens: 0,
+                estimatedCost: 0,
+                toolCallCount: 0,
+              };
+              setVariantState((prev) => ({
+                ...prev,
+                isStreaming: false,
+                metrics: finalVariantMetrics,
+              }));
+            }
+          })
+          .catch((err) => {
+            setVariantState((prev) => ({
+              ...prev,
+              isStreaming: false,
+              error: err.message,
+            }));
+          }),
+      ]);
+
+      // Update run status with responses and metrics
+      await fetch(`/api/comparison-run/${runId}/update`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          status: "completed",
+          controlMetrics: finalControlMetrics,
+          variantMetrics: finalVariantMetrics,
+          controlResponse: finalControlResponse,
+          variantResponse: finalVariantResponse,
+        }),
+      });
+    } catch (err) {
+      setError(err instanceof Error ? err.message : "Failed to run comparison");
+    } finally {
+      setIsRunning(false);
+    }
+  }, [run, config, runId]);
+
+  if (loading) {
+    return (
+      <DebuggerLayout title="Comparison" showBackButton backHref="/town-hall">
+        <div className="container mx-auto p-8">
+          <div className="text-muted-foreground">Loading comparison...</div>
+        </div>
+      </DebuggerLayout>
+    );
+  }
+
+  if (error || !run) {
+    return (
+      <DebuggerLayout title="Comparison" showBackButton backHref="/town-hall">
+        <div className="container mx-auto p-8">
+          <div className="text-red-500">Error: {error || "Run not found"}</div>
+        </div>
+      </DebuggerLayout>
+    );
+  }
+
+  const getControlDimensionLabel = () => {
+    if (!config) return "";
+    switch (config.dimension) {
+      case "model":
+        return `Model: ${config.controlModel || "unknown"}`;
+      case "system_prompt":
+        return "System Prompt (original)";
+      case "tools":
+        return "Tools (original)";
+      default:
+        return "";
+    }
+  };
+
+  const getDimensionLabel = () => {
+    if (!config) return "";
+    switch (config.dimension) {
+      case "model":
+        return `Model: ${config.variantModel}`;
+      case "system_prompt":
+        return "System Prompt (modified)";
+      case "tools":
+        return `Tools: ${config.variantTools?.join(", ")}`;
+      default:
+        return "";
+    }
+  };
+
+  return (
+    <DebuggerLayout title="Comparison" showBackButton backHref="/town-hall">
+      <div className="container mx-auto p-4 h-[calc(100vh-4rem)] flex flex-col overflow-hidden">
+        {/* Header */}
+        <div className="flex items-center justify-between mb-4">
+          <div>
+            <h2 className="text-lg font-semibold">A/B Comparison</h2>
+            <p className="text-sm text-muted-foreground">
+              Comparing: {config?.dimension?.replace("_", " ")} -{" "}
+              {getDimensionLabel()}
+            </p>
+          </div>
+          {!hasRun && (
+            <Button onClick={runComparison} disabled={isRunning}>
+              {isRunning ? "Running..." : "Run Comparison"}
+            </Button>
+          )}
+        </div>
+
+        {/* Pre-run state */}
+        {!hasRun && (
+          <div className="flex-1 flex items-center justify-center">
+            <Card className="max-w-md w-full">
+              <CardHeader className="text-center">
+                <CardTitle>Ready to Compare</CardTitle>
+                <CardDescription>
+                  This comparison will send the same prompt to both
+                  configurations and display the results side by side.
+                </CardDescription>
+              </CardHeader>
+              <CardContent className="space-y-4">
+                <div className="bg-muted rounded-lg p-4">
+                  <div className="text-xs font-medium uppercase text-muted-foreground mb-2">
+                    First message
+                  </div>
+                  <div className="text-sm">{run?.firstUserMessage}</div>
+                </div>
+                <div className="grid grid-cols-2 gap-4 text-sm">
+                  <div className="space-y-1">
+                    <div className="flex items-center gap-2">
+                      <span className="w-2 h-2 rounded-full bg-blue-500" />
+                      <span className="font-medium">Control</span>
+                    </div>
+                    <div className="text-muted-foreground text-xs">
+                      {getControlDimensionLabel()}
+                    </div>
+                  </div>
+                  <div className="space-y-1">
+                    <div className="flex items-center gap-2">
+                      <span className="w-2 h-2 rounded-full bg-orange-500" />
+                      <span className="font-medium">Variant</span>
+                    </div>
+                    <div className="text-muted-foreground text-xs">
+                      {getDimensionLabel()}
+                    </div>
+                  </div>
+                </div>
+              </CardContent>
+            </Card>
+          </div>
+        )}
+
+        {/* Side-by-side comparison */}
+        {hasRun && (
+          <div className="grid grid-cols-2 gap-4 flex-1 min-h-0">
+            {/* Control */}
+            <Card className="flex flex-col h-full min-h-0 overflow-hidden">
+              <CardHeader className="py-3 border-b shrink-0">
+                <CardTitle className="text-sm flex items-center gap-2">
+                  <span className="w-2 h-2 rounded-full bg-blue-500" />
+                  Control (Original)
+                </CardTitle>
+                <CardDescription className="text-xs">
+                  {getControlDimensionLabel()}
+                </CardDescription>
+              </CardHeader>
+              <CardContent className="flex-1 overflow-auto py-4">
+                {controlState.messages.map((msg, i) => (
+                  <div
+                    key={i}
+                    className={`mb-4 ${msg.role === "user" ? "text-blue-600 dark:text-blue-400" : ""}`}
+                  >
+                    <div className="text-xs font-medium uppercase mb-1">
+                      {msg.role}
+                    </div>
+                    <div className="text-sm whitespace-pre-wrap">
+                      {msg.content}
+                      {controlState.isStreaming &&
+                        msg.role === "assistant" &&
+                        i === controlState.messages.length - 1 && (
+                          <span className="animate-pulse">▊</span>
+                        )}
+                    </div>
+                  </div>
+                ))}
+                {controlState.error && (
+                  <div className="text-red-500 text-sm">
+                    Error: {controlState.error}
+                  </div>
+                )}
+              </CardContent>
+              {/* Metrics */}
+              {controlState.metrics && (
+                <div className="border-t p-3 shrink-0 bg-muted/50">
+                  <div className="grid grid-cols-4 gap-2 text-xs">
+                    <div>
+                      <span className="text-muted-foreground">Duration:</span>{" "}
+                      {formatDuration(controlState.metrics.durationMs)}
+                    </div>
+                    <div>
+                      <span className="text-muted-foreground">Tokens:</span>{" "}
+                      {formatTokens(controlState.metrics.totalTokens)}
+                    </div>
+                    <div>
+                      <span className="text-muted-foreground">Cost:</span>{" "}
+                      {formatCost(controlState.metrics.estimatedCost)}
+                    </div>
+                    <div>
+                      <span className="text-muted-foreground">Tools:</span>{" "}
+                      {controlState.metrics.toolCallCount}
+                    </div>
+                  </div>
+                </div>
+              )}
+            </Card>
+
+            {/* Variant */}
+            <Card className="flex flex-col h-full min-h-0 overflow-hidden">
+              <CardHeader className="py-3 border-b shrink-0">
+                <CardTitle className="text-sm flex items-center gap-2">
+                  <span className="w-2 h-2 rounded-full bg-orange-500" />
+                  Variant
+                </CardTitle>
+                <CardDescription className="text-xs">
+                  {getDimensionLabel()}
+                </CardDescription>
+              </CardHeader>
+              <CardContent className="flex-1 overflow-auto py-4">
+                {variantState.messages.map((msg, i) => (
+                  <div
+                    key={i}
+                    className={`mb-4 ${msg.role === "user" ? "text-orange-600 dark:text-orange-400" : ""}`}
+                  >
+                    <div className="text-xs font-medium uppercase mb-1">
+                      {msg.role}
+                    </div>
+                    <div className="text-sm whitespace-pre-wrap">
+                      {msg.content}
+                      {variantState.isStreaming &&
+                        msg.role === "assistant" &&
+                        i === variantState.messages.length - 1 && (
+                          <span className="animate-pulse">▊</span>
+                        )}
+                    </div>
+                  </div>
+                ))}
+                {variantState.error && (
+                  <div className="text-red-500 text-sm">
+                    Error: {variantState.error}
+                  </div>
+                )}
+              </CardContent>
+              {/* Metrics */}
+              {variantState.metrics && (
+                <div className="border-t p-3 shrink-0 bg-muted/50">
+                  <div className="grid grid-cols-4 gap-2 text-xs">
+                    <div>
+                      <span className="text-muted-foreground">Duration:</span>{" "}
+                      {formatDuration(variantState.metrics.durationMs)}
+                    </div>
+                    <div>
+                      <span className="text-muted-foreground">Tokens:</span>{" "}
+                      {formatTokens(variantState.metrics.totalTokens)}
+                    </div>
+                    <div>
+                      <span className="text-muted-foreground">Cost:</span>{" "}
+                      {formatCost(variantState.metrics.estimatedCost)}
+                    </div>
+                    <div>
+                      <span className="text-muted-foreground">Tools:</span>{" "}
+                      {variantState.metrics.toolCallCount}
+                    </div>
+                  </div>
+                </div>
+              )}
+            </Card>
+          </div>
+        )}
+      </div>
+    </DebuggerLayout>
+  );
+}