@townco/debugger 0.1.28 → 0.1.29

package/src/pages/ComparisonView.tsx

@@ -112,6 +112,14 @@ export function ComparisonView({ runId }: ComparisonViewProps) {
       });
   }, [runId]);
 
+  const generateRequestId = (prefix: string, sessionId?: string) => {
+    const randomPart =
+      typeof crypto !== "undefined" && "randomUUID" in crypto
+        ? crypto.randomUUID()
+        : `${Math.random().toString(16).slice(2)}-${Date.now().toString(16)}`;
+    return `${prefix}-${sessionId ? `${sessionId}-` : ""}${randomPart}`;
+  };
+
   // Create a new session with the agent server
   const createSession = async (
     configOverrides?: Record<string, unknown>,
@@ -121,7 +129,7 @@ export function ComparisonView({ runId }: ComparisonViewProps) {
       headers: { "Content-Type": "application/json" },
       body: JSON.stringify({
         jsonrpc: "2.0",
-        id: `init-${Date.now()}`,
+        id: generateRequestId("init"),
         method: "initialize",
         params: {
           protocolVersion: 1,
@@ -136,7 +144,7 @@ export function ComparisonView({ runId }: ComparisonViewProps) {
       headers: { "Content-Type": "application/json" },
       body: JSON.stringify({
         jsonrpc: "2.0",
-        id: `session-${Date.now()}`,
+        id: generateRequestId("session"),
         method: "session/new",
         params: {
           cwd: "/",
@@ -154,7 +162,7 @@ export function ComparisonView({ runId }: ComparisonViewProps) {
     sessionId: string,
     message: string,
     onUpdate: (content: string) => void,
-  ): Promise<void> => {
+  ): Promise<string> => {
     let accumulatedContent = "";
     let abortController: AbortController | null = new AbortController();
 
@@ -228,7 +236,7 @@ export function ComparisonView({ runId }: ComparisonViewProps) {
       headers: { "Content-Type": "application/json" },
       body: JSON.stringify({
         jsonrpc: "2.0",
-        id: `prompt-${Date.now()}`,
+        id: generateRequestId("prompt", sessionId),
         method: "session/prompt",
         params: {
           sessionId,
@@ -243,6 +251,9 @@ export function ComparisonView({ runId }: ComparisonViewProps) {
     // Abort the SSE connection since we're done
     abortController.abort();
     abortController = null;
+
+    // Return the accumulated content
+    return accumulatedContent;
   };
 
   // Run the comparison
@@ -271,15 +282,16 @@ export function ComparisonView({ runId }: ComparisonViewProps) {
     });
 
     try {
-      // Build config overrides based on dimension
+      // Build config overrides based on all selected dimensions
       const variantOverrides: Record<string, unknown> = {};
-      if (config.dimension === "model" && config.variantModel) {
+      const dimensions = config.dimensions || [];
+      if (dimensions.includes("model") && config.variantModel) {
         variantOverrides.model = config.variantModel;
       }
-      if (config.dimension === "system_prompt" && config.variantSystemPrompt) {
+      if (dimensions.includes("system_prompt") && config.variantSystemPrompt) {
         variantOverrides.systemPrompt = config.variantSystemPrompt;
       }
-      if (config.dimension === "tools" && config.variantTools) {
+      if (dimensions.includes("tools") && config.variantTools) {
         variantOverrides.tools = config.variantTools;
       }
 
@@ -307,123 +319,157 @@ export function ComparisonView({ runId }: ComparisonViewProps) {
       const startTime = Date.now();
 
       // Track final responses and metrics
-      let finalControlResponse = "";
-      let finalVariantResponse = "";
-      let finalControlMetrics: SessionMetrics | null = null;
-      let finalVariantMetrics: SessionMetrics | null = null;
-
-      await Promise.all([
-        // Control session
-        sendMessageAndCollect(controlSessionId, firstMessage, (content) => {
-          finalControlResponse = content;
-          setControlState((prev) => ({
-            ...prev,
-            messages: [
-              { role: "user", content: firstMessage },
-              { role: "assistant", content },
-            ],
-          }));
-        })
-          .then(async () => {
-            const duration = Date.now() - startTime;
-            // Wait for telemetry data to be written to the database
-            await new Promise((r) => setTimeout(r, 2000));
-            // Fetch metrics - use control model for cost calculation
-            const controlModel =
-              config.controlModel || "claude-sonnet-4-5-20250929";
-            try {
-              const metricsRes = await fetch(
-                `/api/session-metrics/${controlSessionId}?model=${encodeURIComponent(controlModel)}`,
-              );
-              const metrics = await metricsRes.json();
-              metrics.durationMs = duration;
-              finalControlMetrics = metrics;
-              setControlState((prev) => ({
-                ...prev,
-                isStreaming: false,
-                metrics,
-              }));
-            } catch {
-              finalControlMetrics = {
+      let finalControlMetrics: SessionMetrics = {
+        durationMs: 0,
+        inputTokens: 0,
+        outputTokens: 0,
+        totalTokens: 0,
+        estimatedCost: 0,
+        toolCallCount: 0,
+      };
+      let finalVariantMetrics: SessionMetrics = {
+        durationMs: 0,
+        inputTokens: 0,
+        outputTokens: 0,
+        totalTokens: 0,
+        estimatedCost: 0,
+        toolCallCount: 0,
+      };
+
+      // Helper to run a session and fetch metrics
+      const runSession = async (
+        sessionId: string,
+        model: string,
+        setState: typeof setControlState,
+        onContentUpdate: (content: string) => void,
+      ): Promise<{ response: string; metrics: SessionMetrics }> => {
+        try {
+          const response = await sendMessageAndCollect(
+            sessionId,
+            firstMessage,
+            onContentUpdate,
+          );
+
+          const duration = Date.now() - startTime;
+
+          // Poll metrics until they stabilize or we hit a max wait window.
+          const fetchMetricsWithRetry = async (): Promise<SessionMetrics> => {
+            const maxWaitMs = 60_000;
+            const pollIntervalMs = 2_000;
+            let elapsed = 0;
+            let previousTokens = -1;
+            let previousTools = -1;
+            let lastMetrics: SessionMetrics | null = null;
+
+            while (elapsed <= maxWaitMs) {
+              try {
+                const metricsRes = await fetch(
+                  `/api/session-metrics/${sessionId}?model=${encodeURIComponent(model)}`,
+                );
+                const metrics = await metricsRes.json();
+                lastMetrics = { ...metrics, durationMs: duration };
+
+                // If tokens/tool calls stopped changing and we have data, treat as final.
+                if (
+                  metrics.totalTokens > 0 &&
+                  metrics.totalTokens === previousTokens &&
+                  metrics.toolCallCount === previousTools
+                ) {
+                  return lastMetrics!;
+                }
+
+                previousTokens = metrics.totalTokens ?? 0;
+                previousTools = metrics.toolCallCount ?? 0;
+              } catch {
+                // swallow and retry
+              }
+
+              await new Promise((r) => setTimeout(r, pollIntervalMs));
+              elapsed += pollIntervalMs;
+            }
+
+            // Return whatever we last saw (or zeros if nothing ever arrived)
+            return (
+              lastMetrics ?? {
                 durationMs: duration,
                 inputTokens: 0,
                 outputTokens: 0,
                 totalTokens: 0,
                 estimatedCost: 0,
                 toolCallCount: 0,
-              };
-              setControlState((prev) => ({
-                ...prev,
-                isStreaming: false,
-                metrics: finalControlMetrics,
-              }));
-            }
-          })
-          .catch((err) => {
-            setControlState((prev) => ({
-              ...prev,
-              isStreaming: false,
-              error: err.message,
-            }));
-          }),
+              }
+            );
+          };
+
+          const metrics = await fetchMetricsWithRetry();
 
-        // Variant session
-        sendMessageAndCollect(variantSessionId, firstMessage, (content) => {
-          finalVariantResponse = content;
-          setVariantState((prev) => ({
+          setState((prev) => ({
             ...prev,
-            messages: [
-              { role: "user", content: firstMessage },
-              { role: "assistant", content },
-            ],
+            isStreaming: false,
+            metrics,
           }));
-        })
-          .then(async () => {
-            const duration = Date.now() - startTime;
-            // Wait for telemetry data to be written to the database
-            await new Promise((r) => setTimeout(r, 2000));
-            // Fetch metrics - use variant model for cost calculation
-            const variantModel =
-              config.variantModel ||
-              config.controlModel ||
-              "claude-sonnet-4-5-20250929";
-            try {
-              const metricsRes = await fetch(
-                `/api/session-metrics/${variantSessionId}?model=${encodeURIComponent(variantModel)}`,
-              );
-              const metrics = await metricsRes.json();
-              metrics.durationMs = duration;
-              finalVariantMetrics = metrics;
-              setVariantState((prev) => ({
-                ...prev,
-                isStreaming: false,
-                metrics,
-              }));
-            } catch {
-              finalVariantMetrics = {
-                durationMs: duration,
-                inputTokens: 0,
-                outputTokens: 0,
-                totalTokens: 0,
-                estimatedCost: 0,
-                toolCallCount: 0,
-              };
-              setVariantState((prev) => ({
-                ...prev,
-                isStreaming: false,
-                metrics: finalVariantMetrics,
-              }));
-            }
-          })
-          .catch((err) => {
+
+          return { response, metrics };
+        } catch (err) {
+          setState((prev) => ({
+            ...prev,
+            isStreaming: false,
+            error: err instanceof Error ? err.message : "Unknown error",
+          }));
+          return {
+            response: "",
+            metrics: {
+              durationMs: 0,
+              inputTokens: 0,
+              outputTokens: 0,
+              totalTokens: 0,
+              estimatedCost: 0,
+              toolCallCount: 0,
+            },
+          };
+        }
+      };
+
+      const controlModel = config.controlModel || "claude-sonnet-4-5-20250929";
+      const variantModel =
+        config.variantModel ||
+        config.controlModel ||
+        "claude-sonnet-4-5-20250929";
+
+      const [controlResult, variantResult] = await Promise.all([
+        runSession(
+          controlSessionId,
+          controlModel,
+          setControlState,
+          (content) => {
+            setControlState((prev) => ({
+              ...prev,
+              messages: [
+                { role: "user", content: firstMessage },
+                { role: "assistant", content },
+              ],
+            }));
+          },
+        ),
+        runSession(
+          variantSessionId,
+          variantModel,
+          setVariantState,
+          (content) => {
             setVariantState((prev) => ({
               ...prev,
-              isStreaming: false,
-              error: err.message,
+              messages: [
+                { role: "user", content: firstMessage },
+                { role: "assistant", content },
+              ],
             }));
-          }),
+          },
+        ),
       ]);
 
+      finalControlMetrics = controlResult.metrics;
+      finalVariantMetrics = variantResult.metrics;
+
       // Update run status with responses and metrics
       await fetch(`/api/comparison-run/${runId}/update`, {
         method: "POST",
@@ -432,8 +478,8 @@ export function ComparisonView({ runId }: ComparisonViewProps) {
           status: "completed",
           controlMetrics: finalControlMetrics,
           variantMetrics: finalVariantMetrics,
-          controlResponse: finalControlResponse,
-          variantResponse: finalVariantResponse,
+          controlResponse: controlResult.response,
+          variantResponse: variantResult.response,
         }),
       });
     } catch (err) {
@@ -464,31 +510,97 @@ export function ComparisonView({ runId }: ComparisonViewProps) {
   }
 
   const getControlDimensionLabel = () => {
-    if (!config) return "";
-    switch (config.dimension) {
-      case "model":
-        return `Model: ${config.controlModel || "unknown"}`;
-      case "system_prompt":
-        return "System Prompt (original)";
-      case "tools":
-        return "Tools (original)";
-      default:
-        return "";
+    if (!config || !config.dimensions || config.dimensions.length === 0)
+      return "";
+    const labels: string[] = [];
+    for (const dim of config.dimensions) {
+      switch (dim) {
+        case "model":
+          labels.push(`Model: ${config.controlModel || "original"}`);
+          break;
+        case "system_prompt":
+          labels.push("System Prompt: original");
+          break;
+        case "tools":
+          labels.push("Tools: original");
+          break;
+      }
     }
+    return labels.join(" | ");
   };
 
   const getDimensionLabel = () => {
-    if (!config) return "";
-    switch (config.dimension) {
-      case "model":
-        return `Model: ${config.variantModel}`;
-      case "system_prompt":
-        return "System Prompt (modified)";
-      case "tools":
-        return `Tools: ${config.variantTools?.join(", ")}`;
-      default:
-        return "";
+    if (!config || !config.dimensions || config.dimensions.length === 0)
+      return "";
+    const labels: string[] = [];
+    for (const dim of config.dimensions) {
+      switch (dim) {
+        case "model":
+          labels.push(`Model: ${config.variantModel}`);
+          break;
+        case "system_prompt":
+          labels.push("System Prompt: modified");
+          break;
+        case "tools":
+          labels.push(`Tools: ${config.variantTools?.join(", ")}`);
+          break;
+      }
     }
+    return labels.join(" | ");
+  };
+
+  const getDimensionsSummary = () => {
+    if (!config || !config.dimensions || config.dimensions.length === 0)
+      return "";
+    return config.dimensions.map((d) => d.replace("_", " ")).join(", ");
+  };
+
+  const formatToolTime = (ns?: number) => {
+    if (!ns) return "";
+    return new Date(ns / 1_000_000).toLocaleTimeString();
+  };
+
+  const renderToolCalls = (toolCalls?: SessionMetrics["toolCalls"]) => {
+    if (!toolCalls || toolCalls.length === 0) {
+      return <div className="text-xs text-muted-foreground">No tool calls</div>;
+    }
+
+    return (
+      <div className="space-y-2">
+        {toolCalls.map((call, idx) => (
+          <details
+            key={`${call.name}-${call.startTimeUnixNano ?? idx}`}
+            className="rounded-md border px-3 py-2 bg-muted/50"
+          >
+            <summary className="text-xs font-medium cursor-pointer flex items-center justify-between">
+              <span>
+                {call.name}{" "}
+                {call.startTimeUnixNano ? (
+                  <span className="text-muted-foreground">
+                    @ {formatToolTime(call.startTimeUnixNano)}
+                  </span>
+                ) : null}
+              </span>
+              <span className="text-muted-foreground text-[11px]">view</span>
+            </summary>
+            <div className="mt-2 text-[11px] space-y-1 break-words">
+              <div>
+                <span className="font-semibold">Args:</span>{" "}
+                <code className="break-words">
+                  {JSON.stringify(call.input, null, 2)}
+                </code>
+              </div>
+              <div>
+                <span className="font-semibold">Result:</span>{" "}
+                <code className="break-words">
+                  {JSON.stringify(call.output, null, 2)}
+                </code>
+              </div>
+            </div>
+          </details>
+        ))}
+      </div>
+    );
   };
 
   return (
@@ -499,8 +611,7 @@ export function ComparisonView({ runId }: ComparisonViewProps) {
           <div>
             <h2 className="text-lg font-semibold">A/B Comparison</h2>
             <p className="text-sm text-muted-foreground">
-              Comparing: {config?.dimension?.replace("_", " ")} -{" "}
-              {getDimensionLabel()}
+              Comparing: {getDimensionsSummary()}
             </p>
           </div>
           {!hasRun && (
@@ -613,6 +724,12 @@ export function ComparisonView({ runId }: ComparisonViewProps) {
                       {controlState.metrics.toolCallCount}
                     </div>
                   </div>
+                  <div className="mt-3">
+                    <div className="text-[11px] font-semibold mb-1">
+                      Tool calls
+                    </div>
+                    {renderToolCalls(controlState.metrics.toolCalls)}
+                  </div>
                 </div>
               )}
             </Card>
@@ -674,6 +791,12 @@ export function ComparisonView({ runId }: ComparisonViewProps) {
                       {variantState.metrics.toolCallCount}
                     </div>
                   </div>
+                  <div className="mt-3">
+                    <div className="text-[11px] font-semibold mb-1">
+                      Tool calls
+                    </div>
+                    {renderToolCalls(variantState.metrics.toolCalls)}
+                  </div>
                 </div>
               )}
             </Card>