@townco/debugger 0.1.23 → 0.1.24

package/src/db.ts

@@ -80,4 +80,75 @@ export class DebuggerDb {
       )
       .all(limit, offset);
   }
+
+  /**
+   * Get all spans that have a specific session ID in their attributes.
+   * This is more reliable than querying by trace.session_id because
+   * concurrent sessions can cause race conditions in trace association.
+   */
+  getSpansBySessionAttribute(sessionId: string): Span[] {
+    // Use JSON extract to find spans where agent.session_id matches
+    return this.db
+      .query<Span, [string]>(
+        `
+        SELECT *
+        FROM spans
+        WHERE json_extract(attributes, '$."agent.session_id"') = ?
+        ORDER BY start_time_unix_nano ASC
+        `,
+      )
+      .all(sessionId);
+  }
+
+  /**
+   * Get the first user message for a session by querying span attributes.
+   * The user message is stored in the "user.message" attribute of adapter.prompt spans.
+   */
+  getFirstUserMessageBySession(sessionId: string): string | null {
+    // Use LIKE for pattern matching since json_extract doesn't work reliably
+    // with keys containing dots
+    const span = this.db
+      .query<Span, [string]>(
+        `
+        SELECT *
+        FROM spans
+        WHERE name = 'adapter.prompt'
+          AND attributes LIKE '%"agent.session_id":"' || ? || '"%'
+        ORDER BY start_time_unix_nano ASC
+        LIMIT 1
+        `,
+      )
+      .get(sessionId);
+
+    if (!span || !span.attributes) return null;
+
+    try {
+      const attrs = JSON.parse(span.attributes);
+      return attrs["user.message"] || null;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Get trace IDs for a session by querying span attributes.
+   * This is more reliable than querying by trace.session_id because
+   * concurrent sessions can cause race conditions in trace association.
+   * Returns trace IDs with their earliest span timestamp.
+   */
+  getTraceIdsBySessionAttribute(
+    sessionId: string,
+  ): Array<{ trace_id: string; start_time_unix_nano: number }> {
+    return this.db
+      .query<{ trace_id: string; start_time_unix_nano: number }, [string]>(
+        `
+        SELECT DISTINCT trace_id, MIN(start_time_unix_nano) as start_time_unix_nano
+        FROM spans
+        WHERE json_extract(attributes, '$."agent.session_id"') = ?
+        GROUP BY trace_id
+        ORDER BY start_time_unix_nano ASC
+        `,
+      )
+      .all(sessionId);
+  }
 }

package/src/index.ts

@@ -14,12 +14,14 @@ const otlpPort = Number.parseInt(
 );
 const dbPath = process.env.DB_PATH ?? "./traces.db";
 const agentName = process.env.AGENT_NAME ?? "Agent";
+const agentServerUrl = process.env.AGENT_SERVER_URL ?? "http://localhost:3100";
 
 const { server, otlpServer } = startDebuggerServer({
   port,
   otlpPort,
   dbPath,
   agentName,
+  agentServerUrl,
 });
 
 console.log(`OTLP server running at ${otlpServer.url}`);

package/src/lib/metrics.test.ts

@@ -0,0 +1,51 @@
+import { describe, expect, test } from "bun:test";
+import type { Span } from "../types";
+import { calculateCost, extractMetricsFromSpans } from "./metrics";
+
+describe("metrics", () => {
+  describe("calculateCost", () => {
+    test("calculates cost correctly for known model", () => {
+      // Claude 3.5 Sonnet: $3 input, $15 output per 1M
+      const cost = calculateCost(
+        "claude-sonnet-4-5-20250929",
+        1_000_000,
+        1_000_000,
+      );
+      expect(cost).toBeCloseTo(18.0);
+    });
+
+    test("returns 0 for unknown model", () => {
+      const cost = calculateCost("unknown-model", 1000, 1000);
+      expect(cost).toBe(0);
+    });
+  });
+
+  describe("extractMetricsFromSpans", () => {
+    test("extracts tokens and tool calls", () => {
+      const spans: Span[] = [
+        {
+          name: "gen_ai.generate",
+          attributes: JSON.stringify({
+            "gen_ai.usage.input_tokens": 100,
+            "gen_ai.usage.output_tokens": 50,
+          }),
+        } as any,
+        {
+          name: "tool_call:search",
+          attributes: "{}",
+        } as any,
+      ];
+
+      const metrics = extractMetricsFromSpans(
+        spans,
+        "claude-sonnet-4-5-20250929",
+      );
+
+      expect(metrics.inputTokens).toBe(100);
+      expect(metrics.outputTokens).toBe(50);
+      expect(metrics.totalTokens).toBe(150);
+      expect(metrics.toolCallCount).toBe(1);
+      expect(metrics.estimatedCost).toBeGreaterThan(0);
+    });
+  });
+});

package/src/lib/metrics.ts

@@ -0,0 +1,136 @@
+import type { SessionMetrics, Span, Trace } from "../types";
+import { getModelPricing } from "./pricing";
+
+/**
+ * Calculate estimated cost based on model and token counts
+ */
+export function calculateCost(
+  model: string,
+  inputTokens: number,
+  outputTokens: number,
+): number {
+  const pricing = getModelPricing(model);
+  if (!pricing) {
+    // Unknown model, return 0
+    return 0;
+  }
+  return (
+    (inputTokens * pricing.input + outputTokens * pricing.output) / 1_000_000
+  );
+}
+
+/**
+ * Extract metrics from trace spans
+ */
+export function extractMetricsFromSpans(
+  spans: Span[],
+  model: string,
+): Omit<SessionMetrics, "durationMs"> {
+  let inputTokens = 0;
+  let outputTokens = 0;
+  let toolCallCount = 0;
+
+  for (const span of spans) {
+    const attrs = span.attributes ? JSON.parse(span.attributes) : {};
+
+    // Extract token usage from gen_ai attributes
+    if (attrs["gen_ai.usage.input_tokens"]) {
+      inputTokens += attrs["gen_ai.usage.input_tokens"];
+    }
+    if (attrs["gen_ai.usage.output_tokens"]) {
+      outputTokens += attrs["gen_ai.usage.output_tokens"];
+    }
+
+    // Count tool calls
+    if (span.name.includes("tool_call") || span.name.startsWith("tool:")) {
+      toolCallCount++;
+    }
+  }
+
+  const totalTokens = inputTokens + outputTokens;
+  const estimatedCost = calculateCost(model, inputTokens, outputTokens);
+
+  return {
+    inputTokens,
+    outputTokens,
+    totalTokens,
+    estimatedCost,
+    toolCallCount,
+  };
+}
+
+/**
+ * Extract full session metrics from traces
+ */
+export function extractSessionMetrics(
+  traces: Trace[],
+  spans: Span[],
+  model: string,
+): SessionMetrics {
+  // Calculate total duration from traces
+  let minStartTime = Number.MAX_SAFE_INTEGER;
+  let maxEndTime = 0;
+
+  for (const trace of traces) {
+    if (trace.start_time_unix_nano < minStartTime) {
+      minStartTime = trace.start_time_unix_nano;
+    }
+    if (trace.end_time_unix_nano > maxEndTime) {
+      maxEndTime = trace.end_time_unix_nano;
+    }
+  }
+
+  const durationMs =
+    minStartTime < Number.MAX_SAFE_INTEGER
+      ? (maxEndTime - minStartTime) / 1_000_000
+      : 0;
+
+  // Extract token metrics from spans
+  const tokenMetrics = extractMetricsFromSpans(spans, model);
+
+  return {
+    durationMs,
+    ...tokenMetrics,
+  };
+}
+
+/**
+ * Format duration in a human-readable way
+ */
+export function formatDuration(ms: number | undefined): string {
+  if (ms === undefined || ms === null) {
+    return "0ms";
+  }
+  if (ms < 1000) {
+    return `${Math.round(ms)}ms`;
+  }
+  if (ms < 60000) {
+    return `${(ms / 1000).toFixed(1)}s`;
+  }
+  const minutes = Math.floor(ms / 60000);
+  const seconds = ((ms % 60000) / 1000).toFixed(0);
+  return `${minutes}m ${seconds}s`;
+}
+
+/**
+ * Format cost in USD
+ */
+export function formatCost(cost: number | undefined): string {
+  if (cost === undefined || cost === null) {
+    return "$0.00";
+  }
+  if (cost < 0.01) {
+    return `$${cost.toFixed(4)}`;
+  }
+  return `$${cost.toFixed(2)}`;
+}
+
+/**
+ * Format token count with commas
+ */
+export function formatTokens(tokens: number | undefined): string {
+  if (tokens === undefined || tokens === null) {
+    return "0";
+  }
+  return tokens.toLocaleString();
+}

package/src/lib/pricing.ts

@@ -0,0 +1,23 @@
+export interface ModelPricing {
+  input: number; // Cost per 1M tokens
+  output: number; // Cost per 1M tokens
+}
+
+// Model pricing per 1M tokens (USD)
+export const DEFAULT_MODEL_PRICING: Record<string, ModelPricing> = {
+  // Anthropic models
+  "claude-sonnet-4-5-20250929": { input: 3.0, output: 15.0 },
+  "claude-3-5-haiku-20241022": { input: 0.25, output: 1.25 },
+  "claude-opus-4-5-20251101": { input: 15.0, output: 75.0 },
+  // Google Gemini models
+  "gemini-2.0-flash": { input: 0.075, output: 0.3 },
+  "gemini-1.5-pro": { input: 1.25, output: 5.0 },
+  "gemini-1.5-flash": { input: 0.075, output: 0.3 },
+};
+
+/**
+ * Get pricing for a specific model
+ */
+export function getModelPricing(model: string): ModelPricing | undefined {
+  return DEFAULT_MODEL_PRICING[model];
+}

package/src/lib/turnExtractor.ts

@@ -1,8 +1,9 @@
-import type { Log, Span } from "../types";
+import type { AgentMessage, Log, Span } from "../types";
 
 export interface TurnMessages {
   userInput: string | null;
   llmOutput: string | null;
+  agentMessages: AgentMessage[];
 }
 
 export interface TokenUsage {
@@ -42,6 +43,7 @@ export function extractTurnMessages(spans: Span[], logs?: Log[]): TurnMessages {
   const result: TurnMessages = {
     userInput: null,
     llmOutput: null,
+    agentMessages: [],
   };
 
   // Extract user input from logs
@@ -69,31 +71,70 @@ export function extractTurnMessages(spans: Span[], logs?: Log[]): TurnMessages {
     })
     .sort((a, b) => a.start_time_unix_nano - b.start_time_unix_nano);
 
-  if (chatSpans.length === 0) return result;
-
-  // Extract LLM output from last chat span
+  // Extract LLM output from ALL chat spans and tool calls
   try {
-    const lastSpan = chatSpans[chatSpans.length - 1];
-    if (!lastSpan) return result;
-
-    const attrs = parseAttributes(lastSpan.attributes);
-    const outputMessages = attrs["gen_ai.output.messages"];
-
-    if (outputMessages) {
-      const messages: Message[] =
-        typeof outputMessages === "string"
-          ? JSON.parse(outputMessages)
-          : outputMessages;
-
-      // Look for assistant message - handle both "assistant" (OpenAI) and "ai" (LangChain) roles
-      const assistantMessage = messages
-        .filter((msg) => msg.role === "assistant" || msg.role === "ai")
-        .pop();
-
-      if (assistantMessage) {
-        result.llmOutput = extractMessageContent(assistantMessage);
+    // Find all tool call spans
+    const toolCallSpans = spans
+      .filter((span) => span.name === "agent.tool_call")
+      .sort((a, b) => a.start_time_unix_nano - b.start_time_unix_nano);
+
+    // Combine chat spans and tool calls, sorted by time
+    const allEvents = [
+      ...chatSpans.map((s) => ({ span: s, type: "chat" as const })),
+      ...toolCallSpans.map((s) => ({ span: s, type: "tool_call" as const })),
+    ].sort((a, b) => a.span.end_time_unix_nano - b.span.end_time_unix_nano);
+
+    for (const event of allEvents) {
+      const attrs = parseAttributes(event.span.attributes);
+
+      if (event.type === "chat") {
+        const outputMessages = attrs["gen_ai.output.messages"];
+
+        if (outputMessages) {
+          const messages: Message[] =
+            typeof outputMessages === "string"
+              ? JSON.parse(outputMessages)
+              : outputMessages;
+
+          // Look for assistant message - handle both "assistant" (OpenAI) and "ai" (LangChain) roles
+          const assistantMessage = messages
+            .filter((msg) => msg.role === "assistant" || msg.role === "ai")
+            .pop();
+
+          if (assistantMessage) {
+            const content = extractMessageContent(assistantMessage);
+            // Only include messages with actual text content (not empty or whitespace-only)
+            if (content && content.trim().length > 0) {
+              result.agentMessages.push({
+                content,
+                spanId: event.span.span_id,
+                timestamp: event.span.end_time_unix_nano,
+                type: "chat",
+              });
+            }
+          }
+        }
+      } else if (event.type === "tool_call") {
+        const toolName = attrs["tool.name"] as string;
+        if (toolName) {
+          result.agentMessages.push({
+            content: toolName,
+            spanId: event.span.span_id,
+            timestamp: event.span.end_time_unix_nano,
+            type: "tool_call",
+            toolName,
+          });
+        }
       }
     }
+
+    // Set llmOutput to the last chat message for backwards compatibility
+    const lastChatMessage = result.agentMessages
+      .filter((m) => m.type === "chat")
+      .pop();
+    if (lastChatMessage) {
+      result.llmOutput = lastChatMessage.content;
+    }
   } catch (error) {
     console.error("Failed to extract LLM output:", error);
   }