@tracemarketplace/shared 0.0.6 → 0.0.9

package/src/extractor-codex.test.ts

@@ -0,0 +1,225 @@
+import { describe, expect, it } from "vitest";
+import { extractCodex } from "./extractors/codex.js";
+
+function makeBuffer(lines: Array<Record<string, unknown>>): Buffer {
+  return Buffer.from(lines.map((line) => JSON.stringify(line)).join("\n") + "\n", "utf-8");
+}
+
+describe("extractCodex", () => {
+  it("parses modern Codex response items and token counts", async () => {
+    const trace = await extractCodex(
+      makeBuffer([
+        {
+          timestamp: "2026-03-21T00:00:00.000Z",
+          type: "session_meta",
+          payload: {
+            id: "codex-session-1",
+            timestamp: "2026-03-21T00:00:00.000Z",
+            cwd: "/Users/test/project",
+            cli_version: "0.110.0",
+          },
+        },
+        {
+          timestamp: "2026-03-21T00:00:01.000Z",
+          type: "event_msg",
+          payload: { type: "task_started", turn_id: "turn-1" },
+        },
+        {
+          timestamp: "2026-03-21T00:00:01.000Z",
+          type: "turn_context",
+          payload: { model: "gpt-5.4" },
+        },
+        {
+          timestamp: "2026-03-21T00:00:02.000Z",
+          type: "event_msg",
+          payload: { type: "user_message", message: "Audit the repo" },
+        },
+        {
+          timestamp: "2026-03-21T00:00:03.000Z",
+          type: "response_item",
+          payload: {
+            type: "message",
+            role: "assistant",
+            content: [{ type: "output_text", text: "I will inspect the repo." }],
+          },
+        },
+        {
+          timestamp: "2026-03-21T00:00:04.000Z",
+          type: "response_item",
+          payload: {
+            type: "function_call",
+            name: "exec_command",
+            arguments: JSON.stringify({
+              cmd: "sed -n '1,120p' src/main.ts",
+              workdir: "/Users/test/project",
+            }),
+            call_id: "call-1",
+          },
+        },
+        {
+          timestamp: "2026-03-21T00:00:05.000Z",
+          type: "response_item",
+          payload: {
+            type: "function_call_output",
+            call_id: "call-1",
+            output: "Process exited with code 0\nsrc/main.ts",
+          },
+        },
+        {
+          timestamp: "2026-03-21T00:00:06.000Z",
+          type: "event_msg",
+          payload: {
+            type: "token_count",
+            info: {
+              total_token_usage: {
+                input_tokens: 1200,
+                cached_input_tokens: 200,
+                output_tokens: 300,
+                reasoning_output_tokens: 50,
+                total_tokens: 1500,
+              },
+            },
+          },
+        },
+        {
+          timestamp: "2026-03-21T00:00:07.000Z",
+          type: "event_msg",
+          payload: { type: "agent_message", message: "I found the relevant file." },
+        },
+        {
+          timestamp: "2026-03-21T00:00:08.000Z",
+          type: "event_msg",
+          payload: {
+            type: "task_complete",
+            last_agent_message: "I found the relevant file.",
+          },
+        },
+      ]),
+      "test@example.com",
+    );
+
+    expect(trace.source_tool).toBe("codex_cli");
+    expect(trace.source_session_id).toBe("codex-session-1");
+    expect(trace.source_version).toBe("0.110.0");
+    expect(trace.turn_count).toBe(2);
+    expect(trace.total_input_tokens).toBe(1200);
+    expect(trace.total_output_tokens).toBe(300);
+    expect(trace.total_cache_read_tokens).toBe(200);
+    expect(trace.has_shell_commands).toBe(true);
+    expect(trace.has_tool_calls).toBe(true);
+    expect(trace.cwd_hash).toBeTruthy();
+
+    expect(trace.turns[0]).toMatchObject({
+      role: "user",
+      content: [{ type: "text", text: "Audit the repo" }],
+    });
+
+    expect(trace.turns[1].role).toBe("assistant");
+    expect(trace.turns[1].model).toBe("gpt-5.4");
+    expect(trace.turns[1].content.map((block) => block.type)).toEqual([
+      "text",
+      "tool_use",
+      "tool_result",
+      "text",
+    ]);
+    expect(trace.turns[1].content[0]).toMatchObject({
+      type: "text",
+      text: "I will inspect the repo.",
+    });
+    expect(trace.turns[1].content[1]).toMatchObject({
+      type: "tool_use",
+      tool_name: "exec_command",
+    });
+    expect(trace.turns[1].content[2]).toMatchObject({
+      type: "tool_result",
+      tool_call_id: "call-1",
+      is_error: false,
+      exit_code: 0,
+    });
+    expect(trace.turns[1].content[3]).toMatchObject({
+      type: "text",
+      text: "I found the relevant file.",
+    });
+  });
+
+  it("falls back to response-item user messages and preserves invalid tool arguments", async () => {
+    const trace = await extractCodex(
+      makeBuffer([
+        {
+          timestamp: "2026-03-21T01:00:00.000Z",
+          type: "session_meta",
+          payload: { id: "codex-session-2", timestamp: "2026-03-21T01:00:00.000Z" },
+        },
+        {
+          timestamp: "2026-03-21T01:00:01.000Z",
+          type: "event_msg",
+          payload: { type: "task_started", turn_id: "turn-2" },
+        },
+        {
+          timestamp: "2026-03-21T01:00:02.000Z",
+          type: "response_item",
+          payload: {
+            type: "message",
+            role: "user",
+            content: [{ type: "input_text", text: "Retry the failing command" }],
+          },
+        },
+        {
+          timestamp: "2026-03-21T01:00:03.000Z",
+          type: "response_item",
+          payload: {
+            type: "reasoning",
+            content: [{ type: "text", text: "I should re-run it with verbose output." }],
+          },
+        },
+        {
+          timestamp: "2026-03-21T01:00:04.000Z",
+          type: "response_item",
+          payload: {
+            type: "function_call",
+            name: "exec_command",
+            arguments: "{not-json",
+            call_id: "call-err",
+          },
+        },
+        {
+          timestamp: "2026-03-21T01:00:05.000Z",
+          type: "response_item",
+          payload: {
+            type: "function_call_output",
+            call_id: "call-err",
+            output: "Process exited with code 2\nError: command failed",
+          },
+        },
+        {
+          timestamp: "2026-03-21T01:00:06.000Z",
+          type: "event_msg",
+          payload: { type: "task_complete", last_agent_message: "The command failed again." },
+        },
+      ]),
+      "test@example.com",
+    );
+
+    expect(trace.turn_count).toBe(2);
+    expect(trace.has_thinking_blocks).toBe(true);
+    expect(trace.turns[0]).toMatchObject({
+      role: "user",
+      content: [{ type: "text", text: "Retry the failing command" }],
+    });
+
+    const assistantBlocks = trace.turns[1].content;
+    expect(assistantBlocks[0]).toMatchObject({
+      type: "thinking",
+      text: "I should re-run it with verbose output.",
+    });
+    expect(assistantBlocks[1]).toMatchObject({
+      type: "tool_use",
+      tool_input: { raw: "{not-json" },
+    });
+    expect(assistantBlocks[2]).toMatchObject({
+      type: "tool_result",
+      is_error: true,
+      exit_code: 2,
+    });
+  });
+});

package/src/extractor-cursor.test.ts

@@ -0,0 +1,141 @@
+import { mkdtempSync, rmSync } from "fs";
+import { join } from "path";
+import { tmpdir } from "os";
+import Database from "better-sqlite3";
+import { afterEach, describe, expect, it } from "vitest";
+import { extractCursor } from "./extractors/cursor.js";
+
+const tempDirs: string[] = [];
+
+function makeCursorDb(
+  sessionId: string,
+  headers: Array<{ bubbleId: string; type: number }>,
+  values: Array<[string, unknown]>,
+): string {
+  const dir = mkdtempSync(join(tmpdir(), "tracemp-cursor-"));
+  tempDirs.push(dir);
+
+  const dbPath = join(dir, "state.vscdb");
+  const db = new Database(dbPath);
+  db.exec("CREATE TABLE cursorDiskKV (key TEXT PRIMARY KEY, value TEXT NOT NULL)");
+
+  db.prepare("INSERT INTO cursorDiskKV (key, value) VALUES (?, ?)").run(
+    `composerData:${sessionId}`,
+    JSON.stringify({
+      _v: 14,
+      createdAt: Date.parse("2026-03-21T00:00:00.000Z"),
+      fullConversationHeadersOnly: headers,
+    }),
+  );
+
+  const insert = db.prepare("INSERT INTO cursorDiskKV (key, value) VALUES (?, ?)");
+  for (const [key, value] of values) {
+    insert.run(key, JSON.stringify(value));
+  }
+
+  db.close();
+  return dbPath;
+}
+
+afterEach(() => {
+  while (tempDirs.length > 0) {
+    rmSync(tempDirs.pop()!, { force: true, recursive: true });
+  }
+});
+
+describe("extractCursor", () => {
+  it("supports the modern bubbleId:<session>:<bubble> storage format", async () => {
+    const sessionId = "cursor-modern";
+    const dbPath = makeCursorDb(sessionId, [
+      { bubbleId: "user-1", type: 1 },
+      { bubbleId: "assistant-1", type: 2 },
+    ], [
+      [
+        `bubbleId:${sessionId}:user-1`,
+        {
+          type: 1,
+          text: "Help me debug the repo",
+          createdAt: "2026-03-21T00:00:01.000Z",
+          tokenCount: { inputTokens: 0, outputTokens: 0 },
+        },
+      ],
+      [
+        `bubbleId:${sessionId}:assistant-1`,
+        {
+          type: 2,
+          text: "I will inspect src/main.ts",
+          thinking: { text: "Start with the entrypoint." },
+          createdAt: "2026-03-21T00:00:02.000Z",
+          modelName: "gpt-5.4",
+          tokenCount: { inputTokens: 200, outputTokens: 80 },
+          relevantFiles: [{ path: "src/main.ts" }],
+          workspaceUris: ["file:///Users/test/project"],
+        },
+      ],
+    ]);
+
+    const trace = await extractCursor(dbPath, sessionId, "test@example.com");
+
+    expect(trace.turn_count).toBe(2);
+    expect(trace.source_version).toBe("14");
+    expect(trace.total_input_tokens).toBe(200);
+    expect(trace.total_output_tokens).toBe(80);
+    expect(trace.has_thinking_blocks).toBe(true);
+    expect(trace.turns[0]).toMatchObject({
+      role: "user",
+      content: [{ type: "text", text: "Help me debug the repo" }],
+    });
+    expect(trace.turns[1].role).toBe("assistant");
+    expect(trace.turns[1].model).toBe("gpt-5.4");
+    expect(trace.turns[1].content.map((block) => block.type)).toEqual([
+      "thinking",
+      "text",
+    ]);
+    expect(trace.turns[1].usage).toMatchObject({
+      input_tokens: 200,
+      output_tokens: 80,
+    });
+    expect(trace.env_state?.open_files_in_editor).toEqual(
+      expect.arrayContaining(["src/main.ts", "/Users/test/project"]),
+    );
+  });
+
+  it("falls back to the legacy agentKv:blob:<bubble> storage format", async () => {
+    const sessionId = "cursor-legacy";
+    const dbPath = makeCursorDb(sessionId, [
+      { bubbleId: "user-1", type: 1 },
+      { bubbleId: "assistant-1", type: 2 },
+    ], [
+      [
+        `agentKv:blob:user-1`,
+        {
+          role: "user",
+          type: "user",
+          text: "Open app.py",
+          createdAt: "2026-03-21T01:00:01.000Z",
+        },
+      ],
+      [
+        `agentKv:blob:assistant-1`,
+        {
+          role: "assistant",
+          message: "Opened the file.",
+          createdAt: "2026-03-21T01:00:02.000Z",
+          usage: { promptTokens: 55, completionTokens: 21 },
+          context: { openFiles: [{ path: "/tmp/app.py" }] },
+        },
+      ],
+    ]);
+
+    const trace = await extractCursor(dbPath, sessionId, "test@example.com");
+
+    expect(trace.turn_count).toBe(2);
+    expect(trace.total_input_tokens).toBe(55);
+    expect(trace.total_output_tokens).toBe(21);
+    expect(trace.turns[1]).toMatchObject({
+      role: "assistant",
+      content: [{ type: "text", text: "Opened the file." }],
+    });
+    expect(trace.env_state?.open_files_in_editor).toEqual(["/tmp/app.py"]);
+  });
+});

package/src/extractors/claude-code.ts

@@ -8,6 +8,7 @@ import type {
   ContentBlock,
   TokenUsage,
 } from "../types.js";
+import { deriveTurnActors } from "../turn-actors.js";
 
 export async function extractClaudeCode(
   sessionFilePath: string,
@@ -27,97 +28,207 @@ export async function extractClaudeCode(
 
   const sessionId =
     sessionFilePath.split("/").pop()?.replace(".jsonl", "") ?? randomUUID();
-  const turns: Turn[] = [];
 
+  // Build uuid→entry map for parent lookups
+  const uuidMap = new Map<string, any>();
+  for (const line of lines) {
+    if (line.uuid) uuidMap.set(line.uuid, line);
+  }
+
+  // Filter to lines that carry message content (skip progress, snapshots, etc.)
+  const SKIP_TYPES = new Set([
+    "progress",
+    "file-history-snapshot",
+    "system",
+    "last-prompt",
+  ]);
+  const entries = lines.filter(
+    (l) => !SKIP_TYPES.has(l.type) && l.message
+  );
+
+  // Claude Code splits a single logical turn into multiple chained JSONL entries:
+  //
+  // 1. Assistant turns: thinking → tool_use → text each parent the next
+  //    (assistant→assistant parentage). Merge into one turn.
+  //
+  // 2. User tool_results for parallel tool calls: each result is a separate entry
+  //    whose parentUuid points to a different entry in the same assistant chain.
+  //    Merge into one user turn.
+
+  // For assistant entries: walk up until the parent is NOT an assistant entry.
+  function getAssistantRoot(entry: any): any {
+    let cur = entry;
+    while (cur.message?.role === "assistant") {
+      const parent = uuidMap.get(cur.parentUuid);
+      if (!parent?.message || parent.message.role !== "assistant") break;
+      cur = parent;
+    }
+    return cur;
+  }
+
+  // For a user entry: return the uuid of the assistant group root that it responds to
+  // (null if its parent is not an assistant).
+  function parentAssistantRootUuid(entry: any): string | null {
+    const parent = uuidMap.get(entry.parentUuid);
+    if (!parent?.message || parent.message.role !== "assistant") return null;
+    return getAssistantRoot(parent).uuid ?? null;
+  }
+
+  // Group entries in file order.
+  // Key: root uuid for assistant groups; entry uuid for user groups.
+  const groupMap = new Map<string, any[]>();
+  const rootOrder: string[] = [];
+
+  for (const entry of entries) {
+    const role = entry.message?.role;
+
+    if (role === "assistant") {
+      const root = getAssistantRoot(entry);
+      const rootUuid = root.uuid ?? randomUUID();
+      if (!groupMap.has(rootUuid)) {
+        groupMap.set(rootUuid, [root]);
+        rootOrder.push(rootUuid);
+      } else if (entry !== root) {
+        groupMap.get(rootUuid)!.push(entry);
+      }
+    } else {
+      // User entry: merge with the previous user group if they share the same
+      // parent assistant root (i.e., parallel tool results for the same tool call batch).
+      const myParentRoot = parentAssistantRootUuid(entry);
+      const lastKey = rootOrder[rootOrder.length - 1];
+      const lastGroup = lastKey ? groupMap.get(lastKey) : undefined;
+      const lastRole = lastGroup?.[0]?.message?.role;
+
+      if (
+        myParentRoot &&
+        lastRole === "user" &&
+        parentAssistantRootUuid(lastGroup![0]) === myParentRoot
+      ) {
+        lastGroup!.push(entry);
+      } else {
+        const groupKey = entry.uuid ?? randomUUID();
+        groupMap.set(groupKey, [entry]);
+        rootOrder.push(groupKey);
+      }
+    }
+  }
+
+  const turns: Turn[] = [];
   let totalInputTokens = 0;
   let totalOutputTokens = 0;
   let totalCacheReadTokens = 0;
   let gitBranch: string | null = null;
   let cwdHash: string | null = null;
 
-  for (const line of lines) {
-    if (!line.message) continue;
-    const { role, content, usage, model } = line.message;
-
-    const tokenUsage: TokenUsage | null = usage
-      ? {
-          input_tokens: usage.input_tokens ?? 0,
-          output_tokens: usage.output_tokens ?? 0,
-          cache_read_input_tokens: usage.cache_read_input_tokens ?? null,
-          cache_creation_input_tokens:
-            usage.cache_creation_input_tokens ?? null,
-          reasoning_tokens: null,
-        }
-      : null;
-    if (tokenUsage) {
-      totalInputTokens += tokenUsage.input_tokens;
-      totalOutputTokens += tokenUsage.output_tokens;
-      totalCacheReadTokens += tokenUsage.cache_read_input_tokens ?? 0;
-    }
+  for (const rootUuid of rootOrder) {
+    const group = groupMap.get(rootUuid)!;
+    const root = group[0];
 
-    if (line.gitBranch) gitBranch = line.gitBranch;
-    if (line.cwd && !cwdHash) cwdHash = hashString(line.cwd);
+    if (root.gitBranch) gitBranch = root.gitBranch;
+    if (root.cwd && !cwdHash) cwdHash = hashString(root.cwd);
 
+    const role: "user" | "assistant" =
+      root.message.role === "assistant" ? "assistant" : "user";
+
+    // Collect content blocks from all entries in the group
     const contentBlocks: ContentBlock[] = [];
-    const rawContent = Array.isArray(content)
-      ? content
-      : typeof content === "string"
-        ? [{ type: "text", text: content }]
-        : [];
-
-    for (const block of rawContent) {
-      if (!block || !block.type) continue;
-      if (block.type === "file-history-snapshot") continue;
-      if (block.type === "text") {
-        contentBlocks.push({ type: "text", text: block.text ?? "" });
-      } else if (block.type === "thinking") {
-        contentBlocks.push({
-          type: "thinking",
-          text: block.thinking ?? block.text ?? "",
-        });
-      } else if (block.type === "tool_use") {
-        contentBlocks.push({
-          type: "tool_use",
-          tool_call_id: block.id ?? randomUUID(),
-          tool_name: block.name ?? "",
-          tool_input: block.input ?? {},
-        });
-      } else if (block.type === "tool_result") {
-        const resultContent = Array.isArray(block.content)
-          ? block.content.map((c: any) => c.text ?? "").join("\n")
-          : block.content ?? null;
-        contentBlocks.push({
-          type: "tool_result",
-          tool_call_id: block.tool_use_id ?? "",
-          is_error: block.is_error ?? false,
-          result_content: resultContent,
-          exit_code: null,
-        });
+    for (const entry of group) {
+      const raw = entry.message.content;
+      const rawContent = Array.isArray(raw)
+        ? raw
+        : typeof raw === "string"
+          ? [{ type: "text", text: raw }]
+          : [];
+
+      for (const block of rawContent) {
+        if (!block || !block.type) continue;
+        if (block.type === "file-history-snapshot") continue;
+        if (block.type === "text") {
+          contentBlocks.push({ type: "text", text: block.text ?? "" });
+        } else if (block.type === "thinking") {
+          contentBlocks.push({
+            type: "thinking",
+            text: block.thinking ?? block.text ?? "",
+          });
+        } else if (block.type === "tool_use") {
+          contentBlocks.push({
+            type: "tool_use",
+            tool_call_id: block.id ?? randomUUID(),
+            tool_name: block.name ?? "",
+            tool_input: block.input ?? {},
+          });
+        } else if (block.type === "tool_result") {
+          const resultContent = Array.isArray(block.content)
+            ? block.content.map((c: any) => c.text ?? "").join("\n")
+            : block.content ?? null;
+          contentBlocks.push({
+            type: "tool_result",
+            tool_call_id: block.tool_use_id ?? "",
+            is_error: block.is_error ?? false,
+            result_content: resultContent,
+            exit_code: null,
+          });
+        }
       }
     }
 
     if (contentBlocks.length === 0) continue;
 
+    // Token accounting for assistant turns:
+    // - input/cache tokens come from the root entry only (same API call repeated on each chain entry)
+    // - output tokens must be summed across all chain entries (each has its own generated content)
+    let tokenUsage: TokenUsage | null = null;
+    if (role === "assistant") {
+      const rootUsage = root.message.usage;
+      if (rootUsage) {
+        const inputTokens = rootUsage.input_tokens ?? 0;
+        const cacheCreate = rootUsage.cache_creation_input_tokens ?? 0;
+        const cacheRead = rootUsage.cache_read_input_tokens ?? 0;
+        // Sum output tokens across all chain entries
+        const outputTokens = group.reduce((sum: number, entry: any) => {
+          return sum + (entry.message.usage?.output_tokens ?? 0);
+        }, 0);
+
+        tokenUsage = {
+          input_tokens: inputTokens + cacheCreate + cacheRead,
+          output_tokens: outputTokens,
+          cache_read_input_tokens: cacheRead,
+          cache_creation_input_tokens: cacheCreate,
+          reasoning_tokens: null,
+        };
+
+        totalInputTokens += inputTokens + cacheCreate + cacheRead;
+        totalOutputTokens += outputTokens;
+        totalCacheReadTokens += cacheRead;
+      }
+    }
+
     turns.push({
-      turn_id: line.uuid ?? randomUUID(),
-      parent_turn_id: line.parentUuid ?? null,
-      role: role === "assistant" ? "assistant" : "user",
-      timestamp: line.timestamp ?? null,
+      turn_id: root.uuid ?? randomUUID(),
+      parent_turn_id: root.parentUuid ?? null,
+      role,
+      timestamp: root.timestamp ?? null,
       content: contentBlocks,
-      model: model ?? null,
+      model: root.message.model ?? null,
       usage: tokenUsage,
       source_metadata: {
-        uuid: line.uuid,
-        parentUuid: line.parentUuid,
-        gitBranch: line.gitBranch,
+        uuid: root.uuid,
+        parentUuid: root.parentUuid,
+        gitBranch: root.gitBranch,
       },
     });
   }
 
-  const startedAt = turns[0]?.timestamp ?? new Date().toISOString();
-  const endedAt = turns[turns.length - 1]?.timestamp ?? new Date().toISOString();
+  const turnActors = deriveTurnActors(turns);
+  const normalizedTurns = turns.map((turn) => ({
+    ...turn,
+    actor: turnActors[turn.turn_id],
+  }));
+
+  const startedAt = normalizedTurns[0]?.timestamp ?? new Date().toISOString();
+  const endedAt = normalizedTurns[normalizedTurns.length - 1]?.timestamp ?? new Date().toISOString();
 
-  const allBlocks = turns.flatMap((t) => t.content);
+  const allBlocks = normalizedTurns.flatMap((t) => t.content);
   const toolCallCount = allBlocks.filter((b) => b.type === "tool_use").length;
   const hasFileChanges = allBlocks.some(
     (b) =>
@@ -146,8 +257,8 @@ export async function extractClaudeCode(
     working_language: null,
     started_at: startedAt,
     ended_at: endedAt,
-    turns,
-    turn_count: turns.length,
+    turns: normalizedTurns,
+    turn_count: normalizedTurns.length,
     tool_call_count: toolCallCount,
     has_tool_calls: toolCallCount > 0,
     has_thinking_blocks: hasThinking,