@tracemarketplace/shared 0.0.6 → 0.0.9

package/src/extractors/cursor.ts

@@ -1,6 +1,250 @@
 import { randomUUID } from "crypto";
 import { hashString, computeContentHash } from "../hash.js";
-import type { NormalizedTrace, Turn, TokenUsage } from "../types.js";
+import type { ContentBlock, NormalizedTrace, Turn, TokenUsage } from "../types.js";
+import {
+  collectTraceMetrics,
+  createPassiveEnvState,
+  extractTextFragments,
+  normalizeTimestamp,
+  pushUniqueTextBlock,
+} from "./common.js";
+
+type CursorHeader = {
+  bubbleId?: string;
+  id?: string;
+  type?: number;
+  createdAt?: string | number | null;
+};
+
+type CursorBlob = {
+  type?: number | string;
+  role?: string;
+  text?: string;
+  content?: string;
+  message?: string;
+  markdown?: string;
+  thinking?: unknown;
+  usage?: {
+    promptTokens?: number;
+    completionTokens?: number;
+  };
+  tokenCount?: {
+    inputTokens?: number;
+    outputTokens?: number;
+  };
+  createdAt?: string | number | null;
+  model?: string;
+  modelName?: string;
+  context?: {
+    openFiles?: unknown[];
+  };
+  openFiles?: unknown[];
+  relevantFiles?: unknown[];
+  recentlyViewedFiles?: unknown[];
+  workspaceUris?: unknown[];
+  attachedFolders?: unknown[];
+  attachedFoldersNew?: unknown[];
+};
+
+type CursorDb = {
+  prepare(sql: string): {
+    get(param: string): { value: string } | undefined;
+  };
+  close(): void;
+};
+
+function toNumber(value: unknown): number | null {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+function hasCursorConversationPayload(blob: CursorBlob): boolean {
+  return Boolean(
+    blob.role ||
+    blob.text ||
+    blob.content ||
+    blob.message ||
+    blob.markdown ||
+    blob.thinking ||
+    blob.usage ||
+    blob.tokenCount ||
+    blob.model ||
+    blob.modelName ||
+    blob.context?.openFiles ||
+    blob.openFiles ||
+    blob.relevantFiles ||
+    blob.recentlyViewedFiles ||
+    blob.workspaceUris ||
+    blob.attachedFolders ||
+    blob.attachedFoldersNew,
+  );
+}
+
+function readCursorBlob(
+  db: CursorDb,
+  sessionId: string,
+  bubbleId: string,
+): CursorBlob | null {
+  const keys = [
+    `bubbleId:${sessionId}:${bubbleId}`,
+    `agentKv:blob:${bubbleId}`,
+  ];
+
+  for (const key of keys) {
+    const row = db
+      .prepare("SELECT value FROM cursorDiskKV WHERE key = ?")
+      .get(key) as { value: string } | undefined;
+    if (!row) continue;
+
+    try {
+      const blob = JSON.parse(row.value) as CursorBlob;
+      if (hasCursorConversationPayload(blob)) return blob;
+    } catch {
+      return null;
+    }
+  }
+
+  return null;
+}
+
+function cursorRole(header: CursorHeader, blob: CursorBlob): "user" | "assistant" {
+  const role = blob.role?.toLowerCase();
+  if (role === "user" || role === "assistant") return role;
+
+  const type = typeof blob.type === "number" ? blob.type : header.type;
+  return type === 1 ? "user" : "assistant";
+}
+
+function extractCursorTokenUsage(blob: CursorBlob): TokenUsage | null {
+  const usage = blob.usage;
+  const tokenCount = blob.tokenCount;
+
+  const inputTokens =
+    toNumber(usage?.promptTokens) ?? toNumber(tokenCount?.inputTokens);
+  const outputTokens =
+    toNumber(usage?.completionTokens) ?? toNumber(tokenCount?.outputTokens);
+
+  if (inputTokens === null && outputTokens === null) {
+    return null;
+  }
+
+  return {
+    input_tokens: inputTokens ?? 0,
+    output_tokens: outputTokens ?? 0,
+    cache_read_input_tokens: null,
+    cache_creation_input_tokens: null,
+    reasoning_tokens: null,
+  };
+}
+
+function normalizeCursorPath(value: string): string {
+  if (!value.startsWith("file://")) return value;
+
+  try {
+    return decodeURIComponent(new URL(value).pathname);
+  } catch {
+    return value;
+  }
+}
+
+function extractCursorPaths(value: unknown): string[] {
+  if (typeof value === "string") {
+    return value ? [normalizeCursorPath(value)] : [];
+  }
+
+  if (Array.isArray(value)) {
+    return value.flatMap((item) => extractCursorPaths(item));
+  }
+
+  if (!value || typeof value !== "object") {
+    return [];
+  }
+
+  const record = value as Record<string, unknown>;
+  const paths: string[] = [];
+  for (const key of [
+    "path",
+    "uri",
+    "absolutePath",
+    "relativeWorkspacePath",
+    "relativePath",
+    "filePath",
+  ]) {
+    if (key in record) {
+      paths.push(...extractCursorPaths(record[key]));
+    }
+  }
+
+  return paths;
+}
+
+function collectCursorOpenFiles(blob: CursorBlob): string[] {
+  const paths = [
+    blob.context?.openFiles,
+    blob.openFiles,
+    blob.relevantFiles,
+    blob.recentlyViewedFiles,
+    blob.workspaceUris,
+    blob.attachedFolders,
+    blob.attachedFoldersNew,
+  ].flatMap((value) => extractCursorPaths(value));
+
+  return Array.from(new Set(paths.filter(Boolean)));
+}
+
+function extractCursorBlocks(blob: CursorBlob): ContentBlock[] {
+  const blocks: ContentBlock[] = [];
+
+  const thinkingText = extractTextFragments(blob.thinking).join("\n\n");
+  pushUniqueTextBlock(blocks, "thinking", thinkingText);
+
+  const messageText = [
+    blob.text,
+    blob.content,
+    blob.message,
+    blob.markdown,
+  ].find((value): value is string => typeof value === "string" && value.trim().length > 0);
+  pushUniqueTextBlock(blocks, "text", messageText ?? null);
+
+  return blocks;
+}
+
+function mergeUsage(
+  left: TokenUsage | null,
+  right: TokenUsage | null,
+): TokenUsage | null {
+  if (!left && !right) return null;
+
+  return {
+    input_tokens: (left?.input_tokens ?? 0) + (right?.input_tokens ?? 0),
+    output_tokens: (left?.output_tokens ?? 0) + (right?.output_tokens ?? 0),
+    cache_read_input_tokens: null,
+    cache_creation_input_tokens: null,
+    reasoning_tokens: null,
+  };
+}
+
+function appendCursorTurn(turns: Turn[], nextTurn: Turn): void {
+  const previousTurn = turns[turns.length - 1];
+  if (!previousTurn || previousTurn.role !== nextTurn.role) {
+    turns.push(nextTurn);
+    return;
+  }
+
+  const previousBubbleIds = Array.isArray(previousTurn.source_metadata.bubbleIds)
+    ? previousTurn.source_metadata.bubbleIds
+    : [previousTurn.source_metadata.bubbleId].filter(Boolean);
+
+  turns[turns.length - 1] = {
+    ...previousTurn,
+    content: [...previousTurn.content, ...nextTurn.content],
+    usage: mergeUsage(previousTurn.usage, nextTurn.usage),
+    model: previousTurn.model ?? nextTurn.model,
+    source_metadata: {
+      ...previousTurn.source_metadata,
+      bubbleIds: [...previousBubbleIds, nextTurn.source_metadata.bubbleId].filter(Boolean),
+    },
+  };
+}
 
 export async function extractCursor(
   dbPath: string,
@@ -8,7 +252,7 @@ export async function extractCursor(
   submittedBy = "unknown"
 ): Promise<NormalizedTrace> {
   const Database = (await import("better-sqlite3")).default;
-  const db = new Database(dbPath, { readonly: true });
+  const db: CursorDb = new Database(dbPath, { readonly: true });
 
   try {
     const composerRow = db
@@ -17,63 +261,66 @@ export async function extractCursor(
     if (!composerRow) throw new Error(`Session ${sessionId} not found in cursor DB`);
 
     const composerData = JSON.parse(composerRow.value);
-    const headers: any[] = composerData.fullConversationHeadersOnly ?? [];
+    const headers = (composerData.fullConversationHeadersOnly ?? []) as CursorHeader[];
 
     const turns: Turn[] = [];
     const openFiles: string[] = [];
+    let totalInputTokens = 0;
+    let totalOutputTokens = 0;
 
     for (const header of headers) {
       const bubbleId = header.bubbleId ?? header.id;
       if (!bubbleId) continue;
 
-      const blobRow = db
-        .prepare("SELECT value FROM cursorDiskKV WHERE key = ?")
-        .get(`agentKv:blob:${bubbleId}`) as { value: string } | undefined;
-      if (!blobRow) continue;
-
-      const blob = JSON.parse(blobRow.value);
-      const role: "user" | "assistant" =
-        blob.type === "user" || blob.role === "user" ? "user" : "assistant";
-      const text: string = blob.text ?? blob.content ?? blob.message ?? "";
+      const blob = readCursorBlob(db, sessionId, bubbleId);
+      if (!blob) continue;
 
-      if (blob.context?.openFiles) {
-        openFiles.push(
-          ...blob.context.openFiles.map((f: any) => (typeof f === "string" ? f : f.path ?? ""))
-        );
+      const role = cursorRole(header, blob);
+      const tokenUsage = extractCursorTokenUsage(blob);
+      if (tokenUsage) {
+        totalInputTokens += tokenUsage.input_tokens;
+        totalOutputTokens += tokenUsage.output_tokens;
       }
 
-      const tokenUsage: TokenUsage | null = blob.usage
-        ? {
-            input_tokens: blob.usage.promptTokens ?? 0,
-            output_tokens: blob.usage.completionTokens ?? 0,
-            cache_read_input_tokens: null,
-            cache_creation_input_tokens: null,
-            reasoning_tokens: null,
-          }
-        : null;
-
-      turns.push({
+      openFiles.push(...collectCursorOpenFiles(blob));
+
+      const content = extractCursorBlocks(blob);
+      if (content.length === 0) continue;
+
+      appendCursorTurn(turns, {
         turn_id: bubbleId,
         parent_turn_id: null,
         role,
-        timestamp: blob.createdAt ?? header.createdAt ?? null,
-        content: [{ type: "text", text }],
-        model: blob.model ?? null,
+        timestamp: normalizeTimestamp(blob.createdAt ?? header.createdAt),
+        content,
+        model: blob.model ?? blob.modelName ?? null,
         usage: tokenUsage,
-        source_metadata: { bubbleId, type: blob.type },
+        source_metadata: { bubbleId, type: blob.type ?? header.type ?? null },
       });
     }
 
-    db.close();
+    if (turns.length === 0) {
+      throw new Error(`Session ${sessionId} contained no readable conversation bubbles`);
+    }
 
-    const startedAt = turns[0]?.timestamp ?? new Date().toISOString();
-    const endedAt = turns[turns.length - 1]?.timestamp ?? new Date().toISOString();
+    const metrics = collectTraceMetrics(turns);
+    const startedAt =
+      turns[0]?.timestamp ??
+      normalizeTimestamp(composerData.createdAt) ??
+      new Date().toISOString();
+    const endedAt =
+      turns[turns.length - 1]?.timestamp ??
+      normalizeTimestamp(composerData.createdAt) ??
+      new Date().toISOString();
 
     const partialTrace: Omit<NormalizedTrace, "trace_id"> = {
       schema_version: "1.0",
       source_tool: "cursor",
       source_session_id: sessionId,
-      source_version: null,
+      source_version:
+        typeof composerData._v === "number" || typeof composerData._v === "string"
+          ? String(composerData._v)
+          : null,
       submitted_by: submittedBy,
       submitted_at: new Date().toISOString(),
       extracted_at: new Date().toISOString(),
@@ -84,24 +331,18 @@ export async function extractCursor(
       ended_at: endedAt,
       turns,
       turn_count: turns.length,
-      tool_call_count: 0,
-      has_tool_calls: false,
-      has_thinking_blocks: false,
-      has_file_changes: false,
-      has_shell_commands: false,
-      total_input_tokens: null,
-      total_output_tokens: null,
+      tool_call_count: metrics.toolCallCount,
+      has_tool_calls: metrics.toolCallCount > 0,
+      has_thinking_blocks: metrics.hasThinkingBlocks,
+      has_file_changes: metrics.hasFileChanges,
+      has_shell_commands: metrics.hasShellCommands,
+      total_input_tokens: totalInputTokens || null,
+      total_output_tokens: totalOutputTokens || null,
       total_cache_read_tokens: null,
       content_fidelity: "chat_only",
-      env_state: {
-        git_branch: null,
-        inferred_file_tree: null,
-        inferred_changed_files: null,
-        inferred_error_files: null,
-        shell_exit_codes: null,
-        open_files_in_editor: openFiles.length > 0 ? openFiles : null,
-        extraction_method: "passive",
-      },
+      env_state: createPassiveEnvState({
+        open_files_in_editor: openFiles.length > 0 ? Array.from(new Set(openFiles)) : null,
+      }),
       score: null,
       raw_r2_key: "",
       normalized_r2_key: "",
@@ -112,7 +353,8 @@ export async function extractCursor(
 
     return { ...partialTrace, trace_id: traceId };
   } catch (err) {
-    db.close();
     throw err;
+  } finally {
+    db.close();
   }
 }

package/src/hash.test.ts

@@ -42,10 +42,10 @@ describe("computeContentHash", () => {
     expect(computeContentHash(trace)).toBe(computeContentHash(trace));
   });
 
-  it("returns different hashes for different session_ids", () => {
+  it("returns the same hash for different session_ids when content is identical", () => {
     const a = makeTrace({ source_session_id: "session-aaa" });
     const b = makeTrace({ source_session_id: "session-bbb" });
-    expect(computeContentHash(a)).not.toBe(computeContentHash(b));
+    expect(computeContentHash(a)).toBe(computeContentHash(b));
   });
 
   it("returns different hashes for different turns", () => {
@@ -65,6 +65,35 @@ describe("computeContentHash", () => {
     expect(computeContentHash(a)).not.toBe(computeContentHash(b));
   });
 
+  it("ignores volatile turn metadata like timestamps and turn ids", () => {
+    const a = makeTrace({
+      turns: [{
+        turn_id: "turn-a",
+        parent_turn_id: null,
+        role: "assistant",
+        timestamp: "2024-01-01T00:00:00Z",
+        content: [{ type: "text", text: "hello" }],
+        model: "gpt-test",
+        usage: { input_tokens: 1, output_tokens: 2, cache_read_input_tokens: null, cache_creation_input_tokens: null, reasoning_tokens: null },
+        source_metadata: { foo: "bar" },
+      }],
+    });
+    const b = makeTrace({
+      turns: [{
+        turn_id: "turn-b",
+        parent_turn_id: "parent-b",
+        role: "assistant",
+        timestamp: "2024-01-05T00:00:00Z",
+        content: [{ type: "text", text: "hello" }],
+        model: "gpt-test",
+        usage: { input_tokens: 999, output_tokens: 999, cache_read_input_tokens: 1, cache_creation_input_tokens: 1, reasoning_tokens: 1 },
+        source_metadata: { baz: "qux" },
+      }],
+    });
+
+    expect(computeContentHash(a)).toBe(computeContentHash(b));
+  });
+
   it("output is a 64-char hex string", () => {
     const hash = computeContentHash(makeTrace());
     expect(hash).toMatch(/^[0-9a-f]{64}$/);

package/src/hash.ts

@@ -1,11 +1,46 @@
 import { createHash } from "crypto";
-import type { NormalizedTrace } from "./types.js";
+import type { ContentBlock, NormalizedTrace, Turn } from "./types.js";
+
+function canonicalizeBlock(block: ContentBlock) {
+  switch (block.type) {
+    case "text":
+    case "thinking":
+      return { type: block.type, text: block.text };
+    case "tool_use":
+      return {
+        type: block.type,
+        tool_name: block.tool_name,
+        tool_input: block.tool_input,
+      };
+    case "tool_result":
+      return {
+        type: block.type,
+        is_error: block.is_error,
+        result_content: block.result_content,
+        exit_code: block.exit_code,
+      };
+    case "image":
+      return {
+        type: block.type,
+        media_type: block.media_type,
+        data_r2_key: block.data_r2_key,
+      };
+  }
+}
+
+function canonicalizeTurn(turn: Turn) {
+  return {
+    role: turn.role,
+    model: turn.model,
+    content: turn.content.map(canonicalizeBlock),
+  };
+}
 
 export function computeContentHash(trace: NormalizedTrace): string {
   const content = JSON.stringify({
     source_tool: trace.source_tool,
-    source_session_id: trace.source_session_id,
-    turns: trace.turns,
+    content_fidelity: trace.content_fidelity,
+    turns: trace.turns.map(canonicalizeTurn),
   });
   return createHash("sha256").update(content).digest("hex");
 }

package/src/index.ts

@@ -5,6 +5,7 @@ export * from "./utils.js";
 export * from "./validators.js";
 export * from "./redact.js";
 export * from "./chunker.js";
+export * from "./turn-actors.js";
 export { extractClaudeCode } from "./extractors/claude-code.js";
 export { extractCodex } from "./extractors/codex.js";
 export { extractCursor } from "./extractors/cursor.js";

package/src/redact.test.ts

@@ -0,0 +1,100 @@
+import { describe, expect, it } from "vitest";
+import type { NormalizedTrace } from "./types.js";
+import { redactTraceWithStats } from "./redact.js";
+
+function makeTrace(): NormalizedTrace {
+  return {
+    trace_id: "trace-1",
+    schema_version: "1.0",
+    source_tool: "claude_code",
+    source_session_id: "session-1",
+    source_version: null,
+    submitted_by: "user@example.com",
+    submitted_at: "2024-01-01T00:00:00.000Z",
+    extracted_at: "2024-01-01T00:00:00.000Z",
+    git_branch: "main",
+    cwd_hash: null,
+    working_language: null,
+    started_at: "2024-01-01T00:00:00.000Z",
+    ended_at: "2024-01-01T00:00:01.000Z",
+    turns: [
+      {
+        turn_id: "t1",
+        parent_turn_id: null,
+        role: "assistant",
+        timestamp: "2024-01-01T00:00:00.000Z",
+        model: "test",
+        usage: null,
+        source_metadata: {},
+        content: [
+          {
+            type: "tool_use",
+            tool_call_id: "c1",
+            tool_name: "Bash",
+            tool_input: {
+              command: "echo sk-proj-secret-secret-secret-secret",
+              nested: {
+                path: "/Users/tester/project/.env",
+              },
+            },
+          },
+          {
+            type: "tool_result",
+            tool_call_id: "c1",
+            is_error: false,
+            result_content: "Bearer abc.def.ghi",
+            exit_code: 0,
+          },
+        ],
+      },
+    ],
+    turn_count: 1,
+    tool_call_count: 1,
+    has_tool_calls: true,
+    has_thinking_blocks: false,
+    has_file_changes: false,
+    has_shell_commands: true,
+    total_input_tokens: null,
+    total_output_tokens: null,
+    total_cache_read_tokens: null,
+    content_fidelity: "full",
+    env_state: {
+      git_branch: "main",
+      inferred_file_tree: ["/Users/tester/project/.env"],
+      inferred_changed_files: null,
+      inferred_error_files: null,
+      shell_exit_codes: [0],
+      open_files_in_editor: null,
+      extraction_method: "passive",
+    },
+    score: null,
+    raw_r2_key: "",
+    normalized_r2_key: "",
+  };
+}
+
+describe("redactTraceWithStats", () => {
+  it("redacts nested strings and reports redaction stats", () => {
+    const result = redactTraceWithStats(makeTrace(), { homeDir: "/Users/tester" });
+
+    const toolUse = result.trace.turns[0]?.content[0];
+    const toolResult = result.trace.turns[0]?.content[1];
+
+    expect(result.trace.submitted_by).toBe("[redacted]");
+    expect(toolUse?.type).toBe("tool_use");
+    if (!toolUse || toolUse.type !== "tool_use") {
+      throw new Error("expected a tool_use block");
+    }
+    expect(toolUse.tool_input.command).toContain("[OPENAI_KEY]");
+    expect((toolUse.tool_input.nested as { path: string }).path).toBe("~/project/.env");
+    expect(toolResult?.type).toBe("tool_result");
+    if (!toolResult || toolResult.type !== "tool_result") {
+      throw new Error("expected a tool_result block");
+    }
+    expect(toolResult.result_content).toBe("Bearer [BEARER_TOKEN]");
+    expect(result.trace.env_state?.inferred_file_tree?.[0]).toBe("~/project/.env");
+    expect(result.stats.changed).toBe(true);
+    expect(result.stats.piiMatches).toBe(0);
+    expect(result.stats.totalMatches).toBeGreaterThanOrEqual(3);
+  });
+});