@tracemarketplace/shared 0.0.1

package/src/scoring.ts

@@ -0,0 +1,149 @@
+import type { NormalizedTrace, TraceScore, FailureMode } from "./types.js";
+
+export function detectFailureModes(trace: NormalizedTrace): FailureMode[] {
+  const modes = new Set<FailureMode>();
+  const allBlocks = trace.turns.flatMap((t) => t.content);
+
+  // tool_call_failure: any tool_result with is_error=true
+  const hasToolError = allBlocks.some(
+    (b) => b.type === "tool_result" && b.is_error
+  );
+  if (hasToolError) modes.add("tool_call_failure");
+
+  // repeated_tool_calls: same tool_name 3+ times in a row
+  const toolUses = allBlocks.filter((b) => b.type === "tool_use") as Array<{
+    type: "tool_use";
+    tool_call_id: string;
+    tool_name: string;
+    tool_input: Record<string, unknown>;
+  }>;
+  let streak = 1;
+  for (let i = 1; i < toolUses.length; i++) {
+    if (toolUses[i].tool_name === toolUses[i - 1].tool_name) {
+      streak++;
+      if (streak >= 3) {
+        modes.add("repeated_tool_calls");
+        break;
+      }
+    } else {
+      streak = 1;
+    }
+  }
+
+  // context_limit_approached: text mentioning context/limit
+  const contextLimitRegex = /context.*(limit|window|maximum)|context limit/i;
+  const hasContextLimit = trace.turns.some((t) =>
+    t.content.some(
+      (b) =>
+        b.type === "text" && contextLimitRegex.test(b.text)
+    )
+  );
+  if (hasContextLimit) modes.add("context_limit_approached");
+
+  // graceful_recovery: tool errors followed by recovery text
+  if (hasToolError) {
+    const recoveryRegex = /let me try|instead|alternative|another approach|different way/i;
+    const laterTurns = trace.turns.slice(Math.floor(trace.turns.length / 2));
+    const hasRecovery = laterTurns.some((t) =>
+      t.content.some(
+        (b) => b.type === "text" && recoveryRegex.test(b.text)
+      )
+    );
+    if (hasRecovery) modes.add("graceful_recovery");
+  }
+
+  // repeated_tool_calls → graceful_recovery if later success
+  if (modes.has("repeated_tool_calls")) {
+    const lastTurn = trace.turns[trace.turns.length - 1];
+    if (lastTurn?.role === "assistant") {
+      const hasSuccessText = lastTurn.content.some(
+        (b) => b.type === "text" && b.text.length > 50
+      );
+      if (hasSuccessText) modes.add("graceful_recovery");
+    }
+  }
+
+  // catastrophic_failure: last 3+ turns are all errors with no recovery
+  const lastTurns = trace.turns.slice(-3);
+  const allLastAreErrors =
+    lastTurns.length >= 2 &&
+    lastTurns.every((t) =>
+      t.content.some((b) => b.type === "tool_result" && b.is_error)
+    );
+  if (allLastAreErrors && !modes.has("graceful_recovery")) {
+    modes.add("catastrophic_failure");
+  }
+
+  if (modes.size === 0) modes.add("no_failure");
+
+  return Array.from(modes);
+}
+
+export function checkCompleteness(
+  trace: NormalizedTrace
+): "complete" | "incomplete" | "malformed" {
+  if (trace.turns.length === 0) return "malformed";
+
+  // malformed: any turn with empty content
+  if (trace.turns.some((t) => t.content.length === 0)) return "malformed";
+
+  const lastTurn = trace.turns[trace.turns.length - 1];
+  if (!lastTurn) return "malformed";
+
+  // complete: last assistant turn ends with text
+  if (lastTurn.role === "assistant") {
+    const hasText = lastTurn.content.some((b) => b.type === "text");
+    if (hasText) return "complete";
+    // ends with tool_use but no following result → incomplete
+    const lastBlock = lastTurn.content[lastTurn.content.length - 1];
+    if (lastBlock?.type === "tool_use") return "incomplete";
+  }
+
+  // last turn is user (e.g., tool_result with no following assistant) → incomplete
+  return "incomplete";
+}
+
+export function scoreTrace(trace: NormalizedTrace): TraceScore {
+  const completeness = checkCompleteness(trace);
+  const failureModes = detectFailureModes(trace);
+
+  const fidelityBase = trace.content_fidelity === "full" ? 0.4 : 0.15;
+
+  let interestBonus = 0;
+  if (failureModes.includes("graceful_recovery")) interestBonus += 0.3;
+  if (failureModes.includes("repeated_tool_calls")) interestBonus += 0.2;
+  if (failureModes.includes("catastrophic_failure")) interestBonus += 0.15;
+  if (
+    failureModes.includes("tool_call_failure") &&
+    !failureModes.includes("graceful_recovery")
+  )
+    interestBonus += 0.1;
+
+  const totalTokens =
+    (trace.total_input_tokens ?? 0) + (trace.total_output_tokens ?? 0);
+  const lengthBonus = Math.min(
+    0.15,
+    Math.log10(Math.max(1, totalTokens / 1000)) * 0.05
+  );
+
+  const total = Math.min(1.0, fidelityBase + interestBonus + lengthBonus);
+  const payoutCents = Math.min(500, Math.round(total * 500));
+
+  return {
+    completeness,
+    failure_modes: failureModes,
+    has_error_recovery: failureModes.includes("graceful_recovery"),
+    has_repeated_calls: failureModes.includes("repeated_tool_calls"),
+    content_fidelity: trace.content_fidelity,
+    total,
+    payout_cents: payoutCents,
+    failure_taxonomy_label: null,
+    failure_taxonomy_explanation: null,
+    rarity_score: null,
+    cluster_id: null,
+    is_duplicate: false,
+    duplicate_of: null,
+    scored_at: new Date().toISOString(),
+    scorer_version: "v0-heuristic",
+  };
+}

package/src/types.ts

@@ -0,0 +1,96 @@
+export type SourceTool = "claude_code" | "codex_cli" | "cursor";
+
+export type FailureMode =
+  | "tool_call_failure"
+  | "repeated_tool_calls"
+  | "context_limit_approached"
+  | "catastrophic_failure"
+  | "graceful_recovery"
+  | "unexpected_capability"
+  | "wrong_tool_sequence"
+  | "no_failure";
+
+export interface TokenUsage {
+  input_tokens: number;
+  output_tokens: number;
+  cache_read_input_tokens: number | null;
+  cache_creation_input_tokens: number | null;
+  reasoning_tokens: number | null;
+}
+
+export type ContentBlock =
+  | { type: "text"; text: string }
+  | { type: "thinking"; text: string }
+  | { type: "tool_use"; tool_call_id: string; tool_name: string; tool_input: Record<string, unknown> }
+  | { type: "tool_result"; tool_call_id: string; is_error: boolean; result_content: string | null; exit_code: number | null }
+  | { type: "image"; media_type: string; data_r2_key: string };
+
+export interface Turn {
+  turn_id: string;
+  parent_turn_id: string | null;
+  role: "user" | "assistant";
+  timestamp: string | null;
+  content: ContentBlock[];
+  model: string | null;
+  usage: TokenUsage | null;
+  source_metadata: Record<string, unknown>;
+}
+
+export interface EnvState {
+  git_branch: string | null;
+  inferred_file_tree: string[] | null;
+  inferred_changed_files: string[] | null;
+  inferred_error_files: string[] | null;
+  shell_exit_codes: number[] | null;
+  open_files_in_editor: string[] | null;
+  extraction_method: "passive" | "active";
+}
+
+export interface TraceScore {
+  completeness: "complete" | "incomplete" | "malformed";
+  failure_modes: FailureMode[];
+  has_error_recovery: boolean;
+  has_repeated_calls: boolean;
+  content_fidelity: "full" | "chat_only";
+  total: number;
+  payout_cents: number;
+  failure_taxonomy_label: string | null;
+  failure_taxonomy_explanation: string | null;
+  rarity_score: number | null;
+  cluster_id: string | null;
+  is_duplicate: boolean;
+  duplicate_of: string | null;
+  scored_at: string;
+  scorer_version: string;
+}
+
+export interface NormalizedTrace {
+  trace_id: string;
+  schema_version: string;
+  source_tool: SourceTool;
+  source_session_id: string;
+  source_version: string | null;
+  submitted_by: string;
+  submitted_at: string;
+  extracted_at: string;
+  git_branch: string | null;
+  cwd_hash: string | null;
+  working_language: string | null;
+  started_at: string;
+  ended_at: string;
+  turns: Turn[];
+  turn_count: number;
+  tool_call_count: number;
+  has_tool_calls: boolean;
+  has_thinking_blocks: boolean;
+  has_file_changes: boolean;
+  has_shell_commands: boolean;
+  total_input_tokens: number | null;
+  total_output_tokens: number | null;
+  total_cache_read_tokens: number | null;
+  content_fidelity: "full" | "chat_only";
+  env_state: EnvState | null;
+  score: TraceScore | null;
+  raw_r2_key: string;
+  normalized_r2_key: string;
+}

package/src/utils.ts

@@ -0,0 +1,9 @@
+export function formatCents(cents: number | null): string {
+  if (cents === null) return "—";
+  return `$${(cents / 100).toFixed(2)}`;
+}
+
+export function formatTimestamp(ts: number | null): string {
+  if (!ts) return "—";
+  return new Date(ts * 1000).toLocaleString();
+}

package/src/validators.test.ts

@@ -0,0 +1,61 @@
+import { describe, it, expect } from "vitest";
+import { BatchSubmitSchema, NormalizedTraceSchema } from "./validators.js";
+
+function makeTrace(overrides: Record<string, unknown> = {}) {
+  return {
+    source_tool: "claude_code",
+    source_session_id: "session-abc-123",
+    ...overrides,
+  };
+}
+
+describe("NormalizedTraceSchema", () => {
+  it("valid minimal trace → success", () => {
+    const result = NormalizedTraceSchema.safeParse(makeTrace());
+    expect(result.success).toBe(true);
+  });
+
+  it("missing source_tool → ZodError", () => {
+    const result = NormalizedTraceSchema.safeParse({ source_session_id: "abc" });
+    expect(result.success).toBe(false);
+  });
+
+  it("missing source_session_id → ZodError", () => {
+    const result = NormalizedTraceSchema.safeParse({ source_tool: "claude_code" });
+    expect(result.success).toBe(false);
+  });
+
+  it("invalid content_fidelity → ZodError", () => {
+    const result = NormalizedTraceSchema.safeParse(makeTrace({ content_fidelity: "high_fidelity" }));
+    expect(result.success).toBe(false);
+  });
+
+  it("negative turn_count → ZodError", () => {
+    const result = NormalizedTraceSchema.safeParse(makeTrace({ turn_count: -1 }));
+    expect(result.success).toBe(false);
+  });
+});
+
+describe("BatchSubmitSchema", () => {
+  it("valid batch → success", () => {
+    const result = BatchSubmitSchema.safeParse({ traces: [makeTrace()] });
+    expect(result.success).toBe(true);
+  });
+
+  it("empty traces array → ZodError", () => {
+    const result = BatchSubmitSchema.safeParse({ traces: [] });
+    expect(result.success).toBe(false);
+  });
+
+  it("missing traces → ZodError", () => {
+    const result = BatchSubmitSchema.safeParse({});
+    expect(result.success).toBe(false);
+  });
+
+  it("invalid source_tool in batch → ZodError", () => {
+    const result = BatchSubmitSchema.safeParse({
+      traces: [{ source_tool: "vscode", source_session_id: "abc" }],
+    });
+    expect(result.success).toBe(false);
+  });
+});

package/src/validators.ts

@@ -0,0 +1,41 @@
+import { z } from "zod";
+
+export const NormalizedTraceSchema = z.object({
+  trace_id: z.string().optional(),
+  schema_version: z.string().optional(),
+  source_tool: z.enum(["claude_code", "codex_cli", "cursor"]),
+  source_session_id: z.string().min(1),
+  source_version: z.string().nullable().optional(),
+  submitted_by: z.string().optional(),
+  submitted_at: z.string().optional(),
+  extracted_at: z.string().optional(),
+  git_branch: z.string().nullable().optional(),
+  cwd_hash: z.string().nullable().optional(),
+  working_language: z.string().nullable().optional(),
+  started_at: z.string().optional(),
+  ended_at: z.string().optional(),
+  turns: z.array(z.any()).optional(),
+  turn_count: z.number().int().nonnegative().optional(),
+  tool_call_count: z.number().int().nonnegative().optional(),
+  has_tool_calls: z.boolean().optional(),
+  has_thinking_blocks: z.boolean().optional(),
+  has_file_changes: z.boolean().optional(),
+  has_shell_commands: z.boolean().optional(),
+  total_input_tokens: z.number().nullable().optional(),
+  total_output_tokens: z.number().nullable().optional(),
+  total_cache_read_tokens: z.number().nullable().optional(),
+  content_fidelity: z.enum(["full", "chat_only"]).optional(),
+  env_state: z.any().nullable().optional(),
+  score: z.any().nullable().optional(),
+  raw_r2_key: z.string().optional(),
+  normalized_r2_key: z.string().optional(),
+});
+
+export type NormalizedTraceInput = z.infer<typeof NormalizedTraceSchema>;
+
+export const BatchSubmitSchema = z.object({
+  traces: z.array(NormalizedTraceSchema).min(1),
+  source_tool: z.enum(["claude_code", "codex_cli", "cursor"]).optional(),
+});
+
+export type BatchSubmitInput = z.infer<typeof BatchSubmitSchema>;

package/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "include": ["src"]
+}

package/vitest.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    environment: "node",
+    include: ["src/**/*.test.ts"],
+  },
+});