create-walle 0.9.21 → 0.9.23

package/template/wall-e/eval/cc-replay.js

@@ -1,719 +0,0 @@
-'use strict';
-
-/**
- * Claude Code session replay — runs past Claude Code sessions through
- * the wall-e coding agent in isolated git worktrees, scores wall-e's
- * output against Claude's actual tool calls + diff for the same task
- * on the same repo state.
- *
- * Pipeline:
- *   1. pickRecentSessions  — walks ~/.claude/projects, filters by cwd + mtime
- *   2. recoverRepoState    — finds git commit at session start time
- *   3. makeReplaySandbox   — git worktree add at recovered commit
- *   4. replayAndScore      — runs wall-e on concatenated user messages,
- *                            scores tool/file Jaccard + LLM judge
- *   5. removeReplaySandbox — git worktree remove
- *
- * v1 uses concatenated-prompt scripted multi-turn (joins user messages
- * with `\n\n[user follow-up]\n`) — coding-orchestrator's resumeFromCheckpoint
- * does not natively support injecting new user messages mid-thread.
- */
-
-const fs = require('fs');
-const path = require('path');
-const os = require('os');
-const { execFileSync } = require('child_process');
-
-const {
-  findJsonlFiles,
-  extractContent,
-  extractToolCalls,
-  findCommitsInWindow,
-  getCommitDiff,
-} = require('./harvester');
-
-const CLAUDE_PROJECTS_DIR = path.join(os.homedir(), '.claude', 'projects');
-const SANDBOX_PREFIX = '/tmp/cc-replay-';
-const MIN_PROMPT_CHARS = 20;
-
-// ---------------------------------------------------------------------------
-// pickRecentSessions
-// ---------------------------------------------------------------------------
-
-/**
- * Walk Claude Code session JSONLs and return candidates suitable for replay.
- *
- * @param {object} opts
- * @param {string}  opts.repoPath   - Filter to sessions whose `cwd` is under this path
- * @param {number} [opts.sinceDays] - Only consider files mtime within this window
- * @param {number} [opts.limit]     - Max sessions to return (sorted by mtime desc)
- * @returns {Array<{
- *   sessionId: string, jsonlPath: string, cwd: string, gitBranch: string|null,
- *   userMessages: string[], tsStart: string, tsEnd: string, turnCount: number,
- *   claudeToolCalls: string[], claudeFilesEdited: string[]
- * }>}
- */
-function pickRecentSessions({ repoPath, sinceDays = 14, limit = 5 } = {}) {
-  if (!repoPath) throw new Error('repoPath is required');
-  if (!fs.existsSync(CLAUDE_PROJECTS_DIR)) return [];
-
-  const sinceMs = Date.now() - sinceDays * 24 * 60 * 60 * 1000;
-  const candidates = [];
-
-  for (const jsonlPath of findJsonlFiles(CLAUDE_PROJECTS_DIR)) {
-    let stat;
-    try { stat = fs.statSync(jsonlPath); } catch { continue; }
-    if (stat.mtime.getTime() < sinceMs) continue;
-
-    const session = parseSessionJsonl(jsonlPath, repoPath);
-    if (!session) continue;
-    if (session.userMessages.length < 1) continue;
-    if (session.userMessages[0].length < MIN_PROMPT_CHARS) continue;
-
-    candidates.push({ ...session, _mtime: stat.mtime.getTime() });
-  }
-
-  candidates.sort((a, b) => b._mtime - a._mtime);
-  return candidates.slice(0, limit).map(({ _mtime: _, ...rest }) => rest);
-}
-
-/**
- * Parse a JSONL session file. Returns null if the session's cwd does not
- * fall under repoPath, or if the file has no usable user messages.
- */
-function parseSessionJsonl(jsonlPath, repoPath) {
-  let lines;
-  try { lines = fs.readFileSync(jsonlPath, 'utf8').split('\n').filter(Boolean); }
-  catch { return null; }
-
-  const userMessages = [];
-  const claudeToolCalls = [];
-  const claudeFilesEditedSet = new Set();
-  let cwd = null;
-  let gitBranch = null;
-  let tsStart = null;
-  let tsEnd = null;
-  const sessionId = path.basename(jsonlPath, '.jsonl');
-
-  for (const line of lines) {
-    let evt;
-    try { evt = JSON.parse(line); } catch { continue; }
-
-    if (evt.cwd && !cwd) cwd = evt.cwd;
-    if (evt.gitBranch && !gitBranch) gitBranch = evt.gitBranch;
-    if (evt.timestamp) {
-      if (!tsStart) tsStart = evt.timestamp;
-      tsEnd = evt.timestamp;
-    }
-
-    if (evt.type === 'user' && evt.message?.role === 'user') {
-      const text = extractContent(evt.message);
-      if (text && text.trim().length >= MIN_PROMPT_CHARS && !looksLikeToolResult(text)) {
-        userMessages.push(text);
-      }
-    }
-
-    if (evt.type === 'assistant' && evt.message?.role === 'assistant') {
-      for (const tc of extractToolCalls(evt.message)) {
-        if (tc.name) claudeToolCalls.push(tc.name);
-        // Only count files actually MODIFIED — read_file/glob/grep don't
-        // count toward claudeFilesEdited.
-        if (!isEditToolName(tc.name)) continue;
-        const fp = editedFileFromToolInput(tc.input || {});
-        if (fp) claudeFilesEditedSet.add(normalizeSessionFile(fp, cwd));
-      }
-    }
-  }
-
-  if (!cwd) return null;
-  if (!cwd.startsWith(repoPath.replace(/\/+$/, ''))) return null;
-
-  return {
-    sessionId,
-    jsonlPath,
-    cwd,
-    gitBranch,
-    userMessages,
-    tsStart: tsStart || new Date(0).toISOString(),
-    tsEnd: tsEnd || new Date(0).toISOString(),
-    turnCount: userMessages.length,
-    claudeToolCalls,
-    claudeFilesEdited: [...claudeFilesEditedSet],
-  };
-}
-
-function looksLikeToolResult(text) {
-  if (!text) return true;
-  const head = text.slice(0, 400);
-  // Wrappers Claude Code injects around system/command/tool plumbing —
-  // these are not real user requests, just transcript markers.
-  const wrapperPatterns = [
-    /^\s*\[Request interrupted/i,
-    /^\s*<tool_use_error>/i,
-    /^\s*<system-reminder>/i,
-    /^\s*<local-command-caveat>/i,
-    /^\s*<local-command-stdout>/i,
-    /^\s*<local-command-stderr>/i,
-    /^\s*<command-message>/i,
-    /^\s*<command-name>/i,
-    /^\s*<command-args>/i,
-    // Skill bodies that Claude Code injects when a slash-command runs.
-    // Wall-e doesn't have the same skill, so replaying the body is meaningless.
-    /^\s*Base directory for this skill:/i,
-  ];
-  if (wrapperPatterns.some((re) => re.test(head))) return true;
-  // Pure slash-command invocation with no follow-up prose
-  if (/^\s*\/[a-z][\w:-]+\s*$/i.test(text.trim())) return true;
-  return false;
-}
-
-function isEditToolName(name) {
-  return /^(Edit|Write|MultiEdit|NotebookEdit|Patch|ApplyPatch|edit_file|write_file|apply_patch|multi_edit|str_replace|create_file)$/i.test(name || '');
-}
-
-function editedFileFromToolInput(input) {
-  return input.path
-    || input.file_path
-    || input.filePath
-    || input.notebook_path
-    || input.notebookPath
-    || null;
-}
-
-function normalizeSessionFile(filePath, cwd) {
-  if (!filePath || !cwd || !path.isAbsolute(filePath)) return filePath;
-  const rel = path.relative(cwd, filePath);
-  return rel && !rel.startsWith('..') && !path.isAbsolute(rel) ? rel : filePath;
-}
-
-// ---------------------------------------------------------------------------
-// recoverRepoState
-// ---------------------------------------------------------------------------
-
-/**
- * Find the git commit that was HEAD at session-start time.
- * Strategy: `git log --before=<tsStart> -n 1` — picks the most recent
- * commit reachable from any branch whose date is <= session start.
- * Returns null if the path isn't a git repo or has no commits at all.
- */
-function recoverRepoState(repoPath, tsStart) {
-  if (!fs.existsSync(path.join(repoPath, '.git'))) return null;
-  try {
-    const sha = execFileSync(
-      'git', ['log', '--all', `--before=${new Date(tsStart).toISOString()}`, '-n', '1', '--format=%H'],
-      { cwd: repoPath, encoding: 'utf8', timeout: 10000, stdio: ['pipe', 'pipe', 'pipe'] }
-    ).trim();
-    if (!sha) return null;
-    return { commitSha: sha, branch: null };
-  } catch {
-    return null;
-  }
-}
-
-// ---------------------------------------------------------------------------
-// sandbox lifecycle (git worktree)
-// ---------------------------------------------------------------------------
-
-function makeReplaySandbox(repoPath, commitSha, sessionId) {
-  const sandbox = `${SANDBOX_PREFIX}${sessionId}`;
-  if (fs.existsSync(sandbox)) removeReplaySandbox(sandbox, repoPath);
-  execFileSync('git', ['worktree', 'add', '--detach', sandbox, commitSha], {
-    cwd: repoPath, stdio: 'pipe', timeout: 30000,
-  });
-  return sandbox;
-}
-
-function removeReplaySandbox(sandbox, repoPath) {
-  try {
-    execFileSync('git', ['worktree', 'remove', '--force', sandbox], {
-      cwd: repoPath, stdio: 'pipe', timeout: 30000,
-    });
-  } catch {
-    // fall through to fs cleanup
-  }
-  try {
-    if (fs.existsSync(sandbox)) fs.rmSync(sandbox, { recursive: true, force: true });
-  } catch { /* best-effort */ }
-  try {
-    execFileSync('git', ['worktree', 'prune'], { cwd: repoPath, stdio: 'pipe', timeout: 10000 });
-  } catch { /* best-effort */ }
-}
-
-/**
- * Sweep stale `cc-replay-*` worktrees from /tmp. Run at start of a batch
- * to clean up after crashes.
- */
-function gitWorktreeCleanup(repoPath) {
-  try {
-    const list = execFileSync('git', ['worktree', 'list', '--porcelain'], {
-      cwd: repoPath, encoding: 'utf8', timeout: 10000,
-    });
-    const stale = (list.match(new RegExp(`worktree (${SANDBOX_PREFIX}[^\\n]+)`, 'g')) || [])
-      .map(l => l.replace(/^worktree /, ''));
-    for (const sb of stale) removeReplaySandbox(sb, repoPath);
-  } catch { /* best-effort */ }
-}
-
-// ---------------------------------------------------------------------------
-// replayAndScore
-// ---------------------------------------------------------------------------
-
-/**
- * Replay a Claude Code session through wall-e and score the result.
- *
- * @param {object} session       - From pickRecentSessions
- * @param {string} sandbox       - Path returned by makeReplaySandbox
- * @param {Function} runAgentLoop - From require('../coding-orchestrator')
- * @param {object} [opts]
- * @param {boolean} [opts.useLlmJudge=true]
- * @param {string}  [opts.model]
- * @param {object}  [opts.provider]
- * @param {number}  [opts.timeoutMs=600000]
- * @returns {Promise<object>} {sessionId, scores, walleTools, claudeTools, walleDiff, claudeDiff, latencyMs, error?}
- */
-async function replayAndScore(session, sandbox, runAgentLoop, opts = {}) {
-  const { useLlmJudge = true, model, provider, timeoutMs = 600_000 } = opts;
-  const startTime = Date.now();
-
-  const concatenatedPrompt = session.userMessages.length === 1
-    ? session.userMessages[0]
-    : session.userMessages
-        .map((msg, i) => i === 0 ? msg : `\n\n[user follow-up ${i + 1}]\n${msg}`)
-        .join('');
-
-  let agentResult;
-  try {
-    const agentPromise = runAgentLoop(concatenatedPrompt, {
-      cwd: sandbox,
-      timeoutMs,
-      provider,
-      model,
-      mode: 'build',
-      persistTranscript: false,
-    });
-    const hardTimeout = new Promise((_, reject) =>
-      setTimeout(() => reject(new Error('cc-replay hard timeout exceeded')), timeoutMs + 60_000)
-    );
-    agentResult = await Promise.race([agentPromise, hardTimeout]);
-  } catch (e) {
-    return {
-      sessionId: session.sessionId,
-      turnCount: session.turnCount,
-      error: e.message,
-      latencyMs: Date.now() - startTime,
-      scores: { composite: 0, tool_jaccard: 0, file_jaccard: 0, judge_score: 0 },
-      walleTools: [],
-      claudeTools: session.claudeToolCalls,
-      walleDiff: '',
-      claudeDiff: '',
-    };
-  }
-
-  const latencyMs = Date.now() - startTime;
-  const walleTools = extractWalleTools(agentResult);
-  const claudeTools = canonicalizeToolList(session.claudeToolCalls);
-  const walleFiles = extractWalleFiles(agentResult, sandbox);
-  const walleDiff = computeSandboxDiff(sandbox);
-  const claudeDiff = computeClaudeDiff(session);
-
-  // ---- scoring -----------------------------------------------------------
-  const tool_jaccard = jaccard(new Set(walleTools), new Set(claudeTools));
-  const file_jaccard = jaccard(
-    canonicalizeReplayFiles(walleFiles),
-    canonicalizeReplayFiles(session.claudeFilesEdited),
-  );
-
-  let judge_score = null;
-  let judge_reason = null;
-  if (useLlmJudge && (walleDiff || claudeDiff)) {
-    try {
-      const judged = await scoreLlmJudge({
-        prompt: session.userMessages[0],
-        walleDiff,
-        claudeDiff,
-        model,
-      });
-      judge_score = judged.score;
-      judge_reason = judged.reason;
-    } catch (e) {
-      judge_reason = `judge failed: ${e.message}`;
-    }
-  }
-
-  // Composite: 0.25·tool + 0.25·file + 0.50·judge (when judge available);
-  // otherwise 0.5/0.5 split between tool+file.
-  let composite;
-  if (judge_score != null) {
-    composite = 0.25 * tool_jaccard + 0.25 * file_jaccard + 0.50 * judge_score;
-  } else {
-    composite = 0.5 * tool_jaccard + 0.5 * file_jaccard;
-  }
-
-  return {
-    sessionId: session.sessionId,
-    turnCount: session.turnCount,
-    latencyMs,
-    scores: { composite, tool_jaccard, file_jaccard, judge_score, judge_reason },
-    walleTools,
-    walleFiles,
-    claudeTools,
-    claudeFiles: session.claudeFilesEdited,
-    walleDiff: walleDiff.slice(0, 8000),
-    claudeDiff: (claudeDiff || '').slice(0, 8000),
-    usage: agentResult.usage || null,
-    agentResult: summarizeAgentResult(agentResult),
-  };
-}
-
-// ---------------------------------------------------------------------------
-// scoring helpers
-// ---------------------------------------------------------------------------
-
-function jaccard(setA, setB) {
-  if (!setA.size && !setB.size) return 1;
-  const intersection = [...setA].filter(x => setB.has(x)).length;
-  const union = new Set([...setA, ...setB]).size;
-  return union === 0 ? 0 : intersection / union;
-}
-
-function canonicalizeReplayFiles(files = []) {
-  return new Set((files || []).map(canonicalReplayFilePath).filter(Boolean));
-}
-
-function canonicalReplayFilePath(filePath) {
-  const normalized = String(filePath || '').trim().replace(/\\/g, '/').replace(/^\.\//, '');
-  const templatePrefix = 'create-walle/template/';
-  if (normalized.startsWith(templatePrefix)) {
-    return normalized.slice(templatePrefix.length);
-  }
-  return normalized;
-}
-
-function extractWalleTools(agentResult) {
-  const log = agentResult?.log || [];
-  const tools = [];
-  for (const turn of log) {
-    for (const tc of turn.toolCalls || []) {
-      const canonical = canonicalToolName(tc.name);
-      if (canonical) tools.push(canonical);
-    }
-  }
-  return tools;
-}
-
-function canonicalizeToolList(tools = []) {
-  return (tools || []).map(canonicalToolName).filter(Boolean);
-}
-
-function canonicalToolName(name) {
-  const raw = String(name || '').trim();
-  if (!raw) return null;
-  if (/^(read|read_file)$/i.test(raw)) return 'read_file';
-  if (/^(bash|run_shell|exec_command|shell)$/i.test(raw)) return 'run_shell';
-  if (/^(grep|glob|toolsearch|list_directory|grep_files|search|rg)$/i.test(raw)) return 'search';
-  if (/^(edit|write|multiedit|notebookedit|patch|applypatch|apply_patch|edit_file|write_file|multi_edit|str_replace|create_file)$/i.test(raw)) return 'edit';
-  if (/^(agent|task|taskoutput)$/i.test(raw)) return 'agent';
-  if (/^askuserquestion$/i.test(raw)) return 'ask_user';
-  if (/^skill$/i.test(raw)) return 'skill';
-  if (/^mcp__/i.test(raw)) return 'mcp';
-  return raw.toLowerCase();
-}
-
-function extractWalleFiles(agentResult, sandbox) {
-  const files = new Set();
-  const log = agentResult?.log || [];
-  for (const turn of log) {
-    for (const tc of turn.toolCalls || []) {
-      if (!isEditToolName(tc.name)) continue;
-      const input = tc.input || {};
-      const fp = editedFileFromToolInput(input);
-      if (fp) {
-        files.add(normalizeWalleFilePath(fp, sandbox));
-      }
-      for (const patchFile of extractPatchFilePaths(input.patch_text || input.patchText || input.patch || '')) {
-        files.add(normalizeWalleFilePath(patchFile, sandbox));
-      }
-    }
-  }
-  return [...files];
-}
-
-function normalizeWalleFilePath(filePath, sandbox) {
-  if (!filePath || !sandbox) return filePath;
-  if (!path.isAbsolute(filePath)) return filePath.replace(/^\.\//, '');
-  const fileVariants = pathVariants(filePath);
-  const sandboxVariants = pathVariants(sandbox);
-  for (const candidate of fileVariants) {
-    for (const root of sandboxVariants) {
-      const rel = relativeIfUnder(candidate, root);
-      if (rel) return rel;
-    }
-  }
-  return filePath;
-}
-
-function extractPatchFilePaths(patchText) {
-  if (!patchText || typeof patchText !== 'string') return [];
-  const files = [];
-  for (const line of patchText.split('\n')) {
-    const match = line.match(/^\*\*\* (?:Add|Update|Delete) File:\s+(.+?)\s*$/);
-    if (match?.[1]) files.push(match[1].trim());
-    const moveMatch = line.match(/^\*\*\* Move to:\s+(.+?)\s*$/);
-    if (moveMatch?.[1]) files.push(moveMatch[1].trim());
-  }
-  return [...new Set(files)];
-}
-
-function pathVariants(filePath) {
-  const variants = new Set([filePath]);
-  try { variants.add(path.resolve(filePath)); } catch {}
-  try { variants.add(fs.realpathSync(filePath)); } catch {}
-  for (const value of [...variants]) {
-    if (value.startsWith('/private/tmp/')) variants.add(value.replace(/^\/private\/tmp\//, '/tmp/'));
-    if (value.startsWith('/tmp/')) variants.add(value.replace(/^\/tmp\//, '/private/tmp/'));
-  }
-  return [...variants].filter(Boolean);
-}
-
-function relativeIfUnder(filePath, root) {
-  if (!filePath || !root || !path.isAbsolute(filePath) || !path.isAbsolute(root)) return null;
-  const rel = path.relative(root, filePath);
-  if (!rel || rel.startsWith('..') || path.isAbsolute(rel)) return null;
-  return rel.replace(/\\/g, '/');
-}
-
-function computeSandboxDiff(sandbox) {
-  try {
-    return execFileSync('git', ['diff', 'HEAD'], {
-      cwd: sandbox, encoding: 'utf8', timeout: 15000, maxBuffer: 10 * 1024 * 1024,
-    });
-  } catch {
-    return '';
-  }
-}
-
-function summarizeAgentResult(agentResult) {
-  if (!agentResult || typeof agentResult !== 'object') return null;
-  return {
-    success: agentResult.success,
-    exitCode: agentResult.exitCode,
-    stderr: agentResult.stderr ? String(agentResult.stderr).slice(0, 1000) : '',
-    output: agentResult.output ? String(agentResult.output).slice(0, 2000) : '',
-    logLength: Array.isArray(agentResult.log) ? agentResult.log.length : 0,
-    sessionId: agentResult.sessionId || null,
-  };
-}
-
-/**
- * Best-effort: find the commit just AFTER session end and use its diff
- * as Claude's ground-truth output. If no commit landed, return null.
- */
-function computeClaudeDiff(session) {
-  try {
-    const commits = findCommitsInWindow(session.cwd, session.tsStart, session.tsEnd);
-    if (!commits.length) return null;
-    return getCommitDiff(session.cwd, commits[0].hash) || null;
-  } catch {
-    return null;
-  }
-}
-
-/**
- * LLM-as-judge: ask Claude Haiku whether wall-e's diff achieves the
- * same intent as Claude's diff. Returns {score, reason}.
- */
-async function scoreLlmJudge({ prompt, walleDiff, claudeDiff, model }) {
-  const { getDefaultClient } = require('../llm/client');
-  const client = getDefaultClient();
-  const judgePrompt = [
-    `User's request: ${(prompt || '').slice(0, 1000)}`,
-    '',
-    `--- Diff A (Claude Code's actual output, ground truth) ---`,
-    (claudeDiff || '<no commit landed for this session>').slice(0, 4000),
-    '',
-    `--- Diff B (wall-e's output) ---`,
-    (walleDiff || '<empty>').slice(0, 4000),
-    '',
-    'Score 0.0 to 1.0: how well does Diff B accomplish the same intent as Diff A',
-    'for the user\'s request? Consider goal achievement, not byte equality. If both',
-    'diffs are empty/missing, score 0.5. Reply with ONLY a single line:',
-    'SCORE: <0.0–1.0>',
-    'REASON: <one sentence>',
-  ].join('\n');
-
-  const resp = await client.chat({
-    model: model || 'claude-haiku-4-5-20251001',
-    messages: [{ role: 'user', content: judgePrompt }],
-    maxTokens: 200,
-  });
-  const text = (resp.content || '').trim();
-  const scoreMatch = text.match(/SCORE:\s*([0-9]*\.?[0-9]+)/i);
-  const reasonMatch = text.match(/REASON:\s*(.+)/i);
-  let score = scoreMatch ? parseFloat(scoreMatch[1]) : 0.5;
-  if (!isFinite(score) || score < 0) score = 0;
-  if (score > 1) score = 1;
-  return { score, reason: reasonMatch ? reasonMatch[1].trim() : null };
-}
-
-// ---------------------------------------------------------------------------
-// Reap mode — harvest sessions into the persistent benchmark catalog
-// ---------------------------------------------------------------------------
-
-const crypto = require('crypto');
-const CODING_AGENT_BENCHMARKS_PATH = path.join(__dirname, 'benchmarks', 'coding-agent.json');
-
-/**
- * Convert a picked session into a self-contained benchmark catalog entry.
- * The entry embeds everything needed to replay later (userMessages, recovered
- * commit, claudeTools, claudeFiles) so the original JSONL can be deleted
- * without breaking the entry.
- */
-function sessionToCatalogEntry(session, recoveredCommit) {
-  const sha = crypto.createHash('sha256').update(session.sessionId + '|' + (session.userMessages[0] || '')).digest('hex').slice(0, 8);
-  const id = `agent-cc-${sha}`;
-  const tools = session.claudeToolCalls || [];
-  const traits = ['has code block'];
-  if (tools.some((t) => /read_file|glob|grep|Read|Grep/.test(t))) traits.push('reads before writing');
-  if (tools.some((t) => /edit_file|Edit|str_replace/.test(t))) traits.push('uses edit over write');
-  if (tools.some((t) => /bash|Bash/.test(t))) traits.push('runs commands');
-
-  const turnCount = session.turnCount || 1;
-  const difficulty = turnCount > 10 ? 'hard' : turnCount > 4 ? 'medium' : 'easy';
-
-  return {
-    id,
-    prompt: session.userMessages[0] || '',
-    taskType: 'coding-agent',
-    difficulty,
-    expectedTraits: traits,
-    agentExpectations: {
-      expectedToolCalls: [...new Set(tools)].slice(0, 12),
-      maxTurns: Math.min(turnCount * 2, 50),
-      expectedFileChanges: session.claudeFilesEdited || [],
-    },
-    ccReplay: {
-      sourceSessionId: session.sessionId,
-      cwd: session.cwd,
-      gitBranch: session.gitBranch || null,
-      tsStart: session.tsStart,
-      tsEnd: session.tsEnd,
-      recoveredCommit,
-      userMessages: session.userMessages,
-      turnCount,
-      claudeToolCalls: tools,
-      claudeFilesEdited: session.claudeFilesEdited || [],
-      reapedAt: new Date().toISOString(),
-    },
-  };
-}
-
-/**
- * Harvest fresh Claude Code sessions and append them to the benchmark
- * catalog (coding-agent.json). Dedups against existing `agent-cc-*` and
- * `agent-session-*` IDs. Returns the entries that were added.
- *
- * @param {object} opts
- * @param {string} opts.repoPath
- * @param {number} [opts.limit]      — max NEW entries to add this run
- * @param {number} [opts.sinceDays]  — only consider JSONLs with this mtime window
- * @param {string} [opts.benchmarksPath] — override for tests
- * @returns {{added: object[], skipped: number, total: number}}
- */
-function reapSessions({ repoPath, limit = 5, sinceDays = 14, benchmarksPath = CODING_AGENT_BENCHMARKS_PATH } = {}) {
-  if (!repoPath) throw new Error('repoPath is required');
-
-  // Load existing IDs for dedup
-  let existing = [];
-  try { existing = JSON.parse(fs.readFileSync(benchmarksPath, 'utf8')); }
-  catch { existing = []; }
-  const existingIds = new Set(existing.map((e) => e.id));
-  const existingSourceIds = new Set(
-    existing.filter((e) => e.ccReplay && e.ccReplay.sourceSessionId).map((e) => e.ccReplay.sourceSessionId)
-  );
-
-  // Pick more candidates than `limit` so we can skip non-replayable ones
-  const candidates = pickRecentSessions({ repoPath, sinceDays, limit: limit * 4 });
-
-  const added = [];
-  let skipped = 0;
-  for (const session of candidates) {
-    if (added.length >= limit) break;
-
-    if (existingSourceIds.has(session.sessionId)) { skipped++; continue; }
-
-    const repo = recoverRepoState(session.cwd, session.tsStart);
-    if (!repo) { skipped++; continue; }   // can't reconstruct repo state
-
-    const entry = sessionToCatalogEntry(session, repo.commitSha);
-    if (existingIds.has(entry.id)) { skipped++; continue; }
-
-    existing.push(entry);
-    existingIds.add(entry.id);
-    existingSourceIds.add(session.sessionId);
-    added.push(entry);
-  }
-
-  if (added.length > 0) {
-    fs.writeFileSync(benchmarksPath, JSON.stringify(existing, null, 2) + '\n');
-  }
-
-  return { added, skipped, total: existing.length };
-}
-
-/**
- * Replay a catalog entry that has an embedded ccReplay block. No live JSONL
- * required — everything needed (userMessages, claudeTools, claudeFiles,
- * recoveredCommit) is in the entry itself. Used by the standard
- * /eval/coding-agent/run path when bench.id starts with `agent-cc-`.
- *
- * @param {object} bench         — catalog entry with bench.ccReplay
- * @param {Function} runAgentLoop
- * @param {object} [opts]
- * @returns {Promise<object>} same shape as replayAndScore
- */
-async function replayFromCatalog(bench, runAgentLoop, opts = {}) {
-  if (!bench.ccReplay) throw new Error(`Benchmark ${bench.id} has no ccReplay block`);
-  const cc = bench.ccReplay;
-  const session = {
-    sessionId: cc.sourceSessionId,
-    cwd: cc.cwd,
-    gitBranch: cc.gitBranch,
-    userMessages: cc.userMessages || [bench.prompt],
-    tsStart: cc.tsStart,
-    tsEnd: cc.tsEnd,
-    turnCount: cc.turnCount || (cc.userMessages || [bench.prompt]).length,
-    claudeToolCalls: cc.claudeToolCalls || [],
-    claudeFilesEdited: cc.claudeFilesEdited || [],
-  };
-
-  let sandbox = null;
-  try {
-    sandbox = makeReplaySandbox(cc.cwd, cc.recoveredCommit, session.sessionId);
-    return await replayAndScore(session, sandbox, runAgentLoop, opts);
-  } finally {
-    if (sandbox) {
-      try { removeReplaySandbox(sandbox, cc.cwd); }
-      catch (e) { console.warn('[cc-replay] catalog sandbox cleanup failed:', e.message); }
-    }
-  }
-}
-
-module.exports = {
-  pickRecentSessions,
-  recoverRepoState,
-  makeReplaySandbox,
-  removeReplaySandbox,
-  gitWorktreeCleanup,
-  replayAndScore,
-  reapSessions,
-  replayFromCatalog,
-  sessionToCatalogEntry,
-  // exposed for tests
-  parseSessionJsonl,
-  jaccard,
-  canonicalReplayFilePath,
-  canonicalToolName,
-  canonicalizeToolList,
-  extractWalleFiles,
-  extractPatchFilePaths,
-};