create-walle 0.9.21 → 0.9.23

package/template/wall-e/eval/coding-agent-real.js

@@ -1,570 +0,0 @@
-'use strict';
-
-const crypto = require('crypto');
-const fs = require('fs');
-const os = require('os');
-const path = require('path');
-const { execFileSync } = require('child_process');
-
-const {
-  findTranscriptJsonlFiles,
-  parseTranscriptJsonl,
-} = require('./session-transcripts');
-
-const {
-  recoverRepoState,
-  makeReplaySandbox,
-  removeReplaySandbox,
-  replayAndScore,
-} = require('./cc-replay');
-
-const DEFAULT_REAL_CATALOG_PATH = path.join(__dirname, 'benchmarks', 'coding-agent-real.json');
-const DEFAULT_RESULTS_DIR = path.join(os.homedir(), '.walle', 'eval-results', 'coding-agent-real');
-
-function buildReplayCatalog({
-  repoPath,
-  roots,
-  source = 'all',
-  sinceDays = 14,
-  limit = 20,
-  catalogPath = DEFAULT_REAL_CATALOG_PATH,
-  requireEdits = true,
-  requireCodingIntent = true,
-} = {}) {
-  if (!repoPath) throw new Error('repoPath is required');
-
-  const files = findTranscriptJsonlFiles({ roots, source, sinceDays, limit: limit * 8 });
-  const existing = loadCatalog(catalogPath);
-  const existingIds = new Set(existing.map((entry) => entry.id));
-  const existingSourceKeys = new Set(existing.map(sourceKeyForEntry).filter(Boolean));
-
-  const added = [];
-  const skipped = {
-    unreadable: 0,
-    outOfRepo: 0,
-    noPrompt: 0,
-    nonCodingPrompt: 0,
-    nonReplayableConversation: 0,
-    noReferenceEdits: 0,
-    externalReferenceEdits: 0,
-    noCommit: 0,
-    duplicate: 0,
-    limit: 0,
-  };
-
-  for (const file of files) {
-    if (added.length >= limit) { skipped.limit += 1; continue; }
-
-    const session = parseTranscriptJsonl(file, { repoPath });
-    if (!session) { skipped.outOfRepo += 1; continue; }
-    if (!session.userMessages.length) { skipped.noPrompt += 1; continue; }
-    if (requireCodingIntent && !isStandaloneCodingPrompt(session.userMessages[0])) { skipped.nonCodingPrompt += 1; continue; }
-    if (requireCodingIntent && !isReplayConversationSelfContained(session)) { skipped.nonReplayableConversation += 1; continue; }
-    if (requireEdits && !session.filesEdited.length) { skipped.noReferenceEdits += 1; continue; }
-    if (requireEdits && !hasOnlyRepoRelativeEdits(session.filesEdited)) { skipped.externalReferenceEdits += 1; continue; }
-
-    const sourceKey = `${session.source}:${session.sessionId}`;
-    if (existingSourceKeys.has(sourceKey)) { skipped.duplicate += 1; continue; }
-
-    const recovered = recoverRepoState(session.cwd, session.tsStart);
-    if (!recovered?.commitSha) { skipped.noCommit += 1; continue; }
-
-    const entry = sessionToRealCatalogEntry(session, recovered.commitSha);
-    if (existingIds.has(entry.id)) { skipped.duplicate += 1; continue; }
-
-    existing.push(entry);
-    existingIds.add(entry.id);
-    existingSourceKeys.add(sourceKey);
-    added.push(entry);
-  }
-
-  if (catalogPath) {
-    fs.mkdirSync(path.dirname(catalogPath), { recursive: true });
-    fs.writeFileSync(catalogPath, JSON.stringify(existing, null, 2) + '\n');
-  }
-
-  return {
-    added,
-    skipped,
-    total: existing.length,
-    scanned: files.length,
-    catalogPath,
-  };
-}
-
-function sessionToRealCatalogEntry(session, recoveredCommit) {
-  const prompt = session.userMessages[0] || '';
-  const referenceToolCalls = session.toolCalls.map((call) => call.name).filter(Boolean);
-  const referenceFilesEdited = session.filesEdited || [];
-  const turnCount = session.turnCount || session.userMessages.length || 1;
-  const idHash = crypto
-    .createHash('sha256')
-    .update([session.source, session.sessionId, session.cwd, prompt].join('|'))
-    .digest('hex')
-    .slice(0, 10);
-  const difficulty = classifyDifficulty({ turnCount, files: referenceFilesEdited.length, tools: referenceToolCalls.length });
-
-  return {
-    id: `agent-real-${session.source}-${idHash}`,
-    prompt,
-    taskType: 'coding-agent-real',
-    difficulty,
-    expectedTraits: expectedTraitsFor(referenceToolCalls, referenceFilesEdited),
-    agentExpectations: {
-      expectedToolCalls: [...new Set(referenceToolCalls)].slice(0, 16),
-      expectedFileChanges: referenceFilesEdited,
-      maxTurns: Math.max(12, Math.min(turnCount * 4 + 8, 80)),
-      testCommand: null,
-    },
-    realReplay: {
-      source: session.source,
-      sourceSessionId: session.sessionId,
-      sourceJsonlPath: session.jsonlPath,
-      cwd: session.cwd,
-      gitBranch: session.gitBranch || null,
-      tsStart: session.tsStart,
-      tsEnd: session.tsEnd,
-      recoveredCommit: recoveredCommit || null,
-      userMessages: session.userMessages,
-      assistantMessageCount: session.assistantMessages.length,
-      turnCount,
-      referenceToolCalls,
-      referenceFilesEdited,
-      shellCommands: session.shellCommands || [],
-      rawEventCount: session.rawEventCount || 0,
-      reapedAt: new Date().toISOString(),
-    },
-  };
-}
-
-function isStandaloneCodingPrompt(prompt) {
-  const text = String(prompt || '').trim();
-  if (text.length < 25) return false;
-  const lower = text.toLowerCase();
-
-  // These are often useful conversations, but they do not make good replay
-  // prompts because the expected output is explanation or shared context, not
-  // a concrete codebase action.
-  if (/\b(let'?s discuss|let us discuss|let'?s have a discussion|discussion\s*\/\s*plan|walk me through|how should i|what do you think|take a close look)\b/.test(lower)) {
-    return false;
-  }
-  if (/^\s*(go ahead|goa ahead|proceed|do it|yes[, ]|yep[, ]|ok[, ]|okay[, ]|everything in main)\b/.test(lower)) {
-    return false;
-  }
-
-  return /\b(fix|implement|add|change|update|refactor|test|merge|commit|build|debug|make|write|delete|remove|harden|wire|land|ship|bug|failing|error|regression|feature|endpoint|api|ui|server|component|tests?|code review|review.*code)\b/.test(lower);
-}
-
-function isReplayConversationSelfContained(session) {
-  const messages = session?.userMessages || [];
-  if (!messages.length) return false;
-  if (!isStandaloneCodingPrompt(messages[0])) return false;
-  return messages.every((message, index) => isReplayableCodingTurn(message, index));
-}
-
-function isReplayableCodingTurn(message, index = 0) {
-  const text = String(message || '').trim();
-  if (!text) return false;
-  const lower = text.toLowerCase();
-
-  if (/\[note:\s*user originally attached[\s\S]*not visible in this replay/i.test(text)) return false;
-  if (/\[image:/i.test(text) || /<image\b/i.test(text)) return false;
-  if (/^continue from where you left off\b/i.test(text)) return false;
-  if (/^\s*(go ahead|yes|yep|ok|okay|check again|have you committed|everything in main)\b/i.test(lower)) return false;
-  if (/\b(i have ran|i ran it|check the log|pending fixes|auto approver missed|look at recent changes)\b/i.test(lower)) return false;
-
-  // Follow-ups are replayable only when they carry fresh standalone coding
-  // instructions; otherwise they depend on invisible prior UI/session state.
-  if (index > 0 && !isStandaloneCodingPrompt(text)) return false;
-  return true;
-}
-
-function hasOnlyRepoRelativeEdits(files = []) {
-  return (files || []).every((file) => {
-    if (!file || typeof file !== 'string') return false;
-    if (path.isAbsolute(file)) return false;
-    return !file.split(/[\\/]+/).includes('..');
-  });
-}
-
-function preflightCatalogEntry(entry) {
-  const issues = [];
-  const rr = entry?.realReplay || null;
-  const prompt = entry?.prompt || '';
-
-  if (!rr) issues.push('missing_replay');
-  if (!prompt.trim()) issues.push('no_prompt');
-  if (rr && !(rr.userMessages || []).length) issues.push('no_user_messages');
-  if (rr && !(rr.referenceFilesEdited || []).length) issues.push('no_reference_edits');
-  if (rr && !(rr.referenceToolCalls || []).length) issues.push('no_reference_tools');
-
-  let repoRoot = null;
-  if (rr) {
-    const userMessages = (rr.userMessages || []).length ? rr.userMessages : [prompt];
-    if (!isReplayConversationSelfContained({ userMessages })) issues.push('non_replayable_conversation');
-
-    const expectedFiles = rr.referenceFilesEdited || entry.agentExpectations?.expectedFileChanges || [];
-    if (!hasOnlyRepoRelativeEdits(expectedFiles)) issues.push('external_reference_edits');
-
-    if (!rr.cwd || !fs.existsSync(rr.cwd)) {
-      issues.push('missing_repo');
-    } else {
-      repoRoot = gitRoot(rr.cwd);
-      if (!repoRoot) issues.push('missing_git_repo');
-    }
-
-    if (!rr.recoveredCommit) {
-      issues.push('missing_commit');
-    } else if (repoRoot && !gitHasCommit(repoRoot, rr.recoveredCommit)) {
-      issues.push('missing_commit');
-    }
-  }
-
-  const status = issues.length ? issues[0] : 'replayable_git';
-  return {
-    ok: issues.length === 0,
-    status,
-    issues: [...new Set(issues)],
-    repoRoot,
-    recoveredCommit: rr?.recoveredCommit || null,
-  };
-}
-
-async function runReplayEntry(entry, {
-  runAgentLoop,
-  provider = null,
-  model = null,
-  dryRun = false,
-  record = false,
-  resultsDir = DEFAULT_RESULTS_DIR,
-  timeoutMs = 600_000,
-  keepFailures = false,
-} = {}) {
-  const startedAt = new Date().toISOString();
-  const startMs = Date.now();
-  const preflight = preflightCatalogEntry(entry);
-  const base = {
-    id: entry?.id || 'unknown',
-    prompt: entry?.prompt || '',
-    dryRun,
-    preflight,
-    startedAt,
-    provider: provider?.type || provider || null,
-    model: model || null,
-  };
-
-  if (!preflight.ok) {
-    const result = {
-      ...base,
-      success: false,
-      failureType: classifyFailure({ preflight }),
-      latencyMs: Date.now() - startMs,
-    };
-    if (record) result.artifactPath = writeResultArtifact(result, { resultsDir });
-    return result;
-  }
-
-  const rr = entry.realReplay;
-  const session = catalogEntryToReplaySession(entry);
-  let sandbox = null;
-  let keepSandbox = false;
-  try {
-    sandbox = makeReplaySandbox(rr.cwd, rr.recoveredCommit, safeId(`${rr.source}-${rr.sourceSessionId}`));
-
-    if (dryRun) {
-      const result = {
-        ...base,
-        success: true,
-        status: 'dry_run_ok',
-        sandboxCreated: sandbox,
-        latencyMs: Date.now() - startMs,
-      };
-      if (record) result.artifactPath = writeResultArtifact(result, { resultsDir });
-      return result;
-    }
-
-    if (!runAgentLoop) throw new Error('runAgentLoop function is required for non-dry-run replay');
-    const replay = await replayAndScore(session, sandbox, runAgentLoop, {
-      provider,
-      model,
-      timeoutMs,
-      useLlmJudge: false,
-    });
-    const failureType = classifyFailure({
-      ...replay,
-      expectedFiles: session.claudeFilesEdited,
-    });
-    const success = !replay.error && replay.scores?.composite >= 0.35 && failureType === null;
-    const result = {
-      ...base,
-      success,
-      failureType,
-      latencyMs: Date.now() - startMs,
-      replay,
-    };
-    if (!success) keepSandbox = !!keepFailures;
-    if (record) result.artifactPath = writeResultArtifact(result, { resultsDir });
-    return result;
-  } catch (err) {
-    keepSandbox = !!keepFailures;
-    const result = {
-      ...base,
-      success: false,
-      failureType: classifyFailure({ error: err.message, preflight }),
-      error: err.message,
-      latencyMs: Date.now() - startMs,
-    };
-    if (record) result.artifactPath = writeResultArtifact(result, { resultsDir });
-    return result;
-  } finally {
-    if (sandbox && !keepSandbox) {
-      try { removeReplaySandbox(sandbox, rr.cwd); } catch { /* best-effort */ }
-    }
-  }
-}
-
-function defaultReplayTimeoutMs(entry = {}) {
-  const expectedFiles = entry.agentExpectations?.expectedFileChanges || entry.realReplay?.referenceFilesEdited || [];
-  const expectedTools = entry.agentExpectations?.expectedToolCalls || entry.realReplay?.referenceToolCalls || [];
-  if (entry.difficulty === 'hard' || expectedFiles.length > 4 || expectedTools.length > 30) return 1_200_000;
-  if (entry.difficulty === 'medium' || expectedFiles.length > 1 || expectedTools.length > 10) return 900_000;
-  return 600_000;
-}
-
-function catalogEntryToReplaySession(entry) {
-  const rr = entry.realReplay;
-  return {
-    sessionId: rr.sourceSessionId,
-    cwd: rr.cwd,
-    gitBranch: rr.gitBranch,
-    userMessages: rr.userMessages?.length ? rr.userMessages : [entry.prompt],
-    tsStart: rr.tsStart,
-    tsEnd: rr.tsEnd,
-    turnCount: rr.turnCount || rr.userMessages?.length || 1,
-    claudeToolCalls: rr.referenceToolCalls || entry.agentExpectations?.expectedToolCalls || [],
-    claudeFilesEdited: rr.referenceFilesEdited || entry.agentExpectations?.expectedFileChanges || [],
-  };
-}
-
-function classifyFailure(resultLike = {}) {
-  const preflight = resultLike.preflight || null;
-  if (preflight && !preflight.ok) {
-    if (preflight.issues?.includes('non_replayable_conversation') || preflight.issues?.includes('external_reference_edits')) return 'bad_replay_entry';
-    if (preflight.issues?.includes('missing_repo') || preflight.issues?.includes('missing_git_repo')) return 'path_blocked';
-    if (preflight.issues?.includes('missing_commit')) return 'missing_commit';
-    if (preflight.issues?.includes('no_prompt') || preflight.issues?.includes('no_user_messages')) return 'planning_failed';
-    if (preflight.issues?.includes('no_reference_edits')) return 'no_reference_edits';
-    return 'preflight_failed';
-  }
-
-  const text = [
-    resultLike.error,
-    resultLike.message,
-    resultLike.stderr,
-    resultLike.replay?.error,
-    resultLike.agentResult?.stderr,
-    resultLike.agentResult?.output,
-    resultLike.replay?.agentResult?.stderr,
-    resultLike.replay?.agentResult?.output,
-  ].filter(Boolean).join('\n').toLowerCase();
-
-  if (/timeout|timed out|hard timeout|aborted|aborterror/.test(text)) return 'timeout';
-  if (/api key|unauthorized|forbidden|401|403|invalid auth|authentication/.test(text)) return 'provider_auth';
-  if (/quota|rate limit|429|insufficient credits|billing/.test(text)) return 'provider_quota';
-  if (/econn|enotfound|network|socket hang up|fetch failed|dns/.test(text)) return 'provider_network';
-  if (/permission denied|operation not permitted|sandbox|worktree add|outside project|path.*blocked|not allowed/.test(text)) return 'path_blocked';
-  if (/git add -a|git add --all|unsafe commit|git commit/.test(text)) return 'unsafe_commit';
-  if (/test failed|tests failed|assertionerror|npm test.*failed/.test(text)) return 'tests_failed';
-
-  const tools = resultLike.walleTools || resultLike.actualToolCalls || resultLike.replay?.walleTools || [];
-  const files = resultLike.walleFiles || resultLike.actualFileChanges || resultLike.replay?.walleFiles || [];
-  const expectedFiles = resultLike.expectedFiles || resultLike.claudeFiles || resultLike.replay?.claudeFiles || [];
-  const score = resultLike.scores || resultLike.score || resultLike.replay?.scores || {};
-
-  if (Array.isArray(tools) && tools.length === 0) return 'no_tool_calls';
-  if (Array.isArray(files) && Array.isArray(expectedFiles) && expectedFiles.length > 0 && files.length === 0) return 'no_file_changes';
-  if (expectedFiles.length > 0 && score.file_jaccard === 0) return 'wrong_files';
-  if (expectedFiles.length > 0 && typeof score.file_jaccard === 'number' && score.file_jaccard < 0.5) return 'partial_file_coverage';
-  if (typeof score.composite === 'number' && score.composite < 0.35) return 'low_score';
-
-  if (resultLike.success === false) return 'unknown';
-  return null;
-}
-
-function summarizeResults(results) {
-  const total = results.length;
-  const passed = results.filter((r) => r.success).length;
-  const failureCounts = {};
-  let compositeTotal = 0;
-  let compositeCount = 0;
-
-  for (const result of results) {
-    const score = result.replay?.scores?.composite;
-    if (typeof score === 'number') {
-      compositeTotal += score;
-      compositeCount += 1;
-    }
-    if (!result.success) {
-      const key = result.failureType || 'unknown';
-      failureCounts[key] = (failureCounts[key] || 0) + 1;
-    }
-  }
-
-  return {
-    total,
-    passed,
-    failed: total - passed,
-    passRate: total ? passed / total : 0,
-    avgComposite: compositeCount ? compositeTotal / compositeCount : null,
-    failureCounts,
-  };
-}
-
-function writeResultArtifact(result, { resultsDir = DEFAULT_RESULTS_DIR } = {}) {
-  fs.mkdirSync(resultsDir, { recursive: true });
-  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
-  const file = path.join(resultsDir, `${stamp}-${safeId(result.id || 'result')}.json`);
-  fs.writeFileSync(file, JSON.stringify(result, null, 2) + '\n');
-  return file;
-}
-
-function loadCatalog(catalogPath = DEFAULT_REAL_CATALOG_PATH) {
-  if (!catalogPath || !fs.existsSync(catalogPath)) return [];
-  try {
-    const parsed = JSON.parse(fs.readFileSync(catalogPath, 'utf8'));
-    return Array.isArray(parsed) ? parsed : [];
-  } catch {
-    return [];
-  }
-}
-
-function loadProviderKey(providerType) {
-  const envByProvider = {
-    anthropic: ['ANTHROPIC_API_KEY', 'ANTHROPIC_AUTH_TOKEN'],
-    openai: ['OPENAI_API_KEY'],
-    google: ['GOOGLE_API_KEY', 'GEMINI_API_KEY'],
-    deepseek: ['DEEPSEEK_API_KEY'],
-    moonshot: ['MOONSHOT_API_KEY'],
-  };
-  for (const key of envByProvider[providerType] || []) {
-    if (process.env[key]) return { key: process.env[key], source: key };
-  }
-
-  try {
-    const brain = require('../brain');
-    const row = brain.getDb().prepare(
-      'SELECT api_key_encrypted FROM model_providers WHERE type = ? AND enabled = 1 AND api_key_encrypted IS NOT NULL ORDER BY updated_at DESC LIMIT 1'
-    ).get(providerType);
-    if (row?.api_key_encrypted) return { key: row.api_key_encrypted, source: 'brain_db' };
-  } catch {}
-
-  return { key: null, source: null };
-}
-
-function preflightProvider(providerType, { dryRun = false } = {}) {
-  if (!providerType) return { ok: true, status: dryRun ? 'default_provider_deferred' : 'default_provider' };
-  if (dryRun) return { ok: true, status: 'provider_deferred_dry_run', providerType };
-  if (providerType === 'ollama' || providerType === 'mlx') return { ok: true, status: 'local_provider', providerType };
-  if (providerType === 'claude-cli') return commandPreflight('claude', providerType);
-  if (providerType === 'codex-cli') return commandPreflight('codex', providerType);
-
-  const { key, source } = loadProviderKey(providerType);
-  if (!key) return { ok: false, status: 'missing_provider_key', providerType };
-  return { ok: true, status: 'provider_key_found', providerType, keySource: source };
-}
-
-function commandPreflight(command, providerType) {
-  try {
-    execFileSync('which', [command], { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 5000 });
-    return { ok: true, status: 'provider_cli_found', providerType, command };
-  } catch {
-    return { ok: false, status: 'missing_provider_cli', providerType, command };
-  }
-}
-
-function createProviderClient(providerType) {
-  if (!providerType) return null;
-  const { createClient } = require('../llm/client');
-  if (providerType === 'ollama' || providerType === 'mlx' || providerType === 'claude-cli' || providerType === 'codex-cli') {
-    return createClient(providerType, {});
-  }
-  const { key } = loadProviderKey(providerType);
-  const config = {};
-  if (key) config.apiKey = key;
-  if (providerType === 'openai' && process.env.OPENAI_BASE_URL) config.baseUrl = process.env.OPENAI_BASE_URL;
-  if (providerType === 'deepseek' && process.env.DEEPSEEK_BASE_URL) config.baseUrl = process.env.DEEPSEEK_BASE_URL;
-  if (providerType === 'moonshot' && process.env.MOONSHOT_BASE_URL) config.baseUrl = process.env.MOONSHOT_BASE_URL;
-  if (providerType === 'google' && process.env.GOOGLE_AUTH_MODE === 'oauth') {
-    config.authMode = 'oauth';
-    config.refreshToken = process.env.GOOGLE_REFRESH_TOKEN;
-  }
-  return createClient(providerType, config);
-}
-
-function classifyDifficulty({ turnCount, files, tools }) {
-  if (turnCount > 8 || files > 4 || tools > 30) return 'hard';
-  if (turnCount > 3 || files > 1 || tools > 10) return 'medium';
-  return 'easy';
-}
-
-function expectedTraitsFor(tools, files) {
-  const traits = ['real transcript'];
-  if (tools.some((t) => /read|grep|glob|list|search/i.test(t))) traits.push('reads before writing');
-  if (tools.some((t) => /bash|exec|shell|command/i.test(t))) traits.push('runs commands');
-  if (files.length > 0) traits.push('edits files');
-  if (files.length > 1) traits.push('multi-file change');
-  return traits;
-}
-
-function sourceKeyForEntry(entry) {
-  const rr = entry.realReplay;
-  return rr?.source && rr?.sourceSessionId ? `${rr.source}:${rr.sourceSessionId}` : null;
-}
-
-function gitRoot(cwd) {
-  try {
-    return execFileSync('git', ['-C', cwd, 'rev-parse', '--show-toplevel'], {
-      encoding: 'utf8',
-      stdio: ['pipe', 'pipe', 'pipe'],
-      timeout: 10000,
-    }).trim();
-  } catch {
-    return null;
-  }
-}
-
-function gitHasCommit(repoRoot, commit) {
-  try {
-    execFileSync('git', ['-C', repoRoot, 'cat-file', '-e', `${commit}^{commit}`], {
-      stdio: ['pipe', 'pipe', 'pipe'],
-      timeout: 10000,
-    });
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-function safeId(value) {
-  return String(value || 'unknown').replace(/[^a-z0-9_.-]+/gi, '-').slice(0, 80);
-}
-
-module.exports = {
-  DEFAULT_REAL_CATALOG_PATH,
-  DEFAULT_RESULTS_DIR,
-  buildReplayCatalog,
-  sessionToRealCatalogEntry,
-  preflightCatalogEntry,
-  runReplayEntry,
-  catalogEntryToReplaySession,
-  defaultReplayTimeoutMs,
-  classifyFailure,
-  summarizeResults,
-  writeResultArtifact,
-  loadCatalog,
-  preflightProvider,
-  createProviderClient,
-  isStandaloneCodingPrompt,
-  isReplayConversationSelfContained,
-  hasOnlyRepoRelativeEdits,
-};