create-walle 0.9.21 → 0.9.23

package/template/wall-e/eval/codex-cli-baseline.js

@@ -1,669 +0,0 @@
-'use strict';
-
-const crypto = require('crypto');
-const fs = require('fs');
-const os = require('os');
-const path = require('path');
-const { execFile, execFileSync, spawn } = require('child_process');
-const { promisify } = require('util');
-
-const execFileAsync = promisify(execFile);
-
-const {
-  cleanupSandbox,
-  countTests,
-  scoreAgentResult,
-  setupSandbox,
-} = require('./agent-runner');
-const { decorateBenchmarkResult, DEFAULT_SCORER_VERSION } = require('./manifest');
-const { testCommandAllowed } = require('./allowed-test-commands');
-
-const DEFAULT_RESULTS_DIR = path.join(os.homedir(), '.walle', 'eval-results', 'codex-cli-baseline');
-const DEFAULT_TIMEOUT_MS = 600_000;
-
-function buildCodexPrompt(benchmark = {}, opts = {}) {
-  const expectations = benchmark.agentExpectations || {};
-  const parts = [
-    benchmark.prompt || '',
-    '',
-    'You are running as an external Codex CLI baseline for a Wall-E coding-agent benchmark.',
-    'Work only in this repository. Do not commit changes. Make the requested code changes directly.',
-    'Use a tight OpenCode-style loop: inspect the smallest relevant files, edit directly, run targeted validation, and stop.',
-  ];
-  if (opts.retryReason) {
-    parts.push(`Previous baseline attempt failed due to ${opts.retryReason}. Be more direct and avoid broad exploration.`);
-  }
-  if (expectations.testCommand) {
-    parts.push(`After editing, run this validation command if practical: ${expectations.testCommand}`);
-  }
-  if (Array.isArray(expectations.expectedFileChanges) && expectations.expectedFileChanges.length > 0) {
-    parts.push(`Expected changed file(s), when appropriate: ${expectations.expectedFileChanges.join(', ')}`);
-  }
-  return parts.filter(Boolean).join('\n');
-}
-
-function buildCodexExecArgs({
-  prompt,
-  cwd,
-  model,
-  outFile,
-  fullAuto = true,
-  sandbox = null,
-  json = true,
-  dangerouslyBypassSandbox = false,
-  configOverrides = [],
-} = {}) {
-  if (!cwd) throw new Error('cwd is required');
-  if (!outFile) throw new Error('outFile is required');
-  const args = [
-    'exec',
-    '--skip-git-repo-check',
-    '--ephemeral',
-    '--output-last-message', outFile,
-    '-C', cwd,
-  ];
-  if (model) args.push('-m', model);
-  if (json) args.push('--json');
-  for (const override of configOverrides || []) {
-    if (override) args.push('-c', override);
-  }
-  if (dangerouslyBypassSandbox) {
-    args.push('--dangerously-bypass-approvals-and-sandbox');
-  } else if (fullAuto) {
-    args.push('--full-auto');
-  } else if (sandbox) {
-    args.push('--sandbox', sandbox);
-  }
-  args.push(prompt || '');
-  return args;
-}
-
-function codexEnv({ useEnvOpenAIKey = false, env = process.env } = {}) {
-  const childEnv = { ...env };
-  if (!useEnvOpenAIKey) delete childEnv.OPENAI_API_KEY;
-  if (!childEnv.OTEL_SDK_DISABLED) childEnv.OTEL_SDK_DISABLED = 'true';
-  return childEnv;
-}
-
-async function runCodexExec({
-  prompt,
-  cwd,
-  model,
-  timeoutMs = DEFAULT_TIMEOUT_MS,
-  useEnvOpenAIKey = false,
-  fullAuto = true,
-  sandbox = null,
-  dangerouslyBypassSandbox = false,
-  json = true,
-  allowMcp = false,
-  disableMcpServers = null,
-  configOverrides = [],
-} = {}) {
-  const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'walle-codex-baseline-'));
-  const outFile = path.join(tmpDir, 'last-message.txt');
-  const effectiveConfigOverrides = [
-    ...configOverrides,
-    ...buildMcpDisableConfigOverrides({ allowMcp, disableMcpServers }),
-  ];
-  const args = buildCodexExecArgs({
-    prompt,
-    cwd,
-    model,
-    outFile,
-    fullAuto,
-    sandbox,
-    json,
-    dangerouslyBypassSandbox,
-    configOverrides: effectiveConfigOverrides,
-  });
-  const started = Date.now();
-
-  try {
-    const result = await spawnCodex(args, {
-      cwd,
-      env: codexEnv({ useEnvOpenAIKey }),
-      timeoutMs,
-    });
-    let output = '';
-    try { output = fs.readFileSync(outFile, 'utf8'); } catch {}
-    const telemetry = json ? parseCodexJsonEvents(result.stdout) : parseCodexFormattedOutput(result.stderr || result.stdout);
-    return {
-      ...result,
-      args,
-      ...telemetry,
-      output: output || result.stdout || '',
-      latencyMs: Date.now() - started,
-    };
-  } finally {
-    fs.rmSync(tmpDir, { recursive: true, force: true });
-  }
-}
-
-function spawnCodex(args, { cwd, env, timeoutMs } = {}) {
-  return new Promise((resolve, reject) => {
-    let proc;
-    try {
-      proc = spawn('codex', args, {
-        cwd,
-        env,
-        stdio: ['ignore', 'pipe', 'pipe'],
-      });
-    } catch (err) {
-      reject(err);
-      return;
-    }
-
-    let stdout = '';
-    let stderr = '';
-    let timedOut = false;
-    let killTimer = null;
-    const timer = setTimeout(() => {
-      timedOut = true;
-      try { proc.kill('SIGTERM'); } catch {}
-      killTimer = setTimeout(() => {
-        try { proc.kill('SIGKILL'); } catch {}
-      }, 10_000);
-      if (typeof killTimer.unref === 'function') killTimer.unref();
-    }, timeoutMs || DEFAULT_TIMEOUT_MS);
-    if (typeof timer.unref === 'function') timer.unref();
-
-    proc.stdout.on('data', (chunk) => { stdout += chunk.toString(); });
-    proc.stderr.on('data', (chunk) => { stderr += chunk.toString(); });
-    proc.on('error', (err) => {
-      clearTimeout(timer);
-      if (killTimer) clearTimeout(killTimer);
-      reject(err);
-    });
-    proc.on('close', (code, signal) => {
-      clearTimeout(timer);
-      if (killTimer) clearTimeout(killTimer);
-      resolve({
-        code,
-        signal,
-        timedOut,
-        stdout,
-        stderr,
-      });
-    });
-  });
-}
-
-async function getModifiedFiles(dir) {
-  try {
-    const { stdout } = await execFileAsync('git', ['diff', '--name-only', 'HEAD'], { cwd: dir });
-    const untracked = (await execFileAsync('git', ['ls-files', '--others', '--exclude-standard'], { cwd: dir })).stdout;
-    return [...stdout.trim().split('\n'), ...untracked.trim().split('\n')].filter(Boolean);
-  } catch {
-    return [];
-  }
-}
-
-function buildMcpDisableConfigOverrides({ allowMcp = false, disableMcpServers = null } = {}) {
-  if (allowMcp) return [];
-  const servers = disableMcpServers || discoverEnabledCodexMcpServers();
-  return [...new Set(servers)]
-    .filter(Boolean)
-    .map((name) => `mcp_servers.${tomlPathSegment(name)}.enabled=false`);
-}
-
-function discoverEnabledCodexMcpServers() {
-  try {
-    const stdout = execFileSync('codex', ['mcp', 'list'], {
-      encoding: 'utf8',
-      stdio: ['ignore', 'pipe', 'ignore'],
-      timeout: 10_000,
-    });
-    return parseEnabledCodexMcpServers(stdout);
-  } catch {
-    return [];
-  }
-}
-
-function parseEnabledCodexMcpServers(output = '') {
-  const names = [];
-  for (const line of String(output || '').split(/\r?\n/)) {
-    const trimmed = line.trim();
-    if (!trimmed || /^Name\b/.test(trimmed) || /^WARNING\b/.test(trimmed)) continue;
-    if (!/\benabled\b/.test(trimmed)) continue;
-    const [name] = trimmed.split(/\s+/);
-    if (name) names.push(name);
-  }
-  return names;
-}
-
-function tomlPathSegment(value) {
-  const raw = String(value || '');
-  return /^[A-Za-z_][A-Za-z0-9_]*$/.test(raw) ? raw : JSON.stringify(raw);
-}
-
-function parseCodexJsonEvents(stdout = '') {
-  const events = [];
-  const toolCalls = [];
-  const toolCallDetails = [];
-  const errors = [];
-  const usage = {
-    inputTokens: 0,
-    cachedInputTokens: 0,
-    outputTokens: 0,
-  };
-  let turns = 0;
-
-  for (const line of String(stdout || '').split(/\r?\n/)) {
-    const trimmed = line.trim();
-    if (!trimmed) continue;
-    let event;
-    try {
-      event = JSON.parse(trimmed);
-    } catch {
-      continue;
-    }
-    events.push(event);
-
-    if (event.type === 'turn.completed') {
-      turns += 1;
-      const eventUsage = event.usage || event.turn?.usage || {};
-      usage.inputTokens += Number(eventUsage.input_tokens ?? eventUsage.inputTokens ?? 0);
-      usage.cachedInputTokens += Number(eventUsage.cached_input_tokens ?? eventUsage.cachedInputTokens ?? 0);
-      usage.outputTokens += Number(eventUsage.output_tokens ?? eventUsage.outputTokens ?? 0);
-    }
-
-    if (event.type === 'error') {
-      errors.push(event.message || event.error?.message || event.error || 'codex json error');
-    }
-
-    if (event.type !== 'item.completed' || !event.item) continue;
-    const detail = classifyCodexItem(event.item);
-    if (!detail) continue;
-    toolCalls.push(detail.name);
-    toolCallDetails.push(detail);
-  }
-
-  return {
-    codexEvents: events,
-    codexErrors: errors,
-    toolCalls,
-    toolCallDetails,
-    turns,
-    usage: {
-      inputTokens: usage.inputTokens || null,
-      cachedInputTokens: usage.cachedInputTokens || null,
-      outputTokens: usage.outputTokens || null,
-    },
-  };
-}
-
-function parseCodexFormattedOutput(output = '') {
-  const toolCalls = [];
-  const toolCallDetails = [];
-  for (const line of String(output || '').split(/\r?\n/)) {
-    const commandMatch = line.match(/^\s*(?:exec|shell)\s+(.+)$/i);
-    if (!commandMatch) continue;
-    const detail = classifyCodexCommand(commandMatch[1]);
-    toolCalls.push(detail.name);
-    toolCallDetails.push(detail);
-  }
-  return {
-    toolCalls,
-    toolCallDetails,
-    turns: toolCalls.length ? 1 : 0,
-    usage: {
-      inputTokens: null,
-      cachedInputTokens: null,
-      outputTokens: parseCodexTokenCount(output),
-    },
-  };
-}
-
-function classifyCodexItem(item = {}) {
-  const itemType = String(item.type || item.item_type || '').toLowerCase();
-  if (itemType.includes('command') || itemType === 'exec' || itemType === 'shell') {
-    return classifyCodexCommand(item.command || item.cmd || item.text || item.arguments?.cmd || '', item);
-  }
-  if (itemType.includes('file')) {
-    return {
-      name: 'edit_file',
-      source: 'codex-cli',
-      codexType: item.type || null,
-      path: item.path || item.file || item.file_path || null,
-      status: item.status || null,
-    };
-  }
-  if (itemType.includes('mcp')) {
-    return {
-      name: 'mcp_tool_call',
-      source: 'codex-cli',
-      codexType: item.type || null,
-      tool: item.name || item.tool || null,
-      status: item.status || null,
-    };
-  }
-  if (itemType.includes('web_search') || itemType.includes('web-search')) {
-    return {
-      name: 'web_search',
-      source: 'codex-cli',
-      codexType: item.type || null,
-      status: item.status || null,
-    };
-  }
-  return null;
-}
-
-function classifyCodexCommand(command = '', item = {}) {
-  const raw = String(command || '');
-  const lowered = raw.toLowerCase();
-  let name = 'run_shell';
-  const writesViaRedirection = /\b(tee|cat|printf|echo)\b[\s\S]*[>]/.test(lowered);
-  const patchLike = /\b(apply_patch|git\s+apply|perl\s+-0?pi|python[^\n]*write_text|writefilesync)\b/.test(lowered);
-  if (/\b(npm\s+test|npm\s+run\s+(?:test|typecheck|lint)|pnpm\s+test|yarn\s+test|bun\s+test|node\s+test\.js|node\s+--test|pytest|python\s+-m\s+pytest|make\s+test|(?:npx\s+)?tsc\s+--noemit|go\s+test|cargo\s+test)\b/.test(lowered)) {
-    name = 'run_shell';
-  } else if (patchLike || writesViaRedirection) {
-    name = 'edit_file';
-  } else if (/\b(rg|grep)\b/.test(lowered)) {
-    name = 'grep_files';
-  } else if (/\b(find|ls|tree)\b/.test(lowered)) {
-    name = 'list_directory';
-  } else if (/\b(sed\s+-n|cat|head|tail|awk|nl)\b/.test(lowered)) {
-    name = 'read_file';
-  }
-  return {
-    name,
-    source: 'codex-cli',
-    codexType: item.type || 'command_execution',
-    command: raw,
-    status: item.status || null,
-  };
-}
-
-async function runCodexCliBaselineBenchmark(benchmark, options = {}) {
-  const expectations = benchmark.agentExpectations || {};
-  const fixtureName = expectations.projectFixture || 'express-basic';
-  const timeoutMs = options.timeoutMs || DEFAULT_TIMEOUT_MS;
-  let sandboxDir = null;
-  let keepSandbox = false;
-  const started = Date.now();
-
-  try {
-    sandboxDir = setupSandbox(fixtureName);
-    let testsBefore = null;
-    let totalTests = null;
-    if (testCommandAllowed(expectations.testCommand)) {
-      const beforeCounts = countTests(sandboxDir, expectations.testCommand);
-      testsBefore = beforeCounts.passed;
-      totalTests = beforeCounts.total;
-    }
-
-    if (options.dryRun) {
-      return {
-        benchmarkId: benchmark.id,
-        runner: 'codex-cli-baseline',
-        provider: 'codex-cli-baseline',
-        model: options.model || null,
-        success: true,
-        status: 'dry_run_ok',
-        sandboxCreated: sandboxDir,
-        latencyMs: Date.now() - started,
-      };
-    }
-
-    const runExec = options.runCodexExec || runCodexExec;
-    const maxAttempts = options.retryTimeouts === false
-      ? 1
-      : Math.max(1, options.maxAttempts || 2);
-    let attempts = 0;
-    let retryReason = '';
-    let codex = null;
-    for (attempts = 1; attempts <= maxAttempts; attempts++) {
-      const prompt = buildCodexPrompt(benchmark, { retryReason });
-      const attemptTimeoutMs = attempts === 1
-        ? timeoutMs
-        : Math.ceil(timeoutMs * (options.retryTimeoutMultiplier || 1.5));
-      codex = await runExec({
-        prompt,
-        cwd: sandboxDir,
-        model: options.model,
-        timeoutMs: attemptTimeoutMs,
-        useEnvOpenAIKey: !!options.useEnvOpenAIKey,
-        fullAuto: options.fullAuto !== false,
-        sandbox: options.sandbox || null,
-        dangerouslyBypassSandbox: !!options.dangerouslyBypassSandbox,
-        json: options.json !== false,
-        allowMcp: !!options.allowMcp,
-        disableMcpServers: options.disableMcpServers || null,
-        configOverrides: options.configOverrides || [],
-      });
-      if (!codex.timedOut || attempts >= maxAttempts) break;
-      retryReason = 'a Codex CLI timeout';
-      resetSandboxToHead(sandboxDir);
-    }
-
-    const actualFileChanges = await getModifiedFiles(sandboxDir);
-    let testsPassed = null;
-    let testsAfter = null;
-    if (testCommandAllowed(expectations.testCommand)) {
-      try {
-        execFileSync('sh', ['-c', expectations.testCommand], {
-          cwd: sandboxDir,
-          timeout: options.testTimeoutMs || 30_000,
-          stdio: 'pipe',
-        });
-        testsPassed = true;
-      } catch {
-        testsPassed = false;
-      }
-      const afterCounts = countTests(sandboxDir, expectations.testCommand);
-      testsAfter = afterCounts.passed;
-      if (totalTests == null) totalTests = afterCounts.total;
-    }
-
-    const expectedFileChanges = expectations.expectedFileChanges || [];
-    const missingExpectedWork = expectedFileChanges.length > 0 && actualFileChanges.length === 0;
-    const testRegression = !!(expectations.testCommand && testsPassed === false);
-    const codexErrors = Array.isArray(codex.codexErrors) && codex.codexErrors.length
-      ? codex.codexErrors.join('; ')
-      : null;
-    const exitError = codex.timedOut
-      ? 'codex exec timed out'
-      : codex.code === 0
-        ? codexErrors
-        : `codex exec exited ${codex.code}${codex.signal ? ` (${codex.signal})` : ''}: ${(codex.stderr || codex.stdout || '').slice(-1000)}`;
-    const success = !exitError && !missingExpectedWork && !testRegression;
-    const actualToolCalls = codex.toolCalls || [];
-    const toolCallDetails = codex.toolCallDetails || [];
-    const usage = codex.usage || {};
-
-    let score = scoreAgentResult(benchmark, {
-      actualToolCalls,
-      actualFileChanges,
-      actualTurns: codex.turns || 1,
-      testsPassed,
-      output: codex.output || codex.stdout || '',
-      success,
-      sandboxDir,
-      costDollars: null,
-      testsBefore,
-      testsAfter,
-      totalTests,
-      toolCallDetails,
-    });
-
-    if (!success) {
-      score = {
-        composite: 0,
-        dimensions: {
-          ...(score.dimensions || {}),
-          _zeroed: true,
-          _zeroReason: exitError
-            ? 'codex_error'
-            : testRegression
-              ? 'tests_failed'
-              : 'no_file_changes',
-        },
-      };
-    }
-
-    keepSandbox = !!options.keepFailures && !success;
-    return {
-      benchmarkId: benchmark.id,
-      runner: 'codex-cli-baseline',
-      provider: 'codex-cli-baseline',
-      model: options.model || null,
-      success,
-      score,
-      latencyMs: Date.now() - started,
-      attempts,
-      actualToolCalls,
-      actualFileChanges,
-      actualTurns: codex.turns || 1,
-      testsPassed,
-      testsBefore,
-      testsAfter,
-      totalTests,
-      inputTokens: usage.inputTokens ?? null,
-      outputTokens: usage.outputTokens ?? parseCodexTokenCount(codex.stdout),
-      dimensionsJson: JSON.stringify(score.dimensions || {}),
-      output: (codex.output || codex.stdout || '').slice(0, 2000),
-      stderr: filterNonFatalCodexStderr(codex.stderr || '').slice(0, 2000),
-      error: exitError,
-      sandboxDir: keepSandbox ? sandboxDir : null,
-    };
-  } catch (err) {
-    keepSandbox = !!options.keepFailures;
-    return {
-      benchmarkId: benchmark.id,
-      runner: 'codex-cli-baseline',
-      provider: 'codex-cli-baseline',
-      model: options.model || null,
-      success: false,
-      score: { composite: 0, dimensions: { _zeroed: true, _zeroReason: 'exception' } },
-      latencyMs: Date.now() - started,
-      error: err.message,
-      sandboxDir: keepSandbox ? sandboxDir : null,
-    };
-  } finally {
-    if (sandboxDir && !keepSandbox) cleanupSandbox(sandboxDir);
-  }
-}
-
-function resetSandboxToHead(dir) {
-  if (!dir) return false;
-  try {
-    execFileSync('git', ['reset', '--hard', 'HEAD'], { cwd: dir, stdio: 'pipe' });
-    execFileSync('git', ['clean', '-fd'], { cwd: dir, stdio: 'pipe' });
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-function parseCodexTokenCount(stdout = '') {
-  const match = String(stdout || '').match(/tokens used\s*\n\s*([\d,]+)/i);
-  return match ? parseInt(match[1].replace(/,/g, ''), 10) : null;
-}
-
-function filterNonFatalCodexStderr(stderr = '') {
-  return String(stderr || '')
-    .split(/\r?\n/)
-    .filter((line) => {
-      if (/Reading additional input from stdin/.test(line)) return false;
-      if (/opentelemetry_sdk/.test(line)) return false;
-      if (/BatchSpanProcessor/.test(line)) return false;
-      return true;
-    })
-    .join('\n')
-    .trim();
-}
-
-function summarizeBaselineResults(results = []) {
-  const total = results.length;
-  const passed = results.filter((r) => r.success).length;
-  const avgComposite = total
-    ? results.reduce((sum, r) => sum + (r.score?.composite || 0), 0) / total
-    : 0;
-  const failures = {};
-  for (const result of results) {
-    if (!result.success) {
-      const reason = result.score?.dimensions?._zeroReason || result.error || 'unknown';
-      failures[reason] = (failures[reason] || 0) + 1;
-    }
-  }
-  return { total, passed, failed: total - passed, avgComposite, failures };
-}
-
-function writeBaselineArtifact(result, { resultsDir = DEFAULT_RESULTS_DIR } = {}) {
-  fs.mkdirSync(resultsDir, { recursive: true });
-  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
-  const file = path.join(resultsDir, `${stamp}-${safeId(result.benchmarkId || 'codex-baseline')}.json`);
-  fs.writeFileSync(file, JSON.stringify(result, null, 2) + '\n');
-  return file;
-}
-
-function storeBaselineResult({ brain, runId, benchmark, result, model, timeoutMs }) {
-  if (!brain || typeof brain.insertBenchmarkResult !== 'function') return;
-  const scoringMethod = benchmark.agentExpectations?.testCommand
-    ? 'codex-cli-baseline+tests'
-    : 'codex-cli-baseline';
-  brain.insertBenchmarkResult(decorateBenchmarkResult({
-    runId,
-    suite: 'coding-agent',
-    promptId: benchmark.id,
-    taskType: 'coding-agent',
-    difficulty: benchmark.difficulty,
-    provider: 'codex-cli-baseline',
-    model: model || 'codex-default',
-    prompt: benchmark.prompt,
-    response: result.output || '',
-    traitScore: null,
-    matchedTraits: [],
-    compositeScore: result.score?.composite || 0,
-    latencyMs: result.latencyMs,
-    error: result.error,
-    timestamp: result.timestamp,
-    costDollars: null,
-    testsBefore: result.testsBefore ?? null,
-    testsAfter: result.testsAfter ?? null,
-    totalTests: result.totalTests ?? null,
-    dimensionsJson: result.dimensionsJson || null,
-    inputTokens: null,
-    outputTokens: result.outputTokens ?? null,
-    scorerVersion: DEFAULT_SCORER_VERSION,
-    scoringMethod,
-    trusted: result.success === true && result.testsPassed === true && !result.error,
-    runConfig: { timeoutMs, scoringMethod, externalRunner: 'codex-cli' },
-  }, {
-    suite: 'coding-agent',
-    benchmark,
-    runId,
-    provider: 'codex-cli-baseline',
-    model: model || 'codex-default',
-    scoringMethod,
-    scorerVersion: DEFAULT_SCORER_VERSION,
-    trusted: result.success === true && result.testsPassed === true && !result.error,
-    runConfig: { timeoutMs, scoringMethod, externalRunner: 'codex-cli' },
-  }));
-}
-
-function safeId(value) {
-  return String(value || 'unknown').replace(/[^a-z0-9_.-]+/gi, '-').slice(0, 80);
-}
-
-module.exports = {
-  DEFAULT_RESULTS_DIR,
-  buildCodexExecArgs,
-  buildMcpDisableConfigOverrides,
-  buildCodexPrompt,
-  codexEnv,
-  classifyCodexCommand,
-  classifyCodexItem,
-  discoverEnabledCodexMcpServers,
-  filterNonFatalCodexStderr,
-  parseEnabledCodexMcpServers,
-  parseCodexTokenCount,
-  parseCodexJsonEvents,
-  runCodexCliBaselineBenchmark,
-  runCodexExec,
-  resetSandboxToHead,
-  summarizeBaselineResults,
-  testCommandAllowed,
-  writeBaselineArtifact,
-  storeBaselineResult,
-};