create-walle 0.9.13 → 0.9.15

package/template/wall-e/eval/codex-cli-baseline.js

@@ -0,0 +1,633 @@
+'use strict';
+
+const crypto = require('crypto');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { execFile, execFileSync, spawn } = require('child_process');
+const { promisify } = require('util');
+
+const execFileAsync = promisify(execFile);
+
+const {
+  cleanupSandbox,
+  countTests,
+  scoreAgentResult,
+  setupSandbox,
+} = require('./agent-runner');
+const { decorateBenchmarkResult, DEFAULT_SCORER_VERSION } = require('./manifest');
+
+const DEFAULT_RESULTS_DIR = path.join(os.homedir(), '.walle', 'eval-results', 'codex-cli-baseline');
+const DEFAULT_TIMEOUT_MS = 600_000;
+const ALLOWED_TEST_COMMANDS = /^(npm test|node test\.js|pytest|python -m pytest|make test)$/;
+
+function buildCodexPrompt(benchmark = {}) {
+  const expectations = benchmark.agentExpectations || {};
+  const parts = [
+    benchmark.prompt || '',
+    '',
+    'You are running as an external Codex CLI baseline for a Wall-E coding-agent benchmark.',
+    'Work only in this repository. Do not commit changes. Make the requested code changes directly.',
+  ];
+  if (expectations.testCommand) {
+    parts.push(`After editing, run this validation command if practical: ${expectations.testCommand}`);
+  }
+  if (Array.isArray(expectations.expectedFileChanges) && expectations.expectedFileChanges.length > 0) {
+    parts.push(`Expected changed file(s), when appropriate: ${expectations.expectedFileChanges.join(', ')}`);
+  }
+  return parts.filter(Boolean).join('\n');
+}
+
+function buildCodexExecArgs({
+  prompt,
+  cwd,
+  model,
+  outFile,
+  fullAuto = true,
+  sandbox = null,
+  json = true,
+  dangerouslyBypassSandbox = false,
+  configOverrides = [],
+} = {}) {
+  if (!cwd) throw new Error('cwd is required');
+  if (!outFile) throw new Error('outFile is required');
+  const args = [
+    'exec',
+    '--skip-git-repo-check',
+    '--ephemeral',
+    '--output-last-message', outFile,
+    '-C', cwd,
+  ];
+  if (model) args.push('-m', model);
+  if (json) args.push('--json');
+  for (const override of configOverrides || []) {
+    if (override) args.push('-c', override);
+  }
+  if (dangerouslyBypassSandbox) {
+    args.push('--dangerously-bypass-approvals-and-sandbox');
+  } else if (fullAuto) {
+    args.push('--full-auto');
+  } else if (sandbox) {
+    args.push('--sandbox', sandbox);
+  }
+  args.push(prompt || '');
+  return args;
+}
+
+function codexEnv({ useEnvOpenAIKey = false, env = process.env } = {}) {
+  const childEnv = { ...env };
+  if (!useEnvOpenAIKey) delete childEnv.OPENAI_API_KEY;
+  if (!childEnv.OTEL_SDK_DISABLED) childEnv.OTEL_SDK_DISABLED = 'true';
+  return childEnv;
+}
+
+async function runCodexExec({
+  prompt,
+  cwd,
+  model,
+  timeoutMs = DEFAULT_TIMEOUT_MS,
+  useEnvOpenAIKey = false,
+  fullAuto = true,
+  sandbox = null,
+  dangerouslyBypassSandbox = false,
+  json = true,
+  allowMcp = false,
+  disableMcpServers = null,
+  configOverrides = [],
+} = {}) {
+  const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'walle-codex-baseline-'));
+  const outFile = path.join(tmpDir, 'last-message.txt');
+  const effectiveConfigOverrides = [
+    ...configOverrides,
+    ...buildMcpDisableConfigOverrides({ allowMcp, disableMcpServers }),
+  ];
+  const args = buildCodexExecArgs({
+    prompt,
+    cwd,
+    model,
+    outFile,
+    fullAuto,
+    sandbox,
+    json,
+    dangerouslyBypassSandbox,
+    configOverrides: effectiveConfigOverrides,
+  });
+  const started = Date.now();
+
+  try {
+    const result = await spawnCodex(args, {
+      cwd,
+      env: codexEnv({ useEnvOpenAIKey }),
+      timeoutMs,
+    });
+    let output = '';
+    try { output = fs.readFileSync(outFile, 'utf8'); } catch {}
+    const telemetry = json ? parseCodexJsonEvents(result.stdout) : parseCodexFormattedOutput(result.stderr || result.stdout);
+    return {
+      ...result,
+      args,
+      ...telemetry,
+      output: output || result.stdout || '',
+      latencyMs: Date.now() - started,
+    };
+  } finally {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  }
+}
+
+function spawnCodex(args, { cwd, env, timeoutMs } = {}) {
+  return new Promise((resolve, reject) => {
+    let proc;
+    try {
+      proc = spawn('codex', args, {
+        cwd,
+        env,
+        stdio: ['ignore', 'pipe', 'pipe'],
+      });
+    } catch (err) {
+      reject(err);
+      return;
+    }
+
+    let stdout = '';
+    let stderr = '';
+    let timedOut = false;
+    const timer = setTimeout(() => {
+      timedOut = true;
+      try { proc.kill('SIGTERM'); } catch {}
+    }, timeoutMs || DEFAULT_TIMEOUT_MS);
+    if (typeof timer.unref === 'function') timer.unref();
+
+    proc.stdout.on('data', (chunk) => { stdout += chunk.toString(); });
+    proc.stderr.on('data', (chunk) => { stderr += chunk.toString(); });
+    proc.on('error', (err) => {
+      clearTimeout(timer);
+      reject(err);
+    });
+    proc.on('close', (code, signal) => {
+      clearTimeout(timer);
+      resolve({
+        code,
+        signal,
+        timedOut,
+        stdout,
+        stderr,
+      });
+    });
+  });
+}
+
+async function getModifiedFiles(dir) {
+  try {
+    const { stdout } = await execFileAsync('git', ['diff', '--name-only', 'HEAD'], { cwd: dir });
+    const untracked = (await execFileAsync('git', ['ls-files', '--others', '--exclude-standard'], { cwd: dir })).stdout;
+    return [...stdout.trim().split('\n'), ...untracked.trim().split('\n')].filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+
+function testCommandAllowed(command) {
+  return !!command && ALLOWED_TEST_COMMANDS.test(command);
+}
+
+function buildMcpDisableConfigOverrides({ allowMcp = false, disableMcpServers = null } = {}) {
+  if (allowMcp) return [];
+  const servers = disableMcpServers || discoverEnabledCodexMcpServers();
+  return [...new Set(servers)]
+    .filter(Boolean)
+    .map((name) => `mcp_servers.${tomlPathSegment(name)}.enabled=false`);
+}
+
+function discoverEnabledCodexMcpServers() {
+  try {
+    const stdout = execFileSync('codex', ['mcp', 'list'], {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+      timeout: 10_000,
+    });
+    return parseEnabledCodexMcpServers(stdout);
+  } catch {
+    return [];
+  }
+}
+
+function parseEnabledCodexMcpServers(output = '') {
+  const names = [];
+  for (const line of String(output || '').split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (!trimmed || /^Name\b/.test(trimmed) || /^WARNING\b/.test(trimmed)) continue;
+    if (!/\benabled\b/.test(trimmed)) continue;
+    const [name] = trimmed.split(/\s+/);
+    if (name) names.push(name);
+  }
+  return names;
+}
+
+function tomlPathSegment(value) {
+  const raw = String(value || '');
+  return /^[A-Za-z_][A-Za-z0-9_]*$/.test(raw) ? raw : JSON.stringify(raw);
+}
+
+function parseCodexJsonEvents(stdout = '') {
+  const events = [];
+  const toolCalls = [];
+  const toolCallDetails = [];
+  const errors = [];
+  const usage = {
+    inputTokens: 0,
+    cachedInputTokens: 0,
+    outputTokens: 0,
+  };
+  let turns = 0;
+
+  for (const line of String(stdout || '').split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    let event;
+    try {
+      event = JSON.parse(trimmed);
+    } catch {
+      continue;
+    }
+    events.push(event);
+
+    if (event.type === 'turn.completed') {
+      turns += 1;
+      const eventUsage = event.usage || event.turn?.usage || {};
+      usage.inputTokens += Number(eventUsage.input_tokens ?? eventUsage.inputTokens ?? 0);
+      usage.cachedInputTokens += Number(eventUsage.cached_input_tokens ?? eventUsage.cachedInputTokens ?? 0);
+      usage.outputTokens += Number(eventUsage.output_tokens ?? eventUsage.outputTokens ?? 0);
+    }
+
+    if (event.type === 'error') {
+      errors.push(event.message || event.error?.message || event.error || 'codex json error');
+    }
+
+    if (event.type !== 'item.completed' || !event.item) continue;
+    const detail = classifyCodexItem(event.item);
+    if (!detail) continue;
+    toolCalls.push(detail.name);
+    toolCallDetails.push(detail);
+  }
+
+  return {
+    codexEvents: events,
+    codexErrors: errors,
+    toolCalls,
+    toolCallDetails,
+    turns,
+    usage: {
+      inputTokens: usage.inputTokens || null,
+      cachedInputTokens: usage.cachedInputTokens || null,
+      outputTokens: usage.outputTokens || null,
+    },
+  };
+}
+
+function parseCodexFormattedOutput(output = '') {
+  const toolCalls = [];
+  const toolCallDetails = [];
+  for (const line of String(output || '').split(/\r?\n/)) {
+    const commandMatch = line.match(/^\s*(?:exec|shell)\s+(.+)$/i);
+    if (!commandMatch) continue;
+    const detail = classifyCodexCommand(commandMatch[1]);
+    toolCalls.push(detail.name);
+    toolCallDetails.push(detail);
+  }
+  return {
+    toolCalls,
+    toolCallDetails,
+    turns: toolCalls.length ? 1 : 0,
+    usage: {
+      inputTokens: null,
+      cachedInputTokens: null,
+      outputTokens: parseCodexTokenCount(output),
+    },
+  };
+}
+
+function classifyCodexItem(item = {}) {
+  const itemType = String(item.type || item.item_type || '').toLowerCase();
+  if (itemType.includes('command') || itemType === 'exec' || itemType === 'shell') {
+    return classifyCodexCommand(item.command || item.cmd || item.text || item.arguments?.cmd || '', item);
+  }
+  if (itemType.includes('file')) {
+    return {
+      name: 'edit_file',
+      source: 'codex-cli',
+      codexType: item.type || null,
+      path: item.path || item.file || item.file_path || null,
+      status: item.status || null,
+    };
+  }
+  if (itemType.includes('mcp')) {
+    return {
+      name: 'mcp_tool_call',
+      source: 'codex-cli',
+      codexType: item.type || null,
+      tool: item.name || item.tool || null,
+      status: item.status || null,
+    };
+  }
+  if (itemType.includes('web_search') || itemType.includes('web-search')) {
+    return {
+      name: 'web_search',
+      source: 'codex-cli',
+      codexType: item.type || null,
+      status: item.status || null,
+    };
+  }
+  return null;
+}
+
+function classifyCodexCommand(command = '', item = {}) {
+  const raw = String(command || '');
+  const lowered = raw.toLowerCase();
+  let name = 'run_shell';
+  const writesViaRedirection = /\b(tee|cat|printf|echo)\b[\s\S]*[>]/.test(lowered);
+  const patchLike = /\b(apply_patch|git\s+apply|perl\s+-0?pi|python[^\n]*write_text|writefilesync)\b/.test(lowered);
+  if (/\b(npm\s+test|node\s+test\.js|pytest|python\s+-m\s+pytest|make\s+test)\b/.test(lowered)) {
+    name = 'run_shell';
+  } else if (patchLike || writesViaRedirection) {
+    name = 'edit_file';
+  } else if (/\b(rg|grep)\b/.test(lowered)) {
+    name = 'grep_files';
+  } else if (/\b(find|ls|tree)\b/.test(lowered)) {
+    name = 'list_directory';
+  } else if (/\b(sed\s+-n|cat|head|tail|awk|nl)\b/.test(lowered)) {
+    name = 'read_file';
+  }
+  return {
+    name,
+    source: 'codex-cli',
+    codexType: item.type || 'command_execution',
+    command: raw,
+    status: item.status || null,
+  };
+}
+
+async function runCodexCliBaselineBenchmark(benchmark, options = {}) {
+  const expectations = benchmark.agentExpectations || {};
+  const fixtureName = expectations.projectFixture || 'express-basic';
+  const timeoutMs = options.timeoutMs || DEFAULT_TIMEOUT_MS;
+  let sandboxDir = null;
+  let keepSandbox = false;
+  const started = Date.now();
+
+  try {
+    sandboxDir = setupSandbox(fixtureName);
+    let testsBefore = null;
+    let totalTests = null;
+    if (testCommandAllowed(expectations.testCommand)) {
+      const beforeCounts = countTests(sandboxDir, expectations.testCommand);
+      testsBefore = beforeCounts.passed;
+      totalTests = beforeCounts.total;
+    }
+
+    if (options.dryRun) {
+      return {
+        benchmarkId: benchmark.id,
+        runner: 'codex-cli-baseline',
+        provider: 'codex-cli-baseline',
+        model: options.model || null,
+        success: true,
+        status: 'dry_run_ok',
+        sandboxCreated: sandboxDir,
+        latencyMs: Date.now() - started,
+      };
+    }
+
+    const prompt = buildCodexPrompt(benchmark);
+    const codex = await (options.runCodexExec || runCodexExec)({
+      prompt,
+      cwd: sandboxDir,
+      model: options.model,
+      timeoutMs,
+      useEnvOpenAIKey: !!options.useEnvOpenAIKey,
+      fullAuto: options.fullAuto !== false,
+      sandbox: options.sandbox || null,
+      dangerouslyBypassSandbox: !!options.dangerouslyBypassSandbox,
+      json: options.json !== false,
+      allowMcp: !!options.allowMcp,
+      disableMcpServers: options.disableMcpServers || null,
+      configOverrides: options.configOverrides || [],
+    });
+
+    const actualFileChanges = await getModifiedFiles(sandboxDir);
+    let testsPassed = null;
+    let testsAfter = null;
+    if (testCommandAllowed(expectations.testCommand)) {
+      try {
+        execFileSync('sh', ['-c', expectations.testCommand], {
+          cwd: sandboxDir,
+          timeout: options.testTimeoutMs || 30_000,
+          stdio: 'pipe',
+        });
+        testsPassed = true;
+      } catch {
+        testsPassed = false;
+      }
+      const afterCounts = countTests(sandboxDir, expectations.testCommand);
+      testsAfter = afterCounts.passed;
+      if (totalTests == null) totalTests = afterCounts.total;
+    }
+
+    const expectedFileChanges = expectations.expectedFileChanges || [];
+    const missingExpectedWork = expectedFileChanges.length > 0 && actualFileChanges.length === 0;
+    const testRegression = !!(expectations.testCommand && testsPassed === false);
+    const codexErrors = Array.isArray(codex.codexErrors) && codex.codexErrors.length
+      ? codex.codexErrors.join('; ')
+      : null;
+    const exitError = codex.timedOut
+      ? 'codex exec timed out'
+      : codex.code === 0
+        ? codexErrors
+        : `codex exec exited ${codex.code}${codex.signal ? ` (${codex.signal})` : ''}: ${(codex.stderr || codex.stdout || '').slice(-1000)}`;
+    const success = !exitError && !missingExpectedWork && !testRegression;
+    const actualToolCalls = codex.toolCalls || [];
+    const toolCallDetails = codex.toolCallDetails || [];
+    const usage = codex.usage || {};
+
+    let score = scoreAgentResult(benchmark, {
+      actualToolCalls,
+      actualFileChanges,
+      actualTurns: codex.turns || 1,
+      testsPassed,
+      output: codex.output || codex.stdout || '',
+      success,
+      sandboxDir,
+      costDollars: null,
+      testsBefore,
+      testsAfter,
+      totalTests,
+      toolCallDetails,
+    });
+
+    if (!success) {
+      score = {
+        composite: 0,
+        dimensions: {
+          ...(score.dimensions || {}),
+          _zeroed: true,
+          _zeroReason: exitError
+            ? 'codex_error'
+            : testRegression
+              ? 'tests_failed'
+              : 'no_file_changes',
+        },
+      };
+    }
+
+    keepSandbox = !!options.keepFailures && !success;
+    return {
+      benchmarkId: benchmark.id,
+      runner: 'codex-cli-baseline',
+      provider: 'codex-cli-baseline',
+      model: options.model || null,
+      success,
+      score,
+      latencyMs: codex.latencyMs || (Date.now() - started),
+      actualToolCalls,
+      actualFileChanges,
+      actualTurns: codex.turns || 1,
+      testsPassed,
+      testsBefore,
+      testsAfter,
+      totalTests,
+      inputTokens: usage.inputTokens ?? null,
+      outputTokens: usage.outputTokens ?? parseCodexTokenCount(codex.stdout),
+      dimensionsJson: JSON.stringify(score.dimensions || {}),
+      output: (codex.output || codex.stdout || '').slice(0, 2000),
+      stderr: filterNonFatalCodexStderr(codex.stderr || '').slice(0, 2000),
+      error: exitError,
+      sandboxDir: keepSandbox ? sandboxDir : null,
+    };
+  } catch (err) {
+    keepSandbox = !!options.keepFailures;
+    return {
+      benchmarkId: benchmark.id,
+      runner: 'codex-cli-baseline',
+      provider: 'codex-cli-baseline',
+      model: options.model || null,
+      success: false,
+      score: { composite: 0, dimensions: { _zeroed: true, _zeroReason: 'exception' } },
+      latencyMs: Date.now() - started,
+      error: err.message,
+      sandboxDir: keepSandbox ? sandboxDir : null,
+    };
+  } finally {
+    if (sandboxDir && !keepSandbox) cleanupSandbox(sandboxDir);
+  }
+}
+
+function parseCodexTokenCount(stdout = '') {
+  const match = String(stdout || '').match(/tokens used\s*\n\s*([\d,]+)/i);
+  return match ? parseInt(match[1].replace(/,/g, ''), 10) : null;
+}
+
+function filterNonFatalCodexStderr(stderr = '') {
+  return String(stderr || '')
+    .split(/\r?\n/)
+    .filter((line) => {
+      if (/Reading additional input from stdin/.test(line)) return false;
+      if (/opentelemetry_sdk/.test(line)) return false;
+      if (/BatchSpanProcessor/.test(line)) return false;
+      return true;
+    })
+    .join('\n')
+    .trim();
+}
+
+function summarizeBaselineResults(results = []) {
+  const total = results.length;
+  const passed = results.filter((r) => r.success).length;
+  const avgComposite = total
+    ? results.reduce((sum, r) => sum + (r.score?.composite || 0), 0) / total
+    : 0;
+  const failures = {};
+  for (const result of results) {
+    if (!result.success) {
+      const reason = result.score?.dimensions?._zeroReason || result.error || 'unknown';
+      failures[reason] = (failures[reason] || 0) + 1;
+    }
+  }
+  return { total, passed, failed: total - passed, avgComposite, failures };
+}
+
+function writeBaselineArtifact(result, { resultsDir = DEFAULT_RESULTS_DIR } = {}) {
+  fs.mkdirSync(resultsDir, { recursive: true });
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const file = path.join(resultsDir, `${stamp}-${safeId(result.benchmarkId || 'codex-baseline')}.json`);
+  fs.writeFileSync(file, JSON.stringify(result, null, 2) + '\n');
+  return file;
+}
+
+function storeBaselineResult({ brain, runId, benchmark, result, model, timeoutMs }) {
+  if (!brain || typeof brain.insertBenchmarkResult !== 'function') return;
+  const scoringMethod = benchmark.agentExpectations?.testCommand
+    ? 'codex-cli-baseline+tests'
+    : 'codex-cli-baseline';
+  brain.insertBenchmarkResult(decorateBenchmarkResult({
+    runId,
+    suite: 'coding-agent',
+    promptId: benchmark.id,
+    taskType: 'coding-agent',
+    difficulty: benchmark.difficulty,
+    provider: 'codex-cli-baseline',
+    model: model || 'codex-default',
+    prompt: benchmark.prompt,
+    response: result.output || '',
+    traitScore: null,
+    matchedTraits: [],
+    compositeScore: result.score?.composite || 0,
+    latencyMs: result.latencyMs,
+    error: result.error,
+    timestamp: result.timestamp,
+    costDollars: null,
+    testsBefore: result.testsBefore ?? null,
+    testsAfter: result.testsAfter ?? null,
+    totalTests: result.totalTests ?? null,
+    dimensionsJson: result.dimensionsJson || null,
+    inputTokens: null,
+    outputTokens: result.outputTokens ?? null,
+    scorerVersion: DEFAULT_SCORER_VERSION,
+    scoringMethod,
+    trusted: result.success === true && result.testsPassed === true && !result.error,
+    runConfig: { timeoutMs, scoringMethod, externalRunner: 'codex-cli' },
+  }, {
+    suite: 'coding-agent',
+    benchmark,
+    runId,
+    provider: 'codex-cli-baseline',
+    model: model || 'codex-default',
+    scoringMethod,
+    scorerVersion: DEFAULT_SCORER_VERSION,
+    trusted: result.success === true && result.testsPassed === true && !result.error,
+    runConfig: { timeoutMs, scoringMethod, externalRunner: 'codex-cli' },
+  }));
+}
+
+function safeId(value) {
+  return String(value || 'unknown').replace(/[^a-z0-9_.-]+/gi, '-').slice(0, 80);
+}
+
+module.exports = {
+  DEFAULT_RESULTS_DIR,
+  buildCodexExecArgs,
+  buildMcpDisableConfigOverrides,
+  buildCodexPrompt,
+  codexEnv,
+  classifyCodexCommand,
+  classifyCodexItem,
+  discoverEnabledCodexMcpServers,
+  filterNonFatalCodexStderr,
+  parseEnabledCodexMcpServers,
+  parseCodexTokenCount,
+  parseCodexJsonEvents,
+  runCodexCliBaselineBenchmark,
+  runCodexExec,
+  summarizeBaselineResults,
+  writeBaselineArtifact,
+  storeBaselineResult,
+};

package/template/wall-e/eval/debug-agent003.js

@@ -31,6 +31,7 @@ if (!bench) { console.error('No benchmark:', benchId); process.exit(1); }
       cwd: dir,
       timeoutMs: 120000,
       mode: 'build',
+      persistTranscript: false,
       onProgress: (e) => {
         if (e.message) {
           const detail = e.detail ? ` ${JSON.stringify(e.detail).slice(0, 120)}` : '';

package/template/wall-e/eval/eval-orchestrator.js

@@ -5,7 +5,7 @@ const path = require('path');
 const fs = require('fs');
 
 const { pLimit, getAvailableProviders } = require('./head-to-head');
-const { runAgentBenchmark, runMultiTurnBenchmark } = require('./agent-runner');
+const { runAgentBenchmark, runMultiTurnBenchmark, isTrustedAgentResult } = require('./agent-runner');
 const { createClient } = require('../llm/client');
 const { createAnthropicFromEnv } = require('../llm/anthropic');
 const { decorateBenchmarkResult, DEFAULT_SCORER_VERSION } = require('./manifest');
@@ -566,7 +566,7 @@ class EvalOrchestrator extends EventEmitter {
         outputTokens: result.outputTokens ?? null,
         scorerVersion: DEFAULT_SCORER_VERSION,
         scoringMethod,
-        trusted: !result.error && result.testsPassed === true,
+        trusted: isTrustedAgentResult(result),
         runConfig: {
           timeoutMs: this.timeoutMs,
           concurrency: this.concurrency,
@@ -581,7 +581,7 @@ class EvalOrchestrator extends EventEmitter {
         model: item.model,
         scoringMethod,
         scorerVersion: DEFAULT_SCORER_VERSION,
-        trusted: !result.error && result.testsPassed === true,
+        trusted: isTrustedAgentResult(result),
         runConfig: {
           timeoutMs: this.timeoutMs,
           concurrency: this.concurrency,