create-walle 0.9.21 → 0.9.23

package/template/wall-e/eval/ab-test.js

@@ -1,203 +0,0 @@
-'use strict';
-
-/**
- * A/B testing for orchestrator changes.
- *
- * Runs the same benchmark(s) twice — once with the baseline orchestrator
- * and once with the candidate — then compares scores side-by-side.
- *
- * Usage:
- *   node eval/ab-test.js --baseline HEAD~1 --candidate HEAD --id agent-001
- *   node eval/ab-test.js --id agent-001   # compares current code vs last commit
- */
-
-const { execFileSync } = require('child_process');
-const path = require('path');
-const crypto = require('crypto');
-
-const WALL_E_ROOT = path.join(__dirname, '..');
-
-/**
- * Run benchmarks at a specific git ref by checking out the ref in a temporary worktree.
- * Returns the benchmark results.
- */
-async function runAtRef(ref, benchmarkIds, opts = {}) {
-  const { timeoutMs = 600000 } = opts;
-  const worktreeDir = path.join(WALL_E_ROOT, '..', `.ab-worktree-${crypto.randomUUID().slice(0, 8)}`);
-
-  try {
-    // Create git worktree
-    execFileSync('git', ['worktree', 'add', '--detach', worktreeDir, ref], {
-      cwd: WALL_E_ROOT,
-      stdio: 'pipe',
-      timeout: 30000,
-    });
-
-    // Run benchmarks in the worktree
-    const args = ['eval/run-agent-benchmarks.js'];
-    if (benchmarkIds.length === 1) {
-      args.push('--id', benchmarkIds[0]);
-    }
-
-    const output = execFileSync(process.execPath, args, {
-      cwd: path.join(worktreeDir, 'wall-e'),
-      stdio: 'pipe',
-      timeout: timeoutMs,
-      env: { ...process.env, AB_TEST_MODE: '1' },
-    });
-
-    return parseRunnerOutput(output.toString());
-  } finally {
-    // Clean up worktree
-    try {
-      execFileSync('git', ['worktree', 'remove', '--force', worktreeDir], {
-        cwd: WALL_E_ROOT,
-        stdio: 'pipe',
-        timeout: 10000,
-      });
-    } catch { /* best effort cleanup */ }
-  }
-}
-
-/**
- * Parse benchmark runner stdout to extract results.
- * Falls back to a summary object if output can't be parsed.
- */
-function parseRunnerOutput(output) {
-  const lines = output.split('\n');
-  const results = [];
-  let currentBenchmark = null;
-
-  for (const line of lines) {
-    const idMatch = line.match(/^--- ([\w-]+)/);
-    if (idMatch) {
-      currentBenchmark = { benchmarkId: idMatch[1], success: false, score: 0 };
-      results.push(currentBenchmark);
-      continue;
-    }
-    if (currentBenchmark) {
-      const successMatch = line.match(/Success:\s*(true|false)/);
-      if (successMatch) currentBenchmark.success = successMatch[1] === 'true';
-
-      const scoreMatch = line.match(/Score:\s*([\d.]+)/);
-      if (scoreMatch) currentBenchmark.score = parseFloat(scoreMatch[1]);
-    }
-  }
-
-  // Extract summary
-  const avgMatch = output.match(/Average composite score:\s*([\d.]+)/);
-  const passMatch = output.match(/Passed:\s*(\d+)\/(\d+)/);
-
-  return {
-    results,
-    avgScore: avgMatch ? parseFloat(avgMatch[1]) : 0,
-    passed: passMatch ? parseInt(passMatch[1], 10) : 0,
-    total: passMatch ? parseInt(passMatch[2], 10) : results.length,
-  };
-}
-
-/**
- * Compare two sets of results side-by-side.
- */
-function compareResults(baselineResults, candidateResults) {
-  const comparison = {
-    baselineAvg: baselineResults.avgScore,
-    candidateAvg: candidateResults.avgScore,
-    delta: candidateResults.avgScore - baselineResults.avgScore,
-    deltaPct: baselineResults.avgScore > 0
-      ? ((candidateResults.avgScore - baselineResults.avgScore) / baselineResults.avgScore * 100).toFixed(1)
-      : 'N/A',
-    verdict: 'neutral',
-    perBenchmark: [],
-  };
-
-  if (comparison.delta > 0.02) comparison.verdict = 'candidate_better';
-  else if (comparison.delta < -0.02) comparison.verdict = 'baseline_better';
-
-  // Per-benchmark comparison
-  const candidateMap = new Map(candidateResults.results.map(r => [r.benchmarkId, r]));
-  for (const base of baselineResults.results) {
-    const cand = candidateMap.get(base.benchmarkId);
-    comparison.perBenchmark.push({
-      benchmarkId: base.benchmarkId,
-      baselineScore: base.score,
-      candidateScore: cand?.score || 0,
-      delta: (cand?.score || 0) - base.score,
-      baselineSuccess: base.success,
-      candidateSuccess: cand?.success || false,
-    });
-  }
-
-  return comparison;
-}
-
-/**
- * Run A/B test comparing two git refs.
- * @param {Object} opts
- * @param {string} opts.baseline - Git ref for baseline (default: HEAD~1)
- * @param {string} opts.candidate - Git ref for candidate (default: HEAD)
- * @param {string[]} opts.benchmarkIds - Benchmark IDs to test
- * @param {number} [opts.timeoutMs=600000] - Timeout per ref
- * @returns {Promise<Object>} Comparison results
- */
-async function runABTest(opts) {
-  const { baseline = 'HEAD~1', candidate = 'HEAD', benchmarkIds, timeoutMs = 600000 } = opts;
-
-  console.log(`Running baseline (${baseline})...`);
-  const baselineResults = await runAtRef(baseline, benchmarkIds, { timeoutMs });
-
-  console.log(`Running candidate (${candidate})...`);
-  const candidateResults = await runAtRef(candidate, benchmarkIds, { timeoutMs });
-
-  return compareResults(baselineResults, candidateResults);
-}
-
-// CLI
-if (require.main === module) {
-  const args = process.argv.slice(2);
-  const getArg = (flag, def) => {
-    const idx = args.indexOf(flag);
-    return idx >= 0 ? args[idx + 1] : def;
-  };
-
-  const baseline = getArg('--baseline', 'HEAD~1');
-  const candidate = getArg('--candidate', 'HEAD');
-  const id = getArg('--id', null);
-
-  if (!id) {
-    console.error('Usage: node eval/ab-test.js --id <benchmark-id> [--baseline ref] [--candidate ref]');
-    process.exit(1);
-  }
-
-  console.log('=== A/B Test ===');
-  console.log(`Baseline:  ${baseline}`);
-  console.log(`Candidate: ${candidate}`);
-  console.log(`Benchmark: ${id}`);
-  console.log('');
-
-  runABTest({ baseline, candidate, benchmarkIds: [id] })
-    .then(comparison => {
-      console.log('=== RESULTS ===');
-      console.log(`Baseline avg:  ${comparison.baselineAvg.toFixed(3)}`);
-      console.log(`Candidate avg: ${comparison.candidateAvg.toFixed(3)}`);
-      console.log(`Delta:         ${comparison.delta >= 0 ? '+' : ''}${comparison.delta.toFixed(3)} (${comparison.deltaPct}%)`);
-      console.log(`Verdict:       ${comparison.verdict}`);
-      console.log('');
-
-      if (comparison.perBenchmark.length > 0) {
-        console.log('Per-benchmark:');
-        for (const b of comparison.perBenchmark) {
-          const arrow = b.delta > 0 ? '+' : '';
-          console.log(`  ${b.benchmarkId}: ${b.baselineScore.toFixed(3)} -> ${b.candidateScore.toFixed(3)} (${arrow}${b.delta.toFixed(3)})`);
-        }
-      }
-
-      process.exit(comparison.verdict === 'baseline_better' ? 1 : 0);
-    })
-    .catch(err => {
-      console.error('Fatal:', err.message);
-      process.exit(1);
-    });
-}
-
-module.exports = { runABTest, runAtRef, compareResults, parseRunnerOutput };