@evomap/evolver 1.89.2 → 1.89.4

package/src/experiment/agentRunner.js

@@ -0,0 +1,229 @@
+// src/experiment/agentRunner.js
+//
+// The ONLY module in src/experiment that touches a subprocess. It runs a single
+// arm of a comparison by invoking a headless coding-agent CLI -- by default
+// `claude -p "<prompt>" --output-format json` -- and maps its JSON envelope
+// onto the canonical AgentResult shape (duration / rounds / tokens / cost).
+//
+// Security: the prompt is passed as an argv element with shell:false, so quotes,
+// `$`, backticks etc. in the task can never be interpreted by a shell (same
+// injection posture as src/gep/validator/sandboxExecutor.js). Unlike the
+// sandbox, the real user env IS passed through -- the agent CLI needs PATH and
+// its own auth/credentials to run.
+'use strict';
+
+const { spawn } = require('child_process');
+
+const DEFAULT_CMD = 'claude';
+const DEFAULT_TIMEOUT_MS = 300000; // 5 min
+const MIN_TIMEOUT_MS = 1000;
+const MAX_TIMEOUT_MS = 1800000; // 30 min hard cap
+const RESULT_CAP = 4000;
+const MAX_STDOUT_BYTES = 10 * 1024 * 1024; // 10 MB — a JSON envelope is tiny; cap a chatty/malformed CLI before it OOMs.
+
+function num(v, fallback) {
+  const n = Number(v);
+  return Number.isFinite(n) ? n : (fallback === undefined ? 0 : fallback);
+}
+
+function resolveAgentCommand(opts) {
+  if (opts && opts.command) return String(opts.command);
+  const env = process.env.EVOLVER_EXPERIMENT_AGENT_CMD;
+  return env && env.trim() ? env.trim() : DEFAULT_CMD;
+}
+
+function resolveExtraArgs(opts) {
+  if (opts && Array.isArray(opts.extraArgs)) return opts.extraArgs.map(String);
+  const env = process.env.EVOLVER_EXPERIMENT_AGENT_ARGS;
+  if (!env || !env.trim()) return [];
+  return env.trim().split(/\s+/);
+}
+
+function resolveTimeout(opts) {
+  let t = opts && Number.isFinite(Number(opts.timeoutMs)) ? Number(opts.timeoutMs) : null;
+  if (t == null) {
+    const e = Number(process.env.EVOLVER_EXPERIMENT_TIMEOUT_MS);
+    t = Number.isFinite(e) && e > 0 ? e : DEFAULT_TIMEOUT_MS;
+  }
+  return Math.min(Math.max(MIN_TIMEOUT_MS, t), MAX_TIMEOUT_MS);
+}
+
+// Parse the `--output-format json` envelope. Tolerant of leading/trailing
+// non-JSON noise by falling back to the outermost {...} slice.
+function parseAgentJson(stdout) {
+  const text = String(stdout || '').trim();
+  if (!text) return null;
+  try {
+    return JSON.parse(text);
+  } catch (_) { /* fall through */ }
+  const start = text.indexOf('{');
+  const end = text.lastIndexOf('}');
+  if (start >= 0 && end > start) {
+    try {
+      return JSON.parse(text.slice(start, end + 1));
+    } catch (_) { /* fall through */ }
+  }
+  return null;
+}
+
+function makeFailure(error, command, extra) {
+  return Object.assign(
+    {
+      ok: false,
+      isError: true,
+      error: error,
+      durationMs: 0,
+      rounds: 0,
+      tokensIn: 0,
+      tokensOut: 0,
+      tokensTotal: 0,
+      costUsd: 0,
+      resultText: '',
+      exitCode: -1,
+      timedOut: false,
+      runnerName: 'claude-cli',
+      agentCommand: command,
+    },
+    extra || {},
+  );
+}
+
+// Map a parsed `claude -p --output-format json` envelope onto AgentResult.
+function mapAgentResult(json, ctx) {
+  const command = ctx.command;
+  const exitCode = num(ctx.exitCode, -1);
+  const timedOut = !!ctx.timedOut;
+  const usage = (json && json.usage) || {};
+  const tokensIn = num(usage.input_tokens);
+  const tokensOut = num(usage.output_tokens);
+  const isError = !!(json && json.is_error);
+  const ok = exitCode === 0 && !isError && !timedOut;
+  let error = null;
+  if (!ok) {
+    if (timedOut) error = 'agent_timeout';
+    else if (isError) error = 'agent_reported_error';
+    else if (exitCode !== 0) error = 'agent_exit_' + exitCode;
+  }
+  // Prefer the agent's self-reported duration; fall back to wall-clock.
+  const durationMs = json && Number.isFinite(Number(json.duration_ms))
+    ? num(json.duration_ms)
+    : num(ctx.durationMs);
+  return {
+    ok,
+    isError,
+    error,
+    durationMs,
+    rounds: num(json && json.num_turns),
+    tokensIn,
+    tokensOut,
+    tokensTotal: tokensIn + tokensOut,
+    costUsd: num(json && json.total_cost_usd),
+    resultText: typeof (json && json.result) === 'string' ? json.result.slice(0, RESULT_CAP) : '',
+    exitCode,
+    timedOut,
+    runnerName: 'claude-cli',
+    agentCommand: command,
+  };
+}
+
+/**
+ * Run one task prompt through a headless agent CLI.
+ *
+ * @param {string} prompt
+ * @param {object} [opts] { command, extraArgs, timeoutMs, cwd }
+ * @returns {Promise<AgentResult>} never rejects; failures resolve as { ok:false, error }
+ */
+function runAgentTask(prompt, opts) {
+  opts = opts || {};
+  const command = resolveAgentCommand(opts);
+  const extraArgs = resolveExtraArgs(opts);
+  const timeoutMs = resolveTimeout(opts);
+  const cwd = opts.cwd || process.cwd();
+  const argv = [...extraArgs, '-p', String(prompt == null ? '' : prompt), '--output-format', 'json'];
+
+  return new Promise((resolve) => {
+    let child;
+    let settled = false;
+    let stdout = '';
+    let stderr = '';
+    const startedAt = Date.now();
+
+    const done = (result) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve(result);
+    };
+
+    const timer = setTimeout(() => {
+      if (child && !child.killed) {
+        try { child.kill('SIGKILL'); } catch (_) { /* noop */ }
+      }
+      done(makeFailure('agent_timeout', command, {
+        durationMs: Date.now() - startedAt,
+        timedOut: true,
+      }));
+    }, timeoutMs);
+
+    try {
+      child = spawn(command, argv, {
+        shell: false,
+        cwd,
+        env: process.env,
+        stdio: ['ignore', 'pipe', 'pipe'],
+      });
+    } catch (e) {
+      done(makeFailure('agent_spawn_failed: ' + (e && e.message ? e.message : String(e)), command));
+      return;
+    }
+
+    child.stdout.on('data', (d) => {
+      stdout += d.toString('utf8');
+      if (stdout.length > MAX_STDOUT_BYTES) {
+        // A well-behaved `--output-format json` envelope is tiny; this much
+        // stdout means a runaway/malformed CLI — kill it rather than OOM.
+        try { child.kill('SIGKILL'); } catch (_) { /* noop */ }
+        done(makeFailure('agent_output_too_large', command, { durationMs: Date.now() - startedAt }));
+      }
+    });
+    child.stderr.on('data', (d) => {
+      stderr += d.toString('utf8');
+      if (stderr.length > MAX_STDOUT_BYTES) stderr = stderr.slice(-RESULT_CAP);
+    });
+
+    child.on('error', (err) => {
+      const msg = err && err.code === 'ENOENT'
+        ? 'agent_cli_not_found: ' + command
+        : 'agent_spawn_error: ' + (err && err.message ? err.message : String(err));
+      done(makeFailure(msg, command, { durationMs: Date.now() - startedAt }));
+    });
+
+    child.on('exit', (code, signal) => {
+      const durationMs = Date.now() - startedAt;
+      // A signal-terminated child reports code=null, and Number(null) is 0 —
+      // num(null, -1) would read it as a CLEAN exit and let a killed arm
+      // score ok:true. Map null/undefined explicitly before the numeric path.
+      const exitCode = (code === null || code === undefined) ? -1 : num(code, -1);
+      const json = parseAgentJson(stdout);
+      if (!json) {
+        done(makeFailure(
+          signal ? 'agent_killed_' + signal : 'agent_output_parse_failed', command, {
+            durationMs,
+            exitCode,
+            resultText: String(stdout || stderr || '').slice(0, RESULT_CAP),
+          }));
+        return;
+      }
+      done(mapAgentResult(json, { exitCode, durationMs, timedOut: false, command }));
+    });
+  });
+}
+
+module.exports = {
+  runAgentTask,
+  parseAgentJson,
+  mapAgentResult,
+  resolveAgentCommand,
+  resolveTimeout,
+  DEFAULT_CMD,
+};

package/src/experiment/cli.js

@@ -0,0 +1,159 @@
+// src/experiment/cli.js
+//
+// CLI surface for the `experiment` subcommand. Mirrors src/atp/cli.js:
+//   parseExperimentArgs(args) -> { ok, opts? , error? }
+//   runExperiment(opts, deps) -> { ok, data?, error?, exitCode }
+// Dependency-injected (comparison / agentRunner / geneLoader / sandbox)
+// so the runner is unit-testable without a real agent CLI.
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const MAX_REQUEST_FILE_BYTES = 1024 * 1024; // 1 MB — a request JSON is tiny
+
+// Minimal `--key=value` / `--flag` parser (no shell, no globbing).
+function parseFlags(args) {
+  const out = {};
+  for (const a of args || []) {
+    if (typeof a !== 'string') continue;
+    const eq = a.indexOf('=');
+    if (a.startsWith('--') && eq > 2) {
+      out[a.slice(2, eq)] = a.slice(eq + 1);
+    } else if (a.startsWith('--')) {
+      out[a.slice(2)] = true;
+    }
+  }
+  return out;
+}
+
+/**
+ * Parse experiment subcommand args.
+ *
+ * Flags: --task= --metric= [--baseline=] [--variant=] [--gene=]
+ *        [--validation="cmd1;;cmd2"] [--timeout=ms] [--request-file=<json>]
+ *
+ * --request-file supplies a JSON base ({task,baseline,variant,metric,geneId,
+ * validationCommands,timeoutMs}); explicit flags override it. This is the only
+ * filesystem read here and lets the desktop pass complex params without argv
+ * escaping headaches.
+ *
+ * @returns {{ ok: true, opts: object } | { ok: false, error: string }}
+ */
+function parseExperimentArgs(args) {
+  const f = parseFlags(args);
+
+  let base = {};
+  if (f['request-file']) {
+    try {
+      // Resolve + stat the path and bound its size before reading: don't open
+      // an arbitrary non-file (device/FIFO) or slurp an unbounded file.
+      const rfPath = path.resolve(String(f['request-file']));
+      const st = fs.statSync(rfPath);
+      if (!st.isFile()) return { ok: false, error: '--request-file must be a regular file' };
+      if (st.size > MAX_REQUEST_FILE_BYTES) return { ok: false, error: '--request-file too large (> 1 MB)' };
+      base = JSON.parse(fs.readFileSync(rfPath, 'utf8'));
+    } catch (e) {
+      return { ok: false, error: 'failed to read --request-file: ' + (e && e.message ? e.message : String(e)) };
+    }
+    if (!base || typeof base !== 'object') {
+      return { ok: false, error: '--request-file must contain a JSON object' };
+    }
+  }
+
+  const pick = (flagVal, baseVal, dflt) => {
+    if (flagVal !== undefined && flagVal !== true) return String(flagVal);
+    if (baseVal !== undefined && baseVal !== null) return String(baseVal);
+    return dflt;
+  };
+
+  const opts = {
+    // trim() so a whitespace-only value fails the required-field check below
+    // (parse-time exit 2 + usage) instead of crashing later in runComparison
+    // (exit 1 with no JSON on stdout for the Go caller).
+    task: pick(f.task, base.task, '').trim(),
+    baseline: pick(f.baseline, base.baseline, 'baseline'),
+    variant: pick(f.variant, base.variant, 'variant'),
+    metric: pick(f.metric, base.metric, '').trim(),
+    geneId: pick(f.gene, base.geneId !== undefined ? base.geneId : base.gene, '') || null,
+    validationCommands: null,
+    timeoutMs: undefined,
+  };
+
+  if (f.validation !== undefined && f.validation !== true) {
+    opts.validationCommands = String(f.validation).split(';;').map((s) => s.trim()).filter(Boolean);
+  } else if (Array.isArray(base.validationCommands)) {
+    opts.validationCommands = base.validationCommands.map(String);
+  }
+
+  const timeoutRaw = f.timeout !== undefined && f.timeout !== true ? Number(f.timeout)
+    : (Number.isFinite(Number(base.timeoutMs)) ? Number(base.timeoutMs) : NaN);
+  if (Number.isFinite(timeoutRaw)) opts.timeoutMs = timeoutRaw;
+
+  if (!opts.task) return { ok: false, error: 'missing required --task (or "task" in --request-file)' };
+  if (!opts.metric) return { ok: false, error: 'missing required --metric (or "metric" in --request-file)' };
+
+  return { ok: true, opts };
+}
+
+/**
+ * Run the comparison. Returns the result object (also written to stdout by the
+ * index.js wrapper) plus an exit code. exitCode 3 == inconclusive (a real,
+ * structured outcome, not a crash).
+ *
+ * @param {object} opts  from parseExperimentArgs
+ * @param {object} [deps] { comparison, agentRunner, geneLoader, sandbox, err }
+ */
+async function runExperiment(opts, deps) {
+  deps = deps || {};
+  const comparison = deps.comparison || require('./comparison');
+  const err = typeof deps.err === 'function' ? deps.err : ((...a) => console.error(...a));
+
+  const params = Object.assign({}, opts);
+  if (deps.agentRunner) params.agentRunner = deps.agentRunner;
+  if (deps.geneLoader) params.geneLoader = deps.geneLoader;
+  if (deps.sandbox) params.sandbox = deps.sandbox;
+
+  try {
+    const raw = await comparison.runComparison(params);
+    // Redact any secrets / API keys an agent's resultText (or a parse-failure
+    // snippet) may carry before the result crosses the process boundary into
+    // the desktop consumer / is persisted to disk.
+    const data = require('../gep/sanitize').sanitizePayload(raw);
+    let exitCode = 0;
+    if (data && data.winner === 'inconclusive') {
+      exitCode = 3;
+      // Two distinct causes share the inconclusive verdict; say which one
+      // happened so operators don't go debugging arm failures that never
+      // occurred (both arms can be ok when pass-rate was simply unmeasured).
+      const bothArmsOk = !!(data.arms && data.arms.baseline && data.arms.baseline.ok
+        && data.arms.variant && data.arms.variant.ok);
+      err(bothArmsOk
+        ? '[Experiment] inconclusive: metric resolved to pass-rate but no validation commands were given, so pass-rate was never measured — pass --validation (or pick another --metric)'
+        : '[Experiment] inconclusive: one or both arms failed — see arms.*.error');
+    }
+    return { ok: true, data, exitCode };
+  } catch (e) {
+    err('[Experiment] error: ' + (e && e.message ? e.message : String(e)));
+    return { ok: false, error: e && e.message ? e.message : String(e), exitCode: 1 };
+  }
+}
+
+function printExperimentUsage() {
+  return [
+    'Usage: node index.js experiment --task="..." --metric="..." [flags]',
+    '  --baseline="..."          对照臂标签 (default: baseline)',
+    '  --variant="..."           实验臂标签 (default: variant)',
+    '  --gene=<geneId>           变体臂复用的基因 id (注入其 strategy)',
+    '  --validation="c1;;c2"     自包含 node 校验命令 (通过率评分; ;; 分隔)',
+    '  --timeout=<ms>            单臂超时',
+    '  --request-file=<path>     JSON 基底 (显式 flag 覆盖之)',
+    '',
+    'Runs the same task twice (baseline vs variant-with-gene) via a headless',
+    'agent CLI and prints a comparison JSON to stdout. Logs go to stderr.',
+    'Env: EVOLVER_EXPERIMENT_AGENT_CMD (default claude),',
+    '     EVOLVER_EXPERIMENT_AGENT_ARGS, EVOLVER_EXPERIMENT_TIMEOUT_MS (300000).',
+  ].join('\n');
+}
+
+module.exports = { parseExperimentArgs, runExperiment, printExperimentUsage, parseFlags };

package/src/experiment/comparison.js

@@ -0,0 +1,233 @@
+// src/experiment/comparison.js
+//
+// Thin orchestrator for a comparative experiment: run the SAME task twice --
+// a baseline arm (plain task) and a variant arm (task + the reused gene's
+// strategy injected) -- through a pluggable agent runner, collect real
+// metrics (duration / rounds / tokens / pass-rate), and emit a versioned
+// comparison result.
+//
+// Design notes:
+//  - This module NEVER requires child_process. The agent runner, gene loader,
+//    and sandbox runner are all injectable, so unit tests stay deterministic
+//    (no LLM, no network, no subprocess). Production defaults are lazy-loaded.
+//  - A failed arm never fabricates a score: if either arm is !ok the winner is
+//    'inconclusive' and improvement is null, while still recording whatever
+//    partial metrics were captured.
+'use strict';
+
+const { deriveMetric, scoreArm, num, round } = require('./metrics');
+
+const SCHEMA = 'evolver.experiment.comparison.v1';
+const RESULT_TEXT_CAP = 2000;
+const EPS = 1e-9;
+
+// Build the variant prompt by appending the reused gene's strategy, mirroring
+// the numbered-list format used in src/gep/prompt.js (`${i+1}. ${s}`).
+function buildVariantPrompt(task, gene) {
+  if (!gene || !Array.isArray(gene.strategy) || gene.strategy.length === 0) return task;
+  const steps = gene.strategy.map((s, i) => `${i + 1}. ${s}`).join('\n');
+  return (
+    task +
+    '\n\n## Reuse the following proven strategy\n' +
+    steps +
+    '\n\nApply the strategy above while completing the task.'
+  );
+}
+
+// Coerce whatever the agent runner returned into the canonical arm shape.
+function normalizeArm(label, raw) {
+  raw = raw || {};
+  const tokensIn = num(raw.tokensIn);
+  const tokensOut = num(raw.tokensOut);
+  const tokensTotal = Number.isFinite(Number(raw.tokensTotal)) ? num(raw.tokensTotal) : tokensIn + tokensOut;
+  return {
+    label: String(label == null ? '' : label),
+    ok: !!raw.ok,
+    error: raw.error != null ? String(raw.error) : null,
+    durationMs: num(raw.durationMs),
+    rounds: num(raw.rounds),
+    tokensIn,
+    tokensOut,
+    tokensTotal,
+    costUsd: num(raw.costUsd),
+    passRate: Number.isFinite(Number(raw.passRate)) ? num(raw.passRate) : (raw.ok ? 1 : 0),
+    resultText: typeof raw.resultText === 'string' ? raw.resultText.slice(0, RESULT_TEXT_CAP) : '',
+    exitCode: Number.isFinite(Number(raw.exitCode)) ? num(raw.exitCode) : null,
+    timedOut: !!raw.timedOut,
+  };
+}
+
+// Pass-rate for ONE arm: run its `node <script>` validation commands INSIDE that
+// arm's own workspace (where its agent just ran), so two arms whose agents
+// produced different output get different pass-rates -- the metric is linked to
+// the arm, not to a shared empty sandbox. Each command is a `node <script>`
+// vetted by sandboxExecutor's allowlist (runSingleCommand rejects anything else).
+async function passRateInDir(commands, cwd, runSingleCommand, timeoutMs, warnings) {
+  let passed = 0;
+  let total = 0;
+  for (const cmd of commands) {
+    total += 1;
+    try {
+      const r = await runSingleCommand(cmd, { cwd, timeoutMs });
+      if (r && r.ok) passed += 1;
+    } catch (e) {
+      warnings.push('passrate_command_error: ' + (e && e.message ? e.message : String(e)));
+    }
+  }
+  return total > 0 ? round(passed / total, 4) : 0;
+}
+
+/**
+ * Run a two-arm comparison.
+ *
+ * @param {object}   params
+ * @param {string}   params.task                自然语言任务(必填)
+ * @param {string}  [params.baseline='baseline'] 对照臂标签
+ * @param {string}  [params.variant='variant']   实验臂标签
+ * @param {string}   params.metric               评估指标(必填)
+ * @param {string}  [params.geneId]              变体臂复用的基因 id
+ * @param {string[]}[params.validationCommands]  自包含 `node <script>` 校验命令
+ * @param {number}  [params.timeoutMs]           单臂超时
+ * @param {function}[params.agentRunner]   (prompt, opts) => Promise<AgentResult>
+ * @param {function}[params.geneLoader]    () => Gene[]
+ * @param {object}  [params.sandbox]    { createSandboxDir, cleanupDir, runSingleCommand } (default: sandboxExecutor)
+ * @returns {Promise<object>} versioned ComparisonResult (see SCHEMA)
+ */
+async function runComparison(params) {
+  const p = params || {};
+  const task = String(p.task == null ? '' : p.task).trim();
+  const baseline = p.baseline ? String(p.baseline) : 'baseline';
+  const variant = p.variant ? String(p.variant) : 'variant';
+  const metric = String(p.metric == null ? '' : p.metric);
+  const geneId = p.geneId ? String(p.geneId) : null;
+  const validationCommands = Array.isArray(p.validationCommands)
+    ? p.validationCommands.filter((c) => typeof c === 'string' && c.trim())
+    : null;
+  const timeoutMs = Number.isFinite(Number(p.timeoutMs)) ? Number(p.timeoutMs) : undefined;
+
+  if (!task) throw new Error('task is required');
+  if (!metric) throw new Error('metric is required');
+
+  const agentRunner = typeof p.agentRunner === 'function'
+    ? p.agentRunner
+    : require('./agentRunner').runAgentTask;
+  const geneLoader = typeof p.geneLoader === 'function'
+    ? p.geneLoader
+    : require('../gep/assetStore').loadGenes;
+  const sandbox = p.sandbox && typeof p.sandbox === 'object'
+    ? p.sandbox
+    : require('../gep/validator/sandboxExecutor');
+
+  const startedAt = new Date().toISOString();
+  const t0 = Date.now();
+  const warnings = [];
+
+  const metricInfo = deriveMetric(metric);
+  if (!metricInfo.recognized) warnings.push('metric_unrecognized: ' + metric);
+
+  // Look up the reused gene (variant arm). Without a resolved gene the variant
+  // prompt is identical to the baseline task, so the two arms are NOT a strategy
+  // comparison -- record an explicit warning so identical arms aren't mistaken
+  // for one.
+  let gene = null;
+  if (geneId) {
+    let genes = [];
+    try {
+      genes = geneLoader() || [];
+    } catch (e) {
+      warnings.push('gene_load_error: ' + (e && e.message ? e.message : String(e)));
+    }
+    gene = genes.find((g) => g && String(g.id) === geneId) || null;
+    if (!gene) warnings.push('gene_not_found: ' + geneId + ' (variant arm equals baseline)');
+  } else {
+    warnings.push('no_gene: variant arm equals baseline (no strategy injected)');
+  }
+
+  const hasValidation = !!(validationCommands && validationCommands.length);
+  if (!hasValidation) warnings.push('passrate_degraded_no_validation');
+
+  let metaRunner = null;
+  let metaCommand = null;
+
+  const runArm = async (label, prompt) => {
+    // Each arm runs in its OWN fresh sandbox dir, so the agent works in
+    // isolation (never the evolver repo / process.cwd()) and its pass-rate
+    // validation reads that arm's own output, not a shared empty directory.
+    const workdir = sandbox.createSandboxDir();
+    let raw;
+    try {
+      raw = await agentRunner(prompt, { timeoutMs, cwd: workdir });
+    } catch (e) {
+      raw = { ok: false, error: 'agent_runner_threw: ' + (e && e.message ? e.message : String(e)) };
+    }
+    if (raw) {
+      if (metaRunner == null && raw.runnerName) metaRunner = String(raw.runnerName);
+      if (metaCommand == null && raw.agentCommand) metaCommand = String(raw.agentCommand);
+    }
+    const arm = normalizeArm(label, raw);
+    if (hasValidation) {
+      arm.passRate = await passRateInDir(validationCommands, workdir, sandbox.runSingleCommand, timeoutMs, warnings);
+    }
+    try { sandbox.cleanupDir(workdir); } catch (_) { /* best-effort cleanup */ }
+    return arm;
+  };
+
+  // Arms run sequentially: two real agent CLIs in parallel would contend for
+  // local resources / provider rate limits and muddy the duration metric.
+  const armBaseline = await runArm(baseline, task);
+  const armVariant = await runArm(variant, buildVariantPrompt(task, gene));
+
+  const baselineScore = scoreArm(armBaseline, metricInfo.metricField);
+  const variantScore = scoreArm(armVariant, metricInfo.metricField);
+
+  // Pass-rate is only a real measurement when validation commands ran. Without
+  // them it's a synthetic ok?1:0, so a pass-rate comparison would falsely tie
+  // (both arms 1.0) — report it as inconclusive instead of a fake tie.
+  const passRateNotMeasured = metricInfo.metricField === 'passRate' && !hasValidation;
+  let winner;
+  let improvement;
+  if (!armBaseline.ok || !armVariant.ok || passRateNotMeasured) {
+    winner = 'inconclusive';
+    improvement = null;
+  } else if (Math.abs(baselineScore - variantScore) <= EPS) {
+    winner = 'tie';
+    improvement = 0;
+  } else {
+    const variantBetter = metricInfo.lowerIsBetter
+      ? variantScore < baselineScore
+      : variantScore > baselineScore;
+    winner = variantBetter ? 'variant' : 'baseline';
+    if (baselineScore === 0) {
+      improvement = null;
+    } else {
+      const ratio = metricInfo.lowerIsBetter
+        ? (baselineScore - variantScore) / Math.abs(baselineScore)
+        : (variantScore - baselineScore) / Math.abs(baselineScore);
+      improvement = round(ratio, 4);
+    }
+  }
+
+  return {
+    schema: SCHEMA,
+    task,
+    metric,
+    metricField: metricInfo.metricField,
+    lowerIsBetter: metricInfo.lowerIsBetter,
+    scoreUnit: metricInfo.scoreUnit,
+    geneId,
+    baselineScore,
+    variantScore,
+    winner,
+    improvement,
+    arms: { baseline: armBaseline, variant: armVariant },
+    meta: {
+      runner: metaRunner || 'unknown',
+      agentCommand: metaCommand || null,
+      startedAt,
+      durationMs: Date.now() - t0,
+      warnings,
+    },
+  };
+}
+
+module.exports = { runComparison, buildVariantPrompt, normalizeArm, SCHEMA };