@evomap/evolver 1.89.20 → 2.0.0-beta.0

package/src/experiment/cli.js

@@ -1,159 +0,0 @@
-// src/experiment/cli.js
-//
-// CLI surface for the `experiment` subcommand. Mirrors src/atp/cli.js:
-//   parseExperimentArgs(args) -> { ok, opts? , error? }
-//   runExperiment(opts, deps) -> { ok, data?, error?, exitCode }
-// Dependency-injected (comparison / agentRunner / geneLoader / sandbox)
-// so the runner is unit-testable without a real agent CLI.
-'use strict';
-
-const fs = require('fs');
-const path = require('path');
-
-const MAX_REQUEST_FILE_BYTES = 1024 * 1024; // 1 MB — a request JSON is tiny
-
-// Minimal `--key=value` / `--flag` parser (no shell, no globbing).
-function parseFlags(args) {
-  const out = {};
-  for (const a of args || []) {
-    if (typeof a !== 'string') continue;
-    const eq = a.indexOf('=');
-    if (a.startsWith('--') && eq > 2) {
-      out[a.slice(2, eq)] = a.slice(eq + 1);
-    } else if (a.startsWith('--')) {
-      out[a.slice(2)] = true;
-    }
-  }
-  return out;
-}
-
-/**
- * Parse experiment subcommand args.
- *
- * Flags: --task= --metric= [--baseline=] [--variant=] [--gene=]
- *        [--validation="cmd1;;cmd2"] [--timeout=ms] [--request-file=<json>]
- *
- * --request-file supplies a JSON base ({task,baseline,variant,metric,geneId,
- * validationCommands,timeoutMs}); explicit flags override it. This is the only
- * filesystem read here and lets the desktop pass complex params without argv
- * escaping headaches.
- *
- * @returns {{ ok: true, opts: object } | { ok: false, error: string }}
- */
-function parseExperimentArgs(args) {
-  const f = parseFlags(args);
-
-  let base = {};
-  if (f['request-file']) {
-    try {
-      // Resolve + stat the path and bound its size before reading: don't open
-      // an arbitrary non-file (device/FIFO) or slurp an unbounded file.
-      const rfPath = path.resolve(String(f['request-file']));
-      const st = fs.statSync(rfPath);
-      if (!st.isFile()) return { ok: false, error: '--request-file must be a regular file' };
-      if (st.size > MAX_REQUEST_FILE_BYTES) return { ok: false, error: '--request-file too large (> 1 MB)' };
-      base = JSON.parse(fs.readFileSync(rfPath, 'utf8'));
-    } catch (e) {
-      return { ok: false, error: 'failed to read --request-file: ' + (e && e.message ? e.message : String(e)) };
-    }
-    if (!base || typeof base !== 'object') {
-      return { ok: false, error: '--request-file must contain a JSON object' };
-    }
-  }
-
-  const pick = (flagVal, baseVal, dflt) => {
-    if (flagVal !== undefined && flagVal !== true) return String(flagVal);
-    if (baseVal !== undefined && baseVal !== null) return String(baseVal);
-    return dflt;
-  };
-
-  const opts = {
-    // trim() so a whitespace-only value fails the required-field check below
-    // (parse-time exit 2 + usage) instead of crashing later in runComparison
-    // (exit 1 with no JSON on stdout for the Go caller).
-    task: pick(f.task, base.task, '').trim(),
-    baseline: pick(f.baseline, base.baseline, 'baseline'),
-    variant: pick(f.variant, base.variant, 'variant'),
-    metric: pick(f.metric, base.metric, '').trim(),
-    geneId: pick(f.gene, base.geneId !== undefined ? base.geneId : base.gene, '') || null,
-    validationCommands: null,
-    timeoutMs: undefined,
-  };
-
-  if (f.validation !== undefined && f.validation !== true) {
-    opts.validationCommands = String(f.validation).split(';;').map((s) => s.trim()).filter(Boolean);
-  } else if (Array.isArray(base.validationCommands)) {
-    opts.validationCommands = base.validationCommands.map(String);
-  }
-
-  const timeoutRaw = f.timeout !== undefined && f.timeout !== true ? Number(f.timeout)
-    : (Number.isFinite(Number(base.timeoutMs)) ? Number(base.timeoutMs) : NaN);
-  if (Number.isFinite(timeoutRaw)) opts.timeoutMs = timeoutRaw;
-
-  if (!opts.task) return { ok: false, error: 'missing required --task (or "task" in --request-file)' };
-  if (!opts.metric) return { ok: false, error: 'missing required --metric (or "metric" in --request-file)' };
-
-  return { ok: true, opts };
-}
-
-/**
- * Run the comparison. Returns the result object (also written to stdout by the
- * index.js wrapper) plus an exit code. exitCode 3 == inconclusive (a real,
- * structured outcome, not a crash).
- *
- * @param {object} opts  from parseExperimentArgs
- * @param {object} [deps] { comparison, agentRunner, geneLoader, sandbox, err }
- */
-async function runExperiment(opts, deps) {
-  deps = deps || {};
-  const comparison = deps.comparison || require('./comparison');
-  const err = typeof deps.err === 'function' ? deps.err : ((...a) => console.error(...a));
-
-  const params = Object.assign({}, opts);
-  if (deps.agentRunner) params.agentRunner = deps.agentRunner;
-  if (deps.geneLoader) params.geneLoader = deps.geneLoader;
-  if (deps.sandbox) params.sandbox = deps.sandbox;
-
-  try {
-    const raw = await comparison.runComparison(params);
-    // Redact any secrets / API keys an agent's resultText (or a parse-failure
-    // snippet) may carry before the result crosses the process boundary into
-    // the desktop consumer / is persisted to disk.
-    const data = require('../gep/sanitize').sanitizePayload(raw);
-    let exitCode = 0;
-    if (data && data.winner === 'inconclusive') {
-      exitCode = 3;
-      // Two distinct causes share the inconclusive verdict; say which one
-      // happened so operators don't go debugging arm failures that never
-      // occurred (both arms can be ok when pass-rate was simply unmeasured).
-      const bothArmsOk = !!(data.arms && data.arms.baseline && data.arms.baseline.ok
-        && data.arms.variant && data.arms.variant.ok);
-      err(bothArmsOk
-        ? '[Experiment] inconclusive: metric resolved to pass-rate but no validation commands were given, so pass-rate was never measured — pass --validation (or pick another --metric)'
-        : '[Experiment] inconclusive: one or both arms failed — see arms.*.error');
-    }
-    return { ok: true, data, exitCode };
-  } catch (e) {
-    err('[Experiment] error: ' + (e && e.message ? e.message : String(e)));
-    return { ok: false, error: e && e.message ? e.message : String(e), exitCode: 1 };
-  }
-}
-
-function printExperimentUsage() {
-  return [
-    'Usage: node index.js experiment --task="..." --metric="..." [flags]',
-    '  --baseline="..."          对照臂标签 (default: baseline)',
-    '  --variant="..."           实验臂标签 (default: variant)',
-    '  --gene=<geneId>           变体臂复用的基因 id (注入其 strategy)',
-    '  --validation="c1;;c2"     自包含 node 校验命令 (通过率评分; ;; 分隔)',
-    '  --timeout=<ms>            单臂超时',
-    '  --request-file=<path>     JSON 基底 (显式 flag 覆盖之)',
-    '',
-    'Runs the same task twice (baseline vs variant-with-gene) via a headless',
-    'agent CLI and prints a comparison JSON to stdout. Logs go to stderr.',
-    'Env: EVOLVER_EXPERIMENT_AGENT_CMD (default claude),',
-    '     EVOLVER_EXPERIMENT_AGENT_ARGS, EVOLVER_EXPERIMENT_TIMEOUT_MS (300000).',
-  ].join('\n');
-}
-
-module.exports = { parseExperimentArgs, runExperiment, printExperimentUsage, parseFlags };

package/src/experiment/comparison.js

@@ -1,233 +0,0 @@
-// src/experiment/comparison.js
-//
-// Thin orchestrator for a comparative experiment: run the SAME task twice --
-// a baseline arm (plain task) and a variant arm (task + the reused gene's
-// strategy injected) -- through a pluggable agent runner, collect real
-// metrics (duration / rounds / tokens / pass-rate), and emit a versioned
-// comparison result.
-//
-// Design notes:
-//  - This module NEVER requires child_process. The agent runner, gene loader,
-//    and sandbox runner are all injectable, so unit tests stay deterministic
-//    (no LLM, no network, no subprocess). Production defaults are lazy-loaded.
-//  - A failed arm never fabricates a score: if either arm is !ok the winner is
-//    'inconclusive' and improvement is null, while still recording whatever
-//    partial metrics were captured.
-'use strict';
-
-const { deriveMetric, scoreArm, num, round } = require('./metrics');
-
-const SCHEMA = 'evolver.experiment.comparison.v1';
-const RESULT_TEXT_CAP = 2000;
-const EPS = 1e-9;
-
-// Build the variant prompt by appending the reused gene's strategy, mirroring
-// the numbered-list format used in src/gep/prompt.js (`${i+1}. ${s}`).
-function buildVariantPrompt(task, gene) {
-  if (!gene || !Array.isArray(gene.strategy) || gene.strategy.length === 0) return task;
-  const steps = gene.strategy.map((s, i) => `${i + 1}. ${s}`).join('\n');
-  return (
-    task +
-    '\n\n## Reuse the following proven strategy\n' +
-    steps +
-    '\n\nApply the strategy above while completing the task.'
-  );
-}
-
-// Coerce whatever the agent runner returned into the canonical arm shape.
-function normalizeArm(label, raw) {
-  raw = raw || {};
-  const tokensIn = num(raw.tokensIn);
-  const tokensOut = num(raw.tokensOut);
-  const tokensTotal = Number.isFinite(Number(raw.tokensTotal)) ? num(raw.tokensTotal) : tokensIn + tokensOut;
-  return {
-    label: String(label == null ? '' : label),
-    ok: !!raw.ok,
-    error: raw.error != null ? String(raw.error) : null,
-    durationMs: num(raw.durationMs),
-    rounds: num(raw.rounds),
-    tokensIn,
-    tokensOut,
-    tokensTotal,
-    costUsd: num(raw.costUsd),
-    passRate: Number.isFinite(Number(raw.passRate)) ? num(raw.passRate) : (raw.ok ? 1 : 0),
-    resultText: typeof raw.resultText === 'string' ? raw.resultText.slice(0, RESULT_TEXT_CAP) : '',
-    exitCode: Number.isFinite(Number(raw.exitCode)) ? num(raw.exitCode) : null,
-    timedOut: !!raw.timedOut,
-  };
-}
-
-// Pass-rate for ONE arm: run its `node <script>` validation commands INSIDE that
-// arm's own workspace (where its agent just ran), so two arms whose agents
-// produced different output get different pass-rates -- the metric is linked to
-// the arm, not to a shared empty sandbox. Each command is a `node <script>`
-// vetted by sandboxExecutor's allowlist (runSingleCommand rejects anything else).
-async function passRateInDir(commands, cwd, runSingleCommand, timeoutMs, warnings) {
-  let passed = 0;
-  let total = 0;
-  for (const cmd of commands) {
-    total += 1;
-    try {
-      const r = await runSingleCommand(cmd, { cwd, timeoutMs });
-      if (r && r.ok) passed += 1;
-    } catch (e) {
-      warnings.push('passrate_command_error: ' + (e && e.message ? e.message : String(e)));
-    }
-  }
-  return total > 0 ? round(passed / total, 4) : 0;
-}
-
-/**
- * Run a two-arm comparison.
- *
- * @param {object}   params
- * @param {string}   params.task                自然语言任务(必填)
- * @param {string}  [params.baseline='baseline'] 对照臂标签
- * @param {string}  [params.variant='variant']   实验臂标签
- * @param {string}   params.metric               评估指标(必填)
- * @param {string}  [params.geneId]              变体臂复用的基因 id
- * @param {string[]}[params.validationCommands]  自包含 `node <script>` 校验命令
- * @param {number}  [params.timeoutMs]           单臂超时
- * @param {function}[params.agentRunner]   (prompt, opts) => Promise<AgentResult>
- * @param {function}[params.geneLoader]    () => Gene[]
- * @param {object}  [params.sandbox]    { createSandboxDir, cleanupDir, runSingleCommand } (default: sandboxExecutor)
- * @returns {Promise<object>} versioned ComparisonResult (see SCHEMA)
- */
-async function runComparison(params) {
-  const p = params || {};
-  const task = String(p.task == null ? '' : p.task).trim();
-  const baseline = p.baseline ? String(p.baseline) : 'baseline';
-  const variant = p.variant ? String(p.variant) : 'variant';
-  const metric = String(p.metric == null ? '' : p.metric);
-  const geneId = p.geneId ? String(p.geneId) : null;
-  const validationCommands = Array.isArray(p.validationCommands)
-    ? p.validationCommands.filter((c) => typeof c === 'string' && c.trim())
-    : null;
-  const timeoutMs = Number.isFinite(Number(p.timeoutMs)) ? Number(p.timeoutMs) : undefined;
-
-  if (!task) throw new Error('task is required');
-  if (!metric) throw new Error('metric is required');
-
-  const agentRunner = typeof p.agentRunner === 'function'
-    ? p.agentRunner
-    : require('./agentRunner').runAgentTask;
-  const geneLoader = typeof p.geneLoader === 'function'
-    ? p.geneLoader
-    : require('../gep/assetStore').loadGenes;
-  const sandbox = p.sandbox && typeof p.sandbox === 'object'
-    ? p.sandbox
-    : require('../gep/validator/sandboxExecutor');
-
-  const startedAt = new Date().toISOString();
-  const t0 = Date.now();
-  const warnings = [];
-
-  const metricInfo = deriveMetric(metric);
-  if (!metricInfo.recognized) warnings.push('metric_unrecognized: ' + metric);
-
-  // Look up the reused gene (variant arm). Without a resolved gene the variant
-  // prompt is identical to the baseline task, so the two arms are NOT a strategy
-  // comparison -- record an explicit warning so identical arms aren't mistaken
-  // for one.
-  let gene = null;
-  if (geneId) {
-    let genes = [];
-    try {
-      genes = geneLoader() || [];
-    } catch (e) {
-      warnings.push('gene_load_error: ' + (e && e.message ? e.message : String(e)));
-    }
-    gene = genes.find((g) => g && String(g.id) === geneId) || null;
-    if (!gene) warnings.push('gene_not_found: ' + geneId + ' (variant arm equals baseline)');
-  } else {
-    warnings.push('no_gene: variant arm equals baseline (no strategy injected)');
-  }
-
-  const hasValidation = !!(validationCommands && validationCommands.length);
-  if (!hasValidation) warnings.push('passrate_degraded_no_validation');
-
-  let metaRunner = null;
-  let metaCommand = null;
-
-  const runArm = async (label, prompt) => {
-    // Each arm runs in its OWN fresh sandbox dir, so the agent works in
-    // isolation (never the evolver repo / process.cwd()) and its pass-rate
-    // validation reads that arm's own output, not a shared empty directory.
-    const workdir = sandbox.createSandboxDir();
-    let raw;
-    try {
-      raw = await agentRunner(prompt, { timeoutMs, cwd: workdir });
-    } catch (e) {
-      raw = { ok: false, error: 'agent_runner_threw: ' + (e && e.message ? e.message : String(e)) };
-    }
-    if (raw) {
-      if (metaRunner == null && raw.runnerName) metaRunner = String(raw.runnerName);
-      if (metaCommand == null && raw.agentCommand) metaCommand = String(raw.agentCommand);
-    }
-    const arm = normalizeArm(label, raw);
-    if (hasValidation) {
-      arm.passRate = await passRateInDir(validationCommands, workdir, sandbox.runSingleCommand, timeoutMs, warnings);
-    }
-    try { sandbox.cleanupDir(workdir); } catch (_) { /* best-effort cleanup */ }
-    return arm;
-  };
-
-  // Arms run sequentially: two real agent CLIs in parallel would contend for
-  // local resources / provider rate limits and muddy the duration metric.
-  const armBaseline = await runArm(baseline, task);
-  const armVariant = await runArm(variant, buildVariantPrompt(task, gene));
-
-  const baselineScore = scoreArm(armBaseline, metricInfo.metricField);
-  const variantScore = scoreArm(armVariant, metricInfo.metricField);
-
-  // Pass-rate is only a real measurement when validation commands ran. Without
-  // them it's a synthetic ok?1:0, so a pass-rate comparison would falsely tie
-  // (both arms 1.0) — report it as inconclusive instead of a fake tie.
-  const passRateNotMeasured = metricInfo.metricField === 'passRate' && !hasValidation;
-  let winner;
-  let improvement;
-  if (!armBaseline.ok || !armVariant.ok || passRateNotMeasured) {
-    winner = 'inconclusive';
-    improvement = null;
-  } else if (Math.abs(baselineScore - variantScore) <= EPS) {
-    winner = 'tie';
-    improvement = 0;
-  } else {
-    const variantBetter = metricInfo.lowerIsBetter
-      ? variantScore < baselineScore
-      : variantScore > baselineScore;
-    winner = variantBetter ? 'variant' : 'baseline';
-    if (baselineScore === 0) {
-      improvement = null;
-    } else {
-      const ratio = metricInfo.lowerIsBetter
-        ? (baselineScore - variantScore) / Math.abs(baselineScore)
-        : (variantScore - baselineScore) / Math.abs(baselineScore);
-      improvement = round(ratio, 4);
-    }
-  }
-
-  return {
-    schema: SCHEMA,
-    task,
-    metric,
-    metricField: metricInfo.metricField,
-    lowerIsBetter: metricInfo.lowerIsBetter,
-    scoreUnit: metricInfo.scoreUnit,
-    geneId,
-    baselineScore,
-    variantScore,
-    winner,
-    improvement,
-    arms: { baseline: armBaseline, variant: armVariant },
-    meta: {
-      runner: metaRunner || 'unknown',
-      agentCommand: metaCommand || null,
-      startedAt,
-      durationMs: Date.now() - t0,
-      warnings,
-    },
-  };
-}
-
-module.exports = { runComparison, buildVariantPrompt, normalizeArm, SCHEMA };

package/src/experiment/metrics.js

@@ -1,75 +0,0 @@
-// src/experiment/metrics.js
-//
-// Pure, table-driven mapping from a human metric label (e.g. "完成耗时 (s)",
-// "轮次", "token", "通过率") onto a per-arm field + comparison direction.
-// No I/O, no side effects -- safe to unit-test in isolation.
-'use strict';
-
-function num(v, fallback) {
-  const n = Number(v);
-  return Number.isFinite(n) ? n : (fallback === undefined ? 0 : fallback);
-}
-
-function round(n, digits) {
-  const f = Math.pow(10, digits);
-  return Math.round((num(n) + Number.EPSILON) * f) / f;
-}
-
-// Ordered rules. The FIRST rule whose any keyword is a (case-insensitive)
-// substring of the metric label wins. Order matters: pass-rate / rounds /
-// tokens / cost are checked before duration so a label like "通过率" is not
-// swallowed by a looser rule.
-const METRIC_RULES = [
-  { keys: ['通过率', 'pass', 'success', 'accuracy', '准确', '正确率'], field: 'passRate', lowerIsBetter: false },
-  { keys: ['轮次', 'turn', 'round', 'step', 'iteration', '迭代'], field: 'rounds', lowerIsBetter: true },
-  { keys: ['token', '令牌'], field: 'tokensTotal', lowerIsBetter: true },
-  { keys: ['成本', 'cost', 'usd', '价格', '费用'], field: 'costUsd', lowerIsBetter: true },
-  { keys: ['耗时', 'duration', 'latency', '延迟', '秒', 'second', '(s)', 'time'], field: 'durationMs', lowerIsBetter: true },
-];
-
-/**
- * Resolve a metric label to the per-arm field used for scoring, the
- * comparison direction, and the display unit.
- *
- * @param {string} metricStr
- * @returns {{ metricField: string, lowerIsBetter: boolean, scoreUnit: string, recognized: boolean }}
- */
-function deriveMetric(metricStr) {
-  const m = String(metricStr || '').toLowerCase();
-  for (const rule of METRIC_RULES) {
-    if (rule.keys.some((k) => m.includes(String(k).toLowerCase()))) {
-      if (rule.field === 'durationMs') {
-        // Seconds-flavoured labels ("(s)", "秒", "seconds") -> report in seconds.
-        if (/\(s\)|秒|second/.test(m)) {
-          return { metricField: 'durationSec', lowerIsBetter: true, scoreUnit: 'seconds', recognized: true };
-        }
-        return { metricField: 'durationMs', lowerIsBetter: true, scoreUnit: 'ms', recognized: true };
-      }
-      return { metricField: rule.field, lowerIsBetter: rule.lowerIsBetter, scoreUnit: 'raw', recognized: true };
-    }
-  }
-  // Unrecognized -> degrade to pass-rate (higher is better). Caller records a warning.
-  return { metricField: 'passRate', lowerIsBetter: false, scoreUnit: 'raw', recognized: false };
-}
-
-/**
- * Pull the scalar score for one arm given the resolved metric field.
- *
- * @param {object} arm  a normalized arm (see comparison.normalizeArm)
- * @param {string} metricField
- * @returns {number}
- */
-function scoreArm(arm, metricField) {
-  if (!arm) return 0;
-  switch (metricField) {
-    case 'durationSec': return round(num(arm.durationMs) / 1000, 2);
-    case 'durationMs': return num(arm.durationMs);
-    case 'rounds': return num(arm.rounds);
-    case 'tokensTotal': return num(arm.tokensTotal);
-    case 'costUsd': return round(num(arm.costUsd), 4);
-    case 'passRate': return round(num(arm.passRate), 4);
-    default: return round(num(arm.passRate), 4);
-  }
-}
-
-module.exports = { deriveMetric, scoreArm, METRIC_RULES, round, num };