clembot-doorman 0.1.0

package/doorman/cli/sandbox.mjs

@@ -0,0 +1,177 @@
+/**
+ * The throwaway Docker sandbox, and the one thing that makes an A/B honest.
+ *
+ * Two arms. They must be byte-identical except for a single install layer, or
+ * the comparison measures the difference between two environments rather than
+ * the difference the candidate makes. So both arms are built FROM THE SAME
+ * base image digest, and the candidate arm is literally the baseline image plus
+ * one `RUN` instruction. Nothing else varies: same task file, same model, same
+ * temperature, same seed order, same network policy.
+ *
+ * `--network none` is the default for the task run itself. A candidate that
+ * needs the network has to declare it, and what it actually reaches is then
+ * observable rather than assumed. That is the hook the security clause in
+ * `verdict.mjs` reads.
+ */
+
+import { execFile } from 'node:child_process';
+import { mkdtemp, writeFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+
+const run = (cmd, args, opts = {}) =>
+  new Promise((resolve) => {
+    const child = execFile(cmd, args, { maxBuffer: 32 * 1024 * 1024, ...opts },
+      (err, stdout, stderr) => resolve({
+        ok: !err, code: err?.code ?? 0, stdout: stdout ?? '', stderr: stderr ?? '',
+      }));
+    if (opts.stdin) { child.stdin.write(opts.stdin); child.stdin.end(); }
+  });
+
+export async function dockerAvailable() {
+  const r = await run('docker', ['version', '--format', '{{.Server.Version}}']);
+  return { ok: r.ok, version: r.stdout.trim(), detail: r.ok ? null : (r.stderr || r.stdout).trim() };
+}
+
+/**
+ * Derive the ONE install instruction for the candidate arm.
+ *
+ * Refuses anything it cannot express as a single reproducible layer, rather
+ * than guessing. A candidate that needs a compose file or a running service is
+ * a different lane (see evals/ROADMAP.md, "Platform-class candidates"), and
+ * pretending otherwise would produce an arm that silently installed nothing.
+ */
+export function installLayer(link) {
+  const l = String(link).trim();
+
+  let m = l.match(/^npm:(.+)$/) || l.match(/^https?:\/\/(?:www\.)?npmjs\.com\/package\/(.+?)\/?$/);
+  if (m) return { kind: 'npm', spec: m[1], run: `npm install -g ${JSON.stringify(m[1])}` };
+
+  m = l.match(/^pip:(.+)$/) || l.match(/^https?:\/\/pypi\.org\/project\/(.+?)\/?$/);
+  if (m) return { kind: 'pip', spec: m[1], run: `pip install --no-cache-dir ${JSON.stringify(m[1])}` };
+
+  m = l.match(/^https?:\/\/github\.com\/([^/]+\/[^/]+?)(?:\.git)?\/?$/);
+  if (m) {
+    return {
+      kind: 'git',
+      spec: m[1],
+      run:
+        `git clone --depth 1 https://github.com/${m[1]}.git /opt/candidate && ` +
+        `cd /opt/candidate && ` +
+        `(test -f package.json && npm install --omit=dev || true) && ` +
+        `(test -f requirements.txt && pip install --no-cache-dir -r requirements.txt || true) && ` +
+        `(test -f pyproject.toml && pip install --no-cache-dir . || true)`,
+    };
+  }
+
+  if (/^https?:\/\//.test(l)) {
+    return {
+      kind: 'unsupported',
+      spec: l,
+      run: null,
+      why:
+        'this link is not something a single reproducible install layer can express. ' +
+        'Supported: an npm package (npm:name), a PyPI package (pip:name), or a GitHub repo url. ' +
+        'A hosted MCP endpoint has nothing to install, so use `doorman report` for its ' +
+        'static layer instead, and see evals/ROADMAP.md for the platform-class lane.',
+    };
+  }
+
+  return {
+    kind: 'unsupported', spec: l, run: null,
+    why: 'not a recognised candidate link',
+  };
+}
+
+/** The base image both arms share. Pinned by digest at build time, not by tag. */
+const BASE_DOCKERFILE = `
+FROM node:22-bookworm-slim
+RUN apt-get update && apt-get install -y --no-install-recommends \\
+      git ca-certificates python3 python3-pip \\
+  && rm -rf /var/lib/apt/lists/*
+RUN ln -sf /usr/bin/python3 /usr/bin/python
+WORKDIR /work
+# The harness that runs one task and prints one JSON line of metrics.
+COPY harness.mjs /opt/harness.mjs
+ENV NODE_ENV=production
+`;
+
+export async function buildArms({ link, harnessSource, log }) {
+  const layer = installLayer(link);
+  if (layer.kind === 'unsupported') {
+    return { ok: false, why: layer.why, layer };
+  }
+
+  const dir = await mkdtemp(path.join(tmpdir(), 'doorman-'));
+  const tag = 'doorman-eval-' + Math.random().toString(16).slice(2, 10);
+  try {
+    await writeFile(path.join(dir, 'harness.mjs'), harnessSource, 'utf8');
+    await writeFile(path.join(dir, 'Dockerfile.base'), BASE_DOCKERFILE, 'utf8');
+
+    log(`building baseline arm (${tag}-base)`);
+    const b = await run('docker', ['build', '-f', 'Dockerfile.base', '-t', `${tag}-base`, '.'], { cwd: dir });
+    if (!b.ok) return { ok: false, why: 'baseline image build failed', detail: b.stderr.slice(-2000), layer };
+
+    // The candidate arm IS the baseline image plus exactly one instruction.
+    // Written this way on purpose: it is impossible for the two arms to drift,
+    // because one is derived from the other rather than built alongside it.
+    const candDockerfile = `FROM ${tag}-base\nRUN ${layer.run}\n`;
+    await writeFile(path.join(dir, 'Dockerfile.cand'), candDockerfile, 'utf8');
+
+    log(`building candidate arm (+1 layer: ${layer.kind} ${layer.spec})`);
+    const c = await run('docker', ['build', '-f', 'Dockerfile.cand', '-t', `${tag}-cand`, '.'], { cwd: dir });
+    if (!c.ok) {
+      return {
+        ok: false,
+        why: `the candidate would not install: ${layer.kind} ${layer.spec}`,
+        detail: c.stderr.slice(-2000),
+        layer,
+        // A candidate that cannot be installed is a real result, not an error.
+        installFailed: true,
+      };
+    }
+
+    return { ok: true, baseTag: `${tag}-base`, candTag: `${tag}-cand`, dir, tag, layer };
+  } catch (e) {
+    return { ok: false, why: e.message, layer };
+  }
+}
+
+/** Run the harness once inside an arm. Returns the parsed metrics line. */
+export async function runOnce({ image, task, apiKey, model, netAllowed, maxTurns = 24,
+                               timeoutMs = 300_000 }) {
+  const args = [
+    'run', '--rm', '-i',
+    '--network', netAllowed ? 'bridge' : 'none',
+    '--memory', '2g', '--cpus', '2',
+    '-e', `DOORMAN_MODEL=${model}`,
+    '-e', `DOORMAN_MAX_TURNS=${maxTurns}`,
+    '-e', 'ANTHROPIC_API_KEY',
+    image, 'node', '/opt/harness.mjs',
+  ];
+  const started = Date.now();
+  const r = await run('docker', args, {
+    stdin: JSON.stringify(task),
+    timeout: timeoutMs,
+    env: { ...process.env, ANTHROPIC_API_KEY: apiKey },
+  });
+  const wall_ms = Date.now() - started;
+
+  const line = r.stdout.split('\n').reverse().find((l) => l.trim().startsWith('{'));
+  if (!line) {
+    return { success: false, wall_ms, error: 'harness produced no metrics line', stderr: r.stderr.slice(-800) };
+  }
+  try {
+    return { ...JSON.parse(line), wall_ms };
+  } catch {
+    return { success: false, wall_ms, error: 'harness metrics line was not JSON' };
+  }
+}
+
+export async function cleanup({ tag, dir }, log) {
+  if (tag) {
+    await run('docker', ['image', 'rm', '-f', `${tag}-cand`, `${tag}-base`]);
+    log('removed both arm images');
+  }
+  if (dir) await rm(dir, { recursive: true, force: true });
+}

package/doorman/cli/task.mjs

@@ -0,0 +1,239 @@
+/**
+ * A strict YAML-subset loader for task files.
+ *
+ * `doorman/package.json` states zero runtime dependencies as a REQUIREMENT, not
+ * an achievement: this half is the giveaway, and every dependency is one more
+ * thing that can fail to install on somebody else's machine. So there is no
+ * yaml package here.
+ *
+ * The danger with a hand-rolled parser is not that it fails. It is that it
+ * SILENTLY MIS-PARSES, and a task file that means something slightly different
+ * from what it reads like would corrupt an eval while every number still looked
+ * plausible. So this parser refuses everything it does not fully understand,
+ * with the line number and the reason. There is no permissive fallback and no
+ * best-effort branch.
+ *
+ * The supported subset, in full:
+ *
+ *   key: scalar            strings, integers, true/false, null
+ *   key: "quoted scalar"   single or double quotes, no escapes beyond \" and \\
+ *   key:                   followed by an indented block map, or
+ *     - list item          a list of scalars, or
+ *     - key: value         a list of single-level maps
+ *   # comment              whole-line only
+ *   |                      literal block scalar, for prompts
+ *
+ * Deliberately NOT supported, and rejected rather than approximated: anchors,
+ * aliases, tags, flow collections, multi-document streams, folded scalars,
+ * nested lists, and any indentation that is not a multiple of two spaces.
+ */
+
+const RESERVED = new Set(['<<', '!', '&', '*', '%']);
+
+export class TaskParseError extends Error {
+  constructor(line, message) {
+    super(`line ${line}: ${message}`);
+    this.name = 'TaskParseError';
+    this.line = line;
+  }
+}
+
+/** Scalars are typed narrowly. An unquoted value that looks numeric IS numeric. */
+function scalar(raw, lineNo) {
+  const v = raw.trim();
+  if (v === '') return '';
+  if (v === 'null' || v === '~') return null;
+  if (v === 'true') return true;
+  if (v === 'false') return false;
+
+  const q = v[0];
+  if (q === '"' || q === "'") {
+    if (v.length < 2 || v[v.length - 1] !== q) {
+      throw new TaskParseError(lineNo, 'unterminated quoted string');
+    }
+    const inner = v.slice(1, -1);
+    if (q === '"') {
+      if (/\\(?!["\\])/.test(inner)) {
+        throw new TaskParseError(lineNo, 'only \\" and \\\\ escapes are supported');
+      }
+      return inner.replace(/\\(["\\])/g, '$1');
+    }
+    return inner;
+  }
+
+  if (RESERVED.has(v[0])) {
+    throw new TaskParseError(lineNo, `YAML feature "${v[0]}" is not supported here`);
+  }
+  if (v.includes(': ') || v.endsWith(':')) {
+    throw new TaskParseError(lineNo, 'unquoted colon in a value: quote it');
+  }
+  if (/^-?\d+$/.test(v)) return Number(v);
+  if (/^-?\d*\.\d+$/.test(v)) return Number(v);
+  return v;
+}
+
+/** Split into meaningful lines, keeping original numbers for error messages. */
+function lines(src) {
+  return src.replace(/\r\n/g, '\n').split('\n').map((text, i) => ({ text, no: i + 1 }));
+}
+
+function indentOf(text, lineNo) {
+  const m = text.match(/^( *)/);
+  const n = m[1].length;
+  if (text.includes('\t')) throw new TaskParseError(lineNo, 'tabs are not valid YAML indentation');
+  if (n % 2 !== 0) throw new TaskParseError(lineNo, `indent of ${n} is not a multiple of two`);
+  return n;
+}
+
+/**
+ * Parse a block starting at `i` with exactly `indent` leading spaces.
+ * Returns [value, nextIndex].
+ */
+function parseBlock(all, i, indent) {
+  // A list?
+  const first = all[i];
+  if (first && indentOf(first.text, first.no) === indent && /^ *- /.test(first.text)) {
+    const out = [];
+    while (i < all.length) {
+      const l = all[i];
+      if (l.text.trim() === '' || l.text.trim().startsWith('#')) { i++; continue; }
+      const ind = indentOf(l.text, l.no);
+      if (ind < indent || !/^ *- /.test(l.text)) break;
+      if (ind > indent) throw new TaskParseError(l.no, 'over-indented list item');
+      const body = l.text.slice(ind + 2);
+      if (/^[A-Za-z_][A-Za-z0-9_-]*:/.test(body)) {
+        // A list of single-level maps: `- key: value`, continued by deeper keys.
+        const map = {};
+        const [k, ...rest] = body.split(':');
+        map[k.trim()] = scalar(rest.join(':'), l.no);
+        i++;
+        while (i < all.length) {
+          const n = all[i];
+          if (n.text.trim() === '' || n.text.trim().startsWith('#')) { i++; continue; }
+          const nInd = indentOf(n.text, n.no);
+          if (nInd !== indent + 2 || /^ *- /.test(n.text)) break;
+          const mm = n.text.trim().match(/^([A-Za-z_][A-Za-z0-9_-]*):(.*)$/);
+          if (!mm) throw new TaskParseError(n.no, 'expected key: value inside a list item');
+          map[mm[1]] = scalar(mm[2], n.no);
+          i++;
+        }
+        out.push(map);
+      } else {
+        out.push(scalar(body, l.no));
+        i++;
+      }
+    }
+    return [out, i];
+  }
+
+  // Otherwise a map.
+  const out = {};
+  let sawKey = false;
+  while (i < all.length) {
+    const l = all[i];
+    if (l.text.trim() === '' || l.text.trim().startsWith('#')) { i++; continue; }
+    const ind = indentOf(l.text, l.no);
+    if (ind < indent) break;
+    if (ind > indent) throw new TaskParseError(l.no, 'unexpected indentation');
+    if (/^ *- /.test(l.text)) break;
+
+    const m = l.text.trim().match(/^([A-Za-z_][A-Za-z0-9_-]*):(.*)$/);
+    if (!m) throw new TaskParseError(l.no, `not a supported key: ${l.text.trim().slice(0, 40)}`);
+    const key = m[1];
+    const rest = m[2].trim();
+    if (Object.prototype.hasOwnProperty.call(out, key)) {
+      throw new TaskParseError(l.no, `duplicate key "${key}"`);
+    }
+    sawKey = true;
+
+    if (rest === '|') {
+      // Literal block scalar. Everything more-indented, verbatim, dedented.
+      i++;
+      const buf = [];
+      let blockIndent = null;
+      while (i < all.length) {
+        const b = all[i];
+        if (b.text.trim() === '') { buf.push(''); i++; continue; }
+        const bInd = indentOf(b.text, b.no);
+        if (bInd <= ind) break;
+        if (blockIndent === null) blockIndent = bInd;
+        if (bInd < blockIndent) break;
+        buf.push(b.text.slice(blockIndent));
+        i++;
+      }
+      while (buf.length && buf[buf.length - 1] === '') buf.pop();
+      out[key] = buf.join('\n');
+      continue;
+    }
+
+    if (rest === '') {
+      i++;
+      const [val, next] = parseBlock(all, i, ind + 2);
+      // An empty nested block is an error, not an empty object: it is almost
+      // always a truncated file rather than an intentional blank.
+      if ((Array.isArray(val) && val.length === 0) || (!Array.isArray(val) && Object.keys(val).length === 0)) {
+        throw new TaskParseError(l.no, `key "${key}" opens a block and nothing follows it`);
+      }
+      out[key] = val;
+      i = next;
+      continue;
+    }
+
+    out[key] = scalar(rest, l.no);
+    i++;
+  }
+  if (!sawKey && i < all.length) throw new TaskParseError(all[i].no, 'expected a mapping');
+  return [out, i];
+}
+
+/** Parse the supported subset, or throw with a line number. Never guesses. */
+export function parseTaskYaml(src) {
+  const all = lines(src);
+  if (all.some((l) => l.text.trim() === '---' || l.text.trim() === '...')) {
+    throw new TaskParseError(
+      all.find((l) => l.text.trim() === '---' || l.text.trim() === '...').no,
+      'document markers are not supported: one task per file, no front matter',
+    );
+  }
+  let i = 0;
+  while (i < all.length && (all[i].text.trim() === '' || all[i].text.trim().startsWith('#'))) i++;
+  const [val] = parseBlock(all, i, 0);
+  return val;
+}
+
+/**
+ * The fields a task must carry for an eval to mean anything.
+ *
+ * `success` is required and must be checkable without a model: an eval whose
+ * pass condition is itself a judgement call cannot produce a comparable number
+ * across arms, which is the entire point of running two.
+ */
+export const REQUIRED = ['id', 'title', 'category', 'prompt', 'success'];
+
+export function validateTask(task, where = 'task') {
+  const problems = [];
+  if (task === null || typeof task !== 'object' || Array.isArray(task)) {
+    return [`${where}: the file must be a mapping at the top level`];
+  }
+  for (const k of REQUIRED) {
+    if (!(k in task)) problems.push(`${where}: missing required key "${k}"`);
+  }
+  if (typeof task.prompt === 'string' && task.prompt.trim().length < 20) {
+    problems.push(`${where}: prompt is too short to be a real task`);
+  }
+  if ('runs' in task && (!Number.isInteger(task.runs) || task.runs < 1 || task.runs > 20)) {
+    problems.push(`${where}: runs must be an integer between 1 and 20`);
+  }
+  const s = task.success;
+  if (s !== undefined) {
+    if (typeof s !== 'object' || Array.isArray(s)) {
+      problems.push(`${where}: success must be a mapping of checkable conditions`);
+    } else if (!('file_exists' in s) && !('contains' in s) && !('sections' in s)) {
+      problems.push(
+        `${where}: success needs at least one machine-checkable condition ` +
+        `(file_exists, contains, or sections). A model-judged pass cannot be compared across arms.`,
+      );
+    }
+  }
+  return problems;
+}

package/doorman/cli/verdict.mjs

@@ -0,0 +1,199 @@
+/**
+ * ADOPT / DECLINE / INCONCLUSIVE.
+ *
+ * The whole reason to run two arms is to get a number that a code review cannot
+ * produce. That only works if the verdict refuses to overclaim, so the rules
+ * here are deliberately conservative in one direction and not the other:
+ *
+ *   - DECLINE is cheap to reach. Making a candidate worse, or tripping a
+ *     security signal, is enough on its own.
+ *   - ADOPT is expensive to reach. It needs a real effect AND enough runs to
+ *     tell that effect apart from noise.
+ *   - INCONCLUSIVE is the default, and it is a RESULT rather than a failure.
+ *
+ * ## Why one run per arm can never be ADOPT
+ *
+ * Agent runs are noisy: the same task, same model, temperature 0, still varies
+ * in turns and tokens because tool output and timing vary. With a single run
+ * per arm there is no way to distinguish a real improvement from one lucky
+ * sample, so `MIN_RUNS_FOR_ADOPT` gates it. A cheap one-run eval is still
+ * useful, because DECLINE and the security clause both remain reachable: it can
+ * tell you a candidate is bad, it just cannot tell you one is good.
+ */
+
+/** Below this, an improvement is not distinguishable from run-to-run noise. */
+export const MIN_RUNS_FOR_ADOPT = 3;
+
+/** A cost or turn delta smaller than this is treated as no change. */
+export const MATERIAL_DELTA = 0.10;
+
+export const VERDICTS = ['ADOPT', 'DECLINE', 'INCONCLUSIVE'];
+
+/**
+ * ClemVault's stricter clause, layered ON TOP of the benchmark numbers.
+ *
+ * `.claude/rules/security.md`: undeclared network access or permission drift
+ * observed in the sandbox is an automatic DECLINE regardless of how good the
+ * numbers are. A tool that makes the agent faster while phoning somewhere it
+ * never declared is not a tool worth adopting, and the benchmark cannot see
+ * that, which is exactly why this is a separate gate rather than a weight.
+ */
+export function securityOverride(observations = {}) {
+  const reasons = [];
+  const net = observations.undeclared_network;
+  if (Array.isArray(net) && net.length) {
+    reasons.push(`undeclared network access to ${net.join(', ')}`);
+  }
+  const perm = observations.permission_escalation;
+  if (Array.isArray(perm) && perm.length) {
+    reasons.push(`permission escalation: ${perm.join(', ')}`);
+  }
+  return reasons;
+}
+
+const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);
+
+/** Summarise one arm's runs into the numbers the report compares. */
+export function summariseArm(runs) {
+  const ok = runs.filter((r) => r.success);
+  return {
+    runs: runs.length,
+    successes: ok.length,
+    success_rate: runs.length ? ok.length / runs.length : 0,
+    // Averaged over SUCCESSFUL runs only. A failed run's turn count is not
+    // comparable: it may have bailed early or spun until a cap, and either way
+    // it is not the cost of doing the job.
+    mean_turns: mean(ok.map((r) => r.turns).filter((n) => typeof n === 'number')),
+    mean_tool_calls: mean(ok.map((r) => r.tool_calls).filter((n) => typeof n === 'number')),
+    mean_tokens: mean(ok.map((r) => r.tokens).filter((n) => typeof n === 'number')),
+    mean_cost_usd: mean(ok.map((r) => r.cost_usd).filter((n) => typeof n === 'number')),
+    mean_wall_ms: mean(ok.map((r) => r.wall_ms).filter((n) => typeof n === 'number')),
+  };
+}
+
+const rel = (base, cand) => {
+  if (typeof base !== 'number' || typeof cand !== 'number' || base === 0) return null;
+  return (cand - base) / base;
+};
+
+/**
+ * Compare two summarised arms and return a verdict with its reasoning.
+ *
+ * `observations` carries anything the sandbox saw that the task did not measure,
+ * and it outranks every number below it.
+ */
+export function decide({ baseline, candidate, observations = {} }) {
+  const why = [];
+
+  const blocked = securityOverride(observations);
+  if (blocked.length) {
+    return {
+      verdict: 'DECLINE',
+      confidence: 'high',
+      reasons: blocked.map((r) => `SECURITY: ${r}`),
+      note:
+        'ClemVault rule: a security signal in the sandbox is an automatic DECLINE ' +
+        'regardless of benchmark numbers. This is not a close call and is not ' +
+        'weighed against the deltas.',
+      deltas: null,
+    };
+  }
+
+  const deltas = {
+    success_rate: candidate.success_rate - baseline.success_rate,
+    turns: rel(baseline.mean_turns, candidate.mean_turns),
+    tool_calls: rel(baseline.mean_tool_calls, candidate.mean_tool_calls),
+    tokens: rel(baseline.mean_tokens, candidate.mean_tokens),
+    cost_usd: rel(baseline.mean_cost_usd, candidate.mean_cost_usd),
+    wall_ms: rel(baseline.mean_wall_ms, candidate.mean_wall_ms),
+  };
+
+  // 1. Made it worse. Cheap to reach, deliberately.
+  if (candidate.success_rate < baseline.success_rate) {
+    return {
+      verdict: 'DECLINE',
+      confidence: candidate.runs >= MIN_RUNS_FOR_ADOPT ? 'high' : 'low',
+      reasons: [
+        `success rate fell from ${(baseline.success_rate * 100).toFixed(0)}% to ` +
+        `${(candidate.success_rate * 100).toFixed(0)}%`,
+      ],
+      note:
+        candidate.runs < MIN_RUNS_FOR_ADOPT
+          ? `Only ${candidate.runs} run(s) per arm, so this is a signal rather than a measurement. ` +
+            'It is still a DECLINE: a candidate has to earn adoption, not merely survive it.'
+          : null,
+      deltas,
+    };
+  }
+
+  // 2. Not enough evidence to say anything positive.
+  if (candidate.runs < MIN_RUNS_FOR_ADOPT || baseline.runs < MIN_RUNS_FOR_ADOPT) {
+    return {
+      verdict: 'INCONCLUSIVE',
+      confidence: 'low',
+      reasons: [
+        `${Math.min(candidate.runs, baseline.runs)} run(s) per arm is below the ` +
+        `${MIN_RUNS_FOR_ADOPT} needed to tell an improvement from run-to-run noise`,
+      ],
+      note:
+        'INCONCLUSIVE never promotes. Re-run with more runs, or with a task that ' +
+        'exercises the candidate harder. A cheap run can still DECLINE, which is ' +
+        'why it was worth doing.',
+      deltas,
+    };
+  }
+
+  // 3. Real improvement.
+  if (candidate.success_rate > baseline.success_rate) {
+    why.push(
+      `success rate rose from ${(baseline.success_rate * 100).toFixed(0)}% to ` +
+      `${(candidate.success_rate * 100).toFixed(0)}%`,
+    );
+    return { verdict: 'ADOPT', confidence: 'medium', reasons: why, note: null, deltas };
+  }
+
+  // 4. Same success, materially cheaper or shorter.
+  const cheaper = ['cost_usd', 'tokens', 'turns'].filter(
+    (k) => typeof deltas[k] === 'number' && deltas[k] <= -MATERIAL_DELTA,
+  );
+  const dearer = ['cost_usd', 'tokens', 'turns'].filter(
+    (k) => typeof deltas[k] === 'number' && deltas[k] >= MATERIAL_DELTA,
+  );
+
+  if (cheaper.length && !dearer.length) {
+    return {
+      verdict: 'ADOPT',
+      confidence: 'medium',
+      reasons: [
+        'same success rate, and materially cheaper on ' +
+        cheaper.map((k) => `${k} ${(deltas[k] * 100).toFixed(0)}%`).join(', '),
+      ],
+      note: null,
+      deltas,
+    };
+  }
+
+  if (dearer.length && !cheaper.length) {
+    return {
+      verdict: 'DECLINE',
+      confidence: 'medium',
+      reasons: [
+        'no success-rate gain, and materially more expensive on ' +
+        dearer.map((k) => `${k} +${(deltas[k] * 100).toFixed(0)}%`).join(', '),
+      ],
+      note: 'A tool that costs more and achieves the same is a regression with a nicer name.',
+      deltas,
+    };
+  }
+
+  return {
+    verdict: 'INCONCLUSIVE',
+    confidence: 'medium',
+    reasons: ['no material difference in success rate, cost, tokens or turns'],
+    note:
+      'The candidate neither helped nor hurt on this task. That is a real finding: ' +
+      'either it is not for this workload, or the task does not exercise it. ' +
+      'INCONCLUSIVE never promotes.',
+    deltas,
+  };
+}