clembot-doorman 0.1.0

package/doorman/cli/doorman.mjs

@@ -0,0 +1,345 @@
+#!/usr/bin/env node
+/**
+ * doorman — measure a candidate tool instead of reviewing it.
+ *
+ *   doorman report <link>              L1, static implementation report. No key needed.
+ *   doorman eval <link> --task <file>  L3, two-arm benchmark in a throwaway sandbox.
+ *   doorman allow <server> [--scope user|project] [--why TEXT] [--dry-run]
+      Trust a server by NAME, which is what the gate shows you when it blocks
+      one. Records a DECISION, not a measurement: basis is operator and the
+      grade stays null, because nothing graded it.
+
+      --scope user (default) writes ~/.doorman/registry and covers every
+      project. --scope project writes ./registry and covers this one.
+      Refuses to reverse a denylist entry.
+
+  doorman needs [path]               read this build's own prompt history, propose servers.
+ *   doorman watch [path]               poll the feed for candidates new to THIS build.
+ *   doorman discover                   sweep a public directory for candidates. Curate, never enqueue.
+ *
+ * Zero runtime dependencies, per the requirement in package.json. That is why
+ * the YAML loader and the arg parser are in-tree rather than installed.
+ */
+
+import { report } from './report.mjs';
+import { evaluate } from './eval.mjs';
+import { doctor, renderDoctor } from './doctor.mjs';
+import { watch, renderWatch, readState, writeState, DEFAULT_API, DEFAULT_STATE } from './watch.mjs';
+import { discover, renderDiscover, writeCandidates } from './discover.mjs';
+import { needs as readNeeds, render as renderNeedsCli } from './needs.mjs';
+import { install, renderInstall } from './install.mjs';
+import { allow, renderAllow, SCOPES } from './allow.mjs';
+
+const VERSION = '0.1.0';
+
+const HELP = `
+doorman ${VERSION} — measure a candidate, do not just read it
+
+  doorman doctor [path]
+      L0. What is in YOUR build: which harness, which MCP servers your agents can
+      reach, how many subagents hold MCP tools, and whether the gate is installed
+      AND wired (those are different, and both are quiet).
+      Read-only, local, free. No model, no container, no network.
+
+  doorman install [path] [--dry-run] [--json]
+      Install the security gate (mcp-gate.sh), doorman subagent, /vet command,
+      and doorman skill into a project, safely wiring .claude/settings.json
+      and initializing the registry without overwriting existing trust lists.
+
+  doorman needs [path] [--history DIR] [--candidates FILE] [--json]
+      L0.5. What this build keeps REACHING for, read from its own prompt
+      history, against what it already has. Then the graded feed, matched on
+      capability text the candidates published about themselves.
+
+      Free, keyless, and the history never leaves the machine: the one request
+      is the same anonymous GET /feed that watch makes.
+
+      A need nothing graded covers is printed as a GAP rather than dropped, and
+      a match is only ever worth-measuring. Nothing here drove anything, so
+      nothing here claims a server will work. Only eval answers that.
+
+  doorman report <link> [--out DIR] [--needed-for TEXT]
+      L1. The static implementation report: protocol, schemas, annotations, and
+      a scan-only pass over every description an agent would read before
+      choosing a tool. Needs no model key.
+
+  doorman eval <link> --task <file> [--runs N] [--max-cost USD] [--out DIR]
+      L3. Runs the task N times in two images that differ by exactly one install
+      layer, and reports success rate, turns, tool calls, tokens, cost and wall
+      time per arm. Ends in ADOPT / DECLINE / INCONCLUSIVE.
+
+      Needs ANTHROPIC_API_KEY and a running Docker. Fewer than 3 runs per arm
+      cannot reach ADOPT: one sample cannot be told apart from luck. DECLINE
+      stays reachable at any run count, so a cheap run is still worth doing.
+
+  doorman eval <link> --task <file> --estimate
+      What it WOULD cost. Spends nothing, needs no key, needs no Docker.
+      Run this first. An agent loop resends the whole conversation every turn,
+      so cost grows with the SQUARE of the turn count: on Sonnet, three runs
+      per arm is $54 at worst. The ceiling exists because of that number.
+
+  Candidate links doorman can install as one layer:
+      npm:<package>            or  https://npmjs.com/package/<name>
+      pip:<package>            or  https://pypi.org/project/<name>
+      https://github.com/<owner>/<repo>
+
+Options
+  --agent NAME     which harness the arms drive: claude-code (default), builtin,
+                   or exec. YOUR harness, YOUR key, YOUR machine. doorman passes
+                   the credential straight into a local container and never
+                   stores, logs or transmits it.
+  --exec CMD       the command to run, with --agent exec
+  --runs N         runs per arm (default 3)
+  --max-cost USD   hard ceiling for the whole eval (default 5). The turn cap is
+                   DERIVED from this rather than the other way round, so the
+                   budget decides how long a run may get. Enforced by the same
+                   permit ledger the doorman uses on its own outbound spend.
+  --estimate       print the cost and exit. Spends nothing.
+  --ledger PATH    permit ledger (default evals/.spend-ledger.jsonl)
+  --out DIR        write reports here
+  --model NAME     default claude-sonnet-5
+  --allow-network  let the sandbox reach the network (default: --network none)
+  --json           print the result object instead of prose
+  --version, -v
+  --help,    -h
+`;
+
+/**
+ * Flags that take NO value. Without this list a boolean flag swallows the
+ * positional after it, so `doorman allow --dry-run myserver` parsed as
+ * `dry-run="myserver"` with no server at all, and `doorman needs --json .`
+ * lost the path and then crashed. The flag-then-path order is the one people
+ * type, and it was the broken one.
+ */
+const BOOLEAN_FLAGS = new Set([
+  'json', 'dry-run', 'all', 'estimate', 'help', 'version', 'allow-network',
+  'static-only', 'no-feed',
+]);
+
+function parseArgs(argv) {
+  const out = { _: [] };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a.startsWith('--')) {
+      const key = a.slice(2);
+      const next = argv[i + 1];
+      if (BOOLEAN_FLAGS.has(key) || next === undefined || next.startsWith('--')) out[key] = true;
+      else { out[key] = next; i++; }
+    } else if (a === '-h') out.help = true;
+    else if (a === '-v') out.version = true;
+    else out._.push(a);
+  }
+  return out;
+}
+
+const log = (m) => console.error(`${new Date().toISOString().slice(11, 19)} ${m}`);
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.version) { console.log(VERSION); return; }
+  if (args.help || args._.length === 0) { console.log(HELP); return; }
+
+  // `doctor` is the one command that needs no link: it reads the project.
+
+
+  const [cmd, link] = args._;
+
+  if (cmd === 'doctor') {
+    const d = await doctor(args._[1] || process.cwd());
+    if (!d.ok) { console.error(`doctor: ${d.why}`); process.exitCode = 1; return; }
+    if (args.json) { console.log(JSON.stringify(d, null, 2)); return; }
+    console.log(renderDoctor(d));
+    return;
+  }
+
+  if (cmd === 'install') {
+    const target = args._[1] || process.cwd();
+    const dryRun = Boolean(args['dry-run']);
+    const r = await install(target, { dryRun });
+    if (!r.ok) { console.error(`install: ${r.why}`); process.exitCode = 1; return; }
+    if (args.json) { console.log(JSON.stringify(r, null, 2)); return; }
+    console.log(renderInstall(r));
+    return;
+  }
+
+  if (cmd === 'allow') {
+    const r = allow(args._[1], {
+      scope: typeof args.scope === 'string' ? args.scope : 'user',
+      root: process.cwd(),
+      why: typeof args.why === 'string' ? args.why : null,
+      dryRun: Boolean(args['dry-run']),
+    });
+    if (!r.ok) { console.error(renderAllow(r)); process.exitCode = 2; return; }
+    if (args.json) console.log(JSON.stringify(r, null, 2));
+    else console.log(renderAllow(r));
+    return;
+  }
+
+  if (cmd === 'needs') {
+    let r;
+    try {
+      r = await readNeeds({
+        root: args._[1] || process.cwd(),
+        api: (args.api || DEFAULT_API).replace(/\/+$/, ''),
+        historyDir: typeof args.history === 'string' ? args.history : undefined,
+        candidateFile: typeof args.candidates === 'string' ? args.candidates : 'candidates/smithery.json',
+      });
+    } catch (e) {
+      console.error(`needs: ${e.message}`);
+      process.exitCode = e.code === 3 ? 3 : 1;
+      return;
+    }
+    if (args.json) console.log(JSON.stringify(r, null, 2));
+    else console.log(renderNeedsCli(r));
+    return;
+  }
+
+  if (cmd === 'discover') {
+    const pages = Number(args.pages) > 0 ? Number(args.pages) : 1;
+    const out = args.out || 'candidates/smithery.json';
+    // Reading the feed is free and only used to avoid re-proposing what is
+    // already graded. --no-feed skips it for an offline sweep.
+    const feedApi = args['no-feed'] ? null : (args.api || DEFAULT_API).replace(/\/+$/, '');
+    let r;
+    try {
+      r = await discover({ pages, feedApi });
+    } catch (e) {
+      console.error(`discover: ${e.message}`);
+      process.exitCode = e.code === 3 ? 3 : 1;
+      return;
+    }
+    writeCandidates(out, r);
+    if (args.json) console.log(JSON.stringify(r, null, 2));
+    else console.log(renderDiscover(r, out));
+    return;
+  }
+
+  if (cmd === 'watch') {
+    const root = args._[1] || process.cwd();
+    const api = (args.api || DEFAULT_API).replace(/\/+$/, '');
+    const stateFile = args.state || DEFAULT_STATE;
+    // --since beats the state file, and --all ignores both. Neither writes a
+    // cursor: a one-off look must not move a subscription's place in the feed.
+    const oneOff = Boolean(args.since || args.all);
+    const since = args.all ? null : (args.since || readState(stateFile).since);
+    const limit = Number(args.limit) > 0 ? Number(args.limit) : 50;
+
+    let r;
+    try {
+      r = await watch({ root, api, since, limit });
+    } catch (e) {
+      console.error(`watch: ${e.message}`);
+      process.exitCode = e.code === 3 ? 3 : 1;
+      return;
+    }
+
+    if (args.json) console.log(JSON.stringify(r, null, 2));
+    else console.log(renderWatch(r));
+
+    // Only advance the cursor when this was a real poll AND the feed moved.
+    // An empty page leaves it alone, so nothing can be skipped by a run that
+    // happened to arrive between two grades.
+    if (!oneOff && !args['dry-run'] && r.next_since) {
+      writeState(stateFile, { since: r.next_since, seen: r.candidates.length, updated: new Date().toISOString() });
+      if (!args.json) console.log(`\ncursor saved to ${stateFile}`);
+    } else if (oneOff && !args.json) {
+      console.log('\nOne-off look: the saved cursor was not moved.');
+    }
+    return;
+  }
+
+  if (!['report', 'eval'].includes(cmd)) {
+    console.error(`unknown command "${cmd}". Try: doorman --help`);
+    process.exitCode = 2;
+    return;
+  }
+  if (!link) {
+    console.error(`${cmd} needs a candidate link. Try: doorman --help`);
+    process.exitCode = 2;
+    return;
+  }
+
+  if (cmd === 'report') {
+    const out = args.out || `./doorman-report-${Date.now()}`;
+    const r = await report({ link, out, neededFor: args['needed-for'], log });
+    if (!r.ok) {
+      console.error(`\nreport failed: ${r.why}`);
+      if (r.hint) console.error(`\n${r.hint}`);
+      if (r.detail) console.error(`\n${r.detail}`);
+      process.exitCode = 1;
+      return;
+    }
+    if (args.json) { console.log(JSON.stringify(r.grade, null, 2)); return; }
+    const g = r.grade;
+    console.log('');
+    console.log(`L1 static report for ${link}`);
+    console.log(`  grade         ${g.grade ?? 'n/a'} ${typeof g.score === 'number' ? `(${g.score})` : ''}`);
+    console.log(`  hard fail     ${g.hard_fail ? 'YES' : 'no'}`);
+    console.log(`  written to    ${r.out}`);
+    console.log('');
+    console.log('This is the static layer only. It says nothing about whether an agent');
+    console.log('can actually use the thing: that is what `doorman eval` measures.');
+    return;
+  }
+
+  // eval
+  if (!args.task) {
+    console.error('eval needs --task <file>. Try: doorman --help');
+    process.exitCode = 2;
+    return;
+  }
+  const runs = Number(args.runs ?? 3);
+  if (!Number.isInteger(runs) || runs < 1 || runs > 20) {
+    console.error('--runs must be an integer between 1 and 20');
+    process.exitCode = 2;
+    return;
+  }
+
+  const maxCostUsd = Number(args['max-cost'] ?? 5);
+  if (!Number.isFinite(maxCostUsd) || maxCostUsd <= 0) {
+    console.error('--max-cost must be a positive number of USD');
+    process.exitCode = 2;
+    return;
+  }
+
+  const r = await evaluate({
+    link,
+    taskPath: args.task,
+    runs,
+    out: args.out,
+    model: args.model || 'claude-sonnet-5',
+    allowNetwork: Boolean(args['allow-network']),
+    maxCostUsd,
+    estimateOnly: Boolean(args.estimate),
+    ledgerPath: args.ledger || 'evals/.spend-ledger.jsonl',
+    log,
+  });
+
+  if (r.ok && r.estimateOnly) {
+    const { renderEstimate } = await import('./cost.mjs');
+    console.log('');
+    console.log(renderEstimate(r.estimate));
+    console.log('');
+    console.log(`Your ceiling is $${maxCostUsd.toFixed(2)}. Nothing was spent to tell you this.`);
+    return;
+  }
+
+  if (!r.ok) {
+    console.error(`\neval did not run:\n\n${r.why}`);
+    if (r.detail) console.error(`\n${r.detail}`);
+    // A refusal for a missing prerequisite is not the same as a crash, and the
+    // pipeline reads the code: 3 means "could not measure", 1 means "broke".
+    process.exitCode = r.blocked ? 3 : 1;
+    return;
+  }
+
+  if (args.json) { console.log(JSON.stringify(r, null, 2)); return; }
+  const { renderMarkdown } = await import('./eval.mjs');
+  console.log('');
+  console.log(renderMarkdown(r));
+}
+
+main().catch((e) => {
+  console.error(`doorman: ${e && e.stack ? e.stack : e}`);
+  process.exitCode = 1;
+});

package/doorman/cli/eval.mjs

@@ -0,0 +1,320 @@
+/**
+ * `doorman eval <link> --task <file>` — L3, the two-arm benchmark.
+ *
+ * Same task, N times, in two images that differ by exactly one install layer.
+ * Reports success rate, turns, tool calls, tokens, cost and wall time per arm,
+ * and ends in ADOPT / DECLINE / INCONCLUSIVE.
+ *
+ * The order of the preflight checks below is deliberate. Every one of them can
+ * fail without spending a cent, and they are ordered cheapest-refusal-first, so
+ * a run that cannot produce a real number never builds an image, never starts a
+ * container, and never bills a token.
+ */
+
+import { readFile, mkdir, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parseTaskYaml, validateTask } from './task.mjs';
+import { buildArms, cleanup, dockerAvailable, installLayer, runOnce } from './sandbox.mjs';
+import { decide, summariseArm, MIN_RUNS_FOR_ADOPT } from './verdict.mjs';
+import { openBudget, BudgetError } from '../src/budget.mjs';
+import { DEFAULTS, estimateEval, priceFor, renderEstimate, worstCaseRunUsd } from './cost.mjs';
+
+/**
+ * Fit the run inside the money, rather than fitting the money to the run.
+ *
+ * The turn cap used to be a constant, which meant the BUDGET was whatever 24
+ * turns happened to cost. Backwards: the ceiling is the input, so the turn cap
+ * is derived from it. Returns the largest turn cap whose worst case still fits
+ * the per-run allowance, or null when even a minimal run does not fit.
+ */
+export function turnsWithinBudget(model, perRunAllowanceUsd, floor = 3) {
+  if (!priceFor(model)) return null;
+  if (worstCaseRunUsd(model, { maxTurns: floor }) > perRunAllowanceUsd) return null;
+  let best = floor;
+  for (let t = floor; t <= DEFAULTS.maxTurns; t++) {
+    if (worstCaseRunUsd(model, { maxTurns: t }) <= perRunAllowanceUsd) best = t;
+    else break;
+  }
+  return best;
+}
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+
+export async function evaluate({ link, taskPath, runs, out, model, log, allowNetwork,
+                                maxCostUsd, estimateOnly, ledgerPath }) {
+  const started = Date.now();
+
+  // 1. The task must parse and be checkable. Free.
+  let task;
+  try {
+    task = parseTaskYaml(await readFile(taskPath, 'utf8'));
+  } catch (e) {
+    return { ok: false, why: `task file did not parse: ${e.message}` };
+  }
+  const problems = validateTask(task, path.basename(taskPath));
+  if (problems.length) return { ok: false, why: 'task file is not usable:\n  ' + problems.join('\n  ') };
+  log(`task ${task.id}: ${task.title}`);
+
+  // 2. The candidate must be expressible as one install layer. Free.
+  const layer = installLayer(link);
+  if (layer.kind === 'unsupported') return { ok: false, blocked: 'no-install-layer', why: layer.why };
+  log(`candidate ${layer.kind}: ${layer.spec}`);
+
+  // 3. What can this cost, and does it fit? Free, and needs no key: a price
+  //    you can only see after configuring a credential is not a price you can
+  //    decide on.
+  const estimate = estimateEval({ model, runs });
+  if (estimateOnly) {
+    return { ok: true, estimateOnly: true, estimate, task, layer };
+  }
+  if (!estimate.priced) {
+    return {
+      ok: false, blocked: 'unpriced-model',
+      why:
+        `model "${model}" has no price in cli/cost.mjs, so no ceiling can be enforced.\n` +
+        'Refusing to run: an unknown price is not a free one. That rule already ' +
+        'governs the doorman spending outward, and it governs spending our own ' +
+        'money too.',
+    };
+  }
+
+  const perRunAllowance = maxCostUsd / (runs * 2);
+  const maxTurns = turnsWithinBudget(model, perRunAllowance);
+  if (maxTurns === null) {
+    const floorCost = worstCaseRunUsd(model, { maxTurns: 3 });
+    return {
+      ok: false, blocked: 'budget-too-small',
+      why:
+        `a ceiling of $${maxCostUsd} over ${runs * 2} runs allows ` +
+        `$${perRunAllowance.toFixed(2)} per run, and even a 3-turn run can cost ` +
+        `$${floorCost.toFixed(2)} at worst on ${model}.\n\n` +
+        'Nothing was spent. Options: raise --max-cost, lower --runs, or use a ' +
+        'cheaper model: --model claude-haiku-4-5-20251001 is about a third of ' +
+        'Sonnet. A local-model backend is designed but NOT built; see ' +
+        'evals/ROADMAP.md for the two blockers found on this machine.',
+    };
+  }
+  const perRunWorst = worstCaseRunUsd(model, { maxTurns });
+  log(`budget $${maxCostUsd} => ${maxTurns} turns/run, worst $${perRunWorst.toFixed(2)}/run`);
+
+  // 4. Docker must be up. Free.
+  const d = await dockerAvailable();
+  if (!d.ok) {
+    // A prerequisite that is absent is NOT the candidate failing, and the exit
+    // code has to say which. adopt.md reads 3 as 'could not measure' and 1 as
+    // 'broke'. Recording a stopped Docker daemon as a DECLINE would libel a
+    // tool that was never run.
+    return {
+      ok: false,
+      blocked: 'no-docker',
+      why:
+        'docker is not available, so no sandbox could be built.\n' + d.detail + '\n\n' +
+        'Nothing was built and nothing was spent. Start Docker Desktop and re-run.',
+    };
+  }
+  log(`docker ${d.version}`);
+
+  // 5. A key must exist, or there is nothing to measure.
+  //    Invariant 9: a missing key stops the run and says so. It does NOT fall
+  //    back to a cheaper proxy, because a number produced by a different method
+  //    than the one the report describes is a fabricated number.
+  const apiKey = process.env.ANTHROPIC_API_KEY;
+  if (!apiKey) {
+    return {
+      ok: false,
+      blocked: 'no-key',
+      why:
+        'ANTHROPIC_API_KEY is not set, so no benchmark can run.\n' +
+        'L3 measures an agent doing a task; with no model there is no agent.\n' +
+        'Nothing was built and nothing was spent.\n\n' +
+        '`doorman report <link>` still works and needs no key: it produces the ' +
+        'static layer, and the pipeline records the candidate as statically ' +
+        'measured and behaviourally unmeasured rather than as failed.',
+    };
+  }
+
+  // 6. Build the arms.
+  const harnessSource = await readFile(path.join(HERE, 'harness.mjs'), 'utf8');
+  const arms = await buildArms({ link, harnessSource, log });
+  if (!arms.ok) {
+    if (arms.installFailed) {
+      // A real result: DECLINE without running anything.
+      return {
+        ok: true,
+        verdict: {
+          verdict: 'DECLINE', confidence: 'high',
+          reasons: [`the candidate does not install: ${arms.why}`],
+          note: 'No benchmark was run. A tool that cannot be installed reproducibly cannot be adopted.',
+          deltas: null,
+        },
+        baseline: null, candidate: null, task, layer,
+        install_error: arms.detail, eval_cost_usd: 0,
+        wall_ms: Date.now() - started,
+      };
+    }
+    return { ok: false, why: arms.why, detail: arms.detail };
+  }
+
+  // 7. Run both arms, interleaved.
+  //    Interleaved rather than all-baseline-then-all-candidate, so that any
+  //    drift over the run window (model-side latency, rate limiting, a noisy
+  //    machine) lands on both arms rather than only the second one.
+  const baselineRuns = [];
+  const candidateRuns = [];
+  const netAllowed = Boolean(allowNetwork ?? task.network);
+  log(`network inside the sandbox: ${netAllowed ? 'bridge (declared by the task)' : 'none'}`);
+
+  // The same permit machinery the doorman uses to stop itself overspending
+  // outward, pointed at our own bill. Reserve the WORST case before a run and
+  // settle the actual after: reserving the expected cost would be a limit that
+  // only holds when nothing goes wrong.
+  let budget;
+  try {
+    budget = openBudget({ ledgerPath, perRunUsdc: perRunWorst, perDayUsdc: maxCostUsd });
+  } catch (e) {
+    if (e instanceof BudgetError) return { ok: false, blocked: 'budget-config', why: e.message };
+    throw e;
+  }
+
+  let stoppedEarly = null;
+  try {
+    for (let i = 1; i <= runs; i++) {
+      for (const [armName, image, bucket] of [
+        ['baseline', arms.baseTag, baselineRuns],
+        ['candidate', arms.candTag, candidateRuns],
+      ]) {
+        let permit;
+        try {
+          permit = budget.reserve({ price_usdc: perRunWorst, server: `${armName} run ${i}` });
+        } catch (e) {
+          // Out of ceiling. Stop cleanly with what we have rather than
+          // half-spending into an arm and reporting a lopsided comparison.
+          stoppedEarly = `stopped after ${baselineRuns.length + candidateRuns.length} run(s): ${e.message}`;
+          log(stoppedEarly);
+          break;
+        }
+        log(`run ${i}/${runs}: ${armName}`);
+        const r = await runOnce({ image, task, apiKey, model, netAllowed, maxTurns });
+        bucket.push(r);
+        // settle() commits the RESERVED price, and the reservation is the worst
+        // case. Settling it directly would charge every run $9 when it cost 30
+        // cents, and the ceiling would be gone after six runs that spent two
+        // dollars. So: release the guard, then reserve and settle the ACTUAL.
+        // The guard did its job by existing before the call; the ledger should
+        // record what happened, not what was feared.
+        budget.release(permit, 'run finished, actual cost known');
+        const actual = typeof r.cost_usd === 'number' ? r.cost_usd : 0;
+        try {
+          budget.settle(budget.reserve({ price_usdc: actual, server: `${armName} run ${i}` }), {});
+        } catch (e) {
+          // The actual came in over what remained. Record it and stop: the money
+          // is already spent, and pretending otherwise would corrupt the ledger.
+          stoppedEarly = `actual spend exceeded the ceiling on ${armName} run ${i}: ${e.message}`;
+          log(stoppedEarly);
+        }
+      }
+      if (stoppedEarly) break;
+    }
+  } finally {
+    await cleanup(arms, log);
+  }
+
+  // An A/B with unequal arms is not an A/B. If the ceiling cut one arm short,
+  // drop the unpaired runs rather than comparing 3 against 2.
+  const paired = Math.min(baselineRuns.length, candidateRuns.length);
+  if (paired < baselineRuns.length || paired < candidateRuns.length) {
+    log(`trimming to ${paired} paired run(s) per arm: an unequal A/B is not a comparison`);
+    baselineRuns.length = paired;
+    candidateRuns.length = paired;
+  }
+  if (paired === 0) {
+    return {
+      ok: false, blocked: 'budget-exhausted',
+      why: stoppedEarly || 'the ceiling did not allow a single paired run.',
+    };
+  }
+
+  const baseline = summariseArm(baselineRuns);
+  const candidate = summariseArm(candidateRuns);
+
+  // Anything the sandbox saw that the benchmark did not measure. Egress
+  // observation is not wired yet (see evals/ROADMAP.md), so this is empty and
+  // the report SAYS it is empty rather than implying the check passed.
+  const observations = {};
+
+  const verdict = decide({ baseline, candidate, observations });
+  const evalCost =
+    [...baselineRuns, ...candidateRuns]
+      .map((r) => (typeof r.cost_usd === 'number' ? r.cost_usd : 0))
+      .reduce((a, b) => a + b, 0);
+
+  const result = {
+    ok: true, verdict, baseline, candidate, task, layer,
+    runs_per_arm: paired,
+    runs_requested: runs,
+    stopped_early: stoppedEarly,
+    max_turns: maxTurns,
+    max_cost_usd: maxCostUsd,
+    spent_today_usd: budget.spentToday(),
+    model,
+    network: netAllowed ? 'bridge' : 'none',
+    egress_observed: false,
+    eval_cost_usd: evalCost,
+    wall_ms: Date.now() - started,
+    raw: { baseline: baselineRuns, candidate: candidateRuns },
+  };
+
+  if (out) {
+    await mkdir(out, { recursive: true });
+    await writeFile(path.join(out, 'eval.json'), JSON.stringify(result, null, 2), 'utf8');
+    await writeFile(path.join(out, 'eval.md'), renderMarkdown(result), 'utf8');
+    log(`wrote ${path.join(out, 'eval.json')} and eval.md`);
+  }
+  return result;
+}
+
+const pct = (n) => (typeof n === 'number' ? `${(n * 100).toFixed(0)}%` : 'n/a');
+const num = (n, d = 1) => (typeof n === 'number' ? n.toFixed(d) : 'n/a');
+const delta = (n) => (typeof n === 'number' ? `${n >= 0 ? '+' : ''}${(n * 100).toFixed(0)}%` : 'n/a');
+
+export function renderMarkdown(r) {
+  const v = r.verdict;
+  const L = [];
+  L.push(`# ${v.verdict}: ${r.layer.spec}`);
+  L.push('');
+  L.push(`**Task** \`${r.task.id}\` · **${r.runs_per_arm} run(s) per arm** · model \`${r.model}\``);
+  L.push(`**Eval cost** $${num(r.eval_cost_usd, 4)} · **wall** ${num(r.wall_ms / 1000, 1)}s`);
+  L.push('');
+  for (const why of v.reasons) L.push(`- ${why}`);
+  if (v.note) { L.push(''); L.push(`> ${v.note}`); }
+
+  if (r.baseline && r.candidate) {
+    L.push('');
+    L.push('| Metric | Baseline | Candidate | Delta |');
+    L.push('|---|---:|---:|---:|');
+    L.push(`| success rate | ${pct(r.baseline.success_rate)} | ${pct(r.candidate.success_rate)} | ${delta(v.deltas?.success_rate)} |`);
+    L.push(`| turns | ${num(r.baseline.mean_turns)} | ${num(r.candidate.mean_turns)} | ${delta(v.deltas?.turns)} |`);
+    L.push(`| tool calls | ${num(r.baseline.mean_tool_calls)} | ${num(r.candidate.mean_tool_calls)} | ${delta(v.deltas?.tool_calls)} |`);
+    L.push(`| tokens | ${num(r.baseline.mean_tokens, 0)} | ${num(r.candidate.mean_tokens, 0)} | ${delta(v.deltas?.tokens)} |`);
+    L.push(`| cost USD | ${num(r.baseline.mean_cost_usd, 4)} | ${num(r.candidate.mean_cost_usd, 4)} | ${delta(v.deltas?.cost_usd)} |`);
+    L.push(`| wall ms | ${num(r.baseline.mean_wall_ms, 0)} | ${num(r.candidate.mean_wall_ms, 0)} | ${delta(v.deltas?.wall_ms)} |`);
+    L.push('');
+    L.push('Means are over SUCCESSFUL runs only. A failed run either bailed early or');
+    L.push('spun to the turn cap, and neither is the cost of doing the job.');
+  }
+
+  L.push('');
+  L.push('## What this did not measure');
+  L.push('');
+  L.push(`- **Network egress was not observed.** The sandbox ran with \`--network ${r.network}\`, but`);
+  L.push('  nothing captured what the candidate actually reached. The automatic-DECLINE');
+  L.push('  security clause therefore had no input on this run and did not pass; it did not run.');
+  if (r.runs_per_arm < MIN_RUNS_FOR_ADOPT) {
+    L.push(`- **${r.runs_per_arm} run(s) per arm is below the ${MIN_RUNS_FOR_ADOPT}** needed to separate a real`);
+    L.push('  effect from noise, so ADOPT was not reachable on this run by construction.');
+  }
+  L.push('');
+  L.push('_Generated by `doorman eval`. Numbers come from the runs; the verdict line is a proposal._');
+  return L.join('\n') + '\n';
+}