@agenticprimitives/devkit 0.0.0-alpha.6 → 0.0.0-alpha.8

package/bin/ap.mjs

@@ -6,10 +6,32 @@
 //   ap upgrade --canary | --pin <version|tag> [--dry-run] [--no-install] [--accept-definitions] [--json]
 //   ap upgrade --pin-definitions <registry> <ctx>/<archetype>[,…]     pin the contracts the product relies on (398 §10.3)
 //   ap conform a2a|mcp <url> [--bearer <token>] [--timeout <ms>] [--json]
+//   ap conform provenance file:<dir>|memory: [--json]     (spec 414 §6 — a provenance store binding, C1–C6)
 //   ap test --live-gates <ledger.json> [--only a,b] [--json]
 //   ap create <dir> --template product-repo [--name <n>] [--pin <version>] [--json]
 //   ap mcp [--root <dir>]                                       the read-only Developer MCP on stdio (spec 398 §10.4)
 //   ap runtime join|run|mcp|status …                            an outside ACP runtime as a workspace member (spec 400 W1)
+//   ap eval <cases.eval.json> --home <url> [--persona <handle> [--client <id>] | --session <bearer>] [--report out.json] [--json]
+//                                                               cases as data against a live Home; deterministic judge; exit 1 on a failing case
+//   ap eval --from-runs --home <url> --persona <handle> [--addressee <name|0x>] [--out cases.eval.json]   a dataset from the agent's live runs
+//   ap eval window open --store <dir> --home <url> --roster <handle,…> --level L0|L1|L2 [--hours 24] [--purpose …]   (spec 414 §8) open a capture window
+//   ap eval window close --store <dir>                          close it: the salt is purged, nothing exported can be re-linked
+//   ap eval rescore --store <dir> --plan-id <id> --replay <set.json> --criterion <gold.json>   score a stored experiment again (the runs are not re-run)
+//   ap eval report --store <dir> --sets <dir> --out <report.json> [--definition <url|file>]   (spec 415 §5b; --definition flags test sentences the skills carry as examples) what was tested, whether each intent got the right skill,
+//                                                               per-skill results, comparisons in words, and recommendations — what the Lab and skill-web render
+//   ap eval gates --store <dir> [--plan-id <id>,… | --all] --sets <dir> --definition <url|file> [--gate rbox|calibration|consistency|recall|behaviour|conformal --map <map.json> [--variant v] | outcome --outcome-gold <file>]
+//   ap eval scorecard --store <dir> --sets <dir> --plan-id a,b [--out f.json]   (spec 416 §4g) the selection scorecard per experiment × variant
+//   ap eval calibrate --store <dir> --plan-id a,b --variant v --sets <dir> --definition <url|file> --alpha 0.05 --out <map.json>   (spec 416 W3) fit a calibration map on labelled runs   (spec 415 A4)
+//                                                               rbox agreement (the exact half re-derives) + judgment calibration (the probabilistic half); exit 1 on FAIL
+//   ap eval diff --store <dir> --before <plan-id> --after <plan-id> --replay <set.json> --criterion <gold.json> [--variant model | --variant-before a --variant-after b] [--repaired-pair a,b]
+//                                                               two stored experiments on the same set: McNemar on the paired cases, Δ macro with its CI, collateral
+//   ap eval compare --replay <set.json> --criterion <gold.json> --variants <name|file,…> --store <dir> --home <url> [--fixtures <starting-states.json>]
+//                   --persona <handle> --addressee <name|0x> [--split held-out|development|all] [--repeat n] [--skill-map <map.json>]
+//                   [--plan-id <id>] [--policy primary|any-fire] [--json]           (spec 415 A4) a pre-registered comparison, judged from the traces
+//                   [--keep-text <dir>]   keep every case's REPLY and ACTIVITY TREE under <dir>/<plan-id>/ — words included, OUTSIDE the
+//                                         store (which never holds them); only when the open window admits the addressee (a sandbox org)
+//   ap generate client|cli|parity --registry <url> --definition <ctx>/<archetype> | --file <definition.json> [--out <path>] [--only a,b]
+//                                                               the typed client / CLI schema / parity cases FROM the compiled contract (spec 398 §7.2)
 //
 // Exit: 0 clean · 1 findings · 2 usage/config · 3 a rule or dispatch errored.
 import { spawnSync } from 'node:child_process';
@@ -18,7 +40,7 @@ import { resolve } from 'node:path';
 import {
   DevkitError, EXIT, RULE_IDS, runDoctor, formatDoctorReport, loadDoctorConfig, loadRulesCorpus, projectRules, rulesLockEntries,
   readLock, writeLock, planUpgrade, applyUpgradePlan, formatUpgradePlan, loadConform, loadLiveGatesRunner, loadCreateApp, loadRuntimeCli, formatConformanceReport,
-  runDeveloperMcpStdio, reviewDefinitions, pinDefinition, definitionKey, formatAuthorityChanges,
+  runDeveloperMcpStdio, reviewDefinitions, pinDefinition, definitionKey, formatAuthorityChanges, loadHarnessProjections, fetchDefinitionFromRegistry, loadEval, loadCompare, loadEvalStore, loadDecideFromGrounding, loadProposeFromGrounding, loadHarnessCalibration,
 } from '../dist/index.js';
 
 const argv = process.argv.slice(2);
@@ -39,7 +61,7 @@ const fail = (e) => {
   process.exit(code === 'E_USAGE' || code === 'E_CONFIG' || code === 'E_RULE_UNKNOWN' ? EXIT.USAGE : EXIT.ERROR);
 };
 
-const USAGE = `ap <doctor|upgrade|conform|test|create|mcp|runtime> … (see the header of bin/ap.mjs). Rules: ${RULE_IDS.join(', ')}`;
+const USAGE = `ap <doctor|upgrade|conform|test|create|mcp|runtime|generate|eval> … (see the header of bin/ap.mjs). Rules: ${RULE_IDS.join(', ')}`;
 
 async function doctor() {
   if (flag('rules')) {
@@ -132,7 +154,7 @@ async function upgrade() {
 
 async function conform() {
   const [kind, url] = positional();
-  if (!['a2a', 'mcp'].includes(kind) || !url) throw new DevkitError('E_USAGE', 'ap conform a2a|mcp <url>');
+  if (!['a2a', 'mcp', 'provenance'].includes(kind) || !url) throw new DevkitError('E_USAGE', 'ap conform a2a|mcp <url> | ap conform provenance file:<dir>|memory:');
   const fn = await loadConform(kind);
   const report = await fn(url, { bearer: opt('bearer') ?? process.env.AP_CONFORM_BEARER, timeoutMs: opt('timeout') ? Number(opt('timeout')) : undefined });
   out({ ok: report.ok, report }, formatConformanceReport(report));
@@ -170,7 +192,562 @@ async function runtime() {
   process.exit(await fn(process.argv.slice(3)));
 }
 
-const commands = { doctor, upgrade, conform, test, create, mcp, runtime };
+// Spec 398 §7.2 — the fourth and fifth surfaces of ONE contract: a typed client operation, a CLI argument schema, and
+// the parity cases — projected from a compiled definition (the registry's, or a file), never authored beside it.
+async function generate() {
+  const [what] = positional();
+  if (!what || !['client', 'cli', 'parity'].includes(what)) throw new DevkitError('E_USAGE', 'ap generate client|cli|parity --registry <url> --definition <ctx>/<archetype> | --file <definition.json> [--out <path>] [--only a,b]');
+  let definition;
+  if (opt('file')) {
+    const raw = JSON.parse(readFileSync(resolve(root, opt('file')), 'utf8'));
+    definition = raw.definition ?? raw;
+  } else {
+    const registry = opt('registry'); const def = opt('definition');
+    if (!registry || !def || !def.includes('/')) throw new DevkitError('E_USAGE', '--registry <url> and --definition <context>/<archetype> (or --file <definition.json>) are required');
+    const [context, archetype] = def.split('/');
+    const r = await fetch(`${registry.replace(/\/$/, '')}/context/contexts/${encodeURIComponent(context)}/archetypes/${encodeURIComponent(archetype)}/definition`, { headers: { accept: 'application/json', 'user-agent': 'ap-devkit' } });
+    if (!r.ok) throw new DevkitError('E_INPUT_MISSING', `${registry}: ${def} answered ${r.status}`);
+    const b = await r.json();
+    if (!b.definition?.tools) throw new DevkitError('E_INPUT_MISSING', `${registry}: ${def} served no definition`);
+    definition = b.definition;
+  }
+  const only = opt('only') ? new Set(opt('only').split(',').map((s) => s.trim()).filter(Boolean)) : undefined;
+  const p = await loadHarnessProjections();
+  let text; let obj;
+  if (what === 'client') { const c = p.generateCapabilityClient(definition, only ? { only } : {}); text = c.source; obj = { ok: true, operations: c.operations, source: c.source }; }
+  else if (what === 'cli') { const tools = (definition.tools ?? []).filter((t) => !only || only.has(t.capability?.id ?? t.id)); obj = { ok: true, schemas: tools.map((t) => p.cliArgumentSchema(t)) }; text = JSON.stringify(obj.schemas, null, 2); }
+  else { const cases = p.generateParityCases(definition, only ? { only } : {}); obj = { ok: true, cases }; text = JSON.stringify(cases, null, 2); }
+  if (opt('out')) { const { writeFileSync, mkdirSync } = await import('node:fs'); const { dirname } = await import('node:path'); const outPath = resolve(root, opt('out')); mkdirSync(dirname(outPath), { recursive: true }); writeFileSync(outPath, text.endsWith('\n') ? text : `${text}\n`); out({ ...obj, out: outPath }, `ap generate ${what} → ${outPath}`); }
+  else out(obj, text);
+}
+
+// Spec 398 §10 / P1.3 — EVALS AS A PRODUCT. Cases are data; the ask goes to a live Home as a persona (a development
+// estate's demo-signin) or a session bearer the developer holds; the judge is deterministic. An eval never signs: an
+// act's right outcome is `authority_required`, asserted as a check.
+async function evalCmd() {
+  const sub = positional()[0];
+  if (sub === 'window') return evalWindow();
+  if (sub === 'compare') return evalCompare();
+  if (sub === 'rescore') return evalRescore();
+  if (sub === 'diff') return evalDiff();
+  if (sub === 'report') return evalReport();
+  if (sub === 'gates') return evalGates();
+  if (sub === 'calibrate') return evalCalibrate();
+  if (sub === 'scorecard') return evalScorecard();
+  if (sub === 'ledger') return evalLedger();
+  const home = (opt('home') ?? process.env.HOME_URL ?? '').replace(/\/$/, '');
+  if (!home) throw new DevkitError('E_USAGE', '--home <url> (or HOME_URL) is required');
+  const ev = await loadEval();
+  const j = async (r) => { const t = await r.text(); try { return JSON.parse(t); } catch { return { _raw: t.slice(0, 200), _status: r.status }; } };
+  const sessions = new Map();
+  const signin = async (handle) => {
+    if (sessions.has(handle)) return sessions.get(handle);
+    const s = await j(await fetch(`${home}/connect/demo-signin`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ handle, client_id: opt('client') ?? 'demo-jp' }) }));
+    if (!s.homeSession) throw new DevkitError('E_INPUT_MISSING', `${home} did not sign ${handle} in (a persona path exists on development estates only): ${JSON.stringify(s).slice(0, 160)}`);
+    const v = { token: String(s.homeSession), agent: String(s.agent).toLowerCase() }; sessions.set(handle, v); return v;
+  };
+  const sessionOf = async (persona) => (opt('session') ? { token: opt('session'), agent: '' } : signin(persona ?? opt('persona') ?? 'alice'));
+  const csrfRes = await fetch(`${home}/a2a/auth/csrf`, { headers: { origin: home } });
+  const csrf = await j(csrfRes);
+  const H = { 'content-type': 'application/json', origin: home, cookie: (csrfRes.headers.get('set-cookie') ?? '').split(';')[0] ?? '', 'x-csrf-token': csrf.token ?? '' };
+  const resolve = async (name, fallback) => {
+    if (!name) return fallback; if (/^0x[0-9a-fA-F]{40}$/.test(name)) return name.toLowerCase();
+    const r = await j(await fetch(`${home}/connect/name-info?name=${encodeURIComponent(name)}`)); if (!r.agent) throw new DevkitError('E_INPUT_MISSING', `${name} does not resolve at ${home}`); return String(r.agent).toLowerCase();
+  };
+  if (flag('from-runs')) {
+    const s = await sessionOf(opt('persona'));
+    const addressee = await resolve(opt('addressee'), s.agent);
+    const out = await j(await fetch(`${home}/a2a/harness/records`, { method: 'POST', headers: H, body: JSON.stringify({ session: s.token, addressee }) }));
+    const file = ev.casesFromRunRecords(out.records ?? [], { persona: opt('persona') ?? 'alice', addressee: opt('addressee') });
+    const text = JSON.stringify(file, null, 2);
+    if (opt('out')) { const { writeFileSync } = await import('node:fs'); writeFileSync(resolve_(opt('out')), `${text}\n`); out_({ ok: true, cases: file.cases.length, out: resolve_(opt('out')) }, `ap eval --from-runs → ${file.cases.length} case skeleton(s) in ${opt('out')} — edit the checks to what must hold`); }
+    else console.log(text);
+    return;
+  }
+  const [path] = positional();
+  if (!path) throw new DevkitError('E_USAGE', 'ap eval <cases.eval.json> --home <url> [--persona <handle> | --session <bearer>] [--report out.json]');
+  const file = ev.parseEvalFile(JSON.parse(readFileSync(resolve_(path), 'utf8')));
+  const report = await ev.runEvalFile(file, async (c) => {
+    const s = await sessionOf(c.persona);
+    const addressee = await resolve(c.addressee, s.agent);
+    const r = await j(await fetch(`${home}/a2a/harness/ask`, { method: 'POST', headers: H, body: JSON.stringify({ session: s.token, addressee, message: c.message, ...(c.plan ? { plan: c.plan } : {}), runRef: `eval-${Date.now().toString(36)}` }) }));
+    return ev.outcomeOfReply(r.reply ?? r);
+  });
+  if (opt('report')) { const { writeFileSync } = await import('node:fs'); writeFileSync(resolve_(opt('report')), `${JSON.stringify({ file: file.name ?? path, home, at: new Date().toISOString(), ...report }, null, 2)}\n`); }
+  out_({ ok: report.failed === 0, passed: report.passed, failed: report.failed, results: report.results.map((r) => ({ id: r.id, passed: r.passed, failures: r.failures, ms: r.ms, kind: r.outcome?.kind })) }, `ap eval · ${file.name ?? path} · ${home}\n${ev.formatTruthReport(report)}`);
+  process.exit(report.failed ? EXIT.FINDINGS : EXIT.OK);
+}
+const resolve_ = (p) => resolve(root, p);
+const out_ = out;
+
+// ── Spec 414 §8 / 415 A4 — THE ESTATE, AS A COMPARISON RUNNER SEES IT: the persona sign-in, the CSRF pair and the three
+// reads (ask · provenance · measures), shared by `compare` and `window`. Names resolve to addresses at the Home; the
+// runner holds no key and signs nothing — a run that needs authority parks at the Home like any other.
+async function estateClient(home) {
+  const j = async (r) => { const t = await r.text(); try { return JSON.parse(t); } catch { return { _raw: t.slice(0, 200), _status: r.status }; } };
+  const sessions = new Map();
+  const signin = async (handle) => {
+    if (sessions.has(handle)) return sessions.get(handle);
+    const s = await j(await fetch(`${home}/connect/demo-signin`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ handle, client_id: opt('client') ?? 'demo-jp' }) }));
+    if (!s.homeSession) throw new DevkitError('E_INPUT_MISSING', `${home} did not sign ${handle} in (a persona path exists on development estates only): ${JSON.stringify(s).slice(0, 160)}`);
+    const v = { token: String(s.homeSession), agent: String(s.agent).toLowerCase() }; sessions.set(handle, v); return v;
+  };
+  // The CSRF pair is cookie-bound and EXPIRES: a comparison longer than its lifetime once recorded 572 runs refused
+  // with "csrf invalid". A refusal naming the CSRF fetches a fresh pair and sends the SAME request once more — a
+  // bounded retry of the same call, never a different mechanism (ADR-0013).
+  const H = { 'content-type': 'application/json', origin: home, cookie: '', 'x-csrf-token': '' };
+  const refreshCsrf = async () => {
+    const csrfRes = await fetch(`${home}/a2a/auth/csrf`, { headers: { origin: home } });
+    const csrf = await j(csrfRes);
+    H.cookie = (csrfRes.headers.get('set-cookie') ?? '').split(';')[0] ?? ''; H['x-csrf-token'] = csrf.token ?? '';
+  };
+  await refreshCsrf();
+  const post = async (path, body) => {
+    const once = async () => j(await fetch(`${home}/a2a${path}`, { method: 'POST', headers: H, body: JSON.stringify(body) }));
+    const r = await once();
+    if (r && r.ok !== true && /csrf/i.test(String(r.error ?? r._raw ?? ''))) { await refreshCsrf(); return once(); }
+    return r;
+  };
+  const resolveName = async (name) => {
+    if (/^0x[0-9a-fA-F]{40}$/.test(name)) return name.toLowerCase();
+    const r = await j(await fetch(`${home}/connect/name-info?name=${encodeURIComponent(name)}`));
+    if (r.agent) return String(r.agent).toLowerCase();
+    // a persona handle: its own agent
+    try { return (await signin(name)).agent; } catch { throw new DevkitError('E_INPUT_MISSING', `${name} does not resolve at ${home}`); }
+  };
+  const comparison = await j(await fetch(`${home}/a2a/harness/comparison`).catch(() => new Response('{}')));
+  return { j, signin, post, resolveName, evalCapture: comparison?.evalCapture === 'on' ? 'on' : 'off', knob: comparison };
+}
+const windowPaths = (store) => ({ dir: resolve_(store.replace(/^file:/, '')), window: resolve(resolve_(store.replace(/^file:/, '')), 'window.json'), salt: resolve(resolve_(store.replace(/^file:/, '')), 'window.salt') });
+
+async function evalWindow() {
+  const [, action] = positional();
+  const store = opt('store');
+  if (!store || !['open', 'close'].includes(action ?? '')) throw new DevkitError('E_USAGE', 'ap eval window open|close --store <dir> …');
+  const { mkdirSync, writeFileSync, unlinkSync, readFileSync: rf, existsSync: ex } = await import('node:fs');
+  const { randomBytes } = await import('node:crypto');
+  const cm = await loadCompare();
+  const paths = windowPaths(store);
+  if (action === 'close') {
+    if (!ex(paths.window)) throw new DevkitError('E_INPUT_MISSING', `no window at ${paths.window}`);
+    const w = JSON.parse(rf(paths.window, 'utf8'));
+    w.expiresAt = new Date().toISOString();
+    writeFileSync(paths.window, `${JSON.stringify(w, null, 2)}\n`);
+    if (ex(paths.salt)) unlinkSync(paths.salt);
+    out_({ ok: true, closed: w.id, saltPurged: true }, `ap eval window · closed ${w.id} — the salt is purged; nothing exported under it can be re-linked`);
+    return;
+  }
+  const home = (opt('home') ?? process.env.HOME_URL ?? '').replace(/\/$/, '');
+  if (!home) throw new DevkitError('E_USAGE', '--home <url> (or HOME_URL) is required');
+  const level = opt('level') ?? 'L1';
+  const roster = (opt('roster') ?? '').split(',').map((s) => s.trim()).filter(Boolean);
+  if (!roster.length) throw new DevkitError('E_USAGE', '--roster <handle|name|0x,…> names the test personas and the sandbox org the window admits');
+  const est = await estateClient(home);
+  const addresses = [];
+  for (const n of roster) addresses.push(await est.resolveName(n));
+  const hours = Number(opt('hours') ?? 24);
+  const opensAt = new Date();
+  const w = { type: 'ap.eval-capture-window.v1', id: `w-${opensAt.toISOString().slice(0, 19).replace(/[-:T]/g, '')}`, store: `file:${paths.dir}`, level, roster: addresses, estate: home, opensAt: opensAt.toISOString(), expiresAt: new Date(opensAt.getTime() + hours * 3600_000).toISOString(), ...(opt('purpose') ? { purpose: opt('purpose') } : {}), ...(flag('synthetic') ? { synthetic: true } : {}) };
+  const v = cm.validateCaptureWindow(w);
+  if (!v.ok) throw new DevkitError('E_USAGE', `the window is not valid: ${v.errors.join('; ')}`);
+  mkdirSync(paths.dir, { recursive: true });
+  writeFileSync(paths.window, `${JSON.stringify(w, null, 2)}\n`);
+  writeFileSync(paths.salt, randomBytes(32).toString('hex'), { mode: 0o600 });
+  const digest = await cm.windowDigest(w);
+  out_({ ok: true, window: w, digest, estateCapture: est.evalCapture }, `ap eval window · opened ${w.id} (${digest.slice(0, 18)}…) at ${level} for ${hours} h · roster ${roster.join(', ')} · estate ${home} (EVAL_CAPTURE ${est.evalCapture})${est.evalCapture !== 'on' ? '\n  ! this estate does not run comparisons: every export will be refused (estate.capture-off) until EVAL_CAPTURE=on' : ''}`);
+}
+
+// Spec 415 A4 — SCORE A STORED EXPERIMENT AGAIN. The runs, the judgments and the export are what they were; only the
+// scoring step re-runs (a scorer fix, a threshold change). A run's evidence KIND is a property of how the trace was
+// read — a stamping harness that selected nothing is a stamped observation of no selection — so a record written before
+// that rule ("none" on a `tn` run) is read under it here, and the record says it was rescored.
+async function evalRescore() {
+  const store = opt('store'); const planId = opt('plan-id');
+  for (const [k, v] of [['store', store], ['plan-id', planId], ['replay', opt('replay')], ['criterion', opt('criterion')]]) if (!v) throw new DevkitError('E_USAGE', `--${k} is required (ap eval rescore --store <dir> --plan-id <id> --replay <set> --criterion <gold>)`);
+  const { readFileSync: rf, writeFileSync } = await import('node:fs');
+  const cm = await loadCompare();
+  const dir = resolve_(store.replace(/^file:/, ''));
+  const path = resolve(dir, 'experiments', `${encodeURIComponent(`experiment:${planId}`)}.jsonld`);
+  const doc = JSON.parse(rf(path, 'utf8'));
+  const rec = doc.record;
+  if (!rec?.plan) throw new DevkitError('E_INPUT_MISSING', `${path} holds no experiment record`);
+  const setR = cm.validateReplaySet(JSON.parse(rf(resolve_(opt('replay')), 'utf8')));
+  if (!setR.ok) throw new DevkitError('E_USAGE', `the replay set is not valid: ${setR.errors.join('; ')}`);
+  const criterion = JSON.parse(rf(resolve_(opt('criterion')), 'utf8'));
+  const setDigest = await cm.replaySetDigest(setR.set);
+  if (rec.plan.replaySetDigest !== setDigest || rec.plan.criterionDigest !== await cm.criterionDigest(criterion)) throw new DevkitError('E_USAGE', 'the set or the criterion is not the one this experiment pre-registered; a rescore never changes what was judged against');
+  const { cases, skillIds } = cm.casesOf(setR.set, criterion, rec.plan.split);
+  const runs = rec.runs.map((r) => (r.status === 'ok' && r.evidence === 'none' ? { ...r, evidence: 'stamped' } : r));
+  const scored = await cm.scoreExperimentRuns({ plan: rec.plan, setDigest, cases, skillIds, pairs: criterion.collisionPairs ?? [], runs, log: (line) => { if (!json) console.error(`  ${line}`); } });
+  const next = { ...rec, runs, ...scored, rescoredAt: new Date().toISOString() };
+  writeFileSync(path, `${JSON.stringify({ ...doc, record: next }, null, 2)}\n`);
+  out_({ ok: true, plan: next.plan, planDigest: next.planDigest, runs: next.runs.length, scores: Object.fromEntries(Object.entries(next.scores).map(([k, v]) => [k, v.aggregate])), perSkill: Object.fromEntries(Object.entries(next.scores).map(([k, v]) => [k, v.perSkill])), verdicts: Object.fromEntries(Object.entries(next.verdicts).map(([k, v]) => [k, { status: v.status, reason: v.binding_reason }])), comparisons: next.comparisons }, `ap eval rescore · ${path}\n${cm.formatExperiment(next)}`);
+}
+
+// Spec 415 §5b — THE REPORT: every experiment in the store, read with the replay sets and criteria it pre-registered
+// (found in --sets by digest), folded into one deterministic file. Numbers, ids, verdicts and the sets' AUTHORED test
+// sentences — the eval store holds no person's words, so neither does this.
+async function evalReport() {
+  const store = opt('store'); const setsDir = opt('sets'); const outPath = opt('out');
+  for (const [k, v] of [['store', store], ['sets', setsDir], ['out', outPath]]) if (!v) throw new DevkitError('E_USAGE', `--${k} is required (ap eval report --store <dir> --sets <dir> --out <report.json>)`);
+  const { readFileSync: rf, readdirSync, writeFileSync, mkdirSync } = await import('node:fs');
+  const { dirname } = await import('node:path');
+  const cm = await loadCompare();
+  const dir = resolve_(store.replace(/^file:/, ''));
+  const expDir = resolve(dir, 'experiments');
+  const experiments = readdirSync(expDir).filter((f) => f.startsWith(encodeURIComponent('experiment:')) && f.endsWith('.jsonld'))
+    .map((f) => JSON.parse(rf(resolve(expDir, f), 'utf8')).record).filter((r) => r?.plan);
+  const sets = {}; const criteria = {};
+  const walk = (d) => { for (const e of readdirSync(d, { withFileTypes: true })) { const p = resolve(d, e.name); if (e.isDirectory()) walk(p); else if (e.name.endsWith('.replay.json') || e.name.endsWith('.gold.json')) { const j = JSON.parse(rf(p, 'utf8')); if (j.type === 'ap.replay-set.v1') sets[p] = j; if (j.type === 'ap.selection-criterion.v1') criteria[j.replaySetDigest] = j; } } };
+  walk(resolve_(setsDir));
+  const byDigest = {};
+  for (const s of Object.values(sets)) byDigest[await cm.replaySetDigest(s)] = s;
+  // The examples the skills under test declare (their `utterances`, as the compiled playbook carries them) — so a test
+  // sentence that nearly repeats one is flagged as measuring the example, not selection. A URL or a file.
+  let examples; let toolToSkillForReport;
+  if (opt('definition')) {
+    const src = opt('definition');
+    const d = /^https?:/.test(src) ? await (await fetch(src, { headers: { 'user-agent': 'Mozilla/5.0 (ap eval report)' } })).json() : JSON.parse(rf(resolve_(src), 'utf8'));
+    const tools = (d.definition ?? d).tools ?? [];
+    examples = tools.flatMap((t) => (t.utterances ?? []).map((u) => ({ skill: t.source?.skillId ?? t.id, says: u.says, ...(u.isNot ? { isNot: true } : {}) })));
+    toolToSkillForReport = Object.fromEntries(tools.filter((t) => t.source?.skillId).map((t) => [t.id, t.source.skillId]));
+  }
+  // `--alias <experiment>:<variant>=<name>[,…]` — a knob value whose meaning changed after the experiment ran is shown
+  // as what actually ran. The record is never edited.
+  const aliases = {};
+  for (const a of (opt('alias') ?? '').split(',').filter(Boolean)) { const m = /^([^:]+):([^=]+)=(.+)$/.exec(a); if (!m) throw new DevkitError('E_USAGE', `--alias ${a}: expected <experiment>:<variant>=<name>`); (aliases[m[1]] ??= {})[m[2]] = m[3]; }
+  const report = cm.skillAssessmentReportOf({ experiments, sets: byDigest, criteria, ...(examples ? { examples } : {}), ...(Object.keys(aliases).length ? { aliases } : {}), ...(toolToSkillForReport ? { toolToSkill: toolToSkillForReport } : {}) });
+  mkdirSync(dirname(resolve_(outPath)), { recursive: true });
+  writeFileSync(resolve_(outPath), `${JSON.stringify(report, null, 2)}\n`);
+  const recs = report.recommendations;
+  out_({ ok: true, out: resolve_(outPath), experiments: report.experiments.length, recommendations: recs.length, headline: report.headline }, `ap eval report · ${report.experiments.length} experiment(s) · ${recs.length} recommendation(s) → ${resolve_(outPath)}\n${report.headline.map((h) => `  ${h}`).join('\n')}\n${recs.filter((r) => r.severity === 'act').slice(0, 8).map((r) => `  ▸ ${r.title}`).join('\n')}`);
+}
+
+// Spec 415 A4 — THE SELECTION GATES over stored experiments: rbox agreement (every ontology decision re-derives from its
+// recorded grounding under the playbook it ran with) and judgment calibration (the judge's confidence against being right).
+// FAIL exits 1; INCOMPLETE (nothing to check, or too few judged runs) is said and exits 0 — never reported as a pass.
+async function evalGates() {
+  const store = opt('store'); const setsDir = opt('sets'); const defSrc = opt('definition');
+  for (const [k, v] of [['store', store], ['sets', setsDir], ['definition', defSrc]]) if (!v) throw new DevkitError('E_USAGE', `--${k} is required (ap eval gates --store <dir> --sets <dir> --definition <url|file> [--plan-id a,b | --all])`);
+  const { readFileSync: rf, readdirSync } = await import('node:fs');
+  const cm = await loadCompare();
+  const decide = await loadDecideFromGrounding();
+  const propose = await loadProposeFromGrounding();
+  const dir = resolve(resolve_(store.replace(/^file:/, '')), 'experiments');
+  const want = opt('plan-id') ? new Set(opt('plan-id').split(',')) : null;
+  const records = readdirSync(dir).filter((f) => f.startsWith(encodeURIComponent('experiment:'))).map((f) => JSON.parse(rf(resolve(dir, f), 'utf8')).record).filter((r) => r?.plan && (!want || want.has(r.plan.id)));
+  const d = /^https?:/.test(defSrc) ? await (await fetch(defSrc, { headers: { 'user-agent': 'Mozilla/5.0 (ap eval gates)' } })).json() : JSON.parse(rf(resolve_(defSrc), 'utf8'));
+  const definition = d.definition ?? d;
+  const toolToSkill = Object.fromEntries((definition.tools ?? []).filter((t) => t.source?.skillId).map((t) => [t.id, t.source.skillId]));
+  const criteria = {};
+  const walk = (p) => { for (const e of readdirSync(p, { withFileTypes: true })) { const q = resolve(p, e.name); if (e.isDirectory()) walk(q); else if (e.name.endsWith('.gold.json')) { const j = JSON.parse(rf(q, 'utf8')); if (j.type === 'ap.selection-criterion.v1') criteria[j.replaySetDigest] = j; } } };
+  walk(resolve_(setsDir));
+  const which = opt('gate');
+  const only = (g) => !which || which === g;
+  const rbox = !only('rbox') ? null : cm.rboxAgreementOf({ experiment: { runs: records.flatMap((r) => r.runs) }, definition, decide, propose });
+  const calRuns = records.flatMap((r) => r.runs.map((x) => ({ ...x, __crit: criteria[r.plan.replaySetDigest] })));
+  const casesUnion = { cases: Object.assign({}, ...Object.values(criteria).map((c) => c.cases)) };
+  // Repeats count only WITHIN one experiment: across experiments the playbook, the judge or the model may differ.
+  const consistency = !only('consistency') ? null : cm.judgmentConsistencyOf({ experiment: { runs: records.flatMap((r) => r.runs.map((x) => ({ ...x, variant: `${r.plan.id}/${x.variant}` }))) } });
+  const calibration = !only('calibration') ? null : cm.judgmentCalibrationOf({ experiment: { runs: calRuns }, criterion: casesUnion, toolToSkill });
+  // Spec 416 W3 — conformal coverage of a fitted map, on runs of ONE variant (the map's binding), through the harness's set.
+  let conformal = null;
+  if (which === 'conformal') {
+    if (!opt('map')) throw new DevkitError('E_USAGE', '--gate conformal needs --map <map.json> (ap eval calibrate writes one)');
+    const map = JSON.parse(rf(resolve_(opt('map')), 'utf8'));
+    const fns = await loadHarnessCalibration();
+    const variant = opt('variant') ?? map.binding?.variant;
+    const fitted = new Set(map.fittedOn ?? []);
+    const overlap = records.filter((r) => fitted.has(r.plan.id)).map((r) => r.plan.id);
+    if (overlap.length) throw new DevkitError('E_USAGE', `coverage must be measured on runs the map was NOT fitted on — ${overlap.join(', ')} ${overlap.length === 1 ? 'is' : 'are'} in its fittedOn`);
+    conformal = cm.conformalCoverageOf({ experiment: { runs: calRuns.filter((x) => !variant || x.variant === variant) }, criterion: casesUnion, toolToSkill, map, fns });
+  }
+  const behaviour = !only('behaviour') ? null : cm.selectionBehaviourOf({ experiment: { runs: calRuns }, criterion: casesUnion });
+  // Spec 417 — OUTCOME PLANS: each variant's chains against gold plans (`--outcome-gold <file>`: { intentId: { plan, missing? } }).
+  // A measurement, not a pass/fail yet: the gate the spec names (plan-exact ≥ 0.8, no needless step on held inputs) is read off it.
+  let outcome = null;
+  if (which === 'outcome') {
+    if (!opt('outcome-gold')) throw new DevkitError('E_USAGE', '--gate outcome needs --outcome-gold <file>');
+    const gold = JSON.parse(rf(resolve_(opt('outcome-gold')), 'utf8')).cases;
+    const rows = cm.outcomePlanScoresOf(records.flatMap((r) => r.runs.map((x) => ({ ...x, variant: `${r.plan.id}/${x.variant}` }))), gold);
+    const pct = (a, b) => (b ? `${a}/${b}` : '—');
+    const planExact = rows.length ? Math.min(...rows.filter((r) => /outcome/.test(r.variant)).map((r) => r.exact / Math.max(1, r.n))) : 0;
+    outcome = { status: rows.length ? 'MEASURED' : 'INCOMPLETE', rows, reason: rows.length ? `outcome-arm plan-exact min ${planExact.toFixed(2)}` : 'no run of a gold intent' };
+  }
+  const lines = [`ap eval gates · ${records.length} experiment(s) · playbook ${String(d.digest ?? '').slice(0, 14)}`];
+  if (rbox) lines.push(`  check:rbox-agreement       ${rbox.status} — ${rbox.reason}${rbox.disagreements.length ? `\n${rbox.disagreements.slice(0, 8).map((x) => `    ✗ ${x.intentId} (${x.variant}) ${x.field}: recorded ${JSON.stringify(x.recorded)} ≠ derived ${JSON.stringify(x.derived)}`).join('\n')}` : ''}`);
+  if (calibration) lines.push(`  check:judgment-calibration ${calibration.status} — ${calibration.reason}${calibration.n ? ` · accuracy ${(calibration.accuracy * 100).toFixed(0)}% at mean confidence ${(calibration.meanConfidence * 100).toFixed(0)}%` : ''}${calibration.overconfident.length ? `\n    confident and wrong (p ≥ 0.8): ${calibration.overconfident.slice(0, 6).map((x) => `${x.intentId}→${x.predicted.split('/').pop()} (${x.p.toFixed(2)})`).join(', ')}` : ''}`);
+  if (consistency) lines.push(`  check:judgment-consistency ${consistency.status} — ${consistency.reason}${consistency.unstable.length ? `\n    changed its answer: ${consistency.unstable.slice(0, 6).map((x) => `${x.intentId} (${x.variant}) ${x.tops.map((t) => t.split('.').pop()).join('/')}`).join(', ')}` : ''}`);
+  const recall = !only('recall') ? null : cm.candidateRecallOf({ experiment: { runs: calRuns }, criterion: casesUnion, toolToSkill });
+  if (recall) lines.push(`  check:candidate-recall     ${recall.status} — ${recall.reason}${recall.n ? ` · mean ${recall.meanProposed.toFixed(1)} proposed · gold reached by ${Object.entries(recall.byVia).map(([k, v]) => `${k} ${v}`).join(', ')}` : ''}${recall.missed.length ? `\n    never proposed: ${recall.missed.slice(0, 8).map((x) => `${x.intentId}→${x.gold.split('/').pop()}`).join(', ')}` : ''}`);
+  if (conformal) lines.push(`  check:conformal-coverage   ${conformal.status} — ${conformal.reason}${conformal.n ? ` · mean set size ${conformal.meanSetSize.toFixed(2)} · sizes ${JSON.stringify(conformal.sizes)}` : ''}${conformal.uncovered.length ? `\n    not covered: ${conformal.uncovered.slice(0, 8).map((x) => `${x.intentId} gold ${x.gold.split('.').pop()} set [${x.set.join(', ')}]`).join('; ')}` : ''}`);
+  if (behaviour) lines.push(`  check:selection-behaviour  ${behaviour.status} — ${behaviour.reason}${behaviour.n ? ` · ${Object.entries(behaviour.counts).map(([k, v]) => `${k} ${v}`).join(', ')}` : ''}${behaviour.wrong.length ? `\n    not right: ${behaviour.wrong.slice(0, 8).map((x) => `${x.intentId} (${x.behaviour}) ${x.outcome}${x.chose ? ` ${x.chose}` : ''}`).join(', ')}` : ''}`);
+  if (outcome) lines.push(`  outcome-plans  ${outcome.status} — ${outcome.reason}\n${outcome.rows.map((r) => `    ${r.variant}: exact ${r.exact}/${r.n} · needless ${r.needless} · short ${r.short} · asked ${r.asked.right}/${r.asked.of} · steps ${r.stepsScore} · ${Object.entries(r.bySteps).map(([k, v]) => `${k} ${v.exact}/${v.n}`).join(', ')}`).join('\n')}`);
+  const failed = [rbox, calibration, consistency, recall, conformal, behaviour].some((g) => g?.status === 'FAIL');
+  out_({ ok: !failed, rbox, calibration, consistency, recall, conformal, behaviour, outcome }, lines.join('\n'));
+  process.exit(failed ? EXIT.FINDINGS : EXIT.OK);
+}
+
+// Spec 416 §4g — THE SELECTION SCORECARD: the same table every iteration (right, declines, ask p50/p90, time to pick,
+// tokens to pick / in total / cached, model calls and plan steps per ask), over stored experiments; `--out` keeps it, so
+// improvements are tracked as data, not remembered.
+// Spec 418 §4 — THE TECHNIQUE LEDGER: every technique's paired comparisons (quality by McNemar, cost by paired
+// bootstrap medians) from the stored experiments, with a declared verdict rule. Gold plans are read from the sets'
+// *.outcome-gold.json files. `ap eval ledger --store <dir> --techniques <file> --sets <dir> --out <file>`.
+async function evalLedger() {
+  const store = opt('store'); const techFile = opt('techniques'); const setsDir = opt('sets'); const outPath = opt('out');
+  for (const [k, v] of [['store', store], ['techniques', techFile], ['sets', setsDir], ['out', outPath]]) if (!v) throw new DevkitError('E_USAGE', `--${k} is required (ap eval ledger --store <dir> --techniques <file> --sets <dir> --out <file>)`);
+  const { readFileSync: rf, readdirSync, writeFileSync, mkdirSync } = await import('node:fs');
+  const { dirname } = await import('node:path');
+  const cm = await loadCompare();
+  const dir = resolve(resolve_(store.replace(/^file:/, '')), 'experiments');
+  const records = readdirSync(dir).filter((f) => f.startsWith(encodeURIComponent('experiment:'))).map((f) => JSON.parse(rf(resolve(dir, f), 'utf8')).record).filter((r) => r?.plan);
+  const goldBySet = {};
+  const walk = (p) => { for (const e of readdirSync(p, { withFileTypes: true })) { const q = resolve(p, e.name); if (e.isDirectory()) walk(q); else if (e.name.endsWith('.outcome-gold.json')) { const j = JSON.parse(rf(q, 'utf8')); if (j.replaySetDigest) goldBySet[j.replaySetDigest] = j.cases; } } };
+  walk(resolve_(setsDir));
+  const techniques = JSON.parse(rf(resolve_(techFile), 'utf8')).techniques;
+  const ledger = cm.techniqueLedgerOf({ records, techniques, goldBySet });
+  mkdirSync(dirname(resolve_(outPath)), { recursive: true });
+  writeFileSync(resolve_(outPath), `${JSON.stringify(ledger, null, 2)}\n`);
+  const f = (c) => (c?.deltaPct === null || c?.deltaPct === undefined ? '—' : `${c.deltaPct > 0 ? '+' : ''}${c.deltaPct}%`);
+  const lines = ledger.rows.map((r) => `  ${r.verdict.padEnd(24)} ${r.technique.name}${r.technique.ontologyDriven ? ' ◆' : ''}${r.comparisons.map((c) => `\n      ${c.experiment} ${c.baseline}→${c.treatment} n=${c.n} · ${c.quality.metric} ${c.quality.base}→${c.quality.treat} (+${c.quality.fixed}/−${c.quality.broken} p=${c.quality.p}) · pick tok ${f(c.cost.pickTokens)} · pick ms ${f(c.cost.pickMs)} · ask ms ${f(c.cost.askMs)} · ${c.verdict}`).join('')}${(r.pooled ?? []).map((q) => `\n      POOLED vs ${q.baseline} (${q.experiments} experiments, n=${q.n}): +${q.fixed}/−${q.broken} p=${q.p} · pick tok ${q.pickTokensPct ?? '—'}% · ask ms ${q.askMsPct ?? '—'}% · ${q.verdict}`).join('')}${r.missing.length ? `\n      missing: ${r.missing.join(', ')}` : ''}`);
+  out_({ ok: true, out: resolve_(outPath), rows: ledger.rows.length }, `ap eval ledger · ${ledger.rows.length} technique(s) → ${resolve_(outPath)}  (◆ ontology-driven)\n${lines.join('\n')}\n  rule: ${ledger.rule}`);
+}
+
+async function evalScorecard() {
+  const store = opt('store'); const setsDir = opt('sets'); const planIds = opt('plan-id');
+  for (const [k, v] of [['store', store], ['sets', setsDir], ['plan-id', planIds]]) if (!v) throw new DevkitError('E_USAGE', `--${k} is required (ap eval scorecard --store <dir> --sets <dir> --plan-id a,b [--out <file.json>])`);
+  const { readFileSync: rf, readdirSync, writeFileSync, mkdirSync } = await import('node:fs');
+  const { dirname } = await import('node:path');
+  const cm = await loadCompare();
+  const dir = resolve(resolve_(store.replace(/^file:/, '')), 'experiments');
+  const order = planIds.split(',');
+  const records = order.map((id) => { const f = resolve(dir, `${encodeURIComponent('experiment:' + id)}.jsonld`); try { return JSON.parse(rf(f, 'utf8')).record; } catch { throw new DevkitError('E_INPUT_MISSING', `no stored experiment ${id}`); } });
+  const criteria = {};
+  const walk = (p) => { for (const e of readdirSync(p, { withFileTypes: true })) { const q = resolve(p, e.name); if (e.isDirectory()) walk(q); else if (e.name.endsWith('.gold.json')) { const j = JSON.parse(rf(q, 'utf8')); if (j.type === 'ap.selection-criterion.v1') Object.assign(criteria, j.cases); } } };
+  walk(resolve_(setsDir));
+  const rows = cm.scorecardOf(records, criteria);
+  const s = (ms) => (ms === null || ms === undefined ? '—' : `${(ms / 1000).toFixed(1)}s`);
+  const k = (n) => (n === null || n === undefined ? '—' : n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(Math.round(n)));
+  const lines = [`ap eval scorecard · ${records.length} experiment(s)`, `  ${'experiment/variant'.padEnd(46)} right   pos    neg   ask p50/p90   pick  pick tok in/out  total in/out  cached calls steps quality`];
+  for (const r of rows) lines.push(`  ${`${r.experiment}/${r.variant}`.slice(0, 46).padEnd(46)} ${`${r.right}/${r.runs}`.padEnd(6)} ${`${r.rightOnPositives}/${r.positives}`.padEnd(6)} ${`${r.declined}/${r.negatives}`.padEnd(6)} ${`${s(r.askP50)}/${s(r.askP90)}`.padEnd(13)} ${s(r.pickP50).padEnd(5)} ${`${k(r.selectionIn)}/${k(r.selectionOut)}`.padEnd(15)} ${`${k(r.totalIn)}/${k(r.totalOut)}`.padEnd(13)} ${k(r.cachedIn).padEnd(6)} ${String(r.modelCalls ?? '—').padEnd(5)} ${String(r.planSteps ?? '—').padEnd(5)} ${r.quality === null || r.quality === undefined ? '—' : `${r.quality.toFixed(2)} (${r.qualityRuns})`}`);
+  for (const r of rows) if (r.preference) lines.push(`    ${r.variant} side by side (${r.preferenceRuns}): ${Object.entries(r.preference).map(([k, v]) => `${k} ${(v * 100).toFixed(0)}%`).join(' · ')}`);
+  for (const r of rows) { const st = Object.entries(r.stagesP50 ?? {}).slice(0, 6); if (st.length) lines.push(`    ${r.variant} stages p50: ${st.map(([n, v]) => `${n} ${s(v)}`).join(' · ')}`); }
+  if (opt('out')) { mkdirSync(dirname(resolve_(opt('out'))), { recursive: true }); writeFileSync(resolve_(opt('out')), `${JSON.stringify({ type: 'ap.selection-scorecard.v1', at: new Date().toISOString(), rows }, null, 2)}\n`); lines.push(`  → ${resolve_(opt('out'))}`); }
+  out_({ ok: true, rows }, lines.join('\n'));
+}
+
+// Spec 416 W3 — FIT A CALIBRATION MAP on labelled runs of one variant (one binding): a temperature and a conformal q̂ at
+// level alpha, through the harness's own math. The map is data with a digest; `ap eval compare` passes it as the variant's
+// `acceptance`, and `ap eval gates --gate conformal --map` measures it on runs it was not fitted on.
+async function evalCalibrate() {
+  const store = opt('store'); const setsDir = opt('sets'); const defSrc = opt('definition'); const planIds = opt('plan-id'); const variant = opt('variant'); const out = opt('out');
+  for (const [k, v] of [['store', store], ['sets', setsDir], ['definition', defSrc], ['plan-id', planIds], ['variant', variant], ['out', out]]) if (!v) throw new DevkitError('E_USAGE', `--${k} is required (ap eval calibrate --store <dir> --plan-id a,b --variant v --sets <dir> --definition <url|file> --alpha 0.05 --out <map.json>)`);
+  const { readFileSync: rf, readdirSync, writeFileSync, mkdirSync } = await import('node:fs');
+  const { dirname } = await import('node:path');
+  const cm = await loadCompare();
+  const fns = await loadHarnessCalibration();
+  const dir = resolve(resolve_(store.replace(/^file:/, '')), 'experiments');
+  const want = new Set(planIds.split(','));
+  const records = readdirSync(dir).filter((f) => f.startsWith(encodeURIComponent('experiment:'))).map((f) => JSON.parse(rf(resolve(dir, f), 'utf8')).record).filter((r) => r?.plan && want.has(r.plan.id));
+  const missing = [...want].filter((id) => !records.some((r) => r.plan.id === id));
+  if (missing.length) throw new DevkitError('E_INPUT_MISSING', `no stored experiment ${missing.join(', ')}`);
+  const providers = [...new Set(records.map((r) => r.plan.variants?.[variant]?.provider ?? null))];
+  if (providers.length > 1) throw new DevkitError('E_USAGE', `the experiments ran ${variant} on different providers (${providers.join(', ')}) — a map is fitted for ONE binding`);
+  const d = /^https?:/.test(defSrc) ? await (await fetch(defSrc, { headers: { 'user-agent': 'Mozilla/5.0 (ap eval calibrate)' } })).json() : JSON.parse(rf(resolve_(defSrc), 'utf8'));
+  const definition = d.definition ?? d;
+  const toolToSkill = Object.fromEntries((definition.tools ?? []).filter((t) => t.source?.skillId).map((t) => [t.id, t.source.skillId]));
+  const criteria = {};
+  const walk = (p) => { for (const e of readdirSync(p, { withFileTypes: true })) { const q = resolve(p, e.name); if (e.isDirectory()) walk(q); else if (e.name.endsWith('.gold.json')) { const j = JSON.parse(rf(q, 'utf8')); if (j.type === 'ap.selection-criterion.v1') criteria[j.replaySetDigest] = j; } } };
+  walk(resolve_(setsDir));
+  const criterion = { cases: Object.assign({}, ...records.map((r) => criteria[r.plan.replaySetDigest]?.cases ?? {})) };
+  const map = await cm.fitConformalMap({ experiments: records, variant, judge: fns.judge, ...(providers[0] ? { provider: providers[0] } : {}), criterion, toolToSkill, alpha: Number(opt('alpha') ?? 0.05), fns });
+  mkdirSync(dirname(resolve_(out)), { recursive: true });
+  writeFileSync(resolve_(out), `${JSON.stringify(map, null, 2)}\n`);
+  out_({ ok: true, out: resolve_(out), map }, `ap eval calibrate · ${map.binding.judge} on ${map.binding.provider ?? 'the default provider'} · ${map.n} labelled runs from ${map.fittedOn.join(', ')}\n  temperature ${map.temperature} · q̂ ${map.qhat} at α ${map.alpha} (set threshold ${(1 - map.qhat).toFixed(3)}) · ${map.mapDigest}\n  → ${resolve_(out)}`);
+}
+
+// Spec 415 A4 / methodology §4.3 — A REPAIR, VALIDATED: the same set judged before and after ONE change (a description,
+// a doctrine line, a playbook version), compared on the paired cases. The two experiments are what they were; the
+// comparison refuses a different set or criterion, and says when the validating split is not locked.
+async function evalDiff() {
+  const store = opt('store');
+  for (const [k, v] of [['store', store], ['before', opt('before')], ['after', opt('after')], ['replay', opt('replay')], ['criterion', opt('criterion')]]) if (!v) throw new DevkitError('E_USAGE', `--${k} is required (ap eval diff --store <dir> --before <plan-id> --after <plan-id> --replay <set> --criterion <gold>)`);
+  const { readFileSync: rf } = await import('node:fs');
+  const cm = await loadCompare();
+  const dir = resolve_(store.replace(/^file:/, ''));
+  const load = (id) => { const path = resolve(dir, 'experiments', `${encodeURIComponent(`experiment:${id}`)}.jsonld`); const doc = JSON.parse(rf(path, 'utf8')); if (!doc.record?.plan) throw new DevkitError('E_INPUT_MISSING', `${path} holds no experiment record`); return doc.record; };
+  const before = load(opt('before')); const after = load(opt('after'));
+  const setR = cm.validateReplaySet(JSON.parse(rf(resolve_(opt('replay')), 'utf8')));
+  if (!setR.ok) throw new DevkitError('E_USAGE', `the replay set is not valid: ${setR.errors.join('; ')}`);
+  const criterion = JSON.parse(rf(resolve_(opt('criterion')), 'utf8'));
+  const setDigest = await cm.replaySetDigest(setR.set); const critDigest = await cm.criterionDigest(criterion);
+  for (const [n, r] of [['before', before], ['after', after]]) if (r.plan.replaySetDigest !== setDigest || r.plan.criterionDigest !== critDigest) throw new DevkitError('E_USAGE', `${n} (${r.plan.id}) pre-registered a different set or criterion; a diff never crosses them`);
+  if (before.plan.split !== after.plan.split) throw new DevkitError('E_USAGE', `the splits differ (${before.plan.split} vs ${after.plan.split}); paired cases need the same split`);
+  const vBefore = opt('variant-before') ?? opt('variant') ?? 'model';
+  const vAfter = opt('variant-after') ?? opt('variant') ?? 'model';
+  for (const [n, r, v] of [['before', before, vBefore], ['after', after, vAfter]]) if (!r.scores?.[v]) throw new DevkitError('E_INPUT_MISSING', `${n} (${r.plan.id}) has no scored variant "${v}" — ap eval rescore it first`);
+  const variant = `${vBefore} → ${vAfter}`;
+  const { cases } = cm.casesOf(setR.set, criterion, before.plan.split);
+  const side = async (r, v) => ({ datasetDigest: setDigest, variantDigest: await cm.variantRequestDigest(r.plan.variants[v]), split: r.plan.split === 'held-out' ? 'held-out' : 'development', score: r.scores[v], cases });
+  const pair = opt('repaired-pair') ? opt('repaired-pair').split(',').map((x) => x.trim()).filter(Boolean) : undefined;
+  const cmp = cm.compareSelectionRuns(await side(before, vBefore), await side(after, vAfter), pair ? { repairedPair: pair } : {});
+  const f = (v) => (v === null || v === undefined ? 'n/a' : Number(v).toFixed(3));
+  const a = before.scores[vBefore].aggregate, b = after.scores[vAfter].aggregate;
+  const line = (k) => `  ${k.padEnd(22)} ${f(a[k]?.value)} → ${f(b[k]?.value)}`;
+  out_({ ok: true, before: before.plan.id, after: after.plan.id, variant, comparison: cmp, aggregates: { before: a, after: b } }, `ap eval diff · ${before.plan.id} → ${after.plan.id} (${variant}, split ${before.plan.split})\n${['macro_ta', 'micro_ta', 'hold_rate', 'routing_purity', 'collision_containment', 'over_fire_rate'].map(line).join('\n')}\n  Δ macro ${f(cmp.delta_macro)} [${f(cmp.ci_delta?.[0])}, ${f(cmp.ci_delta?.[1])}] · McNemar fixed ${cmp.fixed} broken ${cmp.broken} p=${Number(cmp.p_value).toFixed(4)} (${cmp.mcnemar_scope}, n=${cmp.n_paired})${cmp.regression ? ' · REGRESSION' : cmp.degraded ? ' · degraded' : ''}${cmp.warnings.length ? `\n  ! ${cmp.warnings.join('; ')}` : ''}${cmp.note ? `\n  ${cmp.note}` : ''}`);
+}
+
+const VARIANT_PRESETS = { model: {}, declared: { selection: 'declared' }, ontology: { selection: 'ontology' }, judgment: { selection: 'judgment' }, 'ontology+judgment': { selection: 'ontology+judgment' }, 'propose+judgment': { selection: 'propose+judgment' }, 'ontology-first': { selection: 'ontology-first' }, 'framed-judgment': { selection: 'framed-judgment' }, outcome: { selection: 'outcome', judgeProfile: 'fast' }, 'outcome-selective': { selection: 'outcome-selective', judgeProfile: 'fast' }, rules: { plannerKind: 'rule-based' }, 'kb-off': { toggles: { 'retrieval/kb': 'off' } }, 'kb-tool': { toggles: { 'retrieval/kb': 'tool' } }, 'kb-playbook': { toggles: { 'retrieval/kb': 'playbook' } } };
+
+/** THE ACTIVITIES A RUN SPAWNED, from its PROV bundle: the run (ServiceExecution) at the root, under it every
+ *  SkillExecution (a step; wasInformedBy / isStepOfPlan say which) and every ModelInvocation, each with what it
+ *  generated (receipts, results) and who it ran as. A handed-off or routed step appears as its own activity under the
+ *  step that spawned it. Read from `graph`; nothing is inferred that the bundle does not say. */
+function activityTreeOf(bundle, record) {
+  const nodes = Array.isArray(bundle?.graph) ? bundle.graph : Array.isArray(bundle?.['@graph']) ? bundle['@graph'] : [];
+  const stepAt = (id) => { const m = /:s(\d+)$/.exec(String(id)); return m && record?.steps ? record.steps[Number(m[1])] : undefined; };
+  const callAt = (id) => { const m = /:model:(\d+)$/.exec(String(id)); return m && record?.modelCalls ? record.modelCalls[Number(m[1])] : undefined; };
+  const byId = new Map(nodes.map((n) => [n.id ?? n['@id'], n]));
+  const types = (n) => [].concat(n.type ?? n['@type'] ?? []);
+  const acts = nodes.filter((n) => types(n).includes('Activity'));
+  const generated = (id) => nodes.filter((n) => n.wasGeneratedBy === id).map((n) => ({ id: n.id, type: types(n).filter((t) => t !== 'Entity'), ...(n.digest ? { digest: n.digest } : {}), ...(n.commitment ? { commitment: n.commitment } : {}) }));
+  const parentOf = (n) => n.wasInformedBy ?? n.wasStartedBy ?? n.partOf ?? (typeof n.id === 'string' && /:(s\d+|model:\d+)$/.test(n.id) ? n.id.replace(/:(s\d+|model:\d+)$/, '') : undefined);
+  const node = (n) => ({
+    id: n.id, kind: types(n).filter((t) => t !== 'Activity')[0] ?? 'Activity',
+    ...(n.startedAtTime ? { startedAt: n.startedAtTime } : {}), ...(n.endedAtTime ? { endedAt: n.endedAtTime } : {}),
+    ...(n.wasAssociatedWith ? { ranAs: n.wasAssociatedWith } : {}), ...(n.used ? { used: [].concat(n.used) } : {}),
+    ...(n.hadPlan ? { plan: n.hadPlan } : {}), ...(n.model ? { model: n.model } : {}), ...(n.provider ? { provider: n.provider } : {}), ...(n.role ? { role: n.role } : {}),
+    ...(n.status ? { status: n.status } : {}), ...(n.toolId ? { toolId: n.toolId } : {}),
+    ...((s) => s ? { toolId: s.toolId, status: s.status, ...(s.risk ? { risk: s.risk } : {}) } : {})(stepAt(n.id)),
+    ...((c) => c ? { role: c.role, provider: c.provider, model: c.model, ...(c.tokensIn !== undefined ? { tokensIn: c.tokensIn, tokensOut: c.tokensOut } : {}), ...(c.routeReason ? { because: c.routeReason } : {}), ...(c.stepRef ? { forStep: c.stepRef } : {}) } : {})(callAt(n.id)),
+    generated: generated(n.id), children: [],
+  });
+  const built = new Map(acts.map((a) => [a.id, node(a)]));
+  const roots = [];
+  for (const a of acts) { const p = parentOf(a); if (p && built.has(p) && p !== a.id) built.get(p).children.push(built.get(a.id)); else roots.push(built.get(a.id)); }
+  const order = (xs) => { xs.sort((x, y) => String(x.startedAt ?? '').localeCompare(String(y.startedAt ?? ''))); for (const x of xs) order(x.children); return xs; };
+  void byId;
+  return order(roots);
+}
+
+function keptRunMarkdown(planId, entry, tree) {
+  const lines = [`# ${entry.intentId} · ${entry.variant} · repeat ${entry.repeat}`, '', `plan \`${planId}\` · run \`${entry.runRef}\` · selection **${entry.verdict?.outcome ?? '?'}** (${entry.selected.join(', ') || 'nothing chosen'})${entry.outcome ? ` · outcome check **${entry.outcome.score}**` : ''}${entry.quality ? ` · quality ${entry.quality.score}` : ''}`, '', '## The ask', '', `> ${entry.message.replace(/\n/g, '\n> ')}`, ''];
+  if (entry.outcome) lines.push('## Outcome check — classes delivered', '', ...Object.entries(entry.outcome.classes).map(([k, v]) => `- ${v.toFixed(2)} ${k.split('#').pop()}`), '');
+  if (entry.reply?.results && Array.isArray(entry.reply.results)) { lines.push('## Results per step', ''); for (const r of entry.reply.results) lines.push(`### ${r.toolId ?? r.stepRef ?? 'step'}`, '', '```json', JSON.stringify(r.result ?? r, null, 2).slice(0, 20000), '```', ''); }
+  lines.push('## The reply', '', entry.reply?.text ?? entry.reply?.error ?? '(the ask port carried no reply)', '');
+  lines.push('## Activities the run spawned', '');
+  const walk = (xs, depth) => { for (const x of xs) { lines.push(`${'  '.repeat(depth)}- **${x.kind}** \`${String(x.id).replace(/^urn:ap:prov:act:/, '')}\`${x.toolId ? ` · ${x.toolId}${x.status ? ` (${x.status})` : ''}` : ''}${x.role ? ` · ${x.role}` : ''}${x.model ? ` · ${x.provider ?? ''} ${x.model}` : ''}${x.tokensIn !== undefined ? ` · ${x.tokensIn} → ${x.tokensOut} tokens` : ''}${x.startedAt && x.endedAt ? ` · ${Math.max(0, Date.parse(x.endedAt) - Date.parse(x.startedAt))} ms` : ''}${x.generated.length ? ` → generated ${x.generated.map((g) => `${g.type.join('/')} ${String(g.id).replace(/^urn:ap:prov:/, '').slice(0, 40)}…`).join(', ')}` : ''}`); walk(x.children, depth + 1); } };
+  if (tree.length) walk(tree, 0); else lines.push('(no provenance bundle)');
+  lines.push('');
+  return lines.join('\n');
+}
+
+async function evalCompare() {
+  const home = (opt('home') ?? process.env.HOME_URL ?? '').replace(/\/$/, '');
+  const store = opt('store');
+  for (const [k, v] of [['home', home], ['replay', opt('replay')], ['criterion', opt('criterion')], ['variants', opt('variants')], ['store', store], ['persona', opt('persona')], ['addressee', opt('addressee')]]) if (!v) throw new DevkitError('E_USAGE', `--${k} is required (ap eval compare --replay <set> --criterion <gold> --variants a,b --store <dir> --home <url> --persona <handle> --addressee <name>)`);
+  const { existsSync: ex, readFileSync: rf } = await import('node:fs');
+  const cm = await loadCompare();
+  const fileStore = await loadEvalStore();
+  const setR = cm.validateReplaySet(JSON.parse(rf(resolve_(opt('replay')), 'utf8')));
+  if (!setR.ok) throw new DevkitError('E_USAGE', `the replay set is not valid: ${setR.errors.join('; ')}`);
+  const criterion = JSON.parse(rf(resolve_(opt('criterion')), 'utf8'));
+  if (criterion?.type !== 'ap.selection-criterion.v1') throw new DevkitError('E_USAGE', 'the criterion is not an ap.selection-criterion.v1 record');
+  const variants = {};
+  for (const nameOrFile of opt('variants').split(',').map((s) => s.trim()).filter(Boolean)) {
+    if (VARIANT_PRESETS[nameOrFile]) variants[nameOrFile] = VARIANT_PRESETS[nameOrFile];
+    else if (ex(resolve_(nameOrFile))) variants[nameOrFile.replace(/^.*\//, '').replace(/\.json$/, '')] = JSON.parse(rf(resolve_(nameOrFile), 'utf8'));
+    else throw new DevkitError('E_USAGE', `variant "${nameOrFile}" is neither a preset (${Object.keys(VARIANT_PRESETS).join(', ')}) nor a file`);
+  }
+  const split = opt('split') ?? 'held-out';
+  const repeats = Number(opt('repeat') ?? 1);
+  const skillMap = opt('skill-map') ? JSON.parse(rf(resolve_(opt('skill-map')), 'utf8')) : undefined;
+  const actGoldRaw = opt('act-gold') ? JSON.parse(rf(resolve_(opt('act-gold')), 'utf8')) : undefined;
+  if (actGoldRaw && actGoldRaw.type !== 'ap.act-gold.v1') throw new DevkitError('E_USAGE', 'the act gold is not an ap.act-gold.v1 record');
+  const paths = windowPaths(store);
+  const windowRaw = ex(paths.window) ? JSON.parse(rf(paths.window, 'utf8')) : null;
+  const wv = windowRaw ? cm.validateCaptureWindow(windowRaw) : null;
+  if (wv && !wv.ok) throw new DevkitError('E_USAGE', `the window at ${paths.window} is not valid: ${wv.errors.join('; ')}`);
+  const salt = ex(paths.salt) ? rf(paths.salt, 'utf8').trim() : null;
+  const est = await estateClient(home);
+  const me = await est.signin(opt('persona'));
+  const addressee = await est.resolveName(opt('addressee'));
+  // Spec 418 §12 — the act gold names the estate's agents; resolved once, here, by the estate's own resolver.
+  const actGold = actGoldRaw ? await cm.resolveActGold(actGoldRaw, (n) => est.resolveName(n)) : undefined;
+  const plan = cm.experimentPlan({ id: opt('plan-id') ?? `${setR.set.id}@${setR.set.version}-${Object.keys(variants).join('+')}-${split}-r${repeats}`, replaySetDigest: await cm.replaySetDigest(setR.set), criterionDigest: await cm.criterionDigest(criterion), split, repeats, variants, ...(opt('policy') ? { policy: opt('policy') } : {}) });
+  const wait = (ms) => new Promise((r) => setTimeout(r, ms));
+  // --keep-text <dir>: the reply's words and the run's activity tree, per case, on this disk and nowhere else. The store
+  // never holds words (415 S6); this is the one door for them, and it opens only onto a synthetic estate — the open
+  // window must admit the addressee (a domain sandbox org), the same test the capture firewall applies.
+  const keepDir = opt('keep-text') ? resolve_(opt('keep-text')) : null;
+  if (keepDir) {
+    const admitted = wv?.ok && wv.window.roster.map((a) => a.toLowerCase()).includes(addressee.toLowerCase());
+    if (!admitted) throw new DevkitError('E_USAGE', `--keep-text keeps words: the open window at ${paths.window} must admit ${opt('addressee')} (${addressee}) — open one with it on the roster (ap eval window open … --roster <persona>,${opt('addressee')})`);
+  }
+  const ports = {
+    ask: async ({ intentId, message, variant, variantName, repeat }) => {
+      const runRef = `cmp-${Date.now().toString(36)}-${variantName}-${intentId}-${repeat}`.replace(/[^A-Za-z0-9._:-]/g, '-');
+      // Spec 420 W4 — a UX ACTION case supplies the plan its Home button sends; its words ride as the message.
+      const supplied = actGold?.cases?.[intentId]?.supplied;
+      // A LIVE arm (an empty variant `{}`) sends no variant at all: the estate accepts variants only from the agent or its
+      // steward, so a MEMBER's cases (spec 420 set 5a) run as plain asks under the deployment's defaults — paired across two
+      // deployments by case, never within one run.
+      const live = variant && typeof variant === 'object' && Object.keys(variant).length === 0;
+      const r = await est.post('/harness/ask', { session: me.token, addressee, message, ...(live ? {} : { variant }), runRef, ...(supplied ? { plan: supplied } : {}) });
+      if (r?.ok === false || r?.error) return { ok: false, error: `${r.refused ? `${r.refused}: ` : ''}${r.error ?? JSON.stringify(r).slice(0, 160)}` };
+      // A turn the MODEL HOST refused (overloaded, rate-limited) never planned: a failed run, named as the provider's — it
+      // says nothing about the harness under test and must not be scored as its miss (spec 418 §12, seen 2026-09-28).
+      const said = String(r?.reply?.error ?? r?.reply?.text ?? '');
+      if (r?.reply?.kind === 'refused' && /planner_failed:.*HTTP (402|429|5\d\d)/.test(said)) return { ok: false, error: `provider: ${said.slice(0, 160)}` };
+      const rt = r?.hasProvenance?.recordType;
+      return { ok: true, runRef: typeof rt === 'string' && rt.startsWith('run.provenance:') ? rt.slice('run.provenance:'.length) : runRef, ...(r?.reply?.plannerTrace?.selection ? { selection: r.reply.plannerTrace.selection } : {}), ...(r?.reply?.plannerTrace ? { trace: r.reply.plannerTrace } : {}), ...(keepDir && r?.reply ? { reply: r.reply } : {}), ...(actGold ? { act: cm.actCaptureOf(r?.reply) } : {}) };
+    },
+    ...(keepDir ? { keep: async (entry) => {
+      const { mkdirSync, writeFileSync } = await import('node:fs');
+      const dir = `${keepDir}/${plan.id}/${entry.variant}`; mkdirSync(dir, { recursive: true });
+      const safe = (s) => String(s).replace(/[^A-Za-z0-9._-]/g, '-');
+      const base = `${dir}/${safe(entry.intentId)}#${entry.repeat}`;
+      // The record view says what the bundle's activities do not: each step's tool and status, each model call's role,
+      // provider, model and tokens. Merged by position (`:s<n>` ↔ steps[n]; `:model:<n>` ↔ modelCalls[n]).
+      const rv = await est.post('/harness/provenance', { session: me.token, addressee, runRef: entry.runRef, format: 'record' }).catch(() => null);
+      const tree = activityTreeOf(entry.provenance, rv?.ok ? rv.record : null);
+      writeFileSync(`${base}.json`, `${JSON.stringify({ type: 'ap.kept-run.v1', planId: plan.id, ...entry, ...(rv?.ok ? { record: rv.record } : {}), activities: tree }, null, 2)}\n`);
+      writeFileSync(`${base}.md`, keptRunMarkdown(plan.id, entry, tree));
+    } } : {}),
+    readProvenance: async (runRef) => {
+      for (let i = 0; i < 20; i++) {
+        const r = await est.post('/harness/provenance', { session: me.token, addressee, runRef });
+        if (r?.ok && r.provenance) return r.provenance;
+        await wait(3000);
+      }
+      return null;
+    },
+    readMeasures: async (runRef) => {
+      for (let i = 0; i < 10; i++) {
+        const r = await est.post('/harness/provenance', { session: me.token, addressee, runRef, format: 'measures' });
+        if (r?.ok && r.measures) return r.measures;
+        if (r?.status === 'refused') return null;
+        await wait(3000);
+      }
+      return null;
+    },
+  };
+  const rec = await cm.runComparison({ ...(opt('fixtures') ? { fixtures: JSON.parse(rf(resolve_(opt('fixtures')), 'utf8')).fixtures } : {}), 
+    set: setR.set, criterion, plan, window: wv ? wv.window : null, estate: home, estateCapture: est.evalCapture, asker: me.agent, addressee,
+    store: fileStore(paths.dir), pseudo: cm.pseudonymiser(salt ?? 'no-window'), ports, ...(skillMap ? { skillMap } : {}), ...(actGold ? { actGold } : {}),
+    log: (line) => { if (!json) console.error(`  ${line}`); },
+  });
+  // Spec 420 — the act laboratory's verdicts per DOMAIN PRINCIPLE (each case counts toward every principle it cites).
+  if (actGold && !json) {
+    const cov = cm.principleCoverageOf(rec.runs.map((r) => r.act));
+    const rows = Object.entries(cov).sort(([a], [b]) => a.localeCompare(b));
+    if (rows.length) console.error(`  principles: ${rows.map(([p, v]) => `${p} ${v.right}/${v.total}`).join(' · ')}`);
+  }
+  out_({ ok: true, plan: rec.plan, planDigest: rec.planDigest, runs: rec.runs.length, refused: rec.refused, scores: Object.fromEntries(Object.entries(rec.scores).map(([k, v]) => [k, v.aggregate])), verdicts: Object.fromEntries(Object.entries(rec.verdicts).map(([k, v]) => [k, { status: v.status, reason: v.binding_reason }])), comparisons: rec.comparisons }, `ap eval compare · ${home} · store ${paths.dir}\n${cm.formatExperiment(rec)}`);
+}
+
+const commands = { doctor, upgrade, conform, test, create, mcp, runtime, generate, eval: evalCmd };
 try {
   if (!cmd || flag('help') || cmd === '--help' || !commands[cmd]) {
     if (json) console.log(JSON.stringify({ ok: !cmd || cmd === '--help', usage: USAGE }));