@ngockhoale/ukit 3.3.3 → 3.4.0

package/scripts/bench/subagent-orchestrator-eval.mjs

@@ -0,0 +1,565 @@
+// TASK-C89-007 — subagent-orchestrator golden eval + rollout verdict
+// (SPEC §9 FR-07, §12 truth table honesty, §14 rollout, §15 stop/replan).
+//
+// Pure evaluator + thin CLI. The evaluator replays frozen-corpus fixture runs
+// for two arms — baseline (A: current handoff / B: single-strong) vs candidate
+// (C: orchestrator variant) — and applies the pre-registered gates:
+//
+//   1. Per-class quality gate  — computeGateComparison (evaluation.js §5
+//      order: forbidden → unknown → score floor → wall p95) over one
+//      FR-006 report aggregated per task class. Class/critical gating is
+//      expressed by aggregation, so evaluation.js needs no extension.
+//   2. Savings provenance      — computeSavings (TASK-C89-001): a percentage
+//      exists only when BOTH arms carry provider-measured usage; unknown,
+//      estimated, host-measured and local-tokenizer sources all resolve
+//      reportable:false — never 0%, never a fabricated number.
+//   3. Independent reviewer    — every review-class candidate run must carry
+//      reviewer evidence; detected:false is a critical miss (NO_GO), absent
+//      reviewer evidence is missing data (INCOMPARABLE, not a free pass).
+//   4. VM-lane promotion       — ukit vm evidence records (C87 live lane) are
+//      consumed verbatim through promote(); the printed verdict is advisory,
+//      never a write. Stage holds shadow until PROMOTION_CRITERIA and the
+//      BL-031 net-value bar are both satisfied; the operator chooses rollout.
+//
+// Verdicts: 'GO' | 'NO_GO' | 'INCOMPARABLE'. NO_GO beats INCOMPARABLE beats
+// GO. A critical defect is an independent blocker — savings never overrides
+// quality. Missing data is fail-closed: unscored runs, absent comparability
+// coordinates and unknown usage all withhold comparison, they never guess.
+//
+// RESOURCE_SOURCES ↔ usageSource reconciliation (T-002→T-001 casing gap):
+//   telemetry/types.ts provenance is UPPER_SNAKE (PROVIDER, LOCAL_TOKENIZER,
+//   ESTIMATED, UNKNOWN); corpus computeSavings expects lower-kebab
+//   ('provider', 'host-measured', 'estimated', 'unknown').
+//   normalizeUsageSource() is the single mapping point — every other value,
+//   including anything unrecognized, degrades to 'unknown'.
+
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import {
+  loadSubagentCorpus,
+  checkComparability,
+  computeSavings,
+  CORPUS_REVISION,
+  BASELINE_SHA,
+  RUBRIC_VERSION,
+  ARMS,
+} from './subagent-orchestrator-corpus.mjs';
+import {
+  computeGateComparison,
+  COMPARABLE_RUNS,
+  QUALITY_SCORE_FLOOR,
+} from '../../src/core/agentRuntime/evaluation.js';
+import { UNKNOWN } from './decision-runtime-metrics.mjs';
+import { promote, PROMOTION_STAGE_ORDER } from '../../src/core/agentRuntime/promotion.js';
+import { readEvidence } from '../../src/core/agentRuntime/shadowRun.js';
+import { inspectRuntimeConfig } from '../../src/core/runtimeConfig.js';
+
+const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = path.resolve(MODULE_DIR, '..', '..');
+const VERDICT_DOC_REL = path.join('docs', 'AI_HANDOFF', 'benchmark', 'subagent-orchestrator-verdict.md');
+
+export const ROLLBACK_ROUTE = 'promote(config,{rollback:true}) resolves stage "off" instantly — deterministic owners authoritative, zero VM calls';
+function isObj(v) {
+  return v != null && typeof v === 'object' && !Array.isArray(v);
+}
+
+function isNum(v) {
+  return typeof v === 'number' && Number.isFinite(v);
+}
+
+/**
+ * Map any usage-source spelling to the corpus provenance domain
+ * ('provider' | 'host-measured' | 'estimated' | 'local-tokenizer' |
+ * 'unknown'). Unrecognized spellings degrade to 'unknown' — a usage claim
+ * that cannot name its measurement cannot report savings.
+ */
+export function normalizeUsageSource(source) {
+  if (typeof source !== 'string' || source.trim() === '') return 'unknown';
+  const s = source.trim().toLowerCase().replace(/_/g, '-');
+  if (s === 'provider') return 'provider';
+  if (s === 'host-measured' || s === 'host') return 'host-measured';
+  if (s === 'estimated' || s === 'estimate') return 'estimated';
+  if (s === 'local-tokenizer' || s === 'tokenizer') return 'local-tokenizer';
+  if (s === 'unknown') return 'unknown';
+  return 'unknown';
+}
+
+/** Usage pair for one arm: {usageSource, tokens|null}. */
+function usageOf(run) {
+  const u = isObj(run.usage) ? run.usage : {};
+  return {
+    usageSource: normalizeUsageSource(u.source ?? run.usageSource),
+    tokens: isNum(u.tokens) ? u.tokens : (isNum(run.tokens) ? run.tokens : null),
+  };
+}
+
+/**
+ * Aggregate an arm's usage over its scored runs. Any non-provider run
+ * forfeits provider provenance for the whole arm — the most conservative
+ * source wins, never the most flattering.
+ */
+function aggregateUsage(runs) {
+  const ranked = { 'provider': 4, 'host-measured': 3, 'local-tokenizer': 2, 'estimated': 1, 'unknown': 0 };
+  let source = 'provider';
+  let tokens = 0;
+  let allTokens = true;
+  for (const run of runs) {
+    const u = usageOf(run);
+    if ((ranked[u.usageSource] ?? 0) < (ranked[source] ?? 0)) source = u.usageSource;
+    if (isNum(u.tokens)) tokens += u.tokens; else allTokens = false;
+  }
+  return { usageSource: source, tokens: allTokens && runs.length > 0 ? tokens : null };
+}
+
+/** One run's class score: mean of finite rubric dimension scores. */
+function runScore(run, dimensions) {
+  if (!isObj(run.scores)) return null;
+  let sum = 0;
+  let measured = false;
+  for (const dim of dimensions) {
+    const v = run.scores[dim];
+    if (isNum(v)) {
+      measured = true;
+      sum += Math.min(1, Math.max(0, v));
+    }
+    // absent/unscored dimension contributes 0 — fail-closed, never guessed
+  }
+  return measured ? sum / dimensions.length : 0;
+}
+
+/**
+ * Aggregate scored runs into the FR-006 report shape that
+ * computeGateComparison consumes ({summary, runs, env}).
+ */
+function toReport(runs, dimensions) {
+  let satisfied = 0;
+  let eligible = 0;
+  let forbiddenFailures = 0;
+  let unscored = 0;
+  const walls = [];
+  for (const run of runs) {
+    const score = runScore(run, dimensions);
+    if (score === null) {
+      unscored += 1;
+      continue;
+    }
+    satisfied += score * dimensions.length;
+    eligible += dimensions.length;
+    if (run.criticalFailure === true) forbiddenFailures += 1;
+    if (isNum(run.wallMs)) walls.push(run.wallMs);
+  }
+  walls.sort((a, b) => a - b);
+  const rank = (q) => walls[Math.max(0, Math.ceil((q / 100) * walls.length) - 1)];
+  return {
+    summary: {
+      forbiddenFailures,
+      // any unscored run poisons the class score — fail closed to UNKNOWN
+      qualityScore: unscored === 0 && eligible > 0 ? Math.min(1, satisfied / eligible) : UNKNOWN,
+      p50: walls.length > 0 ? rank(50) : UNKNOWN,
+      p95: walls.length > 0 ? rank(95) : UNKNOWN,
+    },
+    runs: runs.length,
+    unscored,
+  };
+}
+
+/**
+ * Golden eval + rollout verdict over frozen-corpus fixture runs.
+ *
+ * @param {object} input
+ * @param {object[]} input.baselineRuns  scored runs for arm A/B (baseline)
+ * @param {object[]} input.candidateRuns scored runs for the candidate arm
+ * @param {object[]} [input.fixtures]    frozen corpus fixtures (defaults to
+ *                                       loadSubagentCorpus().fixtures)
+ * @param {object}   [input.config]      runtime config (promotion lane stage)
+ * @param {object}   [input.vmEvidence]  {key, runs} — ukit vm evidence rows;
+ *                                       replay-comparison artifacts are
+ *                                       reported but never sampled
+ * @param {string}   [input.baselineLabel] default 'A-current-handoff'
+ * @param {string}   [input.candidateLabel] default 'C-no-pruning'
+ * @returns {object} frozen {verdict, perClass, reasons, savings, reviewer,
+ *   promotion, replay, stage, dropped}
+ */
+export function evaluateSubagentOrchestrator(input = {}) {
+  const corpus = loadSubagentCorpus();
+  const fixtures = Array.isArray(input.fixtures) && input.fixtures.length > 0
+    ? input.fixtures
+    : corpus.fixtures;
+  const rubrics = isObj(corpus.rubrics) ? corpus.rubrics : {};
+  const baselineRuns = Array.isArray(input.baselineRuns) ? input.baselineRuns : [];
+  const candidateRuns = Array.isArray(input.candidateRuns) ? input.candidateRuns : [];
+
+  const reasons = [];
+  let dropped = 0;
+
+  // --- stage 1: comparability — drop runs whose frozen coordinates differ --
+  function comparableOnly(runs, armLabel) {
+    const kept = [];
+    for (const run of runs) {
+      if (!isObj(run) || typeof run.fixtureId !== 'string') {
+        dropped += 1;
+        reasons.push(`dropped:${armLabel}:malformed_run`);
+        continue;
+      }
+      const check = checkComparability(run);
+      if (!check.comparable) {
+        dropped += 1;
+        reasons.push(`dropped:${armLabel}:${check.reason}`);
+        continue;
+      }
+      kept.push(run);
+    }
+    return kept;
+  }
+
+  const keptBaseline = comparableOnly(baselineRuns, input.baselineLabel ?? 'A-current-handoff');
+  const keptCandidate = comparableOnly(candidateRuns, input.candidateLabel ?? 'C-no-pruning');
+
+  // --- stage 2: per-class gate via computeGateComparison -------------------
+  const byClass = new Map();
+  for (const f of fixtures) {
+    const cls = typeof f.taskClass === 'string' ? f.taskClass : null;
+    if (cls && !byClass.has(cls)) byClass.set(cls, { fixtures: [], rubric: null });
+    if (cls) byClass.get(cls).fixtures.push(f);
+  }
+  for (const [cls, slot] of byClass) {
+    slot.rubric = rubrics[cls] ?? rubrics[slot.fixtures[0]?.rubric] ?? null;
+  }
+
+  const perClass = {};
+  let anyNonInferior = false;
+  let anyUnknown = false;
+  let anyCoverageMissing = false;
+  const candidateCritical = [];
+
+  for (const [cls, slot] of byClass) {
+    const ids = new Set(slot.fixtures.map((f) => f.id));
+    const bRuns = keptBaseline.filter((r) => ids.has(r.fixtureId));
+    const cRuns = keptCandidate.filter((r) => ids.has(r.fixtureId));
+    const dimensions = slot.rubric && Array.isArray(slot.rubric.dimensions)
+      ? slot.rubric.dimensions
+      : ['correctness', 'evidence', 'completeness'];
+
+    if (bRuns.length === 0 || cRuns.length === 0) {
+      anyCoverageMissing = true;
+      reasons.push(`class:${cls}:coverage_missing`);
+    }
+
+    const gate = computeGateComparison({
+      baseline: toReport(bRuns, dimensions),
+      variant: toReport(cRuns, dimensions),
+      scoreFloor: QUALITY_SCORE_FLOOR,
+    });
+    if (gate.verdict === 'non-inferior') anyNonInferior = true;
+    if (gate.verdict === 'unknown' || gate.comparable !== true) anyUnknown = true;
+
+    const critical = cRuns.filter((r) => r.criticalFailure === true);
+    for (const r of critical) candidateCritical.push(r.fixtureId);
+
+    perClass[cls] = {
+      fixtureCount: slot.fixtures.length,
+      baseline: { runs: bRuns.length, usage: aggregateUsage(bRuns) },
+      candidate: {
+        runs: cRuns.length,
+        unscored: toReport(cRuns, dimensions).unscored,
+        criticalFailures: critical.length,
+        usage: aggregateUsage(cRuns),
+      },
+      gate,
+      savings: computeSavings({
+        baseline: aggregateUsage(bRuns),
+        candidate: aggregateUsage(cRuns),
+      }),
+    };
+  }
+
+  // --- stage 3: independent reviewer (seeded-defect check) ------------------
+  const reviewFixtures = fixtures.filter((f) => f.taskClass === 'review');
+  const reviewIds = new Set(reviewFixtures.map((f) => f.id));
+  const candidateReviewRuns = keptCandidate.filter((r) => reviewIds.has(r.fixtureId));
+  const reviewerEvidence = candidateReviewRuns.filter((r) => isObj(r.reviewer));
+  const reviewerCaught = reviewerEvidence.filter((r) => r.reviewer.detected === true);
+  const reviewerMissed = reviewerEvidence.filter((r) => r.reviewer.detected === false);
+
+  const reviewer = {
+    reviewFixtures: reviewIds.size,
+    runsWithEvidence: reviewerEvidence.length,
+    detected: reviewerCaught.length,
+    missed: reviewerMissed.length,
+    caught: reviewerMissed.length === 0 && reviewerCaught.length > 0,
+    missing: reviewIds.size > 0 && reviewerEvidence.length === 0,
+  };
+  if (reviewer.missed > 0) {
+    reasons.push(`reviewer:missed_seeded_defect:${reviewer.missed}`);
+  }
+  if (reviewer.missing && candidateReviewRuns.length > 0) {
+    reasons.push('reviewer:evidence_missing');
+  }
+
+  // --- stage 4: savings (provenance-gated, verbatim from corpus module) -----
+  const savings = computeSavings({
+    baseline: aggregateUsage(keptBaseline),
+    candidate: aggregateUsage(keptCandidate),
+  });
+  if (!savings.reportable) reasons.push(`savings:${savings.reason}`);
+
+  // --- stage 5: VM-lane promotion verdict (advisory, verbatim) --------------
+  const vmEvidence = isObj(input.vmEvidence) ? input.vmEvidence : { key: 'vm', runs: [] };
+  const vmRuns = Array.isArray(vmEvidence.runs) ? vmEvidence.runs : [];
+  const replay = [...vmRuns].reverse().find((r) => r?.kind === 'replay-comparison') ?? null;
+  const sampled = vmRuns.filter((r) => r?.kind !== 'replay-comparison');
+  const config = isObj(input.config) ? input.config : {};
+  const promotion = promote(config, {
+    key: typeof vmEvidence.key === 'string' ? vmEvidence.key : 'vm',
+    runs: sampled,
+  });
+
+  if (candidateCritical.length > 0 || reviewer.missed > 0) {
+    reasons.push(`critical_failure:blocker:${[...new Set(candidateCritical)].join(',') || 'review'}`);
+  }
+
+  // --- verdict: NO_GO > INCOMPARABLE > GO -----------------------------------
+  let verdict = 'GO';
+  if (anyNonInferior || candidateCritical.length > 0 || reviewer.missed > 0) {
+    verdict = 'NO_GO';
+  } else if (
+    anyUnknown || anyCoverageMissing || !savings.reportable
+    || reviewer.missing
+  ) {
+    verdict = 'INCOMPARABLE';
+  }
+
+  return Object.freeze({
+    verdict,
+    corpus: { revision: CORPUS_REVISION, baselineSha: BASELINE_SHA, rubricVersion: RUBRIC_VERSION, arms: ARMS },
+    baselineLabel: input.baselineLabel ?? 'A-current-handoff',
+    candidateLabel: input.candidateLabel ?? 'C-no-pruning',
+    perClass,
+    savings,
+    reviewer,
+    promotion,
+    replay,
+    sampledRuns: sampled.length,
+    comparableRunsFloor: COMPARABLE_RUNS,
+    dropped,
+    stage: {
+      unchanged: true,
+      grammar: PROMOTION_STAGE_ORDER.join(' → '),
+      rollbackRoute: ROLLBACK_ROUTE,
+      operatorChooses: true,
+    },
+    reasons: Object.freeze(reasons),
+  });
+}
+
+/**
+ * Render the human-readable verdict document. Percentages appear ONLY when
+ * savings.reportable — the doc can never print an unearned cache or cost
+ * number (SPEC §3 FR-01, §12).
+ */
+export function formatVerdictDoc(report, { generatedAt = new Date().toISOString().slice(0, 10) } = {}) {
+  const r = report;
+  const lines = [];
+  lines.push('# Subagent Orchestrator — Golden Eval Verdict (TASK-C89-007)');
+  lines.push('');
+  lines.push(`> ${generatedAt} · corpus \`${r.corpus.revision}\` · baseline sha \`${r.corpus.baselineSha.slice(0, 8)}\` · rubric \`${r.corpus.rubricVersion}\``);
+  lines.push(`> arms: ${r.corpus.arms.map((a) => `\`${a}\``).join(', ')} — compared baseline \`${r.baselineLabel}\` vs candidate \`${r.candidateLabel}\``);
+  lines.push('');
+  lines.push(`## Verdict: **${r.verdict}**`);
+  lines.push('');
+  lines.push('Evidence-backed, not automatic promotion: the evaluator publishes a');
+  lines.push('verdict and reasons only — it never writes config or flips a stage.');
+  lines.push('');
+
+  // Per-class table
+  lines.push('## Per-class quality gate');
+  lines.push('');
+  lines.push('| class | fixtures | baseline runs | candidate runs | gate | comparable | candidate critical | savings |');
+  lines.push('|---|---|---|---|---|---|---|---|');
+  for (const [cls, c] of Object.entries(r.perClass)) {
+    const sav = c.savings.reportable ? `${c.savings.savingsPct.toFixed(1)}%` : `— (${c.savings.reason})`;
+    lines.push(`| ${cls} | ${c.fixtureCount} | ${c.baseline.runs} | ${c.candidate.runs} | ${c.gate.verdict} | ${c.gate.comparable} | ${c.candidate.criticalFailures} | ${sav} |`);
+  }
+  lines.push('');
+
+  // Savings
+  lines.push('## Savings (provenance-gated)');
+  lines.push('');
+  if (r.savings.reportable) {
+    lines.push(`Reportable savings: **${r.savings.savingsPct.toFixed(2)}%** (both arms provider-measured).`);
+  } else {
+    lines.push(`No reportable savings — \`${r.savings.reason}\`. Unknown usage is never reported as zero, never a fabricated number.`);
+  }
+  lines.push('');
+
+  // Reviewer
+  lines.push('## Independent reviewer (seeded defect)');
+  lines.push('');
+  if (r.reviewer.runsWithEvidence === 0) {
+    lines.push(`No reviewer evidence recorded (${r.reviewer.reviewFixtures} review fixture(s)) — evidence missing, not assumed.`);
+  } else {
+    lines.push(`Review runs with evidence: ${r.reviewer.runsWithEvidence}; seeded defect detected: ${r.reviewer.detected}, missed: ${r.reviewer.missed}; caught=${r.reviewer.caught}.`);
+  }
+  lines.push('');
+
+  // Promotion lane
+  lines.push('## VM-lane promotion (advisory)');
+  lines.push('');
+  lines.push(`Sampled runs: ${r.sampledRuns} (floor ${r.comparableRunsFloor}).`);
+  lines.push(`promote() verdict verbatim: stage=\`${r.promotion.stage}\` promoted=\`${r.promotion.promoted}\` reason=\`${r.promotion.reason}\`.`);
+  if (r.replay) {
+    lines.push(`Replay comparison artifact: verdict=\`${r.replay.verdict}\` reason=\`${r.replay.reason}\` defectsCaught=${r.replay.defectsCaught}.`);
+  } else {
+    lines.push('Replay comparison artifact: none recorded.');
+  }
+  lines.push('');
+  lines.push('## Stage & rollout');
+  lines.push('');
+  lines.push(`- Stage grammar: \`${r.stage.grammar}\` — unchanged by this evaluation (${r.stage.unchanged}).`);
+  lines.push(`- Rollback route: ${r.stage.rollbackRoute}.`);
+  lines.push('- The operator, not the evaluator, chooses any rollout. Live `ukit vm run` evidence at canary/default is input, not license.');
+  lines.push('');
+  lines.push('## Context & deferred work');
+  lines.push('');
+  if (r.verdict === 'INCOMPARABLE') {
+    lines.push('Verdict is INCOMPARABLE: no scored fixture runs were recorded on this host');
+    lines.push('(all lanes remain `unsupported-probe` — live in-host E2E not yet proven),');
+    lines.push('and the VM evidence store holds fewer than the COMPARABLE_RUNS floor. This is');
+    lines.push('an evidence verdict, not a failure: quality claims, savings and rollout are all');
+    lines.push('held pending real runs.');
+  } else if (r.verdict === 'NO_GO') {
+    lines.push('Verdict is NO_GO: quality/critical gates failed — see reasons above. Savings,');
+    lines.push('if any, cannot override a quality blocker.');
+  } else {
+    lines.push('Verdict is GO: all pre-registered gates passed on recorded evidence.');
+  }
+  lines.push('');
+  lines.push('SO-008/009 (budget checkpoints, small-model retrieval/extraction, adaptive');
+  lines.push('policies) remain deferred: SPEC §10 FR-08 requires good evidence from this');
+  lines.push('eval before any follow-on task set — INCOMPARABLE provides none, so no');
+  lines.push('follow-on promotion, rollback or adaptive policy work is licensed by this run.');
+  lines.push('');
+
+  // Reasons
+  lines.push('## Reasons');
+  lines.push('');
+  if (r.reasons.length === 0) {
+    lines.push('- none');
+  } else {
+    for (const reason of r.reasons) lines.push(`- ${reason}`);
+  }
+  lines.push('');
+  lines.push(`Dropped runs (non-comparable coordinates): ${r.dropped}.`);
+  lines.push('');
+  return lines.join('\n');
+}
+
+/**
+ * Compose the full evaluation: verdict + rendered document. Pure given its
+ * inputs — the caller supplies runs, config and VM evidence.
+ */
+export function buildEvaluation(input = {}) {
+  const report = evaluateSubagentOrchestrator(input);
+  const markdown = formatVerdictDoc(report, { generatedAt: input.generatedAt });
+  return { report, markdown };
+}
+
+// --- CLI ------------------------------------------------------------------
+
+const USAGE = 'Usage: node scripts/bench/subagent-orchestrator-eval.mjs [options]\n'
+  + '  --runs <file>       JSON {baselineRuns:[], candidateRuns:[], config?, vmEvidence?}\n'
+  + '  --vm-evidence <key> read .ukit/storage/agent-runtime/evidence/<key>.jsonl (default: vm)\n'
+  + '  --write-verdict <path>  write the markdown verdict doc (default: docs/AI_HANDOFF/benchmark/subagent-orchestrator-verdict.md)\n'
+  + '  --json              print the evaluation report JSON to stdout\n'
+  + 'Exit codes: 0 = ok, 1 = usage error, 2 = crash.';
+
+async function loadJsonFile(file) {
+  try {
+    return JSON.parse(await fs.readFile(file, 'utf8'));
+  } catch (err) {
+    return { __error: err?.message ?? String(err) };
+  }
+}
+
+async function main(argv) {
+  const args = argv.slice(2);
+  const flag = (name) => args.find((a) => a === name);
+  const value = (name) => {
+    const i = args.indexOf(name);
+    return i >= 0 && i + 1 < args.length ? args[i + 1] : null;
+  };
+  if (flag('--help') || flag('-h')) {
+    console.log(USAGE);
+    return 0;
+  }
+
+  const runsFile = value('--runs');
+  const vmKey = value('--vm-evidence') ?? 'vm';
+  const writeVerdict = value('--write-verdict');
+  const wantJson = flag('--json') != null;
+
+  let input = {};
+  if (runsFile) {
+    const parsed = await loadJsonFile(path.resolve(runsFile));
+    if (parsed.__error) {
+      console.error(`[UKit] eval: cannot parse runs file — ${parsed.__error}`);
+      return 1;
+    }
+    input = {
+      baselineRuns: parsed.baselineRuns ?? [],
+      candidateRuns: parsed.candidateRuns ?? [],
+      fixtures: parsed.fixtures,
+      config: parsed.config,
+      vmEvidence: parsed.vmEvidence,
+      baselineLabel: parsed.baselineLabel,
+      candidateLabel: parsed.candidateLabel,
+    };
+  }
+
+  // Consumed evidence sources (live lane where available; honest absence
+  // otherwise). readEvidence returns zero runs for a missing file — never a
+  // fabricated sample.
+  const evidence = await readEvidence(REPO_ROOT, { key: vmKey });
+  if (evidence.ok) input.vmEvidence = { key: vmKey, runs: evidence.runs };
+
+  if (input.config == null) {
+    try {
+      const inspection = await inspectRuntimeConfig(REPO_ROOT, { homeDir: os.homedir() });
+      input.config = inspection && typeof inspection.config === 'object' ? inspection.config : {};
+    } catch {
+      input.config = {};
+    }
+  }
+
+  const { report, markdown } = buildEvaluation(input);
+
+  const out = writeVerdict ? path.resolve(writeVerdict) : path.join(REPO_ROOT, VERDICT_DOC_REL);
+  await fs.mkdir(path.dirname(out), { recursive: true });
+  await fs.writeFile(out, markdown, 'utf8');
+
+  if (wantJson) {
+    console.log(JSON.stringify(report, null, 2));
+  } else {
+    console.log(`[UKit] subagent-orchestrator eval: verdict=${report.verdict}`);
+    console.log(`  per-class gates: ${Object.values(report.perClass).map((c) => c.gate.verdict).join(', ')}`);
+    console.log(`  savings: ${report.savings.reportable ? `${report.savings.savingsPct.toFixed(2)}%` : `not reportable (${report.savings.reason})`}`);
+    console.log(`  promote ${vmKey}: stage=${report.promotion.stage} promoted=${report.promotion.promoted} reason=${report.promotion.reason}`);
+    console.log(`  verdict doc: ${path.relative(REPO_ROOT, out)}`);
+  }
+  return 0;
+}
+
+const isMain = process.argv[1]
+  && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+
+if (isMain) {
+  main(process.argv).then((code) => {
+    process.exitCode = code;
+  }).catch((err) => {
+    console.error(`[UKit] subagent-orchestrator eval crashed: ${err?.message ?? err}`);
+    process.exitCode = 2;
+  });
+}