@mutagent/evaluator 0.1.0-alpha.4 → 0.1.0-alpha.6

package/.claude/skills/mutagent-evaluator/scripts/render-build-cards.ts

@@ -83,6 +83,33 @@ export function renderBuildEvalsProgressCard(p: BuildEvalsProgress): string {
   ].join("\n");
 }
 
+// ── EVAL-1 — "what this stage will produce" notice (up-front, at stage start) ─
+export interface StageProducesNotice {
+  subject: string;
+  /** the command about to run (e.g. "*build-dataset" | "*build-evals" | "*eval"). */
+  command: string;
+  /** the named deliverables this stage emits (dataset · criteria · which files). */
+  produces: string[];
+}
+
+/**
+ * EVAL-1 (1) — render the up-front notice that names WHAT this stage will produce,
+ * so a build never runs silently with the operator unaware a dataset/criteria are
+ * coming. Emitted at Step 0, BEFORE the build. PURE. NOT an approval gate — a
+ * notification only (operator constraint).
+ */
+export function renderStageProducesNotice(n: StageProducesNotice): string {
+  const lines = [
+    TOP,
+    row(`▷ ${n.command} — ${n.subject}`),
+    MID,
+    row("this stage will produce:"),
+  ];
+  for (const p of n.produces) lines.push(row(`  • ${p}`));
+  lines.push(BOT);
+  return lines.join("\n");
+}
+
 // ── F22 — dataset entity card (verbose, after *build-dataset) ─────────────────
 export interface DatasetEntity {
   subject: string;
@@ -99,6 +126,12 @@ export function renderDatasetEntityCard(e: DatasetEntity): string {
     MID,
     row(`categories (${e.categories.length}) · ${e.totalItems} items total`),
   ];
+  // EVAL-1 — a build that produced NOTHING must WARN, never render a blank card that
+  // reads as a silent success (the dogfood symptom: a build that reported and a build
+  // that stayed silent looked identical).
+  if (e.categories.length === 0 || e.totalItems === 0) {
+    lines.push(row("⚠ EMPTY — this build produced no dataset items (nothing to evaluate)."));
+  }
   for (const c of e.categories) {
     lines.push(row(`  • ${c.id}: ${c.items} items  (edge: ${c.edgeItems})`));
   }
@@ -129,6 +162,10 @@ export function renderEvalsEntityCard(e: EvalsEntity): string {
     row(`output sink: ${e.outputSink}`),
     row(`criteria (${e.criteria.length}):`),
   ];
+  // EVAL-1 — an eval suite with zero criteria is a silent no-op build; WARN loudly.
+  if (e.criteria.length === 0) {
+    lines.push(row("⚠ EMPTY — this build produced no criteria (the suite judges nothing)."));
+  }
   for (const c of e.criteria) {
     lines.push(row(`  • ${c.id}  [${c.severity}·${c.kind}]`));
   }

package/.claude/skills/mutagent-evaluator/scripts/render-eval-report.ts

@@ -69,6 +69,14 @@ import { assessEmitCompleteness, type EmitCompleteness } from "./emit-completene
 import type { Scorecard } from "./evaluate.ts";
 import type { SourceMap } from "./source-map.ts";
 import { routeFailures, type FailureRef, type HandoverBundle } from "./route-failures.ts";
+// EV-3 — the step<->criterion side-by-side render fns (`cvRefs` + `critiqueBlock`
+// + `sideBySide`, the last carrying its local `refExaminesStep` + `covEntry`) are
+// EXTRACTED VERBATIM into report-fragments.ts and spliced back into the client
+// script IN PLACE below, so this report's emitted HTML is byte-identical to
+// before the extraction. The review UI (`build-review-ui.ts`) imports the SAME
+// constant to render the identical view. Byte-identity is guarded by this file's
+// render tests + a before/after HTML diff.
+import { SIDE_BY_SIDE_FRAGMENT_JS } from "./report-fragments.ts";
 
 const HERE = dirname(fileURLToPath(import.meta.url));
 const BRAND_DIR = join(HERE, "..", "assets", "brand");
@@ -3319,97 +3327,7 @@ function render(){var qi=document.getElementById('q');if(!qi)return;var q=(qi.va
 // critique-before-verdict + grounding refs, read off the verdict file's verdicts[].
 // Shared by BOTH the side-by-side drill (walk traces) and the per-trajectory
 // scorecard drill (no-walk traces) so NO judged trace renders an empty panel.
-function cvRefs(refs){if(!refs||!refs.length)return '';
-  return '<div class="cvrefs">'+refs.map(function(rf){var s=(typeof rf==='string')?rf:[rf.obs,rf.path,rf.value].filter(Boolean).join(' · ');return s?'<span class="cvref">'+esc(s)+'</span>':'';}).join('')+'</div>';}
-function critiqueBlock(cvs){if(!cvs||!cvs.length)return '';
-  var order={fail:0,uncertain:1,indeterminate:1,pass:2,na:3};
-  var rows=cvs.slice().sort(function(a,b){return (order[a.result]==null?9:order[a.result])-(order[b.result]==null?9:order[b.result]);}).map(function(v){
-    var res=v.result||'na';var disp=res==='uncertain'?'indeterminate':res;
-    var conf=(v.confidence!=null)?'<span class="cvconf">conf '+esc(v.confidence)+(v.confidenceBand?' · '+esc(v.confidenceBand):'')+'</span>':'';
-    var crit=v.critique?'<div class="cvcrit">'+esc(v.critique)+'</div>':'<div class="cvcrit dim">— no critique recorded for this criterion</div>';
-    return '<div class="cvrow '+esc(res)+'"><div class="cvh"><span class="cvb '+esc(res)+'">'+esc(disp)+'</span><span class="cvid">'+esc(v.criterionId||'')+'</span>'+conf+'</div>'+crit+cvRefs(v.refs)+'</div>';}).join('');
-  return '<div class="ctx cvblock"><div class="ctx-h">◇ how the judge reasoned — per-criterion verdict · critique-before-verdict · grounding refs ('+cvs.length+')</div><div class="cvbody">'+rows+'</div></div>';}
-function sideBySide(d){var ctx=d.context||{};
-  // Gap A — the §2 "input + scenario" cell renders the RAW triggering INPUT (the
-  // thing that fired the agent) ABOVE the judge's scenario LABEL. Long inputs
-  // collapse into a <details> (lean, no JS) so the cell stays compact; short ones
-  // render inline (clamped). ABSENT input ⇒ the cell shows the scenario alone (or
-  // "—"). Font stays at the --fs-2xs (11px) floor + brand mono.
-  var inputCell=function(raw,scen){
-    var sc=scen?'<div class="iscn">scenario · '+esc(scen)+'</div>':'';
-    if(!raw)return (scen?'<div class="ival">'+esc(scen)+'</div>':'—');
-    var long=String(raw).length>180;
-    var body=long
-      ? '<details class="iexp"><summary>raw input · '+String(raw).length+' chars</summary><pre class="iraw">'+esc(raw)+'</pre></details>'
-      : '<pre class="iraw clamp">'+esc(raw)+'</pre>';
-    return body+sc;};
-  var refStr=function(rf){if(!rf)return '';if(typeof rf==='string')return rf;return [rf.obs,rf.path,rf.value].filter(Boolean).join(':');};
-  // UI-4 — surface the judge's REASONING (why the verdict + why the exit-states were
-  // concluded) from the EXISTING judge walk: an ordered why-chain band + the decide/bind
-  // text inlined where it explains a conclusion. No emit-contract change — read-only over
-  // the judge_steps already on the verdict file.
-  var KORD={gather:0,context:1,examine:2,detect:3,bind:4,ground:5,critique:6,decide:7,verify:8};
-  var allJs=(d.judgeSteps||[]).slice();
-  var stepText=function(kind){return allJs.filter(function(s){return s.kind===kind;}).map(function(s){return s.text||'';}).filter(Boolean).join(' · ');};
-  var decideWhy=stepText('decide')||stepText('critique');
-  var stateWhy=[stepText('bind'),stepText('gather')].filter(Boolean).join(' · ');
-  var chainSteps=allJs.slice().sort(function(a,b){var x=KORD[a.kind];var y=KORD[b.kind];return (x==null?9:x)-(y==null?9:y);});
-  var whyChain=chainSteps.length?('<div class="ctx whychain"><div class="ctx-h">◇ judge reasoning — full why-chain (gather → bind → ground → decide)</div><div class="wc-body">'+chainSteps.map(function(s){var rs=refStr(s.ref);return '<div class="jstep"><span class="k '+esc(s.kind)+'">'+esc(s.kind)+'</span><span class="t">'+esc(s.text||'')+(rs?' <span class=ref>'+esc(rs)+'</span>':'')+'</span></div>';}).join('')+'</div></div>'):'';
-  var verdictWhy=decideWhy?('<div class="band whyverdict"><div class="bh" style="color:var(--primary-soft)">◇ why this verdict</div><div class="wv-t">'+esc(decideWhy)+'</div></div>'):'';
-  // -- §2 judge lane = per-step EVAL COVERAGE --
-  // The judge lane (.step-r) used to filter judgeSteps by anchor === a.n -- but
-  // judges never emit an anchor, so EVERY step rendered a bare dash. Instead we
-  // map each agent step -> the per-criterion verdicts (d.criterionVerdicts) whose
-  // grounding refs EXAMINED that step: a precise ref.obs === step.obs match plus
-  // the tool-name fallback (ref.path === 'name' && ref.value === step.tool).
-  // Each examining criterion renders one compact entry -- result + CODE/JUDGE tag
-  // + criterionId + the judge reasoning (critique). A step no criterion references
-  // says 'not examined by any eval' (honest), never a bare dash. Any judge step
-  // that DOES carry a real anchor is still honored (future-proof).
-  var cvAll=(d.criterionVerdicts||[]);
-  var refExaminesStep=function(rf,a){
-    if(!rf||typeof rf==='string')return false;
-    if(a.obs&&rf.obs&&String(rf.obs)===String(a.obs))return true;
-    if(rf.path==='name'&&rf.value!=null&&a.tool&&String(rf.value)===String(a.tool))return true;
-    return false;};
-  var covEntry=function(v,a){
-    var res=v.result||'na';var disp=res==='uncertain'?'indeterminate':res;
-    // the router carries TWO vocabularies: matrix-derived ('deterministic') and
-    // mined ('code-based'); 'hybrid' is shared. A '[code-eval ...]'-prefixed critique
-    // is the deterministic-logic fallback when the method is unset.
-    var method=((typeof C!=='undefined'&&C[v.criterionId])||{}).m||'';
-    var isCode=method==='deterministic'||method==='code-based'||method==='hybrid'||String(v.critique||'').trim().indexOf('[code-eval')===0;
-    var tag=method==='hybrid'?'HYBRID':(isCode?'CODE':'JUDGE');
-    var matched=(v.refs||[]).filter(function(rf){return typeof rf!=='string'&&refExaminesStep(rf,a);});
-    var crit=v.critique?esc(v.critique):'<span class="dim">— no critique recorded</span>';
-    return '<div class="jcov '+esc(res)+'"><div class="jcov-h"><span class="cvb '+esc(res)+'">'+esc(disp)+'</span><span class="jm '+(isCode?'code':'judge')+'" title="'+(isCode?'deterministically checked by a code-eval (not LLM-judged)':'reasoned by the LLM judge')+'">'+esc(tag)+'</span><span class="jcid">'+esc(v.criterionId||'')+'</span></div><div class="jcrit">'+crit+'</div>'+cvRefs(matched)+'</div>';};
-  var rowsHtml='';(d.agentSteps||[]).forEach(function(a){
-    // future-proof: a judge step that carries a REAL anchor still renders.
-    var js=(d.judgeSteps||[]).filter(function(s){return s.anchor!=null&&String(s.anchor)===String(a.n)&&s.kind!=='context';});
-    var anchoredHtml=js.map(function(s){var rs=refStr(s.ref);return '<div class="jstep"><span class="k '+esc(s.kind)+'">'+esc(s.kind)+'</span><span class="t">'+esc(s.text||'')+' '+(rs?'<span class=ref>'+esc(rs)+'</span>':'')+'</span></div>';}).join('');
-    var examiners=cvAll.filter(function(v){return (v.refs||[]).some(function(rf){return refExaminesStep(rf,a);});});
-    var covHtml=examiners.map(function(v){return covEntry(v,a);}).join('');
-    var jhtml=anchoredHtml+covHtml;
-    if(!jhtml)jhtml='<div class="jstep noexam"><span class="t">— not examined by any eval</span></div>';
-    rowsHtml+='<div class="step-l"><div class="evb"><div class="top"><span class="tool">'+esc(a.tool||'')+'</span><span class="st '+esc(a.status||'')+'">'+esc(a.status||'')+'</span></div><div class="det">'+esc(a.detail||'')+'</div></div></div><div class="node '+esc(a.status||'')+'"><div class="ln"></div><div class="n">'+esc(a.n)+'</div><div class="ln"></div></div><div class="step-r"><div class="evb r">'+jhtml+'</div></div>';});
-  var h=d.health||{};
-  var sp=d.subjectProfile;
-  var spHtml=sp?'<div class="ctx"><div class="ctx-h">◇ judge · subject profile (M1) · '+esc(sp.provenance||'')+'</div><div class="ctx-g"><div class="ctx-c"><div class="l">identity</div><div class="v">'+esc(sp.identity||'—')+'</div></div><div class="ctx-c"><div class="l">purpose</div><div class="v">'+esc(sp.purpose||'—')+'</div></div><div class="ctx-c"><div class="l">scope · harness</div><div class="v">'+esc((sp.scope||'—'))+' · harness: '+esc(sp.harness||'—')+'</div></div></div></div>':'';
-  var u=d.understanding;
-  var uHtml=u?'<div class="band"><div class="bh" style="color:var(--cyan)">◇ node-0 GATHER — understanding (M2)</div><div class="re-rephrase" style="margin-top:4px">“'+esc(u.rephrase||'')+'”</div></div>':'';
-  var et=d.expectedTrajectory||[];
-  var etHtml=et.length?'<div class="band"><div class="bh" style="color:var(--primary-soft)">◇ node-0.5 EXPECTED-TRAJECTORY (M3) — how it SHOULD have acted</div><ol class="re-ul" style="margin-top:4px">'+et.map(function(s,i){return '<li><b>'+esc(s.step||i+1)+'.</b> '+esc(s.expected||'')+(s.rationale?' <span class=re-rat>— '+esc(s.rationale)+'</span>':'')+'</li>';}).join('')+'</ol></div>':'';
-  return '<div class="drillbox"><div style="display:flex;gap:9px;align-items:center;flex-wrap:wrap;margin-bottom:6px"><b class="mono">'+esc(d.traceId)+'</b><span class="chip">'+esc(d.route||'all')+'</span><span class="verd '+(d.verdict=='FAIL'?'fail':d.verdict=='PASS'?'pass':'inc')+'">'+esc(d.verdict)+'</span></div>'+
-    resRouting(d.res||'judge-walk')+
-    spHtml+
-    verdictWhy+
-    critiqueBlock(d.criterionVerdicts)+
-    '<div class="ctx"><div class="ctx-h">◇ judge · gather context</div><div class="ctx-g"><div class="ctx-c"><div class="l">harness</div><div class="v">'+esc(ctx.harness||'—')+'</div></div><div class="ctx-c"><div class="l">input + scenario</div><div class="v">'+inputCell(d.input,ctx.scenario)+'</div></div><div class="ctx-c"><div class="l">exit states</div><div class="v">'+esc(ctx.exitStates||'—')+(stateWhy?'<div class="why-note"><span class="ref">why concluded:</span> '+esc(stateWhy)+'</div>':'')+'</div></div></div></div>'+
-    whyChain+
-    uHtml+etHtml+
-    '<div class="lanehdr"><div class="a">target agent — what it did</div><div class="x">step</div><div class="j">judge — eval coverage (which criteria examined this step)</div></div>'+
-    '<div class="grid2">'+rowsHtml+'<div class="band loc"><div class="bh">↯ localize (root, not symptom)</div><div style="font-size:var(--fs-sm);margin-top:4px">'+esc(d.localize||'—')+'</div></div></div>'+
-    '<div class="health"><div class="hc"><div class="l">context</div><div class="v good">'+(h.contextGathered?'✓':'—')+'</div></div><div class="hc"><div class="l">grounded</div><div class="v good">'+esc(h.grounded||0)+'</div></div><div class="hc"><div class="l">assumed</div><div class="v '+((h.assumed||0)>0?'warn':'good')+'">'+esc(h.assumed||0)+'</div></div><div class="hc"><div class="l">root vs symptom</div><div class="v good">'+(h.stoppedAtSymptom?'symptom':'✓ root')+'</div></div></div></div>';}
+${SIDE_BY_SIDE_FRAGMENT_JS}
 function scorecard(r){var order={fail:0,uncertain:1,indeterminate:1,pass:2,na:3};
   var cells=Object.keys(r.c).sort(function(a,b){return (order[r.c[a]]||9)-(order[r.c[b]]||9);}).map(function(k){var v=r.c[k];var disp=v=='uncertain'?'indeterminate':v;
     return '<div class="scc '+esc(v)+'"><span class="nm" title="'+esc((C[k]||{}).n||k)+'">'+esc(k)+'</span><span style="margin-left:auto;color:var(--muted)">'+esc(disp)+'</span></div>';}).join('');

package/.claude/skills/mutagent-evaluator/scripts/report-fragments.ts

@@ -0,0 +1,173 @@
+/**
+ * scripts/report-fragments.ts — EV-3 shared client-JS render fragments.
+ * ---------------------------------------------------------------------------
+ * The step<->criterion side-by-side render functions, extracted VERBATIM from
+ * `render-eval-report.ts` so BOTH the eval report AND the `*review` UI
+ * (`build-review-ui.ts`) emit the identical browser-side renderer.
+ *
+ * `SIDE_BY_SIDE_FRAGMENT_JS` is the BYTE-IDENTICAL block (`cvRefs` +
+ * `critiqueBlock` + `sideBySide` — the latter carrying its local
+ * `refExaminesStep` + `covEntry`). render-eval-report splices it into its
+ * client script IN PLACE, so the report's emitted HTML is unchanged (guarded by
+ * the render-eval-report byte-identity tests + a before/after HTML diff).
+ *
+ * The fragment is provider-of-record for the render logic; its runtime deps
+ * (`esc`, `resRouting`, `resBadge`, and the globals `C` / `RES`) are
+ * supplied by whichever host script embeds it (render-eval-report defines them
+ * above the splice; the review UI defines compatible copies — see
+ * `REVIEW_SIDE_BY_SIDE_DEPS_JS`).
+ *
+ * `STEP_ROW_FRAGMENT_JS` is the review-UI VIRTUALIZATION mirror: standalone
+ * `refExaminesStepR` / `covEntryR` / `stepRowR` that emit ONE step triplet
+ * with markup IDENTICAL to `sideBySide`'s per-step output — so the review UI
+ * can window a 1,686-step card without materializing the whole DOM. A drift
+ * guard (`tests/report-fragments.test.ts`) evaluates both paths and asserts the
+ * per-step HTML matches, so the mirror can never silently diverge from the
+ * frozen fragment.
+ */
+
+/**
+ * BYTE-IDENTICAL extraction of the eval report's `cvRefs` + `critiqueBlock` +
+ * `sideBySide` client functions (`sideBySide` contains its local
+ * `refExaminesStep` + `covEntry`). Spliced verbatim by render-eval-report and
+ * reused by the review UI. DO NOT EDIT — regenerate from render-eval-report if
+ * that block ever changes, and re-verify byte-identity.
+ */
+export const SIDE_BY_SIDE_FRAGMENT_JS = `function cvRefs(refs){if(!refs||!refs.length)return '';
+  return '<div class="cvrefs">'+refs.map(function(rf){var s=(typeof rf==='string')?rf:[rf.obs,rf.path,rf.value].filter(Boolean).join(' · ');return s?'<span class="cvref">'+esc(s)+'</span>':'';}).join('')+'</div>';}
+function critiqueBlock(cvs){if(!cvs||!cvs.length)return '';
+  var order={fail:0,uncertain:1,indeterminate:1,pass:2,na:3};
+  var rows=cvs.slice().sort(function(a,b){return (order[a.result]==null?9:order[a.result])-(order[b.result]==null?9:order[b.result]);}).map(function(v){
+    var res=v.result||'na';var disp=res==='uncertain'?'indeterminate':res;
+    var conf=(v.confidence!=null)?'<span class="cvconf">conf '+esc(v.confidence)+(v.confidenceBand?' · '+esc(v.confidenceBand):'')+'</span>':'';
+    var crit=v.critique?'<div class="cvcrit">'+esc(v.critique)+'</div>':'<div class="cvcrit dim">— no critique recorded for this criterion</div>';
+    return '<div class="cvrow '+esc(res)+'"><div class="cvh"><span class="cvb '+esc(res)+'">'+esc(disp)+'</span><span class="cvid">'+esc(v.criterionId||'')+'</span>'+conf+'</div>'+crit+cvRefs(v.refs)+'</div>';}).join('');
+  return '<div class="ctx cvblock"><div class="ctx-h">◇ how the judge reasoned — per-criterion verdict · critique-before-verdict · grounding refs ('+cvs.length+')</div><div class="cvbody">'+rows+'</div></div>';}
+function sideBySide(d){var ctx=d.context||{};
+  // Gap A — the §2 "input + scenario" cell renders the RAW triggering INPUT (the
+  // thing that fired the agent) ABOVE the judge's scenario LABEL. Long inputs
+  // collapse into a <details> (lean, no JS) so the cell stays compact; short ones
+  // render inline (clamped). ABSENT input ⇒ the cell shows the scenario alone (or
+  // "—"). Font stays at the --fs-2xs (11px) floor + brand mono.
+  var inputCell=function(raw,scen){
+    var sc=scen?'<div class="iscn">scenario · '+esc(scen)+'</div>':'';
+    if(!raw)return (scen?'<div class="ival">'+esc(scen)+'</div>':'—');
+    var long=String(raw).length>180;
+    var body=long
+      ? '<details class="iexp"><summary>raw input · '+String(raw).length+' chars</summary><pre class="iraw">'+esc(raw)+'</pre></details>'
+      : '<pre class="iraw clamp">'+esc(raw)+'</pre>';
+    return body+sc;};
+  var refStr=function(rf){if(!rf)return '';if(typeof rf==='string')return rf;return [rf.obs,rf.path,rf.value].filter(Boolean).join(':');};
+  // UI-4 — surface the judge's REASONING (why the verdict + why the exit-states were
+  // concluded) from the EXISTING judge walk: an ordered why-chain band + the decide/bind
+  // text inlined where it explains a conclusion. No emit-contract change — read-only over
+  // the judge_steps already on the verdict file.
+  var KORD={gather:0,context:1,examine:2,detect:3,bind:4,ground:5,critique:6,decide:7,verify:8};
+  var allJs=(d.judgeSteps||[]).slice();
+  var stepText=function(kind){return allJs.filter(function(s){return s.kind===kind;}).map(function(s){return s.text||'';}).filter(Boolean).join(' · ');};
+  var decideWhy=stepText('decide')||stepText('critique');
+  var stateWhy=[stepText('bind'),stepText('gather')].filter(Boolean).join(' · ');
+  var chainSteps=allJs.slice().sort(function(a,b){var x=KORD[a.kind];var y=KORD[b.kind];return (x==null?9:x)-(y==null?9:y);});
+  var whyChain=chainSteps.length?('<div class="ctx whychain"><div class="ctx-h">◇ judge reasoning — full why-chain (gather → bind → ground → decide)</div><div class="wc-body">'+chainSteps.map(function(s){var rs=refStr(s.ref);return '<div class="jstep"><span class="k '+esc(s.kind)+'">'+esc(s.kind)+'</span><span class="t">'+esc(s.text||'')+(rs?' <span class=ref>'+esc(rs)+'</span>':'')+'</span></div>';}).join('')+'</div></div>'):'';
+  var verdictWhy=decideWhy?('<div class="band whyverdict"><div class="bh" style="color:var(--primary-soft)">◇ why this verdict</div><div class="wv-t">'+esc(decideWhy)+'</div></div>'):'';
+  // -- §2 judge lane = per-step EVAL COVERAGE --
+  // The judge lane (.step-r) used to filter judgeSteps by anchor === a.n -- but
+  // judges never emit an anchor, so EVERY step rendered a bare dash. Instead we
+  // map each agent step -> the per-criterion verdicts (d.criterionVerdicts) whose
+  // grounding refs EXAMINED that step: a precise ref.obs === step.obs match plus
+  // the tool-name fallback (ref.path === 'name' && ref.value === step.tool).
+  // Each examining criterion renders one compact entry -- result + CODE/JUDGE tag
+  // + criterionId + the judge reasoning (critique). A step no criterion references
+  // says 'not examined by any eval' (honest), never a bare dash. Any judge step
+  // that DOES carry a real anchor is still honored (future-proof).
+  var cvAll=(d.criterionVerdicts||[]);
+  var refExaminesStep=function(rf,a){
+    if(!rf||typeof rf==='string')return false;
+    if(a.obs&&rf.obs&&String(rf.obs)===String(a.obs))return true;
+    if(rf.path==='name'&&rf.value!=null&&a.tool&&String(rf.value)===String(a.tool))return true;
+    return false;};
+  var covEntry=function(v,a){
+    var res=v.result||'na';var disp=res==='uncertain'?'indeterminate':res;
+    // the router carries TWO vocabularies: matrix-derived ('deterministic') and
+    // mined ('code-based'); 'hybrid' is shared. A '[code-eval ...]'-prefixed critique
+    // is the deterministic-logic fallback when the method is unset.
+    var method=((typeof C!=='undefined'&&C[v.criterionId])||{}).m||'';
+    var isCode=method==='deterministic'||method==='code-based'||method==='hybrid'||String(v.critique||'').trim().indexOf('[code-eval')===0;
+    var tag=method==='hybrid'?'HYBRID':(isCode?'CODE':'JUDGE');
+    var matched=(v.refs||[]).filter(function(rf){return typeof rf!=='string'&&refExaminesStep(rf,a);});
+    var crit=v.critique?esc(v.critique):'<span class="dim">— no critique recorded</span>';
+    return '<div class="jcov '+esc(res)+'"><div class="jcov-h"><span class="cvb '+esc(res)+'">'+esc(disp)+'</span><span class="jm '+(isCode?'code':'judge')+'" title="'+(isCode?'deterministically checked by a code-eval (not LLM-judged)':'reasoned by the LLM judge')+'">'+esc(tag)+'</span><span class="jcid">'+esc(v.criterionId||'')+'</span></div><div class="jcrit">'+crit+'</div>'+cvRefs(matched)+'</div>';};
+  var rowsHtml='';(d.agentSteps||[]).forEach(function(a){
+    // future-proof: a judge step that carries a REAL anchor still renders.
+    var js=(d.judgeSteps||[]).filter(function(s){return s.anchor!=null&&String(s.anchor)===String(a.n)&&s.kind!=='context';});
+    var anchoredHtml=js.map(function(s){var rs=refStr(s.ref);return '<div class="jstep"><span class="k '+esc(s.kind)+'">'+esc(s.kind)+'</span><span class="t">'+esc(s.text||'')+' '+(rs?'<span class=ref>'+esc(rs)+'</span>':'')+'</span></div>';}).join('');
+    var examiners=cvAll.filter(function(v){return (v.refs||[]).some(function(rf){return refExaminesStep(rf,a);});});
+    var covHtml=examiners.map(function(v){return covEntry(v,a);}).join('');
+    var jhtml=anchoredHtml+covHtml;
+    if(!jhtml)jhtml='<div class="jstep noexam"><span class="t">— not examined by any eval</span></div>';
+    rowsHtml+='<div class="step-l"><div class="evb"><div class="top"><span class="tool">'+esc(a.tool||'')+'</span><span class="st '+esc(a.status||'')+'">'+esc(a.status||'')+'</span></div><div class="det">'+esc(a.detail||'')+'</div></div></div><div class="node '+esc(a.status||'')+'"><div class="ln"></div><div class="n">'+esc(a.n)+'</div><div class="ln"></div></div><div class="step-r"><div class="evb r">'+jhtml+'</div></div>';});
+  var h=d.health||{};
+  var sp=d.subjectProfile;
+  var spHtml=sp?'<div class="ctx"><div class="ctx-h">◇ judge · subject profile (M1) · '+esc(sp.provenance||'')+'</div><div class="ctx-g"><div class="ctx-c"><div class="l">identity</div><div class="v">'+esc(sp.identity||'—')+'</div></div><div class="ctx-c"><div class="l">purpose</div><div class="v">'+esc(sp.purpose||'—')+'</div></div><div class="ctx-c"><div class="l">scope · harness</div><div class="v">'+esc((sp.scope||'—'))+' · harness: '+esc(sp.harness||'—')+'</div></div></div></div>':'';
+  var u=d.understanding;
+  var uHtml=u?'<div class="band"><div class="bh" style="color:var(--cyan)">◇ node-0 GATHER — understanding (M2)</div><div class="re-rephrase" style="margin-top:4px">“'+esc(u.rephrase||'')+'”</div></div>':'';
+  var et=d.expectedTrajectory||[];
+  var etHtml=et.length?'<div class="band"><div class="bh" style="color:var(--primary-soft)">◇ node-0.5 EXPECTED-TRAJECTORY (M3) — how it SHOULD have acted</div><ol class="re-ul" style="margin-top:4px">'+et.map(function(s,i){return '<li><b>'+esc(s.step||i+1)+'.</b> '+esc(s.expected||'')+(s.rationale?' <span class=re-rat>— '+esc(s.rationale)+'</span>':'')+'</li>';}).join('')+'</ol></div>':'';
+  return '<div class="drillbox"><div style="display:flex;gap:9px;align-items:center;flex-wrap:wrap;margin-bottom:6px"><b class="mono">'+esc(d.traceId)+'</b><span class="chip">'+esc(d.route||'all')+'</span><span class="verd '+(d.verdict=='FAIL'?'fail':d.verdict=='PASS'?'pass':'inc')+'">'+esc(d.verdict)+'</span></div>'+
+    resRouting(d.res||'judge-walk')+
+    spHtml+
+    verdictWhy+
+    critiqueBlock(d.criterionVerdicts)+
+    '<div class="ctx"><div class="ctx-h">◇ judge · gather context</div><div class="ctx-g"><div class="ctx-c"><div class="l">harness</div><div class="v">'+esc(ctx.harness||'—')+'</div></div><div class="ctx-c"><div class="l">input + scenario</div><div class="v">'+inputCell(d.input,ctx.scenario)+'</div></div><div class="ctx-c"><div class="l">exit states</div><div class="v">'+esc(ctx.exitStates||'—')+(stateWhy?'<div class="why-note"><span class="ref">why concluded:</span> '+esc(stateWhy)+'</div>':'')+'</div></div></div></div>'+
+    whyChain+
+    uHtml+etHtml+
+    '<div class="lanehdr"><div class="a">target agent — what it did</div><div class="x">step</div><div class="j">judge — eval coverage (which criteria examined this step)</div></div>'+
+    '<div class="grid2">'+rowsHtml+'<div class="band loc"><div class="bh">↯ localize (root, not symptom)</div><div style="font-size:var(--fs-sm);margin-top:4px">'+esc(d.localize||'—')+'</div></div></div>'+
+    '<div class="health"><div class="hc"><div class="l">context</div><div class="v good">'+(h.contextGathered?'✓':'—')+'</div></div><div class="hc"><div class="l">grounded</div><div class="v good">'+esc(h.grounded||0)+'</div></div><div class="hc"><div class="l">assumed</div><div class="v '+((h.assumed||0)>0?'warn':'good')+'">'+esc(h.assumed||0)+'</div></div><div class="hc"><div class="l">root vs symptom</div><div class="v good">'+(h.stoppedAtSymptom?'symptom':'✓ root')+'</div></div></div></div>';}`;
+
+/**
+ * The runtime deps the extracted `sideBySide` needs — `esc` / `resBadge` /
+ * `resRouting` — copied BYTE-IDENTICALLY from render-eval-report's client script
+ * so the reused `sideBySide` behaves identically wherever it is embedded. The
+ * review UI embeds these (render-eval-report already defines them above its own
+ * splice). `RES` may be an empty map in the review UI — `resRouting`/`resBadge`
+ * degrade to '' when a resolution key is absent.
+ */
+export const REVIEW_SIDE_BY_SIDE_DEPS_JS = `function esc(s){return String(s==null?'':s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;');}
+function resBadge(res){var m=RES[res];if(!m)return '';return '<span class="resbadge '+m.cls+'" title="'+esc(m.routing)+'">'+esc(m.badge)+'</span>';}
+function resRouting(res){var m=RES[res];if(!m)return '';return '<div class="routing '+m.cls+'"><span class="routing-k">resolution</span><span class="routing-b '+m.cls+'">'+esc(m.badge)+'</span><span class="routing-v">'+esc(m.routing)+'</span></div>';}`;
+
+/**
+ * The review-UI VIRTUALIZATION mirror. `stepRowR` emits ONE step triplet
+ * (`.step-l` + `.node` + `.step-r`) whose markup is CHARACTER-IDENTICAL to the
+ * per-step output of the frozen `sideBySide` (compare render-eval-report's
+ * `rowsHtml` loop). This lets the review UI window a 1,686-step card — rendering
+ * only the visible slice — instead of materializing the whole DOM (the EV-3
+ * wall). `refExaminesStepR` / `covEntryR` mirror `sideBySide`'s local
+ * `refExaminesStep` / `covEntry`. Depends on `esc` + `cvRefs` (from the deps +
+ * the frozen fragment). The drift guard in `tests/report-fragments.test.ts`
+ * evaluates both paths and asserts equality, so this mirror can never diverge.
+ */
+export const STEP_ROW_FRAGMENT_JS = `function refStrR(rf){if(!rf)return '';if(typeof rf==='string')return rf;return [rf.obs,rf.path,rf.value].filter(Boolean).join(':');}
+function refExaminesStepR(rf,a){
+  if(!rf||typeof rf==='string')return false;
+  if(a.obs&&rf.obs&&String(rf.obs)===String(a.obs))return true;
+  if(rf.path==='name'&&rf.value!=null&&a.tool&&String(rf.value)===String(a.tool))return true;
+  return false;}
+function covEntryR(v,a){
+  var res=v.result||'na';var disp=res==='uncertain'?'indeterminate':res;
+  var method=((typeof C!=='undefined'&&C&&C[v.criterionId])||{}).m||'';
+  var isCode=method==='deterministic'||method==='code-based'||method==='hybrid'||String(v.critique||'').trim().indexOf('[code-eval')===0;
+  var tag=method==='hybrid'?'HYBRID':(isCode?'CODE':'JUDGE');
+  var matched=(v.refs||[]).filter(function(rf){return typeof rf!=='string'&&refExaminesStepR(rf,a);});
+  var crit=v.critique?esc(v.critique):'<span class="dim">— no critique recorded</span>';
+  return '<div class="jcov '+esc(res)+'"><div class="jcov-h"><span class="cvb '+esc(res)+'">'+esc(disp)+'</span><span class="jm '+(isCode?'code':'judge')+'" title="'+(isCode?'deterministically checked by a code-eval (not LLM-judged)':'reasoned by the LLM judge')+'">'+esc(tag)+'</span><span class="jcid">'+esc(v.criterionId||'')+'</span></div><div class="jcrit">'+crit+'</div>'+cvRefs(matched)+'</div>';}
+function stepRowInnerR(a,cvAll,judgeSteps){
+  var js=(judgeSteps||[]).filter(function(s){return s.anchor!=null&&String(s.anchor)===String(a.n)&&s.kind!=='context';});
+  var anchoredHtml=js.map(function(s){var rs=refStrR(s.ref);return '<div class="jstep"><span class="k '+esc(s.kind)+'">'+esc(s.kind)+'</span><span class="t">'+esc(s.text||'')+' '+(rs?'<span class=ref>'+esc(rs)+'</span>':'')+'</span></div>';}).join('');
+  var examiners=(cvAll||[]).filter(function(v){return (v.refs||[]).some(function(rf){return refExaminesStepR(rf,a);});});
+  var covHtml=examiners.map(function(v){return covEntryR(v,a);}).join('');
+  var jhtml=anchoredHtml+covHtml;
+  if(!jhtml)jhtml='<div class="jstep noexam"><span class="t">— not examined by any eval</span></div>';
+  return '<div class="step-l"><div class="evb"><div class="top"><span class="tool">'+esc(a.tool||'')+'</span><span class="st '+esc(a.status||'')+'">'+esc(a.status||'')+'</span></div><div class="det">'+esc(a.detail||'')+'</div></div></div><div class="node '+esc(a.status||'')+'"><div class="ln"></div><div class="n">'+esc(a.n)+'</div><div class="ln"></div></div><div class="step-r"><div class="evb r">'+jhtml+'</div></div>';}
+function stepExaminedByR(a,cvAll){return (cvAll||[]).some(function(v){return (v.refs||[]).some(function(rf){return refExaminesStepR(rf,a);});});}`;

package/.claude/skills/mutagent-evaluator/scripts/route-failures.ts

@@ -35,12 +35,18 @@ export const AdlStage = {
   Build: "build",
   Evaluate: "evaluate",
   Diagnose: "diagnose",
-  Improve: "improve",
+  Optimize: "optimize",
   Audit: "audit",
 } as const;
 export type AdlStageValue = (typeof AdlStage)[keyof typeof AdlStage];
 
-export const SubjectKind = { Skill: "skill", Agent: "agent" } as const;
+// Mirrors the orchestrator handover-contract SubjectKind. The CANONICAL inter-stage
+// subject vocabulary is `agent | skill | code` (matches config-schema TargetSubject).
+// `code` (Wave-2 follow-up) is ADDITIVE — every existing skill/agent handoff still
+// validates; it only lets a `code` subject route through *evaluate's failure handoff
+// the SAME way it flows through the orchestrator handover-contract (the two are kept
+// in lock-step by design; the evaluator re-implements the shape, never imports it).
+export const SubjectKind = { Skill: "skill", Agent: "agent", Code: "code" } as const;
 export type SubjectKindValue = (typeof SubjectKind)[keyof typeof SubjectKind];
 
 export const ArtifactKind = {
@@ -67,7 +73,11 @@ export type EscalationPolicyValue =
 // ── TypeBox schemas (closed objects — auditable boundary) ───────────────────
 const SubjectSchema = Type.Object(
   {
-    kind: Type.Union([Type.Literal(SubjectKind.Skill), Type.Literal(SubjectKind.Agent)]),
+    kind: Type.Union([
+      Type.Literal(SubjectKind.Skill),
+      Type.Literal(SubjectKind.Agent),
+      Type.Literal(SubjectKind.Code),
+    ]),
     name: Type.String({ minLength: 1 }),
     path: Type.String({ minLength: 1 }),
   },
@@ -132,7 +142,7 @@ const HandoverBundleSchema = Type.Object(
       Type.Literal(AdlStage.Build),
       Type.Literal(AdlStage.Evaluate),
       Type.Literal(AdlStage.Diagnose),
-      Type.Literal(AdlStage.Improve),
+      Type.Literal(AdlStage.Optimize),
       Type.Literal(AdlStage.Audit),
     ]),
     subject: SubjectSchema,

package/.claude/skills/mutagent-evaluator/scripts/run-deterministic.ts

@@ -32,6 +32,7 @@ import {
   type EvalMatrix,
   type RowResultValue,
   type ScorecardCriterion,
+  CheckMethod,
   RowResult,
   Track,
   trackForCheckMethod,
@@ -84,14 +85,96 @@ function isNonEmpty(value: unknown): boolean {
   return true;
 }
 
+/** A STRUCTURED, non-empty value: a non-empty object or array. A present-but-
+ *  scalar/empty artifact is NOT structured — the typebox-schema re-exec fails it
+ *  (a hollow presence-pass under the old proxy). PURE. */
+function isStructuredNonEmpty(value: unknown): boolean {
+  if (Array.isArray(value)) return value.length > 0;
+  if (value !== null && typeof value === "object") return Object.keys(value).length > 0;
+  return false;
+}
+
+/** Scan a produced artifact for an EXPLICIT gate-failure marker: a falsy pass
+ *  flag (`pass`/`ok`/`passed` === false) or a failing `status` stamp
+ *  (`fail`/`failed`/`error`). Conservative + shallow so it stays deterministic
+ *  and does not false-fail on unrelated substrings. PURE. */
+function hasFailureMarker(value: unknown): boolean {
+  if (value === null || typeof value !== "object") return false;
+  const obj = value as Record<string, unknown>;
+  for (const flag of ["pass", "ok", "passed"] as const) {
+    if (obj[flag] === false) return true;
+  }
+  const status = obj["status"];
+  if (typeof status === "string") {
+    const s = status.toLowerCase();
+    if (s === "fail" || s === "failed" || s === "error") return true;
+  }
+  return false;
+}
+
+/**
+ * Item #4 — the REAL live re-execution of a deterministic row. Replaces the
+ * presence-proxy (`the artifact exists → mark it covered`) with an ACTUAL
+ * checkMethod-keyed predicate over the artifact's LIVE content:
+ *   - typebox-schema     : the produced artifact must be a STRUCTURED, non-empty
+ *                          object/array — a present-but-malformed scalar now FAILS
+ *                          instead of hollow-passing (real shape conformance).
+ *   - gate               : the evidence must be present AND carry NO explicit
+ *                          failure marker (fail-loud gate — `hasFailureMarker`).
+ *   - deterministic-script: the produced artifact must be present + non-empty
+ *                          (the script actually emitted its output).
+ * Returns pass/fail when the criterion maps to a live bundle artifact; returns
+ * `null` when NO artifact answers it offline — the caller then records the
+ * documented seam-skip. A genuinely un-runnable row would require executing the
+ * SUBJECT'S OWN script, which the judge-only cell never does (EV-051) and which
+ * dispatches NO sub-agents (PR-ORCH-01). PURE + deterministic (no clock/random/
+ * network) ⇒ C-PIN byte-identical reruns.
+ */
+export function reexecDeterministicCheck(
+  criterion: Criterion,
+  bundle: RunBundle,
+): RowResultValue | null {
+  const artifactKey = evidenceArtifactFor(criterion, bundle);
+  if (artifactKey == null) return null; // no live artifact → not re-executable here
+  const value = bundle.data[artifactKey];
+  // HONESTY on the depth of each re-exec (do NOT mistake these for deep checks):
+  //   - typebox-schema : a real SHAPE upgrade over presence (rejects a present-but-
+  //                      scalar), but it is NOT a full TypeBox `Value.Check` against
+  //                      the produced artifact's declared schema — it asserts
+  //                      "structured + non-empty", not field-level conformance.
+  //   - gate           : `hasFailureMarker` is TOP-LEVEL-ONLY (scans the object's own
+  //                      pass/ok/passed/status keys) — it does NOT recurse into nested
+  //                      gate evidence, so a failure buried in a child object is missed.
+  //   - deterministic-script : semantically ≡ the OLD presence proxy (present +
+  //                      non-empty). It is re-run against LIVE data (so it is a real
+  //                      re-execution, not a cached verdict), but carries no deeper
+  //                      predicate than presence. Deepening any of these to a true
+  //                      re-run of the subject's own script is the documented seam
+  //                      (out of scope for a judge-only / no-sub-agent cell).
+  switch (criterion.checkMethod) {
+    case CheckMethod.TypeboxSchema:
+      return isStructuredNonEmpty(value) ? RowResult.Pass : RowResult.Fail;
+    case CheckMethod.Gate:
+      return isNonEmpty(value) && !hasFailureMarker(value) ? RowResult.Pass : RowResult.Fail;
+    case CheckMethod.DeterministicScript:
+    default:
+      return isNonEmpty(value) ? RowResult.Pass : RowResult.Fail;
+  }
+}
+
 export interface EvaluateOptions {
   /**
-   * When false (default), criteria that need a LIVE re-exec of the subject's own
-   * script (beyond bundle presence/shape) return `skip` with a documented seam
-   * note instead of false-passing. A future integration may set this true and
-   * provide a re-exec callback.
+   * Item #4 — the LIVE RE-EXEC executor. When supplied it RUNS the real
+   * deterministic check against live bundle data (see `reexecDeterministicCheck`,
+   * the default) and its pass/fail return is authoritative — replacing the
+   * presence-proxy that marked a row covered from mere artifact existence. A
+   * `null` return means "this executor cannot answer this row" ⇒ the legacy
+   * (byte-stable) path runs. ABSENT ⇒ no re-exec wired: fully back-compatible,
+   * byte-identical to before (the presence proxy + documented seam-skip). The
+   * executor is PURE + dispatches NO sub-agents (PR-ORCH-01) + never fixes the
+   * subject (EV-051).
    */
-  liveReexec?: (criterion: Criterion) => RowResultValue | null;
+  liveReexec?: (criterion: Criterion, bundle: RunBundle) => RowResultValue | null;
 }
 
 /**
@@ -104,21 +187,31 @@ export function evaluateDeterministic(
   opts: EvaluateOptions = {},
 ): ScorecardCriterion {
   const track = Track.Deterministic;
+
+  // Item #4 — LIVE RE-EXEC FIRST. When an executor is wired it RUNS the real
+  // deterministic check against live data; a pass/fail is authoritative and
+  // REPLACES the presence-proxy (no more "an artifact exists → covered"). A null
+  // return means the executor could not answer this row ⇒ fall through to the
+  // (byte-stable) legacy path below.
+  const live = opts.liveReexec?.(criterion, bundle) ?? null;
+  if (live === RowResult.Pass || live === RowResult.Fail || live === RowResult.Incomplete) {
+    return {
+      dimension: criterion.dimension,
+      severity: criterion.severity,
+      checkMethod: criterion.checkMethod,
+      track,
+      result: live,
+      detail: `live re-exec (${criterion.checkMethod}): real deterministic check over the produced artifact → ${live}`,
+    };
+  }
+
   const artifactKey = evidenceArtifactFor(criterion, bundle);
 
   if (artifactKey == null) {
-    // No bundle artifact answers this row offline -> live-reexec seam.
-    const live = opts.liveReexec?.(criterion) ?? null;
-    if (live) {
-      return {
-        dimension: criterion.dimension,
-        severity: criterion.severity,
-        checkMethod: criterion.checkMethod,
-        track,
-        result: live,
-        detail: "evaluated via live-reexec callback",
-      };
-    }
+    // No bundle artifact answers this row offline -> documented seam-skip. NOT
+    // silent: the coverage-honesty warning (computeCoverage) surfaces it, and
+    // fully executing it would require running the subject's OWN script — which
+    // the judge-only cell never does (EV-051) and which dispatches no sub-agents.
     return {
       dimension: criterion.dimension,
       severity: criterion.severity,
@@ -130,6 +223,9 @@ export function evaluateDeterministic(
     };
   }
 
+  // LEGACY presence proxy — reached ONLY when no live-reexec executor is wired
+  // (kept for byte-identity with pre-Item-#4 callers). With the default executor
+  // wired (audit-run.ts) this branch is never taken for a mapped row.
   const present = isNonEmpty(bundle.data[artifactKey]);
   return {
     dimension: criterion.dimension,
@@ -138,7 +234,7 @@ export function evaluateDeterministic(
     track,
     result: present ? RowResult.Pass : RowResult.Fail,
     detail: present
-      ? `evidence artifact '${artifactKey}' present + non-empty`
+      ? `evidence artifact '${artifactKey}' present + non-empty (presence proxy — no live re-exec wired)`
       : `evidence artifact '${artifactKey}' MISSING/empty in bundle (fail-loud)`,
   };
 }