@mutagent/evaluator 0.1.0-alpha.5 → 0.1.0-alpha.6

package/.claude/skills/mutagent-evaluator/scripts/report-fragments.ts

@@ -0,0 +1,173 @@
+/**
+ * scripts/report-fragments.ts — EV-3 shared client-JS render fragments.
+ * ---------------------------------------------------------------------------
+ * The step<->criterion side-by-side render functions, extracted VERBATIM from
+ * `render-eval-report.ts` so BOTH the eval report AND the `*review` UI
+ * (`build-review-ui.ts`) emit the identical browser-side renderer.
+ *
+ * `SIDE_BY_SIDE_FRAGMENT_JS` is the BYTE-IDENTICAL block (`cvRefs` +
+ * `critiqueBlock` + `sideBySide` — the latter carrying its local
+ * `refExaminesStep` + `covEntry`). render-eval-report splices it into its
+ * client script IN PLACE, so the report's emitted HTML is unchanged (guarded by
+ * the render-eval-report byte-identity tests + a before/after HTML diff).
+ *
+ * The fragment is provider-of-record for the render logic; its runtime deps
+ * (`esc`, `resRouting`, `resBadge`, and the globals `C` / `RES`) are
+ * supplied by whichever host script embeds it (render-eval-report defines them
+ * above the splice; the review UI defines compatible copies — see
+ * `REVIEW_SIDE_BY_SIDE_DEPS_JS`).
+ *
+ * `STEP_ROW_FRAGMENT_JS` is the review-UI VIRTUALIZATION mirror: standalone
+ * `refExaminesStepR` / `covEntryR` / `stepRowR` that emit ONE step triplet
+ * with markup IDENTICAL to `sideBySide`'s per-step output — so the review UI
+ * can window a 1,686-step card without materializing the whole DOM. A drift
+ * guard (`tests/report-fragments.test.ts`) evaluates both paths and asserts the
+ * per-step HTML matches, so the mirror can never silently diverge from the
+ * frozen fragment.
+ */
+
+/**
+ * BYTE-IDENTICAL extraction of the eval report's `cvRefs` + `critiqueBlock` +
+ * `sideBySide` client functions (`sideBySide` contains its local
+ * `refExaminesStep` + `covEntry`). Spliced verbatim by render-eval-report and
+ * reused by the review UI. DO NOT EDIT — regenerate from render-eval-report if
+ * that block ever changes, and re-verify byte-identity.
+ */
+export const SIDE_BY_SIDE_FRAGMENT_JS = `function cvRefs(refs){if(!refs||!refs.length)return '';
+  return '<div class="cvrefs">'+refs.map(function(rf){var s=(typeof rf==='string')?rf:[rf.obs,rf.path,rf.value].filter(Boolean).join(' · ');return s?'<span class="cvref">'+esc(s)+'</span>':'';}).join('')+'</div>';}
+function critiqueBlock(cvs){if(!cvs||!cvs.length)return '';
+  var order={fail:0,uncertain:1,indeterminate:1,pass:2,na:3};
+  var rows=cvs.slice().sort(function(a,b){return (order[a.result]==null?9:order[a.result])-(order[b.result]==null?9:order[b.result]);}).map(function(v){
+    var res=v.result||'na';var disp=res==='uncertain'?'indeterminate':res;
+    var conf=(v.confidence!=null)?'<span class="cvconf">conf '+esc(v.confidence)+(v.confidenceBand?' · '+esc(v.confidenceBand):'')+'</span>':'';
+    var crit=v.critique?'<div class="cvcrit">'+esc(v.critique)+'</div>':'<div class="cvcrit dim">— no critique recorded for this criterion</div>';
+    return '<div class="cvrow '+esc(res)+'"><div class="cvh"><span class="cvb '+esc(res)+'">'+esc(disp)+'</span><span class="cvid">'+esc(v.criterionId||'')+'</span>'+conf+'</div>'+crit+cvRefs(v.refs)+'</div>';}).join('');
+  return '<div class="ctx cvblock"><div class="ctx-h">◇ how the judge reasoned — per-criterion verdict · critique-before-verdict · grounding refs ('+cvs.length+')</div><div class="cvbody">'+rows+'</div></div>';}
+function sideBySide(d){var ctx=d.context||{};
+  // Gap A — the §2 "input + scenario" cell renders the RAW triggering INPUT (the
+  // thing that fired the agent) ABOVE the judge's scenario LABEL. Long inputs
+  // collapse into a <details> (lean, no JS) so the cell stays compact; short ones
+  // render inline (clamped). ABSENT input ⇒ the cell shows the scenario alone (or
+  // "—"). Font stays at the --fs-2xs (11px) floor + brand mono.
+  var inputCell=function(raw,scen){
+    var sc=scen?'<div class="iscn">scenario · '+esc(scen)+'</div>':'';
+    if(!raw)return (scen?'<div class="ival">'+esc(scen)+'</div>':'—');
+    var long=String(raw).length>180;
+    var body=long
+      ? '<details class="iexp"><summary>raw input · '+String(raw).length+' chars</summary><pre class="iraw">'+esc(raw)+'</pre></details>'
+      : '<pre class="iraw clamp">'+esc(raw)+'</pre>';
+    return body+sc;};
+  var refStr=function(rf){if(!rf)return '';if(typeof rf==='string')return rf;return [rf.obs,rf.path,rf.value].filter(Boolean).join(':');};
+  // UI-4 — surface the judge's REASONING (why the verdict + why the exit-states were
+  // concluded) from the EXISTING judge walk: an ordered why-chain band + the decide/bind
+  // text inlined where it explains a conclusion. No emit-contract change — read-only over
+  // the judge_steps already on the verdict file.
+  var KORD={gather:0,context:1,examine:2,detect:3,bind:4,ground:5,critique:6,decide:7,verify:8};
+  var allJs=(d.judgeSteps||[]).slice();
+  var stepText=function(kind){return allJs.filter(function(s){return s.kind===kind;}).map(function(s){return s.text||'';}).filter(Boolean).join(' · ');};
+  var decideWhy=stepText('decide')||stepText('critique');
+  var stateWhy=[stepText('bind'),stepText('gather')].filter(Boolean).join(' · ');
+  var chainSteps=allJs.slice().sort(function(a,b){var x=KORD[a.kind];var y=KORD[b.kind];return (x==null?9:x)-(y==null?9:y);});
+  var whyChain=chainSteps.length?('<div class="ctx whychain"><div class="ctx-h">◇ judge reasoning — full why-chain (gather → bind → ground → decide)</div><div class="wc-body">'+chainSteps.map(function(s){var rs=refStr(s.ref);return '<div class="jstep"><span class="k '+esc(s.kind)+'">'+esc(s.kind)+'</span><span class="t">'+esc(s.text||'')+(rs?' <span class=ref>'+esc(rs)+'</span>':'')+'</span></div>';}).join('')+'</div></div>'):'';
+  var verdictWhy=decideWhy?('<div class="band whyverdict"><div class="bh" style="color:var(--primary-soft)">◇ why this verdict</div><div class="wv-t">'+esc(decideWhy)+'</div></div>'):'';
+  // -- §2 judge lane = per-step EVAL COVERAGE --
+  // The judge lane (.step-r) used to filter judgeSteps by anchor === a.n -- but
+  // judges never emit an anchor, so EVERY step rendered a bare dash. Instead we
+  // map each agent step -> the per-criterion verdicts (d.criterionVerdicts) whose
+  // grounding refs EXAMINED that step: a precise ref.obs === step.obs match plus
+  // the tool-name fallback (ref.path === 'name' && ref.value === step.tool).
+  // Each examining criterion renders one compact entry -- result + CODE/JUDGE tag
+  // + criterionId + the judge reasoning (critique). A step no criterion references
+  // says 'not examined by any eval' (honest), never a bare dash. Any judge step
+  // that DOES carry a real anchor is still honored (future-proof).
+  var cvAll=(d.criterionVerdicts||[]);
+  var refExaminesStep=function(rf,a){
+    if(!rf||typeof rf==='string')return false;
+    if(a.obs&&rf.obs&&String(rf.obs)===String(a.obs))return true;
+    if(rf.path==='name'&&rf.value!=null&&a.tool&&String(rf.value)===String(a.tool))return true;
+    return false;};
+  var covEntry=function(v,a){
+    var res=v.result||'na';var disp=res==='uncertain'?'indeterminate':res;
+    // the router carries TWO vocabularies: matrix-derived ('deterministic') and
+    // mined ('code-based'); 'hybrid' is shared. A '[code-eval ...]'-prefixed critique
+    // is the deterministic-logic fallback when the method is unset.
+    var method=((typeof C!=='undefined'&&C[v.criterionId])||{}).m||'';
+    var isCode=method==='deterministic'||method==='code-based'||method==='hybrid'||String(v.critique||'').trim().indexOf('[code-eval')===0;
+    var tag=method==='hybrid'?'HYBRID':(isCode?'CODE':'JUDGE');
+    var matched=(v.refs||[]).filter(function(rf){return typeof rf!=='string'&&refExaminesStep(rf,a);});
+    var crit=v.critique?esc(v.critique):'<span class="dim">— no critique recorded</span>';
+    return '<div class="jcov '+esc(res)+'"><div class="jcov-h"><span class="cvb '+esc(res)+'">'+esc(disp)+'</span><span class="jm '+(isCode?'code':'judge')+'" title="'+(isCode?'deterministically checked by a code-eval (not LLM-judged)':'reasoned by the LLM judge')+'">'+esc(tag)+'</span><span class="jcid">'+esc(v.criterionId||'')+'</span></div><div class="jcrit">'+crit+'</div>'+cvRefs(matched)+'</div>';};
+  var rowsHtml='';(d.agentSteps||[]).forEach(function(a){
+    // future-proof: a judge step that carries a REAL anchor still renders.
+    var js=(d.judgeSteps||[]).filter(function(s){return s.anchor!=null&&String(s.anchor)===String(a.n)&&s.kind!=='context';});
+    var anchoredHtml=js.map(function(s){var rs=refStr(s.ref);return '<div class="jstep"><span class="k '+esc(s.kind)+'">'+esc(s.kind)+'</span><span class="t">'+esc(s.text||'')+' '+(rs?'<span class=ref>'+esc(rs)+'</span>':'')+'</span></div>';}).join('');
+    var examiners=cvAll.filter(function(v){return (v.refs||[]).some(function(rf){return refExaminesStep(rf,a);});});
+    var covHtml=examiners.map(function(v){return covEntry(v,a);}).join('');
+    var jhtml=anchoredHtml+covHtml;
+    if(!jhtml)jhtml='<div class="jstep noexam"><span class="t">— not examined by any eval</span></div>';
+    rowsHtml+='<div class="step-l"><div class="evb"><div class="top"><span class="tool">'+esc(a.tool||'')+'</span><span class="st '+esc(a.status||'')+'">'+esc(a.status||'')+'</span></div><div class="det">'+esc(a.detail||'')+'</div></div></div><div class="node '+esc(a.status||'')+'"><div class="ln"></div><div class="n">'+esc(a.n)+'</div><div class="ln"></div></div><div class="step-r"><div class="evb r">'+jhtml+'</div></div>';});
+  var h=d.health||{};
+  var sp=d.subjectProfile;
+  var spHtml=sp?'<div class="ctx"><div class="ctx-h">◇ judge · subject profile (M1) · '+esc(sp.provenance||'')+'</div><div class="ctx-g"><div class="ctx-c"><div class="l">identity</div><div class="v">'+esc(sp.identity||'—')+'</div></div><div class="ctx-c"><div class="l">purpose</div><div class="v">'+esc(sp.purpose||'—')+'</div></div><div class="ctx-c"><div class="l">scope · harness</div><div class="v">'+esc((sp.scope||'—'))+' · harness: '+esc(sp.harness||'—')+'</div></div></div></div>':'';
+  var u=d.understanding;
+  var uHtml=u?'<div class="band"><div class="bh" style="color:var(--cyan)">◇ node-0 GATHER — understanding (M2)</div><div class="re-rephrase" style="margin-top:4px">“'+esc(u.rephrase||'')+'”</div></div>':'';
+  var et=d.expectedTrajectory||[];
+  var etHtml=et.length?'<div class="band"><div class="bh" style="color:var(--primary-soft)">◇ node-0.5 EXPECTED-TRAJECTORY (M3) — how it SHOULD have acted</div><ol class="re-ul" style="margin-top:4px">'+et.map(function(s,i){return '<li><b>'+esc(s.step||i+1)+'.</b> '+esc(s.expected||'')+(s.rationale?' <span class=re-rat>— '+esc(s.rationale)+'</span>':'')+'</li>';}).join('')+'</ol></div>':'';
+  return '<div class="drillbox"><div style="display:flex;gap:9px;align-items:center;flex-wrap:wrap;margin-bottom:6px"><b class="mono">'+esc(d.traceId)+'</b><span class="chip">'+esc(d.route||'all')+'</span><span class="verd '+(d.verdict=='FAIL'?'fail':d.verdict=='PASS'?'pass':'inc')+'">'+esc(d.verdict)+'</span></div>'+
+    resRouting(d.res||'judge-walk')+
+    spHtml+
+    verdictWhy+
+    critiqueBlock(d.criterionVerdicts)+
+    '<div class="ctx"><div class="ctx-h">◇ judge · gather context</div><div class="ctx-g"><div class="ctx-c"><div class="l">harness</div><div class="v">'+esc(ctx.harness||'—')+'</div></div><div class="ctx-c"><div class="l">input + scenario</div><div class="v">'+inputCell(d.input,ctx.scenario)+'</div></div><div class="ctx-c"><div class="l">exit states</div><div class="v">'+esc(ctx.exitStates||'—')+(stateWhy?'<div class="why-note"><span class="ref">why concluded:</span> '+esc(stateWhy)+'</div>':'')+'</div></div></div></div>'+
+    whyChain+
+    uHtml+etHtml+
+    '<div class="lanehdr"><div class="a">target agent — what it did</div><div class="x">step</div><div class="j">judge — eval coverage (which criteria examined this step)</div></div>'+
+    '<div class="grid2">'+rowsHtml+'<div class="band loc"><div class="bh">↯ localize (root, not symptom)</div><div style="font-size:var(--fs-sm);margin-top:4px">'+esc(d.localize||'—')+'</div></div></div>'+
+    '<div class="health"><div class="hc"><div class="l">context</div><div class="v good">'+(h.contextGathered?'✓':'—')+'</div></div><div class="hc"><div class="l">grounded</div><div class="v good">'+esc(h.grounded||0)+'</div></div><div class="hc"><div class="l">assumed</div><div class="v '+((h.assumed||0)>0?'warn':'good')+'">'+esc(h.assumed||0)+'</div></div><div class="hc"><div class="l">root vs symptom</div><div class="v good">'+(h.stoppedAtSymptom?'symptom':'✓ root')+'</div></div></div></div>';}`;
+
+/**
+ * The runtime deps the extracted `sideBySide` needs — `esc` / `resBadge` /
+ * `resRouting` — copied BYTE-IDENTICALLY from render-eval-report's client script
+ * so the reused `sideBySide` behaves identically wherever it is embedded. The
+ * review UI embeds these (render-eval-report already defines them above its own
+ * splice). `RES` may be an empty map in the review UI — `resRouting`/`resBadge`
+ * degrade to '' when a resolution key is absent.
+ */
+export const REVIEW_SIDE_BY_SIDE_DEPS_JS = `function esc(s){return String(s==null?'':s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;');}
+function resBadge(res){var m=RES[res];if(!m)return '';return '<span class="resbadge '+m.cls+'" title="'+esc(m.routing)+'">'+esc(m.badge)+'</span>';}
+function resRouting(res){var m=RES[res];if(!m)return '';return '<div class="routing '+m.cls+'"><span class="routing-k">resolution</span><span class="routing-b '+m.cls+'">'+esc(m.badge)+'</span><span class="routing-v">'+esc(m.routing)+'</span></div>';}`;
+
+/**
+ * The review-UI VIRTUALIZATION mirror. `stepRowR` emits ONE step triplet
+ * (`.step-l` + `.node` + `.step-r`) whose markup is CHARACTER-IDENTICAL to the
+ * per-step output of the frozen `sideBySide` (compare render-eval-report's
+ * `rowsHtml` loop). This lets the review UI window a 1,686-step card — rendering
+ * only the visible slice — instead of materializing the whole DOM (the EV-3
+ * wall). `refExaminesStepR` / `covEntryR` mirror `sideBySide`'s local
+ * `refExaminesStep` / `covEntry`. Depends on `esc` + `cvRefs` (from the deps +
+ * the frozen fragment). The drift guard in `tests/report-fragments.test.ts`
+ * evaluates both paths and asserts equality, so this mirror can never diverge.
+ */
+export const STEP_ROW_FRAGMENT_JS = `function refStrR(rf){if(!rf)return '';if(typeof rf==='string')return rf;return [rf.obs,rf.path,rf.value].filter(Boolean).join(':');}
+function refExaminesStepR(rf,a){
+  if(!rf||typeof rf==='string')return false;
+  if(a.obs&&rf.obs&&String(rf.obs)===String(a.obs))return true;
+  if(rf.path==='name'&&rf.value!=null&&a.tool&&String(rf.value)===String(a.tool))return true;
+  return false;}
+function covEntryR(v,a){
+  var res=v.result||'na';var disp=res==='uncertain'?'indeterminate':res;
+  var method=((typeof C!=='undefined'&&C&&C[v.criterionId])||{}).m||'';
+  var isCode=method==='deterministic'||method==='code-based'||method==='hybrid'||String(v.critique||'').trim().indexOf('[code-eval')===0;
+  var tag=method==='hybrid'?'HYBRID':(isCode?'CODE':'JUDGE');
+  var matched=(v.refs||[]).filter(function(rf){return typeof rf!=='string'&&refExaminesStepR(rf,a);});
+  var crit=v.critique?esc(v.critique):'<span class="dim">— no critique recorded</span>';
+  return '<div class="jcov '+esc(res)+'"><div class="jcov-h"><span class="cvb '+esc(res)+'">'+esc(disp)+'</span><span class="jm '+(isCode?'code':'judge')+'" title="'+(isCode?'deterministically checked by a code-eval (not LLM-judged)':'reasoned by the LLM judge')+'">'+esc(tag)+'</span><span class="jcid">'+esc(v.criterionId||'')+'</span></div><div class="jcrit">'+crit+'</div>'+cvRefs(matched)+'</div>';}
+function stepRowInnerR(a,cvAll,judgeSteps){
+  var js=(judgeSteps||[]).filter(function(s){return s.anchor!=null&&String(s.anchor)===String(a.n)&&s.kind!=='context';});
+  var anchoredHtml=js.map(function(s){var rs=refStrR(s.ref);return '<div class="jstep"><span class="k '+esc(s.kind)+'">'+esc(s.kind)+'</span><span class="t">'+esc(s.text||'')+' '+(rs?'<span class=ref>'+esc(rs)+'</span>':'')+'</span></div>';}).join('');
+  var examiners=(cvAll||[]).filter(function(v){return (v.refs||[]).some(function(rf){return refExaminesStepR(rf,a);});});
+  var covHtml=examiners.map(function(v){return covEntryR(v,a);}).join('');
+  var jhtml=anchoredHtml+covHtml;
+  if(!jhtml)jhtml='<div class="jstep noexam"><span class="t">— not examined by any eval</span></div>';
+  return '<div class="step-l"><div class="evb"><div class="top"><span class="tool">'+esc(a.tool||'')+'</span><span class="st '+esc(a.status||'')+'">'+esc(a.status||'')+'</span></div><div class="det">'+esc(a.detail||'')+'</div></div></div><div class="node '+esc(a.status||'')+'"><div class="ln"></div><div class="n">'+esc(a.n)+'</div><div class="ln"></div></div><div class="step-r"><div class="evb r">'+jhtml+'</div></div>';}
+function stepExaminedByR(a,cvAll){return (cvAll||[]).some(function(v){return (v.refs||[]).some(function(rf){return refExaminesStepR(rf,a);});});}`;

package/.claude/skills/mutagent-evaluator/scripts/route-failures.ts

@@ -35,7 +35,7 @@ export const AdlStage = {
   Build: "build",
   Evaluate: "evaluate",
   Diagnose: "diagnose",
-  Improve: "improve",
+  Optimize: "optimize",
   Audit: "audit",
 } as const;
 export type AdlStageValue = (typeof AdlStage)[keyof typeof AdlStage];
@@ -142,7 +142,7 @@ const HandoverBundleSchema = Type.Object(
       Type.Literal(AdlStage.Build),
       Type.Literal(AdlStage.Evaluate),
       Type.Literal(AdlStage.Diagnose),
-      Type.Literal(AdlStage.Improve),
+      Type.Literal(AdlStage.Optimize),
       Type.Literal(AdlStage.Audit),
     ]),
     subject: SubjectSchema,

package/.claude/skills/mutagent-evaluator/scripts/run-pipeline.ts

@@ -22,6 +22,12 @@
 import type { JudgeInvoke } from "./determine-outcome.ts";
 import { determineOutcome, runJudge } from "./judge-prompt-template.ts";
 import { profileSubject, type SubjectProfile } from "./profile-subject.ts";
+import { buildSubjectProfile, deriveSubjectFrame } from "./subject-profile.ts";
+import {
+  normativeCriterionIds,
+  pendingRatifications,
+  type CalibrationDecision,
+} from "./memory/ratify.ts";
 import { splitTrainEval, buildJudgeSpec, type JudgePin } from "./build-evals.ts";
 import { rollupScorecard, type GradedCriterion, type Scorecard } from "./evaluate.ts";
 import {
@@ -64,6 +70,12 @@ export interface PipelineOptions {
   severityById?: Record<string, string>;
   /** artifact refs to enumerate on the handoff (caller-injected locators). */
   artifacts?: ArtifactRef[];
+  /**
+   * EV-4 — prior operator ratify/eliminate decisions (the review UI's `calibration.json`,
+   * routed via `memory/ratify.ts`). A NORMATIVE criterion with no `verify` here is
+   * `needs-ratification` → the verdict is HELD (surfaced in `pendingRatification`).
+   */
+  ratifications?: CalibrationDecision[];
 }
 
 export interface PipelineResult {
@@ -73,6 +85,12 @@ export interface PipelineResult {
   verdicts: CriterionVerdict[];
   scorecard: Scorecard;
   handoff: HandoverBundle;
+  /**
+   * EV-4 — normative criteria still `needs-ratification` (no operator `verify`).
+   * NON-EMPTY ⇒ the verdict is HELD: a value call the judge cannot settle is
+   * unresolved, so the run's verdict is not yet final.
+   */
+  pendingRatification: string[];
 }
 
 /** Map a determiner outcome → a sampling LabeledTrace. */
@@ -101,11 +119,20 @@ export async function runEvalPipeline(
   // EV-049 — subject profile (carries the subject vocab the engine reads).
   const profile = profileSubject(traces, opts.vocab);
 
+  // EV-5 CUT WIRE 2 — build the M1 judge-packet profile ONCE (identity · purpose ·
+  // tools + reversibility). This is `buildSubjectProfile`'s first PRODUCTION caller;
+  // it flows to the per-criterion judge preamble (EV-5). The determiner (EV-1) reads
+  // the SAME frame via `deriveSubjectFrame` so PREP (Stage A) and this pipeline
+  // (Stage B) render byte-identical determiner prompts (cross-stage cache parity).
+  const m1Profile = buildSubjectProfile({ subjectName: opts.subject.name, traces });
+  const subjectFrame = deriveSubjectFrame(opts.subject.name, traces);
+
   // EV-042 — determine an outcome per trace (sequential → deterministic order).
-  // The determiner reads the subject vocab off the profile (EV-002).
+  // The determiner reads the subject vocab off the profile (EV-002) and, EV-1, the
+  // subject frame so it derives THIS subject's expected outcome (not email signals).
   const outcomes: OutcomeResult[] = [];
   for (const t of traces) {
-    outcomes.push(await determineOutcome(t, judge, profile.vocab));
+    outcomes.push(await determineOutcome(t, judge, profile.vocab, subjectFrame));
   }
   const labeled = traces.map((t, i) => toLabeled(t, outcomes[i]));
 
@@ -125,7 +152,7 @@ export async function runEvalPipeline(
     const spec = buildJudgeSpec(criterion, exemplarsFromTrain(train), trainIds);
     // judge a representative held-out subject trace (first eval target).
     const subjectTrace = judgeTargets[0]?.trace ?? sample[0]?.trace ?? traces[0];
-    const verdict = await runJudge(spec, subjectTrace, judge, opts.pin);
+    const verdict = await runJudge(spec, subjectTrace, judge, opts.pin, m1Profile);
     verdicts.push(verdict);
     graded.push({
       criterionId: criterion.id,
@@ -171,5 +198,13 @@ export async function runEvalPipeline(
     producedAt: opts.producedAt,
   });
 
-  return { profile, outcomes, sample, verdicts, scorecard, handoff };
+  // EV-4 — a normative criterion (a value/standard call the judge cannot settle)
+  // must be operator-RATIFIED before its verdict is final. Compute which normative
+  // criteria have no `verify` decision → the verdict is HELD on them.
+  const pendingRatification = pendingRatifications(
+    normativeCriterionIds(verdicts),
+    opts.ratifications ?? [],
+  );
+
+  return { profile, outcomes, sample, verdicts, scorecard, handoff, pendingRatification };
 }

package/.claude/skills/mutagent-evaluator/scripts/subject-profile.ts

@@ -22,11 +22,62 @@
 import {
   PROFILE_UNKNOWN,
   SubjectProfileProvenance,
+  ToolReversibility,
   type SubjectProfile,
+  type ToolReversibilityEntry,
+  type ToolReversibilityValue,
 } from "./contracts/eval-matrix.ts";
 import { inferToolInventory, inferSystemPrompt } from "./profile-subject.ts";
 import type { EvalTrace } from "./contracts/eval-types.ts";
 
+// ── EV-5 — tool reversibility classification ─────────────────────────────────
+//
+// LEAN start (operator: "classify its tools by what they do — irreversible external
+// or not; keep the start lean"). A GIVEN classification (from the subject definition)
+// ALWAYS wins; absent, a conservative NAME heuristic flags well-known irreversible-
+// external verbs and well-known read-only verbs, and marks everything else `unknown`
+// (HONEST — never a false "reversible"). The heavier per-trace side-effect harness
+// (decision option b) can supersede this later without changing the field shape.
+
+/** Verb stems whose presence in a tool name signals an IRREVERSIBLE EXTERNAL action. */
+const IRREVERSIBLE_VERBS = [
+  "send", "email", "post", "publish", "delete", "remove", "destroy", "drop",
+  "charge", "pay", "refund", "transfer", "deploy", "release", "submit", "create",
+  "insert", "write", "update", "patch", "put", "execute", "provision", "terminate",
+  "cancel", "notify", "dispatch", "message", "reply", "comment", "merge", "push",
+];
+/** Verb stems whose presence signals a READ-ONLY / safely-repeatable action. */
+const REVERSIBLE_VERBS = [
+  "get", "list", "read", "search", "fetch", "lookup", "view", "query", "find",
+  "check", "inspect", "describe", "count", "peek", "preview", "validate",
+];
+
+/**
+ * Classify one tool's reversibility (EV-5). A GIVEN `override` (subject definition)
+ * wins; else a conservative name heuristic — irreversible-external verbs first (the
+ * safety-relevant class), then read-only verbs, else `unknown` (never guessed). PURE.
+ */
+export function classifyToolReversibility(
+  name: string,
+  override?: Record<string, ToolReversibilityValue>,
+): ToolReversibilityValue {
+  const given = override?.[name];
+  if (given !== undefined) return given;
+  const n = name.toLowerCase();
+  const hits = (verbs: string[]): boolean => verbs.some((v) => n.includes(v));
+  if (hits(IRREVERSIBLE_VERBS)) return ToolReversibility.IrreversibleExternal;
+  if (hits(REVERSIBLE_VERBS)) return ToolReversibility.Reversible;
+  return ToolReversibility.Unknown;
+}
+
+/** Classify every tool in the inventory (EV-5). Deterministic; preserves order. */
+export function classifyToolList(
+  tools: string[],
+  override?: Record<string, ToolReversibilityValue>,
+): ToolReversibilityEntry[] {
+  return tools.map((name) => ({ name, reversibility: classifyToolReversibility(name, override) }));
+}
+
 /** The GIVEN facts a caller with code/metadata access can supply (M1). Any field
  *  left absent is RECONSTRUCTED from the trace batch or MARKED `unknown`. */
 export interface GivenSubjectFacts {
@@ -41,6 +92,9 @@ export interface GivenSubjectFacts {
   version?: string;
   /** the agent's system prompt (code access). Rendered COLLAPSED; never confabulated. */
   systemPrompt?: string;
+  /** EV-5 — a GIVEN per-tool reversibility classification (from the subject definition);
+   *  WINS over the name heuristic. Any tool not named here falls back to the heuristic. */
+  toolReversibility?: Record<string, ToolReversibilityValue>;
 }
 
 export interface BuildSubjectProfileParams {
@@ -117,10 +171,14 @@ export function buildSubjectProfile(params: BuildSubjectProfileParams): SubjectP
   if (systemPrompt === undefined) systemPrompt = inferSystemPrompt(traces);
   if (given?.systemPrompt === undefined) inferredFields.push("systemPrompt");
 
+  // EV-5 — classify each tool's reversibility (GIVEN override wins; else name heuristic).
+  const toolReversibility = classifyToolList(tools, given?.toolReversibility);
+
   const profile: SubjectProfile = {
     identity,
     purpose,
     tools,
+    toolReversibility,
     scope,
     harness,
     provenance: hasGiven ? SubjectProfileProvenance.Given : SubjectProfileProvenance.Reconstructed,
@@ -132,3 +190,27 @@ export function buildSubjectProfile(params: BuildSubjectProfileParams): SubjectP
   };
   return profile;
 }
+
+/** The compact determiner subject frame (EV-1) — a structural subset of the M1 profile. */
+export interface SubjectFrameShape {
+  identity: string;
+  purpose: string;
+  tools: string[];
+  toolReversibility?: ToolReversibilityEntry[];
+}
+
+/**
+ * Derive the determiner/judge subject frame (EV-1 / EV-5) from a subject name + trace
+ * batch. THE single derivation used by run-pipeline (Stage-B) AND prep (Stage-A) so
+ * the determiner prompt is byte-identical across the two stages (the cross-stage cache
+ * would miss on any divergence). PURE — same inputs ⇒ same frame.
+ */
+export function deriveSubjectFrame(subjectName: string, traces: EvalTrace[]): SubjectFrameShape {
+  const m1 = buildSubjectProfile({ subjectName, traces });
+  return {
+    identity: m1.identity,
+    purpose: m1.purpose,
+    tools: m1.tools,
+    ...(m1.toolReversibility !== undefined ? { toolReversibility: m1.toolReversibility } : {}),
+  };
+}

package/.claude/skills/mutagent-evaluator/scripts/sync-eval-criteria.ts

@@ -2,7 +2,7 @@
  * scripts/sync-eval-criteria.ts — the EVAL-LEG reconcile CONTRACT (Wave-2 W2I5 · KP-003).
  * ---------------------------------------------------------------------------
  * The THIRD leg of the def → impl → eval triad. Today `#sync-spec` reconciles two
- * legs — spec ↔ impl. When an implementation amends (the ⑤ IMPROVE loop's ai-engineer,
+ * legs — spec ↔ impl. When an implementation amends (the ⑤ OPTIMIZE loop's ai-engineer,
  * or a brownfield drift), the eval criteria that GROUND the subject's evaluation can go
  * stale w.r.t. the changed impl. This module is the EVALUATOR-SIDE CONTRACT that the
  * reconcile (ai-architect REASONING, session-dispatched — Model-B) READS and WRITES:
@@ -35,7 +35,7 @@ import { assertMonotonicGrowth } from "./living-suite.ts";
 // ── subject kind → eval leg ──────────────────────────────────────────────────
 
 /** The subject kinds the triad reconciles. `agent|skill|composite` come from the
- *  agentspec `definition.identity.kind`; `code` is the ⑤ IMPROVE code-target subject. */
+ *  agentspec `definition.identity.kind`; `code` is the ⑤ OPTIMIZE code-target subject. */
 export const EvalSubjectKind = {
   Agent: "agent",
   Skill: "skill",

package/.claude/skills/mutagent-evaluator/scripts/unitf-to-evaltrace.ts

@@ -164,6 +164,34 @@ function effectiveRole(span: UnitfSpan): UnitfRole {
   }
 }
 
+// ── Message-text pick (INF-1) ────────────────────────────────────────────────
+//
+// ROOT CAUSE of the Claude Code loss: the Claude Code exporter writes BOTH user
+// and assistant TEXT into `span.output` (a user turn is `kind:"event", role:"user",
+// output:<text>`; an assistant turn is `kind:"llm", role:"assistant", output:<text>`
+// with `input` UNSET). The old projection read a user/system span's `.input` — empty
+// for EVERY Claude Code trace ⇒ `input.prompt` silently empty for all 21/21.
+//
+// This VENDORS the shared reader's role-conditional pick + fallback
+// (@mutagent/tools `messagesView`, derive.ts:173-174): a user/system turn's text is
+// canonically in `input`, everything else in `output`, but the `?? input ?? output`
+// fallback recovers the text WHICHEVER field a producer used. Vendored — never
+// imported: the standalone-publish guard forbids the evaluator reaching into
+// @mutagent/tools (MIGRATION-diagnostics-evaluator.md:417); diagnostics vendors its
+// own copy and this mirrors that.
+
+/**
+ * The span's effective message TEXT: the role-conditional field pick with the
+ * `?? input ?? output` fallback (mirrors @mutagent/tools `messagesView`). Recovers
+ * a Claude Code turn (text in `output`) AND a Langfuse turn (text in `input`).
+ * PURE; "" when the span carries no text.
+ */
+function messageText(span: UnitfSpan): string {
+  const role = effectiveRole(span);
+  const primary = role === "user" || role === "system" ? span.input : span.output;
+  return asText(primary ?? span.input ?? span.output);
+}
+
 // ── Token total fallback (XF-FIX Finding D) ──────────────────────────────────
 //
 // MIRRORS diagnostics `unitf-adapter.ts` `fallbackTotalTokens` so both lanes
@@ -185,10 +213,11 @@ function fallbackTotalTokens(t: UnitfTokens | undefined): number | undefined {
  *
  *   id               ← ut.traceId
  *   name             ← ut.agentName
- *   output           ← first EFFECTIVE-assistant llm span → { response: <text of span.output> }
- *                      (XF-FIX A: role-less GENERATION spans read as assistant via kind)
- *   input            ← first user|system span's input, ELSE the response span's own
- *                      input → { prompt } (XF-FIX A: recovers the Langfuse prompt)
+ *   output           ← LAST EFFECTIVE-assistant llm span WITH text → { response: <messageText> }
+ *                      (XF-FIX A: role-less GENERATION spans read as assistant via kind;
+ *                       INF-1: last-with-text, and messageText recovers Claude Code's `output`)
+ *   input            ← first user|system span's messageText, ELSE the response span's own
+ *                      input → { prompt } (XF-FIX A: Langfuse prompt; INF-1: Claude Code `output`)
  *   observations     ← ut.spans → { type: kind.toUpperCase(), name, input, output }
  *   errored/status   ← computed.hasError ?? (status==="error" || any span error) / ut.status (XF-FIX C)
  *   tokens/totalTokens← ut.tokens split / computed.totalTokens ?? tokens.total ?? in+out (XF-FIX D)
@@ -216,31 +245,45 @@ export function projectUnitfToEvalTrace(ut: UnifiedTraceLike): EvalTrace {
 
   if (ut.agentName !== undefined) out.name = ut.agentName;
 
-  // output ← first assistant LLM span's output, wrapped to { response }. The
-  // assistant role is now EFFECTIVE (XF-FIX Finding A): a role-less GENERATION
-  // span (kind==="llm", the Langfuse shape) reads as assistant via `effectiveRole`,
-  // so its output is picked up instead of being silently dropped.
-  const responseSpan = ut.spans.find(
+  // output ← the LAST assistant LLM span WITH text, wrapped to { response }. The
+  // assistant role is EFFECTIVE (XF-FIX Finding A): a role-less GENERATION span
+  // (kind==="llm", the Langfuse shape) reads as assistant via `effectiveRole`. The
+  // second INF-1 miss: the old code took the FIRST assistant span — on Claude Code
+  // that is usually a tool_use turn with EMPTY output text, so the real reply (a
+  // LATER assistant span) was dropped. Take the LAST assistant span that carries
+  // text; the `messageText` pick recovers it from `output` (Claude Code) or `input`
+  // (whichever field the producer used).
+  const assistantSpans = ut.spans.filter(
     (s) => effectiveRole(s) === "assistant" && s.kind === "llm",
   );
-  if (responseSpan !== undefined && responseSpan.output !== undefined) {
-    out.output = { response: asText(responseSpan.output) };
+  let responseSpan: UnitfSpan | undefined;
+  for (const s of assistantSpans) {
+    if (messageText(s).length > 0) responseSpan = s; // keep the LAST with text
+  }
+  // Fall back to the last assistant span even without text, so the response SLOT is
+  // still derived from a real span (the empty case is WARNED at intake, not here).
+  const lastAssistant =
+    responseSpan ?? (assistantSpans.length > 0 ? assistantSpans[assistantSpans.length - 1] : undefined);
+  if (responseSpan !== undefined) {
+    out.output = { response: messageText(responseSpan) };
   }
 
-  // input ← first user|system span's input, wrapped to { prompt }. A role-less
-  // span never infers to user/system, so when there is no explicit user/system
-  // turn (again the Langfuse shape — one GENERATION span carrying prompt in its
-  // OWN `input` and response in `output`) the prompt is recovered from the
-  // response span's `input`. Without this fallback `input.prompt` was empty for
-  // EVERY Langfuse trace (Finding A).
+  // input ← the first user|system span's TEXT, wrapped to { prompt }. `messageText`
+  // recovers the text from `output` (Claude Code writes the user turn there) or
+  // `input` (Langfuse). When there is no explicit user/system turn (the Langfuse
+  // shape — one GENERATION span carrying the prompt in its OWN `input` and the
+  // response in `output`) the prompt is recovered from the response span's `input`
+  // SPECIFICALLY (not `messageText`, which for an assistant span would return the
+  // response). Without these fallbacks `input.prompt` was empty for every Claude
+  // Code trace (INF-1) and every Langfuse trace (Finding A).
   const promptSpan = ut.spans.find(
     (s) => effectiveRole(s) === "user" || effectiveRole(s) === "system",
   );
-  if (promptSpan !== undefined && promptSpan.input !== undefined) {
-    out.input = { prompt: asText(promptSpan.input) };
-  } else if (responseSpan !== undefined && responseSpan.input !== undefined) {
-    out.input = { prompt: asText(responseSpan.input) };
+  let promptText = promptSpan !== undefined ? messageText(promptSpan) : "";
+  if (promptText.length === 0 && lastAssistant !== undefined && lastAssistant.input !== undefined) {
+    promptText = asText(lastAssistant.input);
   }
+  if (promptText.length > 0) out.input = { prompt: promptText };
 
   // Finding C — carry structured error/status so an error-KIND criterion has a
   // field to read (previously error state survived only as textual span output).

package/bin/mutagent-cli.mjs

@@ -5045,8 +5045,12 @@ var SpanSchema = Type.Object({
   role: Type.Optional(RoleSchema),
   toolName: Type.Optional(Type.String()),
   toolCallId: Type.Optional(Type.String()),
-  input: Type.Optional(Type.Unknown()),
-  output: Type.Optional(Type.Unknown()),
+  input: Type.Optional(Type.Unknown({
+    description: "Message text for the INPUT-side layers (role user\xB7system\xB7tool = user-input / system-input / tool-call arguments), or a chat-messages array on a multi-layer LLM span. See the field-per-layer rule; enforced by checkFieldConformance()."
+  })),
+  output: Type.Optional(Type.Unknown({
+    description: "Message text for the OUTPUT-side layers (role assistant\xB7toolResult = assistant-output / tool-result). See the field-per-layer rule; enforced by checkFieldConformance()."
+  })),
   tokens: Type.Optional(TokensSchema),
   costUsd: Type.Optional(Type.Number()),
   attributes: Type.Optional(Type.Record(Type.String(), Type.Unknown()))
@@ -6527,7 +6531,7 @@ function normalizeRecords(records, malformedIn) {
           if (ev.timestamp)
             span.startTime = ev.timestamp;
           if (text)
-            span.output = text;
+            span.input = text;
           spans.push(span);
           spanIdx++;
         }
@@ -6542,7 +6546,7 @@ function normalizeRecords(records, malformedIn) {
         if (ev.timestamp)
           span.startTime = ev.timestamp;
         if (typeof content === "string" && content)
-          span.output = content;
+          span.input = content;
         spans.push(span);
         spanIdx++;
       }
@@ -6797,7 +6801,7 @@ function normalizeRecords2(records, malformedIn) {
           span.startTime = ts;
         const msg = str2(p, "message");
         if (msg)
-          span.output = msg;
+          span.input = msg;
         spans.push(span);
         spanIdx++;
       } else if (pType === "agent_message") {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mutagent/evaluator",
-  "version": "0.1.0-alpha.5",
+  "version": "0.1.0-alpha.6",
   "description": "mutagent-evaluator: a generic, subject-agnostic AI-agent auditor — reviewer, never executor. Audits any skill/agent against a generated subject profile and emits a 4-tab master-audit report.",
   "license": "Apache-2.0",
   "type": "module",