@ionivetech/mugiwara 0.5.0 → 0.5.1

package/evals/cases/routing-git.json

@@ -0,0 +1,25 @@
+{
+  "name": "routing-git",
+  "skill": "mugiwara-git",
+  "trigger": {
+    "positive": [
+      { "prompt": "commit these changes with atomic commits per logical unit", "top_k": 3 },
+      { "prompt": "split this diff into separate commits by topic", "top_k": 3 },
+      { "prompt": "find which commit introduced this bug using bisect", "top_k": 3 }
+    ],
+    "negative": [
+      { "prompt": "push the branch and open a pull request" },
+      { "prompt": "deploy the release to production" }
+    ]
+  },
+  "behavioral": [
+    {
+      "task": "Split the staged changes into atomic commits: one for the refactor, one for the new feature, one for the tests.",
+      "rubric": [
+        "creates atomic commits per logical change",
+        "detects the repo's commit style",
+        "does not push or open a PR"
+      ]
+    }
+  ]
+}

package/evals/cases/routing-healing.json

@@ -0,0 +1,25 @@
+{
+  "name": "routing-healing",
+  "skill": "mugiwara-healing",
+  "trigger": {
+    "positive": [
+      { "prompt": "the previous execution wave failed, fix what is broken", "top_k": 3 },
+      { "prompt": "there are blockers in the ledger, triage and fix them", "top_k": 3 },
+      { "prompt": "the blocker ledger has entries, stop the line and fix them", "top_k": 3 }
+    ],
+    "negative": [
+      { "prompt": "execute the next wave in the plan" },
+      { "prompt": "add a new feature to the backlog" }
+    ]
+  },
+  "behavioral": [
+    {
+      "task": "The test suite failed after the last wave. The blocker ledger has 3 entries. Fix the failures.",
+      "rubric": [
+        "reads the blocker ledger before taking action",
+        "applies stop-the-line triage",
+        "proves the fix before moving to next wave"
+      ]
+    }
+  ]
+}

package/evals/cases/routing-lessons.json

@@ -0,0 +1,25 @@
+{
+  "name": "routing-lessons",
+  "skill": "mugiwara-lessons",
+  "trigger": {
+    "positive": [
+      { "prompt": "capture what we learned from this mission", "top_k": 3 },
+      { "prompt": "read past lessons before starting the new mission", "top_k": 3 },
+      { "prompt": "update the cross-mission lessons ledger with new insights", "top_k": 3 }
+    ],
+    "negative": [
+      { "prompt": "log this error in the application monitoring system" },
+      { "prompt": "write a retrospective document for the team" }
+    ]
+  },
+  "behavioral": [
+    {
+      "task": "The mission is complete. Capture lessons learned and append them to the cross-mission ledger.",
+      "rubric": [
+        "reads past lessons at triage",
+        "captures new lessons at closure",
+        "appends to the append-only ledger"
+      ]
+    }
+  ]
+}

package/evals/cases/routing-orchestration.json

@@ -0,0 +1,25 @@
+{
+  "name": "routing-orchestration",
+  "skill": "mugiwara-orchestration",
+  "trigger": {
+    "positive": [
+      { "prompt": "triage this new feature request and decide the lane size", "top_k": 3 },
+      { "prompt": "classify this mission and coordinate the execution waves", "top_k": 3 },
+      { "prompt": "captain a new mission — classify, size, dispatch, check in", "top_k": 3 }
+    ],
+    "negative": [
+      { "prompt": "implement the feature described in the ticket" },
+      { "prompt": "write code for the database migration" }
+    ]
+  },
+  "behavioral": [
+    {
+      "task": "A new mission request arrived. Classify it, size the lane, and dispatch the appropriate waves.",
+      "rubric": [
+        "performs 5-way classification before any action",
+        "sizes the lane based on sensitivity and scope",
+        "coordinates waves without implementing code"
+      ]
+    }
+  ]
+}

package/evals/cases/routing-planning.json

@@ -0,0 +1,26 @@
+{
+  "name": "routing-planning",
+  "skill": "mugiwara-planning",
+  "trigger": {
+    "positive": [
+      { "prompt": "turn this spec into an execution plan with parallel waves", "top_k": 3 },
+      { "prompt": "plan the implementation approach before writing any code", "top_k": 3 },
+      { "prompt": "break down the feature into tasks and estimate the effort", "top_k": 3 }
+    ],
+    "negative": [
+      { "prompt": "just write the code for this simple bug fix" },
+      { "prompt": "brainstorm different architecture options" }
+    ]
+  },
+  "behavioral": [
+    {
+      "task": "Turn the approved feature spec into a detailed execution plan with parallel-proof waves and context scan.",
+      "rubric": [
+        "interviews before planning",
+        "performs a full context scan",
+        "produces parallel-proof waves",
+        "scales the plan to the request scope"
+      ]
+    }
+  ]
+}

package/evals/cases/routing-pr.json

@@ -0,0 +1,25 @@
+{
+  "name": "routing-pr",
+  "skill": "mugiwara-pr",
+  "trigger": {
+    "positive": [
+      { "prompt": "push this branch and prepare the pull request description", "top_k": 3 },
+      { "prompt": "write the PR summary with a verdict file", "top_k": 3 },
+      { "prompt": "prepare the branch for review, push it and generate PR material", "top_k": 3 }
+    ],
+    "negative": [
+      { "prompt": "merge this pull request into main" },
+      { "prompt": "create a release tag for this version" }
+    ]
+  },
+  "behavioral": [
+    {
+      "task": "The mission is complete. Push the branch and prepare PR material including a verdict file.",
+      "rubric": [
+        "does a plain git push",
+        "generates a verdict file with ready PR summary",
+        "never creates the PR, merges, or deploys"
+      ]
+    }
+  ]
+}

package/evals/cases/routing-proof-order.json

@@ -0,0 +1,25 @@
+{
+  "name": "routing-proof-order",
+  "skill": "mugiwara-proof-order",
+  "trigger": {
+    "positive": [
+      { "prompt": "write the test first before implementing the function", "top_k": 3 },
+      { "prompt": "use TDD — red green refactor for this feature", "top_k": 3 },
+      { "prompt": "prove the behavior with a failing test, then implement", "top_k": 3 }
+    ],
+    "negative": [
+      { "prompt": "run the existing unit test suite that was written last week" },
+      { "prompt": "run the acceptance tests defined by the user" }
+    ]
+  },
+  "behavioral": [
+    {
+      "task": "Implement a new utility function. Write the test first, then the implementation, then refactor.",
+      "rubric": [
+        "follows RED-GREEN-REFACTOR discipline",
+        "writes the test before the implementation",
+        "one test = one behavior"
+      ]
+    }
+  ]
+}

package/evals/cases/routing-quality.json

@@ -0,0 +1,25 @@
+{
+  "name": "routing-quality",
+  "skill": "mugiwara-quality",
+  "trigger": {
+    "positive": [
+      { "prompt": "run eslint and prettier on the changed files", "top_k": 3 },
+      { "prompt": "format the code and run the linter before committing", "top_k": 3 },
+      { "prompt": "run the project test suite and check for lint errors", "top_k": 3 }
+    ],
+    "negative": [
+      { "prompt": "check if the coverage threshold is met" },
+      { "prompt": "review the diff for breaking changes" }
+    ]
+  },
+  "behavioral": [
+    {
+      "task": "After the checkpoint passes, run the formatter, linter, and unit tests for the project.",
+      "rubric": [
+        "discovers the project's lint and format tooling",
+        "runs the unit test suite",
+        "never weakens or changes existing configs"
+      ]
+    }
+  ]
+}

package/evals/cases/routing-ship.json

@@ -0,0 +1,26 @@
+{
+  "name": "routing-ship",
+  "skill": "mugiwara-ship",
+  "trigger": {
+    "positive": [
+      { "prompt": "run the pre-launch checklist and give a go no-go decision", "top_k": 3 },
+      { "prompt": "prepare the staged rollout plan with a rollback strategy", "top_k": 3 },
+      { "prompt": "verify feature flags and ship checklist before deployment", "top_k": 3 }
+    ],
+    "negative": [
+      { "prompt": "push the branch to GitHub" },
+      { "prompt": "deploy the build artifact to the server" }
+    ]
+  },
+  "behavioral": [
+    {
+      "task": "The mission is complete and all gates passed. Run the pre-launch checklist and give a GO/NO-GO.",
+      "rubric": [
+        "runs the pre-launch checklist",
+        "checks feature flags and staged rollout plan",
+        "requires a mandatory rollback plan",
+        "gives a binary GO/NO-GO verdict"
+      ]
+    }
+  ]
+}

package/evals/cases/routing-sunset.json

@@ -0,0 +1,25 @@
+{
+  "name": "routing-sunset",
+  "skill": "mugiwara-sunset",
+  "trigger": {
+    "positive": [
+      { "prompt": "remove the old v1 API endpoints and retire them safely", "top_k": 3 },
+      { "prompt": "delete the legacy payment module and migrate to v2", "top_k": 3 },
+      { "prompt": "deprecate this old endpoint and phase it out over two releases", "top_k": 3 }
+    ],
+    "negative": [
+      { "prompt": "add a new version of the payment API" },
+      { "prompt": "refactor the v2 endpoint handler for performance" }
+    ]
+  },
+  "behavioral": [
+    {
+      "task": "Retire the legacy v1 payment endpoints. They have been replaced by v2 for 6 months.",
+      "rubric": [
+        "runs the keep-or-retire gate before any removal",
+        "plans a phased cutover with safe DB migrations",
+        "every removal has a tested way back"
+      ]
+    }
+  ]
+}

package/evals/cases/routing-workflow.json

@@ -0,0 +1,25 @@
+{
+  "name": "routing-workflow",
+  "skill": "mugiwara-workflow",
+  "trigger": {
+    "positive": [
+      { "prompt": "start a new mission for this feature — run the full pipeline", "top_k": 3 },
+      { "prompt": "run the complete workflow from triage to closure", "top_k": 3 },
+      { "prompt": "we have a non-trivial task, kick off the 9-wave pipeline", "top_k": 3 }
+    ],
+    "negative": [
+      { "prompt": "fix this one-line typo in the README" },
+      { "prompt": "just run the tests and close the mission" }
+    ]
+  },
+  "behavioral": [
+    {
+      "task": "A new non-trivial feature request arrived. Run the full mugiwara pipeline from triage to closure.",
+      "rubric": [
+        "starts with Luffy triage gateway",
+        "sequences through brainstorm/plan/execute/checkpoint/quality/gates/review/heal/closure waves",
+        "does not skip waves for non-trivial missions"
+      ]
+    }
+  ]
+}

package/evals/floor.json

@@ -0,0 +1,6 @@
+{
+  "rank1": 88.2,
+  "topk": 100,
+  "negatives": 100,
+  "updated": "2026-08-11"
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ionivetech/mugiwara",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "description": "The Straw Hat crew of AI agents and skills: brainstorm, plan, execute, checkpoint, quality, gates, review, security, self-healing. Installs into Claude Code, opencode, Copilot, Gemini, Codex, Cursor, Kimi, pi, Windsurf, Cline, Kilo, Antigravity.",
   "type": "module",
   "license": "MIT",
@@ -43,6 +43,7 @@
     "evals": "bun scripts/run-evals.ts",
     "retrieval-eval": "bun scripts/retrieval-eval.ts",
     "sync-version": "bun scripts/sync-version.ts",
+    "gate": "bun run typecheck && bun run test && bun run build && bun scripts/validate-content.ts --check-manifest --check-docs && bun scripts/run-evals.ts && bun scripts/retrieval-eval.ts",
     "prepack": "bun run build && bun run sync-version"
   },
   "keywords": [

package/scripts/probe.ts

@@ -0,0 +1,40 @@
+#!/usr/bin/env bun
+// Ad-hoc retrieval probe. Usage: bun scripts/probe.ts "your prompt here"
+
+import { readdirSync, readFileSync, existsSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { parseFrontmatter } from '../src/frontmatter.ts';
+
+const skillsDir = join(import.meta.dirname, '..', 'content', 'skills');
+const STOP = ['the', 'and', 'for', 'use', 'when', 'that', 'with', 'this', 'from', 'its', 'not', 'are', 'has'];
+const tok = (t: string) => t.toLowerCase().replace(/[^a-z0-9\s-]/g, ' ')
+  .split(/\s+/).filter(x => x.length > 1 && !STOP.includes(x));
+
+const terms = new Map<string, number>();
+const docs = new Map<string, Map<string, number>>();
+for (const d of readdirSync(skillsDir).filter(d => statSync(join(skillsDir, d)).isDirectory())) {
+  const f = join(skillsDir, d, 'SKILL.md');
+  if (!existsSync(f)) continue;
+  const { data } = parseFrontmatter(readFileSync(f, 'utf8'));
+  const tf = new Map<string, number>();
+  for (const t of tok(data.description ?? '')) {
+    tf.set(t, (tf.get(t) || 0) + 1);
+    terms.set(t, (terms.get(t) || 0) + 1);
+  }
+  docs.set(d, tf);
+}
+
+const N = docs.size;
+const prompt = process.argv.slice(2).join(' ');
+const ranked = [...docs.keys()].map(d => {
+  let s = 0;
+  for (const t of tok(prompt)) {
+    const tf = docs.get(d)!.get(t) || 0;
+    if (tf) s += tf * Math.log(N / (terms.get(t) || 1));
+  }
+  return { d, s };
+}).sort((a, b) => b.s - a.s);
+
+console.log(`"${prompt}"\n`);
+ranked.slice(0, 8).forEach((r, i) =>
+  console.log(`  ${String(i + 1).padStart(2)}. ${r.d.padEnd(28)} ${r.s.toFixed(2)}`));

package/scripts/retrieval-eval.ts

@@ -24,25 +24,25 @@ function tokenize(text: string): string[] {
 function buildIndex(): Index {
   const index: Index = { terms: new Map(), docs: new Map(), docCount: 0 };
   const dirs = readdirSync(skillsDir).filter(d => statSync(join(skillsDir, d)).isDirectory());
-  
+
   for (const dir of dirs) {
     const file = join(skillsDir, dir, 'SKILL.md');
     if (!existsSync(file)) continue;
-    
+
     const { data } = parseFrontmatter(readFileSync(file, 'utf8'));
     const desc = data.description ?? '';
     const tokens = tokenize(desc);
     const tf = new Map<string, number>();
-    
+
     for (const t of tokens) {
       tf.set(t, (tf.get(t) || 0) + 1);
       index.terms.set(t, (index.terms.get(t) || 0) + 1);
     }
-    
+
     index.docs.set(dir, tf);
     index.docCount++;
   }
-  
+
   return index;
 }
 
@@ -56,7 +56,7 @@ function tfidf(index: Index, doc: string, term: string): number {
 function score(index: Index, prompt: string): { skill: string; score: number }[] {
   const promptTerms = tokenize(prompt);
   const results: { skill: string; score: number }[] = [];
-  
+
   for (const [doc] of index.docs) {
     let total = 0;
     for (const t of promptTerms) {
@@ -64,89 +64,198 @@ function score(index: Index, prompt: string): { skill: string; score: number }[]
     }
     results.push({ skill: doc, score: total });
   }
-  
-  return results.sort((a, b) => b.score - a.score);
+
+  return results.sort((a, b) => b.score - a.score || a.skill.localeCompare(b.skill));
 }
 
-// --- eval cases ---
-interface EvalCase {
-  prompt: string;
+// --- case schema ---
+interface CaseFile {
+  name: string;
+  skill: string;
+  type?: string;
+  task: string;
+  rubric: string[];
+  expect_lane?: string;
+  trigger?: {
+    positive?: { prompt: string; top_k?: number }[];
+    negative?: { prompt: string }[];
+  };
+  behavioral?: { task: string; rubric: string[] }[];
+}
+
+interface Probe {
+  kind: 'positive' | 'negative';
   skill: string;
-  type: 'positive' | 'negative';
-  top_k?: number;
+  prompt: string;
+  topK: number;
 }
 
+// --- build index ---
 const index = buildIndex();
 
-// Load existing case files
-const cases: EvalCase[] = [];
-if (existsSync(evalsDir)) {
-  const files = readdirSync(evalsDir).filter(f => f.endsWith('.json'));
-  for (const f of files) {
-    try {
-      const c = JSON.parse(readFileSync(join(evalsDir, f), 'utf8'));
-      if (c.prompt && c.skill) {
-        cases.push({ prompt: c.prompt, skill: c.skill, type: c.type || 'positive', top_k: c.top_k || 3 });
-      }
-    } catch { /* skip */ }
+// --- load cases ---
+const skills = readdirSync(skillsDir).filter(d => statSync(join(skillsDir, d)).isDirectory());
+const files = readdirSync(evalsDir).filter(f => f.endsWith('.json'));
+const probes: Probe[] = [];
+const noSkillProbes: Probe[] = [];
+const covered = new Set<string>();
+
+for (const f of files) {
+  let c: CaseFile;
+  try {
+    c = JSON.parse(readFileSync(join(evalsDir, f), 'utf8'));
+  } catch (e) {
+    throw new Error(`eval case ${f} is not valid JSON: ${e}`);
   }
-}
+  if (!c.skill) throw new Error(`eval case ${f} has no "skill"`);
+  covered.add(c.skill);
 
-// Add negative cases (must NOT rank first for irrelevant prompts)
-const negativeCases: EvalCase[] = [
-  { prompt: "tell me a joke about programmers", skill: "any", type: "negative", top_k: 1 },
-  { prompt: "convert this image to webp format", skill: "any", type: "negative", top_k: 1 },
-  { prompt: "schedule a meeting for next Tuesday", skill: "any", type: "negative", top_k: 1 },
-  { prompt: "translate this document to Japanese", skill: "any", type: "negative", top_k: 1 },
-];
+  const target = c.skill === '_no-skill' ? noSkillProbes : probes;
+  for (const p of c.trigger?.positive ?? [])
+    target.push({ kind: 'positive', skill: c.skill, prompt: p.prompt, topK: p.top_k ?? 3 });
+  for (const p of c.trigger?.negative ?? [])
+    target.push({ kind: 'negative', skill: c.skill, prompt: p.prompt, topK: 3 });
+}
 
-const allCases = [...cases, ...negativeCases];
+// --- coverage gate ---
+const missing = skills.filter(s => !covered.has(s));
+if (missing.length) {
+  console.error(`missing eval cases for ${missing.length} skills:`);
+  for (const s of missing) console.error(`  ${s}`);
+  process.exit(1);
+}
 
-// --- run ---
-let passed = 0;
-let failed = 0;
+// --- run probes ---
+let rank1 = 0, inTopK = 0, negPass = 0, noSkillPass = 0;
+const positives = probes.filter(p => p.kind === 'positive');
+const negatives = probes.filter(p => p.kind === 'negative');
+const nsPositives = noSkillProbes.filter(p => p.kind === 'positive');
+const nsNegatives = noSkillProbes.filter(p => p.kind === 'negative');
+const failures: string[] = [];
 const results: Record<string, { rank: number; score: number; top_k: number; passed: boolean }> = {};
 
-for (const tc of allCases) {
-  const rankings = score(index, tc.prompt);
-  const rank = rankings.findIndex(r => r.skill === tc.skill) + 1;
-  const entry = rankings[0];
-  
-  if (tc.type === 'negative') {
-    const gotMugiwara = entry && entry.skill.startsWith('mugiwara-') && entry.score > 3.5;
-    const p = !gotMugiwara;
-    const k = tc.top_k ?? 1;
-    results[tc.prompt] = { rank: 1, score: entry?.score ?? 0, top_k: k, passed: p };
-    if (p) passed++; else { failed++; console.error(`FAIL negative: "${tc.prompt}" → rank 1: ${entry?.skill} (score ${entry?.score?.toFixed(2)})`); }
+for (const p of positives) {
+  const ranked = score(index, p.prompt);
+  const rank = ranked.findIndex(r => r.skill === p.skill) + 1;
+  const key = `${p.skill}: ${p.prompt}`;
+  const entryScore = rank > 0 ? ranked[rank - 1].score : 0;
+  const topScore = ranked[0]?.score ?? 0;
+  const passed = entryScore > 0 && rank > 0 && rank <= p.topK;
+  results[key] = { rank, score: entryScore, top_k: p.topK, passed };
+  if (passed && rank === 1) rank1++;
+  if (passed) inTopK++;
+  else failures.push(`positive "${p.prompt}" → ${p.skill} ranked ${rank || 'unranked'} (want <=${p.topK}), got ${ranked[0]?.skill ?? 'none'}`);
+}
+
+for (const p of negatives) {
+  const ranked = score(index, p.prompt);
+  const top = ranked[0];
+  const pass = (top?.score ?? 0) === 0 || top?.skill !== p.skill;
+  const key = `!${p.skill}: ${p.prompt}`;
+  results[key] = { rank: 0, score: top?.score ?? 0, top_k: 3, passed: pass };
+  if (pass) negPass++;
+  else failures.push(`negative "${p.prompt}" wrongly ranked ${p.skill} first`);
+}
+
+for (const p of nsPositives) {
+  const ranked = score(index, p.prompt);
+  const top = ranked[0];
+  const pass = !top || top.score < 3.5;
+  const key = `_no-skill+: ${p.prompt}`;
+  results[key] = { rank: 0, score: top?.score ?? 0, top_k: 3, passed: pass };
+  if (pass) noSkillPass++;
+  else failures.push(`no-skill positive "${p.prompt}" → ${top.skill} score ${top.score.toFixed(2)} (want <3.5)`);
+}
+
+for (const p of nsNegatives) {
+  const ranked = score(index, p.prompt);
+  const top = ranked[0];
+  const pass = !!top && top.skill.startsWith('mugiwara-') && top.score >= 3.5;
+  const key = `_no-skill-: ${p.prompt}`;
+  results[key] = { rank: 0, score: top?.score ?? 0, top_k: 3, passed: pass };
+  if (pass) noSkillPass++;
+  else failures.push(`no-skill negative "${p.prompt}" → ${top?.skill ?? 'none'} score ${top?.score?.toFixed(2) ?? '0'} (want mugiwara skill >=3.5)`);
+}
+
+// --- compute rates ---
+const rank1Rate = positives.length ? (rank1 / positives.length) * 100 : 0;
+const topKRate  = positives.length ? (inTopK / positives.length) * 100 : 0;
+const negRate   = negatives.length ? (negPass / negatives.length) * 100 : 0;
+const nsTotal   = nsPositives.length + nsNegatives.length;
+const nsRate    = nsTotal ? (noSkillPass / nsTotal) * 100 : 100;
+
+const allPassed = Object.values(results).filter(r => r.passed).length;
+const allFailed = Object.values(results).filter(r => !r.passed).length;
+const totalProbes = Object.keys(results).length;
+
+// --- floor / ratchet ---
+const floorPath = join(root, 'evals', 'floor.json');
+const updateFloor = process.argv.includes('--update-floor');
+
+if (!existsSync(floorPath)) {
+  if (updateFloor) {
+    const initial = { rank1: Math.round(rank1Rate * 10) / 10, topk: Math.round(topKRate * 10) / 10, negatives: Math.round(negRate * 10) / 10, updated: new Date().toISOString().split('T')[0] };
+    writeJson(floorPath, initial);
+  }
+} else {
+  const floor = JSON.parse(readFileSync(floorPath, 'utf8'));
+  const TOL = 0.5;
+
+  if (updateFloor) {
+    floor.rank1 = Math.round(rank1Rate * 10) / 10;
+    floor.topk = Math.round(topKRate * 10) / 10;
+    floor.negatives = Math.round(negRate * 10) / 10;
+    floor.updated = new Date().toISOString().split('T')[0];
+    writeJson(floorPath, floor);
   } else {
-    const k = tc.top_k ?? 3;
-    const p = rank > 0 && rank <= k;
-    results[`${tc.skill}: ${tc.prompt}`] = { rank, score: rankings[rank - 1]?.score ?? 0, top_k: k, passed: p };
-    if (p) passed++; else { failed++; console.error(`FAIL: "${tc.prompt}" → "${tc.skill}" rank ${rank} (top ${k})`); }
+    const regressions: string[] = [];
+    if (rank1Rate < floor.rank1 - TOL) regressions.push(`rank-1 ${rank1Rate.toFixed(1)}% < floor ${floor.rank1}%`);
+    if (topKRate < floor.topk - TOL) regressions.push(`top-k ${topKRate.toFixed(1)}% < floor ${floor.topk}%`);
+    if (negRate < floor.negatives - TOL) regressions.push(`negatives ${negRate.toFixed(1)}% < floor ${floor.negatives}%`);
+
+    console.log(`rank-1 ${rank1Rate.toFixed(1)}%  top-3 ${topKRate.toFixed(1)}%  negatives ${negRate.toFixed(1)}%  no-skill ${nsRate.toFixed(1)}%  (${positives.length}p / ${negatives.length}n / ${nsTotal}ns over ${skills.length} skills)`);
+    for (const f of failures) console.error(`  FAIL ${f}`);
+    if (regressions.length) {
+      regressions.forEach(r => console.error(`REGRESSION: ${r}`));
+      process.exit(1);
+    }
   }
 }
 
 // --- report ---
-const rank1Count = Object.values(results).filter(r => 'rank' in r && r.rank === 1).length;
-const total = allCases.length;
-const rank1Rate = total > 0 ? (rank1Count / total * 100).toFixed(1) : '0';
-
-const report = {
-  index_size: index.docCount,
-  index_terms: index.terms.size,
-  cases: total,
-  passed,
-  failed,
-  rank1_count: rank1Count,
-  rank1_rate: `${rank1Rate}%`,
-  results,
-};
-
-// Output JSON for CI
 const ciArg = process.argv.indexOf('--json');
 if (ciArg !== -1) {
+  const report = {
+    index_size: index.docCount,
+    index_terms: index.terms.size,
+    probes: totalProbes,
+    positives: positives.length,
+    negatives: negatives.length,
+    no_skill: nsTotal,
+    rank1_count: rank1,
+    rank1_rate: `${rank1Rate.toFixed(1)}%`,
+    topk_count: inTopK,
+    topk_rate: `${topKRate.toFixed(1)}%`,
+    negative_pass: negPass,
+    negative_rate: `${negRate.toFixed(1)}%`,
+    no_skill_pass: noSkillPass,
+    no_skill_rate: `${nsRate.toFixed(1)}%`,
+    passed: allPassed,
+    failed: allFailed,
+    failures: failures.length > 0 ? failures : undefined,
+    results,
+  };
   console.log(JSON.stringify(report, null, 2));
 } else {
-  console.log(`Retrieval eval: ${passed}/${total} passed, ${rank1Rate} rank-1`);
-  if (failed > 0) process.exit(1);
+  console.log(`\nRetrieval eval: ${allPassed}/${totalProbes} passed, rank-1 ${rank1Rate.toFixed(1)}%, top-3 ${topKRate.toFixed(1)}%, neg ${negRate.toFixed(1)}%, ns ${nsRate.toFixed(1)}%`);
+  if (failures.length > 0) {
+    console.error(`\n${failures.length} failures:`);
+    for (const f of failures) console.error(`  ${f}`);
+    process.exit(1);
+  }
+}
+
+function writeJson(path: string, obj: object) {
+  const { writeFileSync } = require('node:fs');
+  writeFileSync(path, JSON.stringify(obj, null, 2) + '\n');
 }