@event4u/agent-config 16.0.0 → 16.2.0

package/src/scripts/measure_rule_budget.ts

@@ -38,6 +38,7 @@ const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
 
 const OVERRIDES_FILE = path.join(REPO_ROOT, 'docs', 'contracts', 'iron-law-overrides.txt');
 const TREND_FILE = path.join(REPO_ROOT, 'agents', 'runtime', '.rule-budget-history.jsonl');
+const AUTO_BUDGET_FILE = path.join(REPO_ROOT, 'src', 'config', 'auto-rule-budget.json');
 
 // Council R2 amendments (2026-05-06) — see docs/contracts/kernel-membership.md § 5.1.
 // Per-rule cap raised 1.5k → 2.5k; warning band raised 1.2k → 2.0k.
@@ -59,6 +60,8 @@ export interface RuleMeasure {
     lines: number;
     tokens_gpt: number;
     tokens_claude: number;
+    /** Iron-Law headings in the body — the obligation count a char count hides. */
+    iron_laws: number;
 }
 
 export interface Aggregate {
@@ -80,6 +83,7 @@ export interface Aggregate {
     per_rule_hard: number;
     per_rule_target: number;
     per_rule_override_ceiling: number;
+    iron_law_total: number;
     oversize_rules: RuleMeasure[];
     top5_largest: RuleMeasure[];
 }
@@ -130,9 +134,32 @@ export function measure_rule(p: string): RuleMeasure {
         lines: _count(body, '\n'),
         tokens_gpt: token_count.gpt_tokens(body).tokens,
         tokens_claude: token_count.claude_tokens(body).tokens,
+        iron_laws: count_iron_laws(body),
     };
 }
 
+/**
+ * Count Iron-Law HEADINGS in a rule body.
+ *
+ * road-to-design-fidelity-proof Phase 3.2. A rule is not an instruction:
+ * `token-efficiency` carries five Iron Laws in one file, so a file count and a
+ * char count both understate what a model is asked to hold at once. The
+ * instruction-following literature measures against the number of SIMULTANEOUS
+ * constraints, and an Iron-Law heading is the closest proxy this tree can count
+ * deterministically.
+ *
+ * Heading forms accepted, matching `preservation-guard`'s own definition: any
+ * level, optional `The`, singular or plural, optional trailing number or
+ * qualifier (`## Iron Law 2`, `## Iron Law — Gate`). A prose mention is NOT a
+ * heading, and a heading that merely contains the words later (`## Why the Iron
+ * Law exists`) is not one either — the phrase must open the heading text, or
+ * every retrospective paragraph would inflate the count.
+ */
+export function count_iron_laws(body: string): number {
+    const re = /^#{1,6}[ \t]+(?:The[ \t]+)?Iron[ \t]+Laws?\b/gim;
+    return (body.match(re) ?? []).length;
+}
+
 /**
  * Collect rule measurements from every source root (multi-root aware).
  * Deduplicates on logical rule id (stem) — first root wins. Mirrors
@@ -198,6 +225,7 @@ export function aggregate(rules: RuleMeasure[]): Aggregate {
         per_rule_hard: PER_RULE_HARD,
         per_rule_target: PER_RULE_TARGET,
         per_rule_override_ceiling: PER_RULE_OVERRIDE_CEILING,
+        iron_law_total: _sum(rules.map((r) => r.iron_laws)),
         oversize_rules: rules
             .filter((r) => r.chars > PER_RULE_HARD)
             .sort((a, b) => _cmp([-a.chars, a.id], [-b.chars, b.id])),
@@ -324,6 +352,59 @@ export function kernel_budget_check(
     return [0, out];
 }
 
+export interface AutoBudgetConfig {
+    baseline_chars: number;
+}
+
+/** Read the auto-bucket baseline. Throws rather than defaulting: a ratchet that
+ * silently falls back to "no bound" is a permanently green gate. */
+export function load_auto_budget(file: string = AUTO_BUDGET_FILE): AutoBudgetConfig {
+    const raw = JSON.parse(fs.readFileSync(file, 'utf-8')) as { baseline_chars?: unknown };
+    if (typeof raw.baseline_chars !== 'number') {
+        throw new Error(`${file}: baseline_chars must be a number`);
+    }
+    return { baseline_chars: raw.baseline_chars };
+}
+
+/**
+ * Enforce the `auto`-bucket growth ratchet. Returns `[exit_code, report_lines]`.
+ *
+ * road-to-design-fidelity-proof Phase 3.1. `kernel_budget_check` above bounds
+ * nine rules; this bounds the other 107, which carry ~93 % of the estate's rule
+ * prose and grew against nothing. Shrink-only: growth fails, a shrink passes and
+ * says how much slack it created, so the number walks down rather than becoming
+ * unused headroom.
+ *
+ * The Iron-Law total is REPORTED, never gated. No measurement in this tree says
+ * how many simultaneous obligations this layer can carry, and a bound nobody
+ * measured would publish a guess as a finding.
+ */
+export function auto_budget_check(agg: Aggregate, cfg: AutoBudgetConfig): [number, string[]] {
+    const out: string[] = [];
+    const delta = agg.auto_chars - cfg.baseline_chars;
+    const signed = delta >= 0 ? `+${delta}` : String(delta);
+    out.push(
+        `auto-bucket: ${agg.auto_chars} / ${cfg.baseline_chars} chars ` +
+            `(${agg.auto_count} rules, ${signed} vs baseline)`,
+    );
+    out.push(`iron-law total: ${agg.iron_law_total} heading(s) — reported, not gated`);
+    out.push('');
+    if (delta > 0) {
+        out.push(`\u274c  auto budget check: the auto bucket grew ${signed} chars`);
+        out.push(
+            '  The baseline may only move DOWN. Walk the bucket back, or state the ' +
+                'compensating reduction — raising it to clear this is config weakening.',
+        );
+        return [1, out];
+    }
+    out.push(
+        delta === 0
+            ? '\u2705  auto budget check: pass (at the baseline)'
+            : `\u2705  auto budget check: pass (${signed} chars of slack — ratchet it down)`,
+    );
+    return [0, out];
+}
+
 /**
  * Append a daily snapshot to agents/runtime/.rule-budget-history.jsonl.
  * Idempotent per UTC day. Mirrors `trend_append`. Returns `[exit, msg]`.
@@ -363,19 +444,22 @@ export function trend_append(agg: Aggregate): [number, string] {
 interface Args {
     json: boolean;
     kernelBudgetCheck: boolean;
+    autoBudgetCheck: boolean;
     trendAppend: boolean;
 }
 
 function parse_args(argv: string[]): Args {
     let json = false;
     let kernelBudgetCheck = false;
+    let autoBudgetCheck = false;
     let trendAppend = false;
     for (const a of argv) {
         if (a === '--json') json = true;
         else if (a === '--kernel-budget-check') kernelBudgetCheck = true;
+        else if (a === '--auto-budget-check') autoBudgetCheck = true;
         else if (a === '--trend-append') trendAppend = true;
     }
-    return { json, kernelBudgetCheck, trendAppend };
+    return { json, kernelBudgetCheck, autoBudgetCheck, trendAppend };
 }
 
 export function main(argv: string[] = process.argv.slice(2)): number {
@@ -384,10 +468,22 @@ export function main(argv: string[] = process.argv.slice(2)): number {
     const rules = collect();
     const agg = aggregate(rules);
 
-    if (args.kernelBudgetCheck) {
-        const overrides = load_overrides();
-        const [code, report] = kernel_budget_check(rules, agg, overrides);
-        process.stdout.write(report.join('\n') + '\n');
+    // Both budget flags may be passed in one invocation, and the CI step does
+    // exactly that — a second task step would be a new gate `check_ci_local_parity`
+    // has to be told about, for two checks over one measurement.
+    if (args.kernelBudgetCheck || args.autoBudgetCheck) {
+        let code = 0;
+        if (args.kernelBudgetCheck) {
+            const overrides = load_overrides();
+            const [kCode, report] = kernel_budget_check(rules, agg, overrides);
+            process.stdout.write(report.join('\n') + '\n');
+            code = Math.max(code, kCode);
+        }
+        if (args.autoBudgetCheck) {
+            const [aCode, report] = auto_budget_check(agg, load_auto_budget());
+            process.stdout.write(report.join('\n') + '\n');
+            code = Math.max(code, aCode);
+        }
         return code;
     }
 

package/src/scripts/measure_skill_ranker_baseline.ts

@@ -7,35 +7,49 @@
  * the corpus's expected skill per line". Building it surfaced two defects in
  * that premise, and this file is written around them rather than over them.
  *
- * DEFECT A — the 496-line corpus has no expected SKILL. `tests/eval/routing-matrix/`
- * is keyed by `rule:` and every prompt is labelled with the RULE it should
- * activate. Scoring a skill ranker against it would need a skill label that
- * does not exist there, so a "hit rate" over it would be invented.
+ * DEFECT A — the routing matrix had no expected SKILL. `tests/eval/routing-matrix/`
+ * is keyed by `rule:` and every prompt was labelled with the RULE it should
+ * activate. Scoring a skill ranker against it would need a skill label that did
+ * not exist there, so a "hit rate" over it would be invented.
  *
- * DEFECT B — 496 is stale. The matrix holds 499 prompts today
- * (`grep -h "^\s*- prompt:" tests/eval/routing-matrix/*.yaml | wc -l`), and 496
- * is still published in six places including `hook-token-budget.json`.
+ * **Defect A is CLOSED as of `road-to-a-menu-whose-precision-is-measured` 1.1.**
+ * Every case in the matrix now carries an `expected_skills` list, hand-written
+ * by seats that had not read this ranker's scoring (the protocol is in
+ * `tests/eval/routing-matrix/README.md`). An EMPTY list is a first-class value
+ * meaning "deliberately no skill expectation", and such a row is excluded from
+ * the accuracy denominator rather than counted as a miss. So the matrix arm now
+ * has ground truth; `--corpus routing-matrix` scores against it.
  *
- * So the baseline is measured against the corpus that DOES carry expected
- * skills — `tests/eval/corpus-{dev,non-dev}.yaml`, 26 labelled prompts — and
- * the routing matrix is reported alongside it as what it can honestly be: a
- * COVERAGE measurement (does the ranker return anything at all, and how
- * confident) with no ground truth and therefore no accuracy.
+ * DEFECT B — the published size is stale, repeatedly. 496 was published while
+ * the matrix held 499, and 499 is still published in several sites while the
+ * matrix holds more than that again. The size is therefore MEASURED on every run
+ * and printed, never asserted from a constant.
  *
- * Both halves are emitted in one JSON object so a later ranker revision can be
- * compared on the same two axes in one run:
+ * THE INTERVAL, AND THE REFUSAL TO PRINT A VERDICT UNDER n = 100.
+ * A hit rate over 26 prompts
+ * moves ~4 points per prompt; quoting it as "61.5 %" invites a comparison the
+ * sample cannot carry. Both arms therefore report a 95 % Wilson score interval
+ * (`_lib/capture_rate.ts` — Wilson rather than the normal approximation for the
+ * same reason stated there: near p = 1 the normal one produces bounds above 1).
  *
- *   ./scripts-run src/scripts/measure_skill_ranker_baseline --json
- *   ./scripts-run src/scripts/measure_skill_ranker_baseline --ranker keyword-v2 --json
+ * Below {@link MIN_POWERED_N} labelled rows the arm's `verdict` is
+ * `underpowered`. That is not a failure and not an exit code — the point
+ * estimate and the interval are still printed, because suppressing them would
+ * replace a wide measurement with no measurement. It is a label that travels
+ * with the number so a later reader cannot quote the point estimate alone.
  *
- * A 26-prompt denominator is small and is stated as such in the output. It is
- * the whole labelled ground truth this repository has.
+ * Usage:
+ *     ./scripts-run src/scripts/measure_skill_ranker_baseline
+ *     ./scripts-run src/scripts/measure_skill_ranker_baseline --corpus routing-matrix
+ *     ./scripts-run src/scripts/measure_skill_ranker_baseline --corpus all
+ *     ./scripts-run src/scripts/measure_skill_ranker_baseline --ranker keyword-v2
  */
 
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { fileURLToPath } from 'node:url';
 
+import { wilsonInterval } from './_lib/capture_rate.js';
 import { rank } from './skill_tools/score_skill_relevance.js';
 import type { RankOptions } from '../shared/skillRanking.js';
 
@@ -45,6 +59,20 @@ const MATRIX_DIR = 'tests/eval/routing-matrix';
 /** Where the projected skills live in this checkout. */
 export const SKILLS_DIR = path.join(REPO, 'src', 'skills');
 
+/**
+ * The smallest labelled corpus this report will call powered.
+ *
+ * 100 is the roadmap's number, not a derived optimum, and it is stated as such:
+ * at n = 100 a point estimate near 0.7 carries a 95 % Wilson half-width of
+ * roughly ±9 points, which is wide but no longer moves on a single prompt the
+ * way a 26-row denominator does. Raise it with a dated reason if a decision
+ * ever needs a tighter bound than that.
+ */
+export const MIN_POWERED_N = 100;
+
+export type CorpusName = 'labelled' | 'routing-matrix' | 'all';
+export const CORPUS_NAMES: readonly CorpusName[] = ['labelled', 'routing-matrix', 'all'];
+
 export interface LabelledPrompt {
     id: string;
     corpus: string;
@@ -94,38 +122,193 @@ export function readLabelledPrompts(repo = REPO): LabelledPrompt[] {
     return out;
 }
 
-/** Every `- prompt:` string in the rule routing matrix. No skill label exists. */
-export function readMatrixPrompts(repo = REPO): string[] {
+export interface MatrixCase {
+    rule: string;
+    section: 'positives' | 'near_misses';
+    ordinal: number;
+    prompt: string;
+    /** `undefined` = the key is missing (a defect the lint catches); `[]` = deliberately unlabelled. */
+    expected?: string[];
+}
+
+/**
+ * Every case in the rule routing matrix, with its `expected_skills` label when
+ * one is present.
+ *
+ * Line-oriented for the same reason as the reader above, and because a YAML
+ * round-trip through this directory would reformat 100+ hand-kept fixtures.
+ * A case is `prompt:` plus the keys indented under it up to the next `- `.
+ */
+export function readMatrixCases(repo = REPO): MatrixCase[] {
     const dir = path.join(repo, MATRIX_DIR);
     if (!fs.existsSync(dir)) return [];
-    const out: string[] = [];
+    const out: MatrixCase[] = [];
     for (const name of fs.readdirSync(dir).sort()) {
         if (!name.endsWith('.yaml')) continue;
+        const rule = name.replace(/\.yaml$/, '');
+        let section: MatrixCase['section'] = 'positives';
+        let ordinal = 0;
+        let current: MatrixCase | undefined;
         for (const raw of fs.readFileSync(path.join(dir, name), 'utf8').split('\n')) {
-            const m = /^\s*-?\s*prompt:\s*"(.*)"\s*$/.exec(raw.replace(/\r$/, ''));
-            if (m && m[1]) out.push(m[1]);
+            const line = raw.replace(/\r$/, '');
+            if (/^positives:\s*$/.test(line)) {
+                section = 'positives';
+                ordinal = 0;
+                current = undefined;
+                continue;
+            }
+            if (/^near_misses:\s*$/.test(line)) {
+                section = 'near_misses';
+                ordinal = 0;
+                current = undefined;
+                continue;
+            }
+            const mPrompt = /^\s*-?\s*prompt:\s*"(.*)"\s*$/.exec(line);
+            if (mPrompt && mPrompt[1] !== undefined) {
+                current = { rule, section, ordinal: ordinal++, prompt: mPrompt[1] };
+                out.push(current);
+                continue;
+            }
+            const mExp = /^\s*expected_skills:\s*\[(.*)\]\s*$/.exec(line);
+            if (mExp && current) {
+                current.expected = mExp[1]!
+                    .split(',')
+                    .map((s) => s.trim())
+                    .filter(Boolean);
+            }
         }
     }
     return out;
 }
 
+/** Every `- prompt:` string in the rule routing matrix, label or no label. */
+export function readMatrixPrompts(repo = REPO): string[] {
+    return readMatrixCases(repo).map((c) => c.prompt);
+}
+
+/** The matrix rows that carry a NON-EMPTY label — the accuracy denominator. */
+export function readMatrixLabelledPrompts(repo = REPO): LabelledPrompt[] {
+    return readMatrixCases(repo)
+        .filter((c) => (c.expected?.length ?? 0) > 0)
+        .map((c) => ({
+            id: `${c.rule}#${c.section}[${c.ordinal}]`,
+            corpus: 'routing-matrix',
+            prompt: c.prompt,
+            expected: c.expected!,
+        }));
+}
+
+export function promptsForCorpus(corpus: CorpusName, repo = REPO): LabelledPrompt[] {
+    if (corpus === 'labelled') return readLabelledPrompts(repo);
+    if (corpus === 'routing-matrix') return readMatrixLabelledPrompts(repo);
+    return [...readLabelledPrompts(repo), ...readMatrixLabelledPrompts(repo)];
+}
+
+/**
+ * The packs a skill declares, read from its own `packs:` frontmatter block.
+ *
+ * Line-oriented, and deliberately NOT read from `src/packs/*\/pack.yaml`: those
+ * manifests are generated FROM this frontmatter, and four of them
+ * (`analytics`, `core`, `memory`, `product-reasoning`) carry no skill at all.
+ * A pack with no skill can never own a labelled prompt, so a census over the
+ * manifest set would state a floor nothing could ever meet.
+ */
+export function packsForSkill(skillsDir: string, name: string): string[] {
+    const file = path.join(skillsDir, name, 'SKILL.md');
+    if (!fs.existsSync(file)) return [];
+    const text = fs.readFileSync(file, 'utf8');
+    if (!text.startsWith('---')) return [];
+    const end = text.indexOf('\n---', 3);
+    if (end === -1) return [];
+    const lines = text.slice(3, end).split('\n');
+    const at = lines.findIndex((l) => /^packs:/u.test(l));
+    if (at === -1) return [];
+    const out: string[] = [];
+    for (let i = at + 1; i < lines.length; i += 1) {
+        const m = /^\s+-\s+(\S+)\s*$/u.exec(lines[i] as string);
+        if (!m) break;
+        out.push(m[1] as string);
+    }
+    return out;
+}
+
+/** Every pack id that at least one shipped skill declares. The census domain. */
+export function allPacks(skillsDir: string): string[] {
+    const packs = new Set<string>();
+    for (const entry of fs.readdirSync(skillsDir, { withFileTypes: true })) {
+        if (!entry.isDirectory()) continue;
+        for (const p of packsForSkill(skillsDir, entry.name)) packs.add(p);
+    }
+    return [...packs].sort();
+}
+
+/**
+ * How many labelled prompts each pack owns, via the packs of its expected skills.
+ *
+ * A prompt counts ONCE per pack however many of that pack's skills it expects —
+ * otherwise a single prompt naming three `laravel` skills would satisfy a
+ * three-prompt floor on its own, which is the shape of coverage this census
+ * exists to refuse.
+ */
+export function packCensus(prompts: readonly LabelledPrompt[], skillsDir: string): Record<string, number> {
+    const out: Record<string, number> = {};
+    for (const p of allPacks(skillsDir)) out[p] = 0;
+    for (const prompt of prompts) {
+        const packs = new Set<string>();
+        for (const skill of prompt.expected) for (const pack of packsForSkill(skillsDir, skill)) packs.add(pack);
+        for (const pack of packs) out[pack] = (out[pack] ?? 0) + 1;
+    }
+    return out;
+}
+
+/** Packs owning fewer than `floor` labelled prompts, sorted by pack id. */
+export function packsBelow(census: Record<string, number>, floor: number): string[] {
+    return Object.keys(census)
+        .filter((p) => (census[p] ?? 0) < floor)
+        .sort();
+}
+
+/** The per-pack floor `road-to-a-menu-whose-precision-is-measured` 1.1 states. */
+export const MIN_PROMPTS_PER_PACK = 3;
+
+export interface Interval {
+    lower: number;
+    upper: number;
+}
+
 export interface AccuracyArm {
+    corpus: CorpusName;
     corpus_prompts: number;
     top1: number;
+    top1_ci95: Interval;
     top3: number;
+    top3_ci95: Interval;
+    /** `underpowered` below MIN_POWERED_N labelled rows; `measured` at or above. */
+    verdict: 'underpowered' | 'measured';
+    min_powered_n: number;
     misses: string[];
     denominator_note: string;
+    /** Pack ids at least one shipped skill declares — the census domain. */
+    packs_total: number;
+    min_prompts_per_pack: number;
+    /** Packs this corpus leaves under the floor. Empty is the passing shape. */
+    packs_below_floor: string[];
+    /** Full census, so a reader can see the distribution and not just the failures. */
+    pack_census: Record<string, number>;
 }
 
 export interface CoverageArm {
     corpus_prompts: number;
+    labelled_prompts: number;
+    unlabelled_prompts: number;
+    missing_label_key: number;
     prompts_with_any_result: number;
     mean_top_score: number;
     note: string;
 }
 
 export interface RankerBaseline {
-    schema: 1;
+    schema: 2;
     ranker: string;
     commit: string;
     skills_dir: string;
@@ -139,28 +322,85 @@ function hitAt(rows: readonly (readonly [string, number, string[]])[], n: number
     return expected.some((e) => top.includes(e));
 }
 
-export function measure(opts: { ranker: string; commit: string; skillsDir?: string; repo?: string }): RankerBaseline {
-    const repo = opts.repo ?? REPO;
-    const skillsDir = opts.skillsDir ?? SKILLS_DIR;
-    // `keyword-v2` is Phase 3.1: the same formula with `triggers:` prose folded
-    // into each skill's term source. Any other label measures v1.
-    const rankOpts: RankOptions = opts.ranker === 'keyword-v2' ? { includeTriggers: true } : {};
-    const labelled = readLabelledPrompts(repo);
+function round3(n: number): number {
+    return Math.round(n * 1000) / 1000;
+}
+
+function ci(successes: number, trials: number): Interval {
+    const raw = wilsonInterval(successes, trials);
+    return { lower: round3(raw.lower), upper: round3(raw.upper) };
+}
+
+const DENOMINATOR_NOTES: Record<CorpusName, string> = {
+    labelled:
+        'tests/eval/corpus-dev.yaml + corpus-non-dev.yaml — the two hand-written eval corpora. ' +
+        'Small; a single prompt moves each rate by ~4 points, which is why the interval is ' +
+        'printed beside the point estimate and not instead of it.',
+    'routing-matrix':
+        'tests/eval/routing-matrix/ rows whose expected_skills list is NON-EMPTY. A row with an ' +
+        'empty list is a deliberate "no skill expectation" and is excluded from the ' +
+        'denominator — never counted as a miss.',
+    all:
+        'Both labelled sources pooled. The two were written under different protocols, so a ' +
+        'pooled rate is a convenience reading and the per-corpus arms are the citable ones.',
+};
+
+export function measureAccuracy(opts: {
+    corpus: CorpusName;
+    repo: string;
+    skillsDir: string;
+    rankOpts: RankOptions;
+}): AccuracyArm {
+    const labelled = promptsForCorpus(opts.corpus, opts.repo);
     let top1 = 0;
     let top3 = 0;
     const misses: string[] = [];
     for (const p of labelled) {
-        const rows = rank(p.prompt, skillsDir, rankOpts);
+        const rows = rank(p.prompt, opts.skillsDir, opts.rankOpts);
         if (hitAt(rows, 1, p.expected)) top1++;
         if (hitAt(rows, 3, p.expected)) top3++;
         else misses.push(p.id);
     }
+    const n = labelled.length;
+    const census = packCensus(labelled, opts.skillsDir);
+    return {
+        corpus: opts.corpus,
+        corpus_prompts: n,
+        top1: n ? round3(top1 / n) : 0,
+        top1_ci95: ci(top1, n),
+        top3: n ? round3(top3 / n) : 0,
+        top3_ci95: ci(top3, n),
+        verdict: n >= MIN_POWERED_N ? 'measured' : 'underpowered',
+        min_powered_n: MIN_POWERED_N,
+        misses,
+        denominator_note: DENOMINATOR_NOTES[opts.corpus],
+        packs_total: Object.keys(census).length,
+        min_prompts_per_pack: MIN_PROMPTS_PER_PACK,
+        packs_below_floor: packsBelow(census, MIN_PROMPTS_PER_PACK),
+        pack_census: census,
+    };
+}
+
+export function measure(opts: {
+    ranker: string;
+    commit: string;
+    corpus?: CorpusName;
+    skillsDir?: string;
+    repo?: string;
+}): RankerBaseline {
+    const repo = opts.repo ?? REPO;
+    const skillsDir = opts.skillsDir ?? SKILLS_DIR;
+    // `keyword-v2` is Phase 3.1: the same formula with `triggers:` prose folded
+    // into each skill's term source. Any other label measures v1.
+    const rankOpts: RankOptions = opts.ranker === 'keyword-v2' ? { includeTriggers: true } : {};
+    const corpus = opts.corpus ?? 'labelled';
+    const accuracy = measureAccuracy({ corpus, repo, skillsDir, rankOpts });
 
-    const matrix = readMatrixPrompts(repo);
+    const cases = readMatrixCases(repo);
     let withResult = 0;
     let scoreSum = 0;
-    for (const prompt of matrix) {
-        const rows = rank(prompt, skillsDir, rankOpts);
+    for (const c of cases) {
+        const rows = rank(c.prompt, skillsDir, rankOpts);
         if (rows.length > 0) {
             withResult++;
             scoreSum += rows[0]![1];
@@ -172,37 +412,51 @@ export function measure(opts: { ranker: string; commit: string; skillsDir?: stri
         : 0;
 
     return {
-        schema: 1,
+        schema: 2,
         ranker: opts.ranker,
         commit: opts.commit,
         skills_dir: path.relative(repo, skillsDir),
         skills_indexed: skillsIndexed,
-        accuracy: {
-            corpus_prompts: labelled.length,
-            top1: labelled.length ? Math.round((top1 / labelled.length) * 1000) / 1000 : 0,
-            top3: labelled.length ? Math.round((top3 / labelled.length) * 1000) / 1000 : 0,
-            misses,
-            denominator_note:
-                'tests/eval/corpus-dev.yaml + corpus-non-dev.yaml — the only expected-skill ground ' +
-                'truth in this tree. Small; a single prompt moves each rate by ~4 points.',
-        },
+        accuracy,
         matrix_coverage: {
-            corpus_prompts: matrix.length,
+            corpus_prompts: cases.length,
+            labelled_prompts: cases.filter((c) => (c.expected?.length ?? 0) > 0).length,
+            unlabelled_prompts: cases.filter((c) => c.expected?.length === 0).length,
+            missing_label_key: cases.filter((c) => c.expected === undefined).length,
             prompts_with_any_result: withResult,
             mean_top_score: withResult ? Math.round((scoreSum / withResult) * 100) / 100 : 0,
             note:
-                'tests/eval/routing-matrix/ is labelled with the expected RULE, never a skill, so ' +
-                'this arm carries no accuracy — only whether the ranker answers at all and how ' +
-                'confident its top answer is. Its size is measured here, not assumed: the roadmap ' +
-                'and five other sites still publish 496.',
+                'Coverage over EVERY matrix case, labelled or not: whether the ranker answers at ' +
+                'all and how confident its top answer is. Accuracy over the labelled subset is ' +
+                'the `accuracy` arm under `--corpus routing-matrix`. Size is measured here, never ' +
+                'asserted: 496 and 499 are both published elsewhere and both stale.',
         },
     };
 }
 
+function parseCorpus(argv: readonly string[]): CorpusName {
+    const i = argv.indexOf('--corpus');
+    if (i === -1) return 'labelled';
+    const raw = argv[i + 1];
+    if (!raw || !CORPUS_NAMES.includes(raw as CorpusName)) {
+        throw new Error(
+            `measure_skill_ranker_baseline: --corpus expects one of ${CORPUS_NAMES.join(' | ')}, got ${raw ?? '(nothing)'}`,
+        );
+    }
+    return raw as CorpusName;
+}
+
 export function main(argv: readonly string[]): number {
     const ranker = argv.includes('--ranker') ? (argv[argv.indexOf('--ranker') + 1] ?? 'keyword-v1') : 'keyword-v1';
     const commit = argv.includes('--commit') ? (argv[argv.indexOf('--commit') + 1] ?? 'unknown') : 'unknown';
-    const out = measure({ ranker, commit });
+    let corpus: CorpusName;
+    try {
+        corpus = parseCorpus(argv);
+    } catch (err) {
+        process.stderr.write(`${(err as Error).message}\n`);
+        return 2;
+    }
+    const out = measure({ ranker, commit, corpus });
     process.stdout.write(`${JSON.stringify(out, null, 2)}\n`);
     return 0;
 }