@event4u/agent-config 14.10.0 → 14.11.0

package/src/scripts/mcp_render.ts

@@ -13,6 +13,15 @@
  *     .windsurf/mcp.json                                     (in-project)
  *     ~/.config/claude-desktop/claude_desktop_config.json    (user, opt-in)
  *
+ * `.mcp.json` — Claude Code's project-scope config — is a CHECK-ONLY target
+ * (Phase 1.3 of `road-to-skill-delivery-over-mcp`). It is deliberately not
+ * rendered: the installer writes it with `merge_json_file`, which touches only
+ * this package's own server key and records it in `agents/installed-tools.lock`
+ * so uninstall subtracts exactly that key. Rendering it here would overwrite a
+ * file the consumer also edits and silently delete servers they added by hand.
+ * `--check` therefore asserts CONTAINMENT of our entry rather than equality with
+ * a rendering, and says so on the same line.
+ *
  * All targets use the same `mcpServers` top-level key. The source file uses
  * `servers` to keep our internal schema stable if a downstream format ever
  * diverges.
@@ -40,6 +49,36 @@ export function default_project_root(): string {
     return fs.realpathSync(process.cwd());
 }
 
+/** The one entry `.mcp.json` must contain. Mirrors `install.ts:MCP_SERVER_KEY`. */
+export const MCP_JSON_SERVER_KEY = 'agent-config';
+
+/** Claude Code's project-scope MCP config. Checked here, written by the installer. */
+export function claude_code_mcp_target(projectRoot: string): string {
+    return path.join(projectRoot, '.mcp.json');
+}
+
+/**
+ * Does `.mcp.json` carry this package's server entry?
+ *
+ * `absent` is a distinct verdict from `missing-entry` on purpose: absent means
+ * the installer has not run for `claude-code` here, which is a hint; a file that
+ * exists WITHOUT our key means something removed it, which is a failure.
+ */
+export function check_mcp_json(projectRoot: string): 'absent' | 'ok' | 'missing-entry' | 'unreadable' {
+    const target = claude_code_mcp_target(projectRoot);
+    if (!fs.existsSync(target)) return 'absent';
+    let parsed: unknown;
+    try {
+        parsed = JSON.parse(fs.readFileSync(target, 'utf-8'));
+    } catch {
+        return 'unreadable';
+    }
+    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) return 'unreadable';
+    const servers = (parsed as Record<string, unknown>)['mcpServers'];
+    if (servers === null || typeof servers !== 'object' || Array.isArray(servers)) return 'missing-entry';
+    return MCP_JSON_SERVER_KEY in (servers as Record<string, unknown>) ? 'ok' : 'missing-entry';
+}
+
 export function in_project_targets(projectRoot: string): Record<string, string> {
     return {
         cursor: path.join(projectRoot, '.cursor', 'mcp.json'),
@@ -274,7 +313,33 @@ function cmd_check(args: ParsedArgs): number {
         return 1;
     }
     process.stdout.write('✅  All MCP targets match source.\n');
-    return 0;
+    return _report_mcp_json(projectRoot);
+}
+
+/**
+ * The `.mcp.json` leg of `--check`. Additive: it can only turn a green run red
+ * when the file EXISTS and has lost our entry, never on a fresh tree where the
+ * installer has not run.
+ */
+function _report_mcp_json(projectRoot: string): number {
+    const verdict = check_mcp_json(projectRoot);
+    const rel = path.relative(projectRoot, claude_code_mcp_target(projectRoot)) || '.mcp.json';
+    if (verdict === 'ok') {
+        process.stdout.write(`✅  ${rel} carries the \`${MCP_JSON_SERVER_KEY}\` server (containment, not equality).\n`);
+        return 0;
+    }
+    if (verdict === 'absent') {
+        process.stdout.write(
+            `ℹ️   ${rel} not present — written by \`agent-config install\` for the ` +
+                'claude-code tool, never rendered from mcp.json.\n',
+        );
+        return 0;
+    }
+    process.stderr.write(
+        `❌  ${rel} exists but does not carry the \`${MCP_JSON_SERVER_KEY}\` server ` +
+            `(${verdict}). Re-run \`agent-config install\`.\n`,
+    );
+    return 1;
 }
 
 class ArgError extends Error {}

package/src/scripts/mcp_tool_fingerprint.ts

@@ -0,0 +1,241 @@
+/**
+ * Fingerprint store for third-party MCP tool definitions.
+ *
+ * Step 1.1 of `road-to-mcp-runtime-integrity`. The threat is a **rug-pull**: a
+ * third-party MCP server whose tool definition is benign at connection time and
+ * mutated afterwards, so the description the model reasons about is no longer the
+ * description that was reviewed. Nothing in this tree detected that:
+ * `lint_mcp_config_security.ts` reads shipped CONFIG for supply-chain smells and
+ * never sees a definition change after connection, and `audit_mcp_tools.ts` is an
+ * inventory generator for THIS package's own consumer catalog.
+ *
+ * WHAT THIS FILE IS NOT, and the distinction is load-bearing. It is a store and a
+ * comparison, bound to **no hook slot**. The roadmap's `## The no-silent-downgrade
+ * rule` forbids shipping a `post_tool_use` or session-start variant — a check that
+ * fires after first use is a post-mortem, not a control, for a tool with
+ * irreversible side effects — and the pre-use slot is gated on a per-turn
+ * composite ceiling that is not armed (`hook-latency-budget.json`,
+ * `observe_only: true`, `p50_ci: null`). So the placement decision is deliberately
+ * absent here rather than quietly resolved: **rug-pull protection remains absent
+ * and the protection level is zero** until an owner records which side of that
+ * trade-off they chose. Wiring this module into a slot without that record is the
+ * silent downgrade the roadmap binds itself against.
+ *
+ * THIRD-PARTY ONLY. This package's own consumer catalog is already covered by
+ * `audit_mcp_tools.ts`; fingerprinting it here would produce two sources of truth
+ * for the same tools.
+ *
+ * KEYED BY SERVER PLUS TOOL NAME, so a renamed tool reads as a NEW tool rather
+ * than as a mutation of the old one. That is the difference between "this server
+ * shipped a new tool" and "this server changed a tool you already approved", and
+ * collapsing them is how a mismatch surface becomes noise.
+ *
+ * A FIRST SIGHTING IS RECORDED, NOT REPORTED. Third-party servers legitimately
+ * add tools; reporting every first sighting with the same weight as a mutation
+ * trains the reader to ignore the surface, and noise on a security surface is
+ * worse than silence because it reads as coverage (Risk 4 in the roadmap).
+ *
+ * PII-exclusion-by-construction: a record holds a server id, a tool name, a hex
+ * digest and an ISO date. There is NO field capable of holding the description
+ * text, the schema, or any free-form content — the digest is one-way and the
+ * inputs are not retained. Never widen it.
+ */
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { update_json_under_lock } from './hooks/state_io.js';
+
+/** Where the store lives, relative to the workspace root. */
+export const FINGERPRINT_STORE = path.join(
+    'agents',
+    'runtime',
+    'state',
+    'mcp-tool-fingerprints.json',
+);
+
+export const SCHEMA_VERSION = 1;
+
+/**
+ * The three fields a rug-pull can mutate to change what the model reasons about.
+ *
+ * `name` is included even though it is also half the key: a tool whose key is
+ * `server/x` and whose declared `name` later reads `y` is a mismatch, not a
+ * rename, and leaving `name` out of the digest would let that pass.
+ */
+export interface McpToolDefinition {
+    readonly name: string;
+    readonly description?: string;
+    readonly inputSchema?: unknown;
+}
+
+export interface FingerprintRecord {
+    /** Hex sha256 of the canonical definition. */
+    readonly digest: string;
+    /** ISO date (day resolution — a timestamp would be a behavioural fingerprint). */
+    readonly first_seen: string;
+}
+
+export type FingerprintOutcome =
+    /** No record existed. Written silently; nothing to report. */
+    | { readonly kind: 'first-sighting'; readonly key: string; readonly digest: string }
+    /** A record existed and matches. Nothing to report. */
+    | { readonly kind: 'unchanged'; readonly key: string; readonly digest: string }
+    /** A record existed and does NOT match. This is the surface. */
+    | {
+          readonly kind: 'mismatch';
+          readonly key: string;
+          readonly digest: string;
+          readonly recorded: string;
+          readonly first_seen: string;
+      };
+
+/**
+ * The store key. Server id and tool name, separated by a character neither may
+ * contain unescaped, so two distinct pairs cannot collide onto one key — the
+ * same failure `session_state_file` exists to prevent for session ids.
+ */
+export function fingerprintKey(server: string, toolName: string): string {
+    const esc = (s: string): string => s.replace(/%/g, '%25').replace(/\//g, '%2F');
+    return `${esc(server)}/${esc(toolName)}`;
+}
+
+/**
+ * Canonical JSON with object keys sorted at every depth.
+ *
+ * Without this the digest is a fingerprint of the SERIALISER's key order rather
+ * than of the definition, and a server that reorders its schema keys between two
+ * connections would read as a rug-pull. Arrays keep their order — order is
+ * meaningful in a schema's `required` list and reordering it IS a change.
+ */
+export function canonicalize(value: unknown): unknown {
+    if (Array.isArray(value)) return value.map(canonicalize);
+    if (value !== null && typeof value === 'object') {
+        const out: Record<string, unknown> = {};
+        for (const k of Object.keys(value as Record<string, unknown>).sort()) {
+            out[k] = canonicalize((value as Record<string, unknown>)[k]);
+        }
+        return out;
+    }
+    return value;
+}
+
+/**
+ * Hex sha256 over the canonical form of the three mutable fields.
+ *
+ * `description` and `inputSchema` are normalised to `null` when absent so that
+ * "absent" and "explicitly null" hash alike — a server toggling between the two
+ * has changed nothing the model can read, and reporting it would be noise.
+ */
+export function fingerprintDefinition(def: McpToolDefinition): string {
+    const canonical = canonicalize({
+        name: def.name,
+        description: def.description ?? null,
+        inputSchema: def.inputSchema ?? null,
+    });
+    return crypto.createHash('sha256').update(JSON.stringify(canonical), 'utf-8').digest('hex');
+}
+
+interface StoreDoc {
+    schema_version: number;
+    tools: Record<string, FingerprintRecord>;
+}
+
+function emptyDoc(): StoreDoc {
+    return { schema_version: SCHEMA_VERSION, tools: {} };
+}
+
+/** Read the store. A missing, unreadable or malformed store reads as empty. */
+export function readStore(workspaceRoot: string): StoreDoc {
+    try {
+        const decoded: unknown = JSON.parse(
+            fs.readFileSync(path.join(workspaceRoot, FINGERPRINT_STORE), 'utf-8'),
+        );
+        if (typeof decoded === 'object' && decoded !== null && !Array.isArray(decoded)) {
+            const tools = (decoded as Record<string, unknown>)['tools'];
+            return {
+                schema_version: SCHEMA_VERSION,
+                tools:
+                    typeof tools === 'object' && tools !== null && !Array.isArray(tools)
+                        ? (tools as Record<string, FingerprintRecord>)
+                        : {},
+            };
+        }
+    } catch {
+        // Absent or malformed both mean "nothing recorded", and the outcome that
+        // follows is `first-sighting` — which reports nothing. A corrupt store
+        // must never be read as a mismatch: that would turn one bad write into a
+        // rug-pull alert on every tool at once.
+    }
+    return emptyDoc();
+}
+
+/**
+ * Record `def` for `server`, and say what the record implies.
+ *
+ * Read-modify-write under ONE lock, not three steps: two hosts connected to the
+ * same workspace fingerprint concurrently, and `atomic_write_json` makes the
+ * publish atomic while leaving load -> merge -> publish racy — which loses a
+ * whole tool's record rather than merely an increment, because this mutator
+ * republishes the `tools` map. Same primitive and same reason as
+ * `rule-trips.json`.
+ *
+ * `today` is injected rather than read from the clock so a caller (and a test)
+ * can pin it; nothing in this module reads wall time on its own.
+ */
+export function recordFingerprint(
+    workspaceRoot: string,
+    server: string,
+    def: McpToolDefinition,
+    today: string,
+): FingerprintOutcome {
+    const key = fingerprintKey(server, def.name);
+    const digest = fingerprintDefinition(def);
+    let outcome: FingerprintOutcome = { kind: 'first-sighting', key, digest };
+
+    update_json_under_lock<StoreDoc>(path.join(workspaceRoot, FINGERPRINT_STORE), (loaded) => {
+        const doc: StoreDoc = { ...emptyDoc(), ...((loaded ?? {}) as Partial<StoreDoc>) };
+        const tools: Record<string, FingerprintRecord> =
+            typeof doc.tools === 'object' && doc.tools !== null && !Array.isArray(doc.tools)
+                ? { ...doc.tools }
+                : {};
+        const prior = tools[key];
+        if (prior === undefined) {
+            tools[key] = { digest, first_seen: today };
+            outcome = { kind: 'first-sighting', key, digest };
+        } else if (prior.digest === digest) {
+            outcome = { kind: 'unchanged', key, digest };
+        } else {
+            // The recorded digest is NOT overwritten. A mismatch is a surface for
+            // a human decision, not a verdict, and silently re-baselining would
+            // make the second read of a mutated tool report `unchanged` — which
+            // is the one outcome a rug-pull must never produce.
+            outcome = {
+                kind: 'mismatch',
+                key,
+                digest,
+                recorded: prior.digest,
+                first_seen: prior.first_seen,
+            };
+        }
+        return { schema_version: SCHEMA_VERSION, tools };
+    });
+
+    return outcome;
+}
+
+/**
+ * Human-readable line for a mismatch, and nothing for the other two outcomes.
+ *
+ * Returning `null` rather than an empty string is deliberate: a caller that
+ * prints whatever it gets back cannot accidentally emit a blank line for a
+ * first sighting, which is the shape of "reported silently".
+ */
+export function describeOutcome(outcome: FingerprintOutcome): string | null {
+    if (outcome.kind !== 'mismatch') return null;
+    return (
+        `mcp tool definition changed since it was first recorded: ${outcome.key} ` +
+        `(recorded ${outcome.recorded.slice(0, 12)} on ${outcome.first_seen}, ` +
+        `now ${outcome.digest.slice(0, 12)}) — review the change before using this tool`
+    );
+}

package/src/scripts/measure_risk_class_drift.ts

@@ -0,0 +1,140 @@
+#!/usr/bin/env tsx
+/**
+ * Standing R3-rate metric over recently merged changes.
+ *
+ * Step 3.1 of `road-to-target-project-assurance-readiness`, and it exists because of
+ * Risk 2 rather than for its own sake: `classify_change_risk` resolves ties and
+ * unknowns **upward**, which is safe per change and corrosive in aggregate — once
+ * most changes read R3, the owed gate set is noise and gets worked around rather
+ * than met. The roadmap's mitigation is explicit about where the defect would live:
+ * **> 40 % of changes classifying R3 is a defect in the override list**, not in the
+ * people meeting the gates. This is the instrument that can say so.
+ *
+ * WHAT IT CANNOT REPORT, and the null is recorded per row rather than omitted.
+ * The step asks for `agreement` and `r3_recall` alongside the rate. Both are
+ * measured against a **human-labelled** corpus, and step 0.2 recorded that no human
+ * labeller is reachable — so those two fields are written as `null` with
+ * `null_reason` naming why. Writing them absent would make a later reader unable to
+ * distinguish "not measured" from "measured as zero", which on a recall metric is
+ * the difference between no data and total failure.
+ *
+ * NIGHTLY, NOT PER-CHANGE. The classifier runs in-session per change; the
+ * *measurement* runs on a schedule. A per-PR trigger would make the metric a gate,
+ * and Phase 2 — the phase that gives the class authority — is cancelled on the
+ * pre-registered null route precisely because the class is unvalidated.
+ */
+import { spawnSync } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import process from 'node:process';
+import { fileURLToPath } from 'node:url';
+
+import { type RiskClass, classifyPaths } from './classify_change_risk.js';
+
+const _HERE = fileURLToPath(import.meta.url);
+const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
+export const DRIFT_LEDGER = path.join('agents', 'evidence', 'risk-classifier-drift.jsonl');
+
+/** The rate above which the override list — not the reviewer — is the defect. */
+export const R3_RATE_DEFECT_THRESHOLD = 0.4;
+
+export interface DriftRow {
+    measured_at: string;
+    window_days: number;
+    commits: number;
+    counts: Record<RiskClass, number>;
+    r3_rate: number | null;
+    r3_rate_over_threshold: boolean | null;
+    agreement: null;
+    r3_recall: null;
+    null_reason: string;
+}
+
+/** One entry per merged commit in the window: its sha and its touched paths. */
+export function commitsInWindow(days: number, cwd = REPO_ROOT): { sha: string; paths: string[] }[] {
+    const log = spawnSync(
+        'git',
+        ['log', '--first-parent', `--since=${String(days)} days ago`, '--format=%H'],
+        { cwd, encoding: 'utf-8', maxBuffer: 64 * 1024 * 1024 },
+    );
+    if (log.status !== 0) return [];
+    const shas = (log.stdout || '').split('\n').map((s) => s.trim()).filter(Boolean);
+    const out: { sha: string; paths: string[] }[] = [];
+    for (const sha of shas) {
+        // `-m --first-parent` so a merge commit reports the paths it BROUGHT IN
+        // rather than nothing at all — a merge-heavy history would otherwise read
+        // as a stream of empty diffs, and an empty diff classifies R3 by the
+        // upward rule, which would invent an R3 rate of 1.0 out of bookkeeping.
+        const d = spawnSync('git', ['show', '--pretty=', '--name-only', '-m', '--first-parent', sha], {
+            cwd,
+            encoding: 'utf-8',
+            maxBuffer: 64 * 1024 * 1024,
+        });
+        const paths = (d.stdout || '').split('\n').map((s) => s.trim()).filter(Boolean);
+        if (paths.length > 0) out.push({ sha, paths });
+    }
+    return out;
+}
+
+export function buildRow(
+    commits: readonly { sha: string; paths: string[] }[],
+    measured_at: string,
+    window_days: number,
+): DriftRow {
+    const counts: Record<RiskClass, number> = { R0: 0, R1: 0, R2: 0, R3: 0 };
+    for (const c of commits) counts[classifyPaths(c.paths).cls] += 1;
+    const n = commits.length;
+    const r3_rate = n === 0 ? null : counts.R3 / n;
+    return {
+        measured_at,
+        window_days,
+        commits: n,
+        counts,
+        r3_rate,
+        // `null` rather than `false` on an empty window: no commits is not evidence
+        // that the rate is acceptable.
+        r3_rate_over_threshold: r3_rate === null ? null : r3_rate > R3_RATE_DEFECT_THRESHOLD,
+        agreement: null,
+        r3_recall: null,
+        null_reason:
+            'no human-labelled corpus: step 0.2 recorded no reachable labeller, so ' +
+            'agreement and r3_recall have no reference standard to be measured against. ' +
+            'Reopens per agents/evidence/risk-classifier-prereg.md § Re-open threshold.',
+    };
+}
+
+export function main(argv: readonly string[]): number {
+    const wIdx = argv.indexOf('--window-days');
+    const window_days = wIdx >= 0 ? Number(argv[wIdx + 1] ?? '30') : 30;
+    const nowIdx = argv.indexOf('--now');
+    // Injected rather than read from the clock so a test can pin the row.
+    const measured_at = nowIdx >= 0 ? String(argv[nowIdx + 1]) : new Date().toISOString();
+    const dry = argv.includes('--dry-run');
+
+    const commits = commitsInWindow(window_days);
+    const row = buildRow(commits, measured_at, window_days);
+    const line = JSON.stringify(row);
+
+    if (dry) {
+        process.stdout.write(line + '\n');
+        return 0;
+    }
+    const target = path.join(REPO_ROOT, DRIFT_LEDGER);
+    fs.mkdirSync(path.dirname(target), { recursive: true });
+    fs.appendFileSync(target, line + '\n', 'utf-8');
+    process.stdout.write(
+        `risk-class drift: ${String(row.commits)} commit(s) over ${String(window_days)}d — ` +
+            `R3 ${String(row.counts.R3)} (rate ${row.r3_rate === null ? 'n/a' : row.r3_rate.toFixed(3)})` +
+            `${row.r3_rate_over_threshold === true ? ' ⚠️  OVER THRESHOLD — the override list is the defect' : ''}\n`,
+    );
+    return 0;
+}
+
+function _isCliEntry(): boolean {
+    const invoked = process.argv[1];
+    return invoked !== undefined && path.resolve(invoked) === path.resolve(_HERE);
+}
+
+if (_isCliEntry()) {
+    process.exit(main(process.argv.slice(2)));
+}

package/src/scripts/measure_skill_ranker_baseline.ts

@@ -0,0 +1,212 @@
+#!/usr/bin/env tsx
+/**
+ * measure_skill_ranker_baseline — the number Phase 3 has to beat.
+ *
+ * `road-to-skill-delivery-over-mcp` Phase 0.3 asked for a top-1 / top-3 hit
+ * rate for the incumbent keyword ranker, "over the 496-line corpus … against
+ * the corpus's expected skill per line". Building it surfaced two defects in
+ * that premise, and this file is written around them rather than over them.
+ *
+ * DEFECT A — the 496-line corpus has no expected SKILL. `tests/eval/routing-matrix/`
+ * is keyed by `rule:` and every prompt is labelled with the RULE it should
+ * activate. Scoring a skill ranker against it would need a skill label that
+ * does not exist there, so a "hit rate" over it would be invented.
+ *
+ * DEFECT B — 496 is stale. The matrix holds 499 prompts today
+ * (`grep -h "^\s*- prompt:" tests/eval/routing-matrix/*.yaml | wc -l`), and 496
+ * is still published in six places including `hook-token-budget.json`.
+ *
+ * So the baseline is measured against the corpus that DOES carry expected
+ * skills — `tests/eval/corpus-{dev,non-dev}.yaml`, 26 labelled prompts — and
+ * the routing matrix is reported alongside it as what it can honestly be: a
+ * COVERAGE measurement (does the ranker return anything at all, and how
+ * confident) with no ground truth and therefore no accuracy.
+ *
+ * Both halves are emitted in one JSON object so a later ranker revision can be
+ * compared on the same two axes in one run:
+ *
+ *   ./scripts-run src/scripts/measure_skill_ranker_baseline --json
+ *   ./scripts-run src/scripts/measure_skill_ranker_baseline --ranker keyword-v2 --json
+ *
+ * A 26-prompt denominator is small and is stated as such in the output. It is
+ * the whole labelled ground truth this repository has.
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { rank } from './skill_tools/score_skill_relevance.js';
+import type { RankOptions } from '../shared/skillRanking.js';
+
+export const REPO = path.resolve(fileURLToPath(import.meta.url), '..', '..', '..');
+const LABELLED_CORPORA = ['tests/eval/corpus-dev.yaml', 'tests/eval/corpus-non-dev.yaml'];
+const MATRIX_DIR = 'tests/eval/routing-matrix';
+/** Where the projected skills live in this checkout. */
+export const SKILLS_DIR = path.join(REPO, 'src', 'skills');
+
+export interface LabelledPrompt {
+    id: string;
+    corpus: string;
+    prompt: string;
+    expected: string[];
+}
+
+/**
+ * Minimal reader for the two eval corpora. They are hand-written YAML with a
+ * fixed two-space list shape; a full YAML parse is avoided so this script has
+ * the same zero-dependency profile as the ranker it measures.
+ */
+export function readLabelledPrompts(repo = REPO): LabelledPrompt[] {
+    const out: LabelledPrompt[] = [];
+    for (const rel of LABELLED_CORPORA) {
+        const file = path.join(repo, rel);
+        if (!fs.existsSync(file)) continue;
+        const corpus = path.basename(rel, '.yaml');
+        let id = '';
+        let prompt = '';
+        for (const raw of fs.readFileSync(file, 'utf8').split('\n')) {
+            const line = raw.replace(/\r$/, '');
+            if (/^\s*#/.test(line)) continue;
+            const mId = /^\s*-\s+id:\s*(\S+)\s*$/.exec(line);
+            if (mId) {
+                id = mId[1]!;
+                prompt = '';
+                continue;
+            }
+            const mPrompt = /^\s*prompt:\s*"(.*)"\s*$/.exec(line);
+            if (mPrompt) {
+                prompt = mPrompt[1]!;
+                continue;
+            }
+            const mExp = /^\s*expected_skills:\s*\[(.*)\]\s*$/.exec(line);
+            if (mExp && id && prompt) {
+                const expected = mExp[1]!
+                    .split(',')
+                    .map((s) => s.trim())
+                    .filter(Boolean);
+                out.push({ id, corpus, prompt, expected });
+                id = '';
+                prompt = '';
+            }
+        }
+    }
+    return out;
+}
+
+/** Every `- prompt:` string in the rule routing matrix. No skill label exists. */
+export function readMatrixPrompts(repo = REPO): string[] {
+    const dir = path.join(repo, MATRIX_DIR);
+    if (!fs.existsSync(dir)) return [];
+    const out: string[] = [];
+    for (const name of fs.readdirSync(dir).sort()) {
+        if (!name.endsWith('.yaml')) continue;
+        for (const raw of fs.readFileSync(path.join(dir, name), 'utf8').split('\n')) {
+            const m = /^\s*-?\s*prompt:\s*"(.*)"\s*$/.exec(raw.replace(/\r$/, ''));
+            if (m && m[1]) out.push(m[1]);
+        }
+    }
+    return out;
+}
+
+export interface AccuracyArm {
+    corpus_prompts: number;
+    top1: number;
+    top3: number;
+    misses: string[];
+    denominator_note: string;
+}
+
+export interface CoverageArm {
+    corpus_prompts: number;
+    prompts_with_any_result: number;
+    mean_top_score: number;
+    note: string;
+}
+
+export interface RankerBaseline {
+    schema: 1;
+    ranker: string;
+    commit: string;
+    skills_dir: string;
+    skills_indexed: number;
+    accuracy: AccuracyArm;
+    matrix_coverage: CoverageArm;
+}
+
+function hitAt(rows: readonly (readonly [string, number, string[]])[], n: number, expected: readonly string[]): boolean {
+    const top = rows.slice(0, n).map((r) => r[0]);
+    return expected.some((e) => top.includes(e));
+}
+
+export function measure(opts: { ranker: string; commit: string; skillsDir?: string; repo?: string }): RankerBaseline {
+    const repo = opts.repo ?? REPO;
+    const skillsDir = opts.skillsDir ?? SKILLS_DIR;
+    // `keyword-v2` is Phase 3.1: the same formula with `triggers:` prose folded
+    // into each skill's term source. Any other label measures v1.
+    const rankOpts: RankOptions = opts.ranker === 'keyword-v2' ? { includeTriggers: true } : {};
+    const labelled = readLabelledPrompts(repo);
+    let top1 = 0;
+    let top3 = 0;
+    const misses: string[] = [];
+    for (const p of labelled) {
+        const rows = rank(p.prompt, skillsDir, rankOpts);
+        if (hitAt(rows, 1, p.expected)) top1++;
+        if (hitAt(rows, 3, p.expected)) top3++;
+        else misses.push(p.id);
+    }
+
+    const matrix = readMatrixPrompts(repo);
+    let withResult = 0;
+    let scoreSum = 0;
+    for (const prompt of matrix) {
+        const rows = rank(prompt, skillsDir, rankOpts);
+        if (rows.length > 0) {
+            withResult++;
+            scoreSum += rows[0]![1];
+        }
+    }
+
+    const skillsIndexed = fs.existsSync(skillsDir)
+        ? fs.readdirSync(skillsDir).filter((s) => fs.existsSync(path.join(skillsDir, s, 'SKILL.md'))).length
+        : 0;
+
+    return {
+        schema: 1,
+        ranker: opts.ranker,
+        commit: opts.commit,
+        skills_dir: path.relative(repo, skillsDir),
+        skills_indexed: skillsIndexed,
+        accuracy: {
+            corpus_prompts: labelled.length,
+            top1: labelled.length ? Math.round((top1 / labelled.length) * 1000) / 1000 : 0,
+            top3: labelled.length ? Math.round((top3 / labelled.length) * 1000) / 1000 : 0,
+            misses,
+            denominator_note:
+                'tests/eval/corpus-dev.yaml + corpus-non-dev.yaml — the only expected-skill ground ' +
+                'truth in this tree. Small; a single prompt moves each rate by ~4 points.',
+        },
+        matrix_coverage: {
+            corpus_prompts: matrix.length,
+            prompts_with_any_result: withResult,
+            mean_top_score: withResult ? Math.round((scoreSum / withResult) * 100) / 100 : 0,
+            note:
+                'tests/eval/routing-matrix/ is labelled with the expected RULE, never a skill, so ' +
+                'this arm carries no accuracy — only whether the ranker answers at all and how ' +
+                'confident its top answer is. Its size is measured here, not assumed: the roadmap ' +
+                'and five other sites still publish 496.',
+        },
+    };
+}
+
+export function main(argv: readonly string[]): number {
+    const ranker = argv.includes('--ranker') ? (argv[argv.indexOf('--ranker') + 1] ?? 'keyword-v1') : 'keyword-v1';
+    const commit = argv.includes('--commit') ? (argv[argv.indexOf('--commit') + 1] ?? 'unknown') : 'unknown';
+    const out = measure({ ranker, commit });
+    process.stdout.write(`${JSON.stringify(out, null, 2)}\n`);
+    return 0;
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
+    process.exit(main(process.argv.slice(2)));
+}