@event4u/agent-config 16.2.0 → 16.4.0

package/src/scripts/report_installed_rule_links.ts

@@ -0,0 +1,274 @@
+#!/usr/bin/env node
+/**
+ * Which links inside an installed rule file do NOT resolve, per host.
+ *
+ * A rule body is authored against the package tree, where every sibling
+ * directory is present. The installer copies a subset
+ * (`GLOBAL_DEPLOY_SOURCES`), and nothing joined the two, so a rule could ship
+ * a link into a directory no consumer has.
+ *
+ * This reporter reads the real deploy plan and the real rule bodies, so its
+ * numbers are the installer's rather than a restatement of it. It also prices
+ * the two repairs step 1.2 of `road-to-rule-triggers-and-links-that-hold`
+ * weighs against each other:
+ *
+ *   · **deploy** — add the directory to the host's plan. Nothing is written
+ *     into a rule body, so the standing text a host loads as instructions does
+ *     not move; the cost is files on disk.
+ *   · **rewrite** — leave the plan alone and make each link an absolute
+ *     package path. Every character lands INSIDE a rule body, which is exactly
+ *     the text the host loads.
+ *
+ * The rewrite price depends on the installed package root's own length, which
+ * is a property of the machine, so `--prefix-chars` is explicit and the report
+ * states the value it used.
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+import {
+    type LinkAuditReport,
+    auditInstalledKindLinks,
+    auditInstalledRuleLinks,
+    deployPlanFrom,
+    rewriteOptionCost,
+} from '../install/installedRuleLinks.js';
+import { loadBaselines } from './_lib/gate_baseline.js';
+import { GLOBAL_DEPLOY_SOURCES } from './install.js';
+
+// ledger-exempt: a REPORTER, not a gate. It has no verdict for a per-target
+// ledger to account for; it prints what the deploy plan and the rule bodies
+// already say.
+
+const _HERE = fileURLToPath(import.meta.url);
+
+/** Hosts whose plan carries a rules directory — the only ones with an answer. */
+function ruleHosts(): [string, ReadonlyArray<readonly [string, string]>][] {
+    return Object.entries(GLOBAL_DEPLOY_SOURCES).filter(([, pairs]) =>
+        pairs.some(([source]) => source.endsWith('/rules')),
+    );
+}
+
+export interface HostLinkReading {
+    host: string;
+    links: number;
+    resolved: number;
+    unresolved: number;
+    rewrite_cost_chars: number;
+    by_directory: LinkAuditReport['by_directory'];
+}
+
+export function readings(packageRoot: string, prefixChars: number): HostLinkReading[] {
+    const out: HostLinkReading[] = [];
+    for (const [host, pairs] of ruleHosts()) {
+        const report = auditInstalledRuleLinks(deployPlanFrom(pairs), packageRoot);
+        out.push({
+            host,
+            links: report.audits.length,
+            resolved: report.counts.resolved,
+            unresolved: report.audits.length - report.counts.resolved,
+            rewrite_cost_chars: rewriteOptionCost(report, prefixChars),
+            by_directory: report.by_directory,
+        });
+    }
+    return out;
+}
+
+/** Installed artefact kinds `--kinds` can select, in print order. */
+export const LINK_KINDS = ['rules', 'skills', 'commands', 'contexts', 'guidelines'] as const;
+export type LinkKind = (typeof LINK_KINDS)[number];
+
+export interface KindLinkReading {
+    kind: LinkKind | 'non-projected';
+    /** False when the host's plan does not deploy this kind at all. */
+    installed: boolean;
+    links: number;
+    unresolved: number;
+}
+
+/**
+ * One reading per selected kind for one host, plus a final `non-projected` row.
+ *
+ * A link into `docs/` (the ADRs live there) is dead in every install by
+ * construction, so it is taken OUT of its kind's deployable count and summed
+ * into its own row instead — mixing it in would make a kind's number move for a
+ * reason no deploy change can address.
+ */
+export function kindReadings(
+    packageRoot: string,
+    pairs: ReadonlyArray<readonly [string, string]>,
+    kinds: ReadonlyArray<LinkKind>,
+): KindLinkReading[] {
+    const plan = deployPlanFrom(pairs);
+    const out: KindLinkReading[] = [];
+    let npLinks = 0;
+    let npUnresolved = 0;
+    for (const kind of kinds) {
+        const report = auditInstalledKindLinks(plan, packageRoot, `dist/agent-src/${kind}`);
+        if (report === null) {
+            out.push({ kind, installed: false, links: 0, unresolved: 0 });
+            continue;
+        }
+        out.push({
+            kind,
+            installed: true,
+            links: report.audits.length,
+            unresolved: report.audits.length - report.counts.resolved,
+        });
+        npLinks += report.non_projected.length;
+        npUnresolved += report.non_projected.filter((a) => a.verdict !== 'resolved').length;
+    }
+    out.push({ kind: 'non-projected', installed: true, links: npLinks, unresolved: npUnresolved });
+    return out;
+}
+
+/** Key of a kind's recorded count in `src/config/gate-violation-baselines.json`. */
+export function kindBaselineKey(host: string, kind: KindLinkReading['kind']): string {
+    return `report_installed_rule_links:kinds:${host}:${kind}`;
+}
+
+/**
+ * The recorded count for each row, or null where none is recorded.
+ *
+ * Read, never written: the baseline only moves by a reviewed commit, so a new
+ * dead link shows as a live count above it rather than as a quietly raised
+ * number. Nothing here fails on the comparison (D2 — report, never fail).
+ */
+export function recordedBaselines(packageRoot: string, host: string, rows: KindLinkReading[]): (number | null)[] {
+    const { gates } = loadBaselines(packageRoot);
+    return rows.map((r) => gates[kindBaselineKey(host, r.kind)]?.count ?? null);
+}
+
+/** `kind: <name> unresolved <n> of <m>` — the line form the roadmap fixed. */
+export function formatKindLine(r: KindLinkReading, host: string, baseline: number | null = null): string {
+    const name = r.kind === 'non-projected' ? 'non-projected (adr, docs/)' : r.kind;
+    const tail = r.installed ? '' : ` (not installed for ${host})`;
+    const base =
+        baseline === null
+            ? ''
+            : ` [baseline ${String(baseline)}${r.unresolved > baseline ? `, ${String(r.unresolved - baseline)} above it` : ''}]`;
+    return `kind: ${name} unresolved ${String(r.unresolved)} of ${String(r.links)}${tail}${base}`;
+}
+
+function parseKinds(raw: string): LinkKind[] | null {
+    if (raw === 'all') return [...LINK_KINDS];
+    const picked = raw.split(',').map((k) => k.trim());
+    if (picked.length === 0 || picked.some((k) => !(LINK_KINDS as readonly string[]).includes(k))) return null;
+    return LINK_KINDS.filter((k) => picked.includes(k));
+}
+
+function flagValue(flag: string): string | null {
+    const argv = process.argv.slice(2);
+    const i = argv.indexOf(flag);
+    return i >= 0 && i + 1 < argv.length ? (argv[i + 1] as string) : null;
+}
+
+const USAGE =
+    'usage: report_installed_rule_links [--root DIR] [--prefix-chars N] [--json]\n' +
+    '                                   [--kinds all|K[,K...]] [--host HOST]\n' +
+    '  --root          package checkout the installer copies FROM (default: cwd)\n' +
+    '  --prefix-chars  length of the installed package root, for the rewrite price\n' +
+    '                  (default 48, a typical `~/.claude/agent-config` expansion)\n' +
+    '  --kinds         one unresolved count per installed kind instead of the rule\n' +
+    `                  reading: ${LINK_KINDS.join(', ')}, or all; docs/ and ADR\n` +
+    '                  targets are printed as their own row. Reports, never fails.\n' +
+    '  --host          host whose deploy plan --kinds reads (default claude-code)\n' +
+    '  --json          machine-readable readings\n';
+
+export function main(): number {
+    const argv = process.argv.slice(2);
+    if (argv.includes('--help') || argv.includes('-h')) {
+        process.stdout.write(USAGE);
+        return 0;
+    }
+    for (const a of argv) {
+        if (a.startsWith('-') && !['--root', '--prefix-chars', '--json', '--kinds', '--host'].includes(a)) {
+            process.stderr.write(`unknown argument: ${a}\n${USAGE}`);
+            return 2;
+        }
+    }
+
+    const root = flagValue('--root') ?? process.cwd();
+    const rawPrefix = flagValue('--prefix-chars');
+    const prefixChars = rawPrefix === null ? 48 : Number.parseInt(rawPrefix, 10);
+    if (!Number.isInteger(prefixChars) || prefixChars <= 0) {
+        process.stderr.write(`--prefix-chars must be a positive integer\n`);
+        return 2;
+    }
+    if (!fs.existsSync(path.join(root, 'dist', 'agent-src', 'rules'))) {
+        process.stderr.write(`no dist/agent-src/rules under ${root} — nothing to read\n`);
+        return 1;
+    }
+
+    const rawKinds = flagValue('--kinds');
+    if (rawKinds !== null || argv.includes('--kinds')) {
+        const kinds = rawKinds === null ? null : parseKinds(rawKinds);
+        if (kinds === null) {
+            process.stderr.write(`--kinds takes all or a comma list of: ${LINK_KINDS.join(', ')}\n`);
+            return 2;
+        }
+        const host = flagValue('--host') ?? 'claude-code';
+        const pairs = GLOBAL_DEPLOY_SOURCES[host];
+        if (pairs === undefined) {
+            process.stderr.write(`unknown --host ${host}; known: ${Object.keys(GLOBAL_DEPLOY_SOURCES).join(', ')}\n`);
+            return 2;
+        }
+        const kindRows = kindReadings(root, pairs, kinds);
+        const baselines = recordedBaselines(root, host, kindRows);
+        if (argv.includes('--json')) {
+            const rows = kindRows.map((r, i) => ({ ...r, baseline: baselines[i] }));
+            process.stdout.write(`${JSON.stringify({ host, kinds: rows }, null, 2)}\n`);
+            return 0;
+        }
+        process.stdout.write(`installed links per kind, host ${host} (report only; counted since 2026-10-06)\n\n`);
+        kindRows.forEach((r, i) => {
+            process.stdout.write(`  ${formatKindLine(r, host, baselines[i] ?? null)}\n`);
+        });
+        return 0;
+    }
+
+    const rows = readings(root, prefixChars);
+
+    if (argv.includes('--json')) {
+        process.stdout.write(`${JSON.stringify({ prefix_chars: prefixChars, hosts: rows }, null, 2)}\n`);
+        return 0;
+    }
+
+    process.stdout.write(`installed rule links, per host (rewrite priced at ${String(prefixChars)} prefix chars)\n\n`);
+    for (const r of rows) {
+        process.stdout.write(
+            `  ${r.host}: ${String(r.links)} link(s), ${String(r.resolved)} resolved, ${String(r.unresolved)} unresolved\n`,
+        );
+        if (r.unresolved > 0) {
+            // Priced over EVERY unresolved link, including the ones a rewrite
+            // to an absolute package path cannot reach either (a target the
+            // projection does not ship). Comparing it against a deploy option
+            // that repairs only a subset overstates the rewrite; the caveat is
+            // printed rather than left for the reader to rediscover.
+            process.stdout.write(
+                `      rewrite option: +${String(r.rewrite_cost_chars)} standing characters ` +
+                    `(all ${String(r.unresolved)} unresolved; a like-for-like figure prices only ` +
+                    `the subset the option being compared against repairs)\n`,
+            );
+            for (const d of r.by_directory) {
+                process.stdout.write(`      ${String(d.count).padStart(4)}  ${d.directory}  (${d.verdict})\n`);
+            }
+        }
+    }
+    return 0;
+}
+
+function _isCliEntry(): boolean {
+    if (process.argv[1] === undefined) return false;
+    if (import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) return true;
+    try {
+        return fs.realpathSync(_HERE) === fs.realpathSync(path.resolve(process.argv[1]));
+    } catch {
+        return false;
+    }
+}
+
+if (_isCliEntry()) {
+    process.exit(main());
+}

package/src/scripts/report_label_agreement.ts

@@ -0,0 +1,233 @@
+#!/usr/bin/env tsx
+/**
+ * report_label_agreement — ranker error, separated from label noise.
+ *
+ * `road-to-a-ranker-that-routes` 1.4. A top-1 of 0.21 is a statement about two
+ * things at once: how often the ranker is wrong, and how often "wrong" is one
+ * reasonable person disagreeing with another about which skill a prompt wants.
+ * The second is bounded by how far two independent labelling seats agree, and
+ * that bound was published as a single aggregate (39 / 55 exact-set agreement)
+ * with no per-row record — so nobody could ask the only question that matters
+ * here: what does the ranker score on the rows the seats actually AGREE on?
+ *
+ * This script answers it, and it needs a second seat's labels per row to do so.
+ * The store it reads is {@link SECOND_SEAT_FILE}: a flat `{ case id: [skill
+ * ids] }` map written by seats that had not read this ranker's scoring, under
+ * the protocol in `tests/eval/routing-matrix/README.md`. The first seat's labels
+ * are the `expected_skills` lists in the matrix itself.
+ *
+ * THE DISAGREEMENTS ARE NOT RECONCILED, and that is the point. Overwriting one
+ * seat with the other would replace the honest width of the ground truth with a
+ * narrower number that looks better and means less.
+ *
+ * THREE AGREEMENT DEFINITIONS, reported side by side because they answer
+ * different questions and the gap between them IS the finding:
+ *
+ *   - `exact`    — the two label sets are identical. The strictest, and the one
+ *                  the published 70.9 % figure used.
+ *   - `overlap`  — the two sets share at least one skill, or are both empty. A
+ *                  hit is scored when ANY labelled skill appears in the top-k,
+ *                  so this is the definition the MEASUREMENT actually cares
+ *                  about: on an overlap row the two seats would score the same
+ *                  top-1 answer the same way.
+ *   - `polarity` — the two seats agree only on whether a skill answer exists at
+ *                  all. The weakest, and the floor under the other two.
+ *
+ * **`polarity` IS ONE-SIDED ON THIS DENOMINATOR and must not be compared with a
+ * two-sided figure.** The first seat's labels arrive through
+ * `readMatrixLabelledPrompts`, which keeps only NON-EMPTY lists, so "seat one
+ * thinks a skill answer exists" is true for every row here by construction and
+ * the `agrees` both-empty branch is unreachable from this caller. The number
+ * therefore reads "the second seat also wrote a non-empty label", not "the two
+ * seats agree about whether an answer exists". It is still worth reporting — it
+ * counts the rows the second seat declined — and it is NOT the published
+ * two-sided agreement figure wearing the same name.
+ *
+ * `second_seat_rows` and `relabelled_and_labelled` differ by exactly the rows
+ * the second seat relabelled that the first seat had left empty: those are
+ * absent from `first` and so cannot be paired.
+ *
+ * Usage:
+ *     ./scripts-run src/scripts/report_label_agreement
+ *     ./scripts-run src/scripts/report_label_agreement --ranker idf
+ *
+ * Output is JSON on stdout, always — there is no second format and therefore no
+ * flag to pick one.
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { wilsonInterval } from './_lib/capture_rate.js';
+import {
+    REPO,
+    SKILLS_DIR,
+    type LabelledPrompt,
+    parseRanker,
+    rankOptionsFor,
+    readMatrixLabelledPrompts,
+    sliceForId,
+} from './measure_skill_ranker_baseline.js';
+import { rank } from './skill_tools/score_skill_relevance.js';
+
+/** Where the second seat's per-row labels live, relative to the repo root. */
+export const SECOND_SEAT_FILE = path.join('tests', 'eval', 'routing-matrix', 'second-seat-2026-10-01.json');
+
+export type AgreementKind = 'exact' | 'overlap' | 'polarity';
+export const AGREEMENT_KINDS: readonly AgreementKind[] = ['exact', 'overlap', 'polarity'];
+
+/**
+ * The store, or `{}` when nobody has relabelled.
+ *
+ * A MALFORMED store is a third state and must not look like either of the other
+ * two: an empty object would read as "nobody has relabelled" and a raw
+ * `SyntaxError` out of `main` would bypass the message that distinguishes them.
+ */
+export function readSecondSeat(repo = REPO): Record<string, string[]> {
+    const file = path.join(repo, SECOND_SEAT_FILE);
+    if (!fs.existsSync(file)) return {};
+    const raw = fs.readFileSync(file, 'utf8');
+    try {
+        return JSON.parse(raw) as Record<string, string[]>;
+    } catch (err) {
+        throw new Error(
+            `report_label_agreement: ${SECOND_SEAT_FILE} is not valid JSON (${(err as Error).message}). ` +
+                'That is a corrupt store, NOT an empty one — the two must not be confused.',
+        );
+    }
+}
+
+export function agrees(a: readonly string[], b: readonly string[], kind: AgreementKind): boolean {
+    if (kind === 'polarity') return (a.length > 0) === (b.length > 0);
+    if (kind === 'overlap') {
+        if (a.length === 0 || b.length === 0) return a.length === b.length;
+        return a.some((s) => b.includes(s));
+    }
+    if (a.length !== b.length) return false;
+    const sa = [...a].sort();
+    const sb = [...b].sort();
+    return sa.every((s, i) => s === sb[i]);
+}
+
+export interface AgreementArm {
+    kind: AgreementKind | 'all-relabelled';
+    n: number;
+    top1: number;
+    top1_ci95: { lower: number; upper: number };
+    top3: number;
+    top3_ci95: { lower: number; upper: number };
+}
+
+function round3(n: number): number {
+    return Math.round(n * 1000) / 1000;
+}
+
+function arm(
+    kind: AgreementArm['kind'],
+    rows: readonly { prompt: LabelledPrompt; ranked: string[] }[],
+): AgreementArm {
+    let t1 = 0;
+    let t3 = 0;
+    for (const r of rows) {
+        if (r.prompt.expected.some((e) => r.ranked.slice(0, 1).includes(e))) t1 += 1;
+        if (r.prompt.expected.some((e) => r.ranked.slice(0, 3).includes(e))) t3 += 1;
+    }
+    const n = rows.length;
+    const ci = (s: number): { lower: number; upper: number } => {
+        const raw = wilsonInterval(s, n);
+        return { lower: round3(raw.lower), upper: round3(raw.upper) };
+    };
+    return {
+        kind,
+        n,
+        top1: n ? round3(t1 / n) : 0,
+        top1_ci95: ci(t1),
+        top3: n ? round3(t3 / n) : 0,
+        top3_ci95: ci(t3),
+    };
+}
+
+export interface AgreementReport {
+    ranker: string;
+    second_seat_rows: number;
+    relabelled_and_labelled: number;
+    agreement: Record<AgreementKind, { agreed: number; share: number }>;
+    arms: AgreementArm[];
+    /** Rows where the two seats share no skill at all — the honest width. */
+    disjoint: { id: string; first: string[]; second: string[] }[];
+}
+
+export function measureAgreement(opts: {
+    repo?: string;
+    skillsDir?: string;
+    ranker: string;
+}): AgreementReport {
+    const repo = opts.repo ?? REPO;
+    const skillsDir = opts.skillsDir ?? SKILLS_DIR;
+    const second = readSecondSeat(repo);
+    const first = readMatrixLabelledPrompts(repo);
+    const rankOpts = rankOptionsFor(opts.ranker);
+
+    const paired = first.filter((p) => second[p.id] !== undefined);
+    const ranked = new Map<string, string[]>();
+    for (const p of paired) ranked.set(p.id, rank(p.prompt, skillsDir, rankOpts).map((r) => r[0]));
+
+    const rows = paired.map((prompt) => ({ prompt, ranked: ranked.get(prompt.id) ?? [] }));
+    const agreement = {} as Record<AgreementKind, { agreed: number; share: number }>;
+    const arms: AgreementArm[] = [arm('all-relabelled', rows)];
+    for (const kind of AGREEMENT_KINDS) {
+        const kept = rows.filter((r) => agrees(r.prompt.expected, second[r.prompt.id] ?? [], kind));
+        agreement[kind] = {
+            agreed: kept.length,
+            share: rows.length ? round3(kept.length / rows.length) : 0,
+        };
+        arms.push(arm(kind, kept));
+    }
+    const disjoint = paired
+        .filter((p) => !agrees(p.expected, second[p.id] ?? [], 'overlap'))
+        .map((p) => ({ id: p.id, first: p.expected, second: second[p.id] ?? [] }));
+
+    return {
+        ranker: opts.ranker,
+        second_seat_rows: Object.keys(second).length,
+        relabelled_and_labelled: paired.length,
+        agreement,
+        arms,
+        disjoint,
+    };
+}
+
+/** Every relabelled row is expected to sit in the tuning slice; this says so. */
+export function relabelledSliceCensus(repo = REPO): Record<string, number> {
+    const out: Record<string, number> = { tuning: 0, sealed: 0 };
+    for (const id of Object.keys(readSecondSeat(repo))) out[sliceForId(id)] = (out[sliceForId(id)] ?? 0) + 1;
+    return out;
+}
+
+export function main(argv: readonly string[]): number {
+    let report: AgreementReport;
+    try {
+        // Same check as `--slice` elsewhere: a present flag with no value is an
+        // error, never a silent substitution of the baseline.
+        report = measureAgreement({ ranker: parseRanker(argv) });
+    } catch (err) {
+        process.stderr.write(`${(err as Error).message}\n`);
+        return 2;
+    }
+    if (report.second_seat_rows === 0) {
+        process.stderr.write(
+            `report_label_agreement: no second-seat labels at ${SECOND_SEAT_FILE} — ` +
+                'this is "nobody has relabelled", not "the seats disagree about nothing".\n',
+        );
+        return 2;
+    }
+    process.stdout.write(
+        `${JSON.stringify({ ...report, relabelled_slice_census: relabelledSliceCensus() }, null, 2)}\n`,
+    );
+    return 0;
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
+    process.exit(main(process.argv.slice(2)));
+}