@event4u/agent-config 16.1.0 → 16.3.0

package/src/scripts/report_single_token_triggers.ts

@@ -0,0 +1,277 @@
+#!/usr/bin/env node
+/**
+ * Which one-token keyword triggers fire on prompts that belong to other rules.
+ *
+ * Step 1.3 of `road-to-rule-triggers-and-links-that-hold`. A single common word
+ * — `rename`, `delete` — is a legitimate trigger for the rule that owns the
+ * subject and a standing cost for every prompt that merely uses the word. This
+ * reports which words sit at which share of the corpus. **Report only.** A
+ * trigger change is a separate decision per rule, because a common word can be
+ * exactly right: `augment-edit-discipline` genuinely wants to fire on a rename.
+ *
+ * The corpus is `tests/eval/routing-matrix/*.yaml`, the frozen set this tree
+ * already evaluates routing against. Each `positives[].prompt` is labelled with
+ * its file's rule. For a one-token keyword `K` on rule `R`, the share is:
+ *
+ *     |{ prompt : label(prompt) != R  and  K occurs as a word }|
+ *     ---------------------------------------------------------
+ *             |{ prompt : label(prompt) != R }|
+ *
+ * `near_misses` are deliberately NOT in the denominator. They carry a negative
+ * label against their own file's rule and no positive label against any other,
+ * so counting them would mix "belongs to another rule" with "belongs to no
+ * rule" — two different claims, and only the first is what a cross-firing
+ * trigger costs.
+ *
+ * The 5 % threshold is a stated default for a report, not a gate (D3). Nothing
+ * acts on the figure automatically.
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+import { parse as parseYaml } from 'yaml';
+
+// ledger-exempt: a REPORTER, not a gate. Its only non-zero exit says the
+// corpus or the rule directory could not be read.
+
+const _HERE = fileURLToPath(import.meta.url);
+
+export const DEFAULT_THRESHOLD = 0.05;
+
+export interface LabelledPrompt {
+    rule: string;
+    prompt: string;
+    /** `positive` — belongs to `rule`. `near_miss` — declared NOT to be `rule`. */
+    kind: 'positive' | 'near_miss';
+}
+
+export interface TriggerRow {
+    rule: string;
+    keyword: string;
+    /** Foreign POSITIVE prompts the keyword occurs in. */
+    hits: number;
+    /** Prompts labelled positive against any other rule. */
+    foreign: number;
+    share: number;
+    /**
+     * Occurrences in a near-miss the rule ITSELF declares — the keyword fires
+     * on a prompt that rule's own fixture says it must not fire on. A declared
+     * false positive, and the sharpest row this report can produce.
+     */
+    own_near_miss_hits: number;
+    own_near_misses: number;
+    /** Occurrences in another rule's near-miss list. */
+    foreign_near_miss_hits: number;
+    /** Which rules those foreign positives belong to, highest first. */
+    top_labels: { rule: string; count: number }[];
+}
+
+/** Every prompt in the matrix, labelled with its file's rule and its kind. */
+export function loadCorpus(matrixDir: string): LabelledPrompt[] {
+    const out: LabelledPrompt[] = [];
+    const files = fs
+        .readdirSync(matrixDir)
+        .filter((n) => n.endsWith('.yaml'))
+        .sort();
+    for (const file of files) {
+        const doc = parseYaml(fs.readFileSync(path.join(matrixDir, file), 'utf-8')) as {
+            rule?: unknown;
+            positives?: { prompt?: unknown }[];
+            near_misses?: { prompt?: unknown }[];
+        } | null;
+        const rule = typeof doc?.rule === 'string' ? doc.rule : '';
+        if (!rule) continue;
+        for (const [items, kind] of [
+            [doc?.positives, 'positive'],
+            [doc?.near_misses, 'near_miss'],
+        ] as const) {
+            if (!Array.isArray(items)) continue;
+            for (const item of items) {
+                if (typeof item?.prompt === 'string' && item.prompt.trim()) {
+                    out.push({ rule, prompt: item.prompt, kind });
+                }
+            }
+        }
+    }
+    return out;
+}
+
+/** One-token keyword triggers per rule, from `src/rules/*.md` frontmatter. */
+export function singleTokenKeywords(rulesDir: string): { rule: string; keyword: string }[] {
+    const out: { rule: string; keyword: string }[] = [];
+    const files = fs
+        .readdirSync(rulesDir)
+        .filter((n) => n.endsWith('.md'))
+        .sort();
+    for (const file of files) {
+        const content = fs.readFileSync(path.join(rulesDir, file), 'utf-8');
+        if (!content.startsWith('---')) continue;
+        const end = content.indexOf('\n---', 3);
+        const block = end === -1 ? content : content.slice(0, end);
+        const rule = file.replace(/\.md$/, '');
+        for (const m of block.matchAll(/^\s*-\s*keyword:\s*["']?([^"'\n]+?)["']?\s*$/gm)) {
+            const kw = (m[1] as string).trim();
+            if (kw === '' || /\s/.test(kw)) continue;
+            out.push({ rule, keyword: kw });
+        }
+    }
+    return out;
+}
+
+/** Word-boundary, case-insensitive, with the keyword's own characters escaped. */
+function occursAsWord(prompt: string, keyword: string): boolean {
+    const esc = keyword.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    return new RegExp(`(?:^|[^\\p{L}\\p{N}_])${esc}(?:$|[^\\p{L}\\p{N}_])`, 'iu').test(prompt);
+}
+
+export function rows(
+    corpus: readonly LabelledPrompt[],
+    keywords: readonly { rule: string; keyword: string }[],
+): TriggerRow[] {
+    const out: TriggerRow[] = [];
+    for (const { rule, keyword } of keywords) {
+        const foreign = corpus.filter((c) => c.rule !== rule && c.kind === 'positive');
+        if (foreign.length === 0) continue;
+        const hitting = foreign.filter((c) => occursAsWord(c.prompt, keyword));
+        const ownNear = corpus.filter((c) => c.rule === rule && c.kind === 'near_miss');
+        const foreignNear = corpus.filter((c) => c.rule !== rule && c.kind === 'near_miss');
+        const labels = new Map<string, number>();
+        for (const h of hitting) labels.set(h.rule, (labels.get(h.rule) ?? 0) + 1);
+        out.push({
+            rule,
+            keyword,
+            hits: hitting.length,
+            foreign: foreign.length,
+            share: hitting.length / foreign.length,
+            own_near_miss_hits: ownNear.filter((c) => occursAsWord(c.prompt, keyword)).length,
+            own_near_misses: ownNear.length,
+            foreign_near_miss_hits: foreignNear.filter((c) => occursAsWord(c.prompt, keyword)).length,
+            top_labels: [...labels.entries()]
+                .map(([r, count]) => ({ rule: r, count }))
+                .sort((a, b) => b.count - a.count || a.rule.localeCompare(b.rule))
+                .slice(0, 5),
+        });
+    }
+    out.sort(
+        (a, b) =>
+            b.share - a.share ||
+            b.own_near_miss_hits - a.own_near_miss_hits ||
+            b.foreign_near_miss_hits - a.foreign_near_miss_hits ||
+            a.rule.localeCompare(b.rule) ||
+            a.keyword.localeCompare(b.keyword),
+    );
+    return out;
+}
+
+export function renderTable(data: readonly TriggerRow[]): string[] {
+    const L: string[] = [];
+    L.push(
+        '| Rule | Keyword | Foreign positives | Share | Own near-miss hits | Foreign near-miss hits | Mostly labelled against |',
+    );
+    L.push('|---|---|---:|---:|---:|---:|---|');
+    for (const r of data) {
+        const labels = r.top_labels.map((l) => `\`${l.rule}\` ${String(l.count)}`).join(', ') || '—';
+        L.push(
+            `| \`${r.rule}\` | \`${r.keyword}\` | ${String(r.hits)}/${String(r.foreign)} | ` +
+                `${(r.share * 100).toFixed(1)} % | ${String(r.own_near_miss_hits)}/${String(r.own_near_misses)} | ` +
+                `${String(r.foreign_near_miss_hits)} | ${labels} |`,
+        );
+    }
+    return L;
+}
+
+const USAGE =
+    'usage: report_single_token_triggers [--root DIR] [--threshold F] [--json | --table] [--all]\n' +
+    '  --threshold  share above which a row is reported (default 0.05)\n' +
+    '  --all        report every one-token keyword, not just those over the threshold\n';
+
+export function main(): number {
+    const argv = process.argv.slice(2);
+    if (argv.includes('--help') || argv.includes('-h')) {
+        process.stdout.write(USAGE);
+        return 0;
+    }
+    const KNOWN = ['--root', '--threshold', '--json', '--table', '--all'];
+    // A silently-ignored `--treshold` would report against a threshold the
+    // operator did not ask for, while still printing the default in the header.
+    for (const a of argv) {
+        if (a.startsWith('-') && !KNOWN.includes(a)) {
+            process.stderr.write(`unknown argument: ${a}\n${USAGE}`);
+            return 2;
+        }
+    }
+    const flagValue = (f: string): string | null => {
+        const i = argv.indexOf(f);
+        return i >= 0 && i + 1 < argv.length ? (argv[i + 1] as string) : null;
+    };
+    const root = flagValue('--root') ?? process.cwd();
+    const rawT = flagValue('--threshold');
+    const threshold = rawT === null ? DEFAULT_THRESHOLD : Number.parseFloat(rawT);
+    if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {
+        process.stderr.write('--threshold must be a fraction between 0 and 1\n');
+        return 2;
+    }
+
+    const matrixDir = path.join(root, 'tests', 'eval', 'routing-matrix');
+    const rulesDir = path.join(root, 'src', 'rules');
+    if (!fs.existsSync(matrixDir) || !fs.existsSync(rulesDir)) {
+        process.stderr.write(`missing ${matrixDir} or ${rulesDir}\n`);
+        return 1;
+    }
+
+    const corpus = loadCorpus(matrixDir);
+    const keywords = singleTokenKeywords(rulesDir);
+    const all = rows(corpus, keywords);
+    // A keyword reaches the report by EITHER axis: the share the step names, or
+    // a hit on a near-miss. Filtering on the share alone would have hidden the
+    // only rows this corpus actually produces — see the report's own § on why
+    // the 5 % threshold finds nothing here.
+    const data = argv.includes('--all')
+        ? all
+        : all.filter((r) => r.share > threshold || r.own_near_miss_hits > 0 || r.foreign_near_miss_hits > 0);
+
+    if (argv.includes('--json')) {
+        process.stdout.write(
+            `${JSON.stringify({ threshold, corpus: corpus.length, keywords: keywords.length, rows: data }, null, 2)}\n`,
+        );
+        return 0;
+    }
+    if (argv.includes('--table')) {
+        process.stdout.write(`${renderTable(data).join('\n')}\n`);
+        return 0;
+    }
+
+    const positives = corpus.filter((c) => c.kind === 'positive').length;
+    process.stdout.write(
+        `corpus ${String(corpus.length)} prompt(s) — ${String(positives)} positive, ` +
+            `${String(corpus.length - positives)} near-miss · ` +
+            `${String(keywords.length)} one-token keyword(s) over ` +
+            `${String(new Set(keywords.map((k) => k.rule)).size)} rule(s)\n` +
+            `${String(all.filter((r) => r.share > threshold).length)} row(s) above ` +
+            `${(threshold * 100).toFixed(1)} % of foreign positives · ` +
+            `${String(data.length)} reportable on either axis\n\n`,
+    );
+    for (const r of data) {
+        process.stdout.write(
+            `  ${(r.share * 100).toFixed(1).padStart(5)} %  ${r.keyword.padEnd(18)} ${r.rule.padEnd(32)} ` +
+                `pos ${String(r.hits)}/${String(r.foreign)}  own-near ${String(r.own_near_miss_hits)}/${String(r.own_near_misses)}  ` +
+                `foreign-near ${String(r.foreign_near_miss_hits)}\n`,
+        );
+    }
+    return 0;
+}
+
+function _isCliEntry(): boolean {
+    if (process.argv[1] === undefined) return false;
+    if (import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) return true;
+    try {
+        return fs.realpathSync(_HERE) === fs.realpathSync(path.resolve(process.argv[1]));
+    } catch {
+        return false;
+    }
+}
+
+if (_isCliEntry()) {
+    process.exit(main());
+}

package/src/scripts/report_skill_menu_census.ts

@@ -53,8 +53,21 @@
  * `command-only` and `flow-only` name a CANDIDATE for `user-invocable: false`,
  * never a decision — see the sole-entry limit above.
  *
+ * THE PER-PROFILE MENU, added by `road-to-a-menu-whose-precision-is-measured`
+ * 4.1. `--profile minimal|balanced|full|all` prints the menu byte count each
+ * shipped install profile delivers. Each profile's preset is read from its own
+ * file, and the answer is that all three are equal: no shipped preset declares
+ * a key that selects skills, so each one ships the whole menu.
+ *
+ * The equality note is WITHHELD when any profile carries a key listed in
+ * {@link SKILL_SELECTING_INI_KEYS}, and the key is named instead. That is the
+ * whole falsifiability claim, stated no wider than it holds: the byte count
+ * would not move — nothing here knows what such a key means — but the report
+ * stops asserting a conclusion whose premise has changed.
+ *
  * Usage:
  *     ./scripts-run src/scripts/report_skill_menu_census                # stdout summary
+ *     ./scripts-run src/scripts/report_skill_menu_census --profile all
  *     ./scripts-run src/scripts/report_skill_menu_census --emit         # write the artifact
  *     ./scripts-run src/scripts/report_skill_menu_census --emit --pin <sha>
  *
@@ -313,8 +326,203 @@ function headSha(root: string): string {
     return execFileSync('git', ['-C', root, 'rev-parse', 'HEAD'], { encoding: 'utf-8' }).trim();
 }
 
+// ── per-profile menu bytes (road-to-a-menu-whose-precision-is-measured 4.1) ──
+
+/** The shipped install presets, mirroring `install.ts` SUPPORTED_PROFILES. */
+export const PROFILES = ['minimal', 'balanced', 'full'] as const;
+export type Profile = (typeof PROFILES)[number];
+
+/**
+ * Preset keys that would narrow the skill menu.
+ *
+ * EMPTY, and the emptiness is the finding rather than a stub. A preset selects
+ * `rule_loading_tier`, `discipline_profile` and `chat_history_frequency`; which
+ * SKILLS are written is decided by `packs:` frontmatter against the active pack
+ * set, which no preset names.
+ *
+ * It is a TRIPWIRE, not a selector, and the distinction is worth stating
+ * plainly because the first version of this comment overclaimed it. Nothing
+ * here knows what such a key would MEAN, so nothing here can apply one: the
+ * byte count would not move. What the list buys is that a preset carrying a
+ * listed key is NAMED in the output and the equality note is withheld — so the
+ * report stops asserting a conclusion whose premise has changed, instead of
+ * printing the same prose over a different tree.
+ */
+export const SKILL_SELECTING_INI_KEYS: readonly string[] = [];
+
+/** Flat `key=value` read of a profile preset; `;` comments and blanks dropped. */
+export function readProfileIni(root: string, profile: Profile): Record<string, string> {
+    const p = path.join(root, 'src', 'config', 'profiles', `${profile}.ini`);
+    const out: Record<string, string> = {};
+    for (const raw of fs.readFileSync(p, 'utf-8').split('\n')) {
+        const line = raw.trim();
+        if (line === '' || line.startsWith(';') || line.startsWith('#') || line.startsWith('[')) continue;
+        const eq = line.indexOf('=');
+        if (eq === -1) continue;
+        out[line.slice(0, eq).trim()] = line.slice(eq + 1).trim();
+    }
+    return out;
+}
+
+export interface ProfileMenu {
+    readonly profile: Profile;
+    /** Skills this profile writes onto the model menu. */
+    readonly menuSkills: number;
+    /** Bytes of `- <name>: <description>\n`, the shape `censusSkillsCatalog` sums. */
+    readonly menuBytes: number;
+    /** chars/4, the estimator the payload budget uses. */
+    readonly tokensEstimate: number;
+    /** `declared_skill_catalogue_tokens` from the preset, or null when absent. */
+    readonly declaredCatalogueTokens: number | null;
+    /** Preset keys found that would narrow the menu. Empty today, by measurement. */
+    readonly selectingKeys: readonly string[];
+}
+
+/**
+ * One profile's menu, computed from its preset plus the skill tree.
+ *
+ * The catalogue LINE is the same shape `preamble_byte_census.censusSkillsCatalog`
+ * sums — `- <name>: <description>\n`. The POPULATION deliberately is not, and the
+ * difference is stated rather than smoothed over: that census sums every
+ * `SKILL.md`, because the payload bucket it feeds is about what the host lists;
+ * this one drops a skill carrying `user-invocable: false` or
+ * `disable-model-invocation: true`, because a menu is what the MODEL may pick
+ * from. Two skills carry such a flag today, so over the same tree and the same
+ * line shape this report reads 311 B below the full-population sum. Expect the
+ * two to differ; a reader who finds them equal has found a defect in one.
+ */
+export function menuForProfile(root: string, profile: Profile): ProfileMenu {
+    const ini = readProfileIni(root, profile);
+    const selectingKeys = SKILL_SELECTING_INI_KEYS.filter((k) => k in ini);
+    const skillsDir = path.join(root, 'src', 'skills');
+    let menuSkills = 0;
+    let menuBytes = 0;
+    for (const entry of fs.readdirSync(skillsDir, { withFileTypes: true })) {
+        if (!entry.isDirectory()) continue;
+        const skillMd = path.join(skillsDir, entry.name, 'SKILL.md');
+        if (!fs.existsSync(skillMd)) continue;
+        const text = fs.readFileSync(skillMd, 'utf-8');
+        const flags = menuFlags(text);
+        if (flags.userInvocable === false || flags.disableModel === true) continue;
+        const name = /^name:\s*(\S.*?)\s*$/mu.exec(text.slice(0, 4000))?.[1] ?? entry.name;
+        const description = frontmatterDescription(text);
+        menuSkills += 1;
+        menuBytes += `- ${name}: ${description}\n`.length;
+    }
+    const declared = ini['declared_skill_catalogue_tokens'];
+    return {
+        profile,
+        menuSkills,
+        menuBytes,
+        tokensEstimate: Math.round(menuBytes / 4),
+        declaredCatalogueTokens: declared === undefined ? null : Number(declared),
+        selectingKeys,
+    };
+}
+
+/**
+ * The `description:` value, folded lines included.
+ *
+ * A `[^\n]+` capture was measured 17.1 % low elsewhere in this tree because it
+ * truncates every folded multi-line description; this reads to the next
+ * top-level key instead.
+ *
+ * A double-quoted value has its escapes DECODED. Leaving them encoded counted
+ * `\"` as two characters and over-reported the menu by 56 B across 13 skills —
+ * small, but the same class of defect as the 17.1 % one above, and a function
+ * whose docstring cites that fix should not carry a second instance of it.
+ */
+export function frontmatterDescription(text: string): string {
+    if (!text.startsWith('---')) return '';
+    const end = text.indexOf('\n---', 3);
+    if (end === -1) return '';
+    const lines = text.slice(3, end).split('\n');
+    const at = lines.findIndex((l) => /^description:/u.test(l));
+    if (at === -1) return '';
+    const parts = [(lines[at] as string).replace(/^description:\s*/u, '')];
+    for (let i = at + 1; i < lines.length; i += 1) {
+        const l = lines[i] as string;
+        if (/^\S/u.test(l)) break;
+        parts.push(l.trim());
+    }
+    const joined = parts.join(' ').trim();
+    if (joined.length >= 2 && joined.startsWith('"') && joined.endsWith('"')) {
+        return joined.slice(1, -1).replace(/\\(["\\])/gu, '$1');
+    }
+    if (joined.length >= 2 && joined.startsWith("'") && joined.endsWith("'")) {
+        return joined.slice(1, -1).replace(/''/gu, "'");
+    }
+    return joined;
+}
+
+/** Render the per-profile menu table. Returns the lines, so tests read them. */
+export function renderProfileMenus(menus: readonly ProfileMenu[]): string[] {
+    const L: string[] = [];
+    L.push('menu bytes per install profile (name + description, the catalogue shape):');
+    L.push('  profile    skills   menu_bytes   ~tokens   declared');
+    for (const m of menus) {
+        L.push(
+            `  ${m.profile.padEnd(10)} ${String(m.menuSkills).padStart(6)} ` +
+                `${String(m.menuBytes).padStart(12)} ${String(m.tokensEstimate).padStart(9)} ` +
+                `${(m.declaredCatalogueTokens === null ? '—' : String(m.declaredCatalogueTokens)).padStart(10)}`,
+        );
+    }
+    const flagged = menus.filter((m) => m.selectingKeys.length > 0);
+    if (flagged.length > 0) {
+        L.push('');
+        L.push('  A preset declares a key that could select skills, so the equality below is NOT');
+        L.push('  asserted — the premise it rested on has changed. Keys found:');
+        for (const m of flagged) L.push(`    ${m.profile}: ${m.selectingKeys.join(', ')}`);
+        return L;
+    }
+    const distinct = new Set(menus.map((m) => m.menuBytes));
+    if (menus.length > 1 && distinct.size === 1) {
+        L.push('');
+        L.push(
+            '  All profiles report the SAME menu_bytes, and that is a measurement, not a',
+        );
+        L.push(
+            '  placeholder: no preset declares a key that selects skills, so each ships the',
+        );
+        L.push(
+            '  whole menu. The preset chooses which RULES activate; which SKILLS are written',
+        );
+        L.push(
+            '  is decided by `packs:` frontmatter against the active pack set. A menu-bytes',
+        );
+        L.push(
+            '  lever therefore has to move packs or the menu flags — changing profile does',
+        );
+        L.push('  nothing. The three presets already record this in their own comments.');
+    }
+    return L;
+}
+
 export function main(argv: readonly string[]): number {
     const root = REPO_ROOT;
+    const profIdx = argv.indexOf('--profile');
+    if (profIdx !== -1) {
+        const want = argv[profIdx + 1] ?? 'all';
+        const chosen: Profile[] =
+            want === 'all' ? [...PROFILES] : PROFILES.includes(want as Profile) ? [want as Profile] : [];
+        if (chosen.length === 0) {
+            process.stderr.write(
+                `report_skill_menu_census: --profile expects ${PROFILES.join(' | ')} | all, got ${want}\n`,
+            );
+            return 2;
+        }
+        if (argv.includes('--emit')) {
+            // Refused rather than ignored: `--emit` writes the census artifact,
+            // `--profile` prints a table and writes nothing, and silently doing
+            // the second when the caller asked for both is how an artifact goes
+            // un-refreshed without anyone noticing.
+            process.stderr.write('report_skill_menu_census: --profile and --emit are separate runs\n');
+            return 2;
+        }
+        const menus = chosen.map((p) => menuForProfile(root, p));
+        process.stdout.write(`${renderProfileMenus(menus).join('\n')}\n`);
+        return 0;
+    }
     const emit = argv.includes('--emit');
     const pinIdx = argv.indexOf('--pin');
     const pin = pinIdx !== -1 && pinIdx + 1 < argv.length ? (argv[pinIdx + 1] as string) : headSha(root);