@event4u/agent-config 12.0.0 → 13.0.0

package/src/scripts/trigger_coverage.ts

@@ -40,6 +40,7 @@ import { fileURLToPath, pathToFileURL } from 'node:url';
 import { parse as parseYaml } from 'yaml';
 
 import { keyword_matches_anchored } from './router_telemetry.js';
+import { match_prompt, type Router as MatchRouter } from './_lib/router_match.js';
 
 const _HERE = fileURLToPath(import.meta.url);
 
@@ -48,6 +49,11 @@ export const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
 export const ROUTER = path.join(REPO_ROOT, 'dist', 'router.json');
 export const CORPUS = path.join(REPO_ROOT, 'tests', 'eval', 'trigger-coverage.yaml');
 
+/** Skill tree the `skill` scope reads `triggers:` frontmatter from. */
+export const SKILLS_DIR = path.join(REPO_ROOT, 'src', 'skills');
+/** The matrix corpus the unintended-activation census is defined over. */
+export const MATRIX_DIR = path.join(REPO_ROOT, 'tests', 'eval', 'routing-matrix');
+
 type Router = {
     kernel?: string[];
     tier_1?: Array<{ id: string; triggers?: Array<Record<string, string>> }>;
@@ -100,6 +106,168 @@ export function fired_rules(prompt: string, router: Router): Set<string> {
     return fired;
 }
 
+/* ------------------------------------------------------------------ *
+ * The SKILL scope.
+ *
+ * It reuses `match_prompt` from `_lib/router_match.ts` — the same matcher the
+ * rule scope's census runs on — by handing it a router-SHAPED view of the
+ * skill catalogue. Forking the matcher would let the two scopes disagree about
+ * what "fires" means, which is the exact drift the shared helper exists to
+ * prevent.
+ *
+ * WHAT THIS SCOPE CANNOT REGRESS, stated because the roadmap's own risk
+ * register assumed otherwise. Risk 2 reads "the corpus counts activations
+ * across the whole router, so a skill-side tranche can regress a rule-side
+ * number". Verified against the tree: `compile_router.ts` contains zero
+ * references to skills, and `dist/router.json` carries exactly
+ * `kernel` + `tier_1` + `tier_2` rule entries. Skill triggers are compiled
+ * into nothing and are matched here over a SEPARATE catalogue, so the rule-side
+ * 433 census is structurally untouched by any skill tranche. The risk it names
+ * is real for a shared router and does not exist in this one — which makes the
+ * skill-side count below a NEW number needing its own ceiling, not a threat to
+ * the old one.
+ * ------------------------------------------------------------------ */
+
+/** One skill that declares activation triggers. */
+export interface SkillTriggerEntry {
+    id: string;
+    triggers: Array<Record<string, string>>;
+}
+
+/** `triggers:` list items from a SKILL.md frontmatter block. */
+function _parseTriggerBlock(frontmatter: string): Array<Record<string, string>> {
+    const out: Array<Record<string, string>> = [];
+    const lines = frontmatter.split('\n');
+    let inside = false;
+    for (const line of lines) {
+        if (/^triggers:\s*$/.test(line)) {
+            inside = true;
+            continue;
+        }
+        if (inside && /^\S/.test(line)) break; // next top-level key ends the block
+        if (!inside) continue;
+        const item = /^\s*-\s*(keyword|phrase|file_pattern|path_prefix|command):\s*(.+?)\s*$/.exec(line);
+        if (item) {
+            out.push({ [item[1]!]: item[2]!.replace(/^["']|["']$/g, '') });
+        }
+    }
+    return out;
+}
+
+/** Every skill under `skillsDir` that declares a non-empty `triggers:` list. */
+export function loadSkillTriggers(skillsDir: string): SkillTriggerEntry[] {
+    let names: string[];
+    try {
+        names = fs.readdirSync(skillsDir);
+    } catch {
+        return [];
+    }
+    const out: SkillTriggerEntry[] = [];
+    for (const name of names.sort()) {
+        const file = path.join(skillsDir, name, 'SKILL.md');
+        let content: string;
+        try {
+            content = fs.readFileSync(file, 'utf-8');
+        } catch {
+            continue;
+        }
+        if (!content.startsWith('---')) continue;
+        const end = content.indexOf('\n---', 3);
+        if (end === -1) continue;
+        const triggers = _parseTriggerBlock(content.slice(3, end));
+        if (triggers.length > 0) out.push({ id: name, triggers });
+    }
+    return out;
+}
+
+/**
+ * Every prompt in the matrix corpus, positives and near-misses alike.
+ *
+ * Parsed with the YAML reader this module already imports, not with a
+ * line regex. A regex anchored on `- prompt: "…"` silently drops any entry
+ * written unquoted or in single quotes, and a corpus that shrinks without
+ * saying so moves every rate computed against it — the denominator would
+ * change while the number kept its old name.
+ */
+export function loadMatrixPrompts(matrixDir: string): string[] {
+    let files: string[];
+    try {
+        files = fs.readdirSync(matrixDir).filter((n) => n.endsWith('.yaml')).sort();
+    } catch {
+        return [];
+    }
+    const out: string[] = [];
+    for (const f of files) {
+        let doc: unknown;
+        try {
+            doc = parseYaml(fs.readFileSync(path.join(matrixDir, f), 'utf-8'), { version: '1.1' });
+        } catch {
+            continue; // an unparseable corpus file is skipped, never guessed at
+        }
+        if (typeof doc !== 'object' || doc === null) continue;
+        const record = doc as Record<string, unknown>;
+        for (const section of ['positives', 'near_misses']) {
+            const entries = record[section];
+            if (!Array.isArray(entries)) continue;
+            for (const raw of entries) {
+                const prompt = (raw as Record<string, unknown> | null)?.['prompt'];
+                if (typeof prompt === 'string' && prompt.trim() !== '') out.push(prompt);
+            }
+        }
+    }
+    return out;
+}
+
+export interface SkillScopeReport {
+    /** Skills declaring at least one trigger. Zero on the day the schema ships. */
+    skills_with_triggers: number;
+    prompts: number;
+    /** Total (prompt, skill) activations — the skill-side census. */
+    activations: number;
+    /** Prompts that activated at least one skill. */
+    prompts_activating: number;
+    /** Per-skill activation counts, descending then by id. */
+    histogram: Array<[string, number]>;
+}
+
+/**
+ * Run the skill catalogue over the matrix corpus with the shared matcher.
+ *
+ * `tier_2` is the carrier for the router-shaped view purely because
+ * `match_prompt` reads its rule lists from `tier_1`/`tier_2`; skills have no
+ * tier and none is implied by the placement.
+ */
+export function runSkillScope(
+    entries: readonly SkillTriggerEntry[],
+    prompts: readonly string[],
+): SkillScopeReport {
+    // `kernel: []` is load-bearing: `match_prompt` appends every kernel id to
+    // `activated_rules` unconditionally, so a non-empty kernel here would count
+    // rule activations as skill ones on every prompt.
+    const view = { kernel: [], tier_1: [], tier_2: entries } as unknown as MatchRouter;
+    const histogram: Record<string, number> = {};
+    let activations = 0;
+    let promptsActivating = 0;
+    for (const prompt of prompts) {
+        const result = match_prompt(view, prompt, 'full', null, null);
+        const ids = result.activated_rules.map((a) => String(a.rule));
+        if (ids.length > 0) promptsActivating += 1;
+        for (const id of ids) {
+            histogram[id] = (histogram[id] ?? 0) + 1;
+            activations += 1;
+        }
+    }
+    return {
+        skills_with_triggers: entries.length,
+        prompts: prompts.length,
+        activations,
+        prompts_activating: promptsActivating,
+        histogram: Object.entries(histogram).sort((a, b) =>
+            b[1] !== a[1] ? b[1] - a[1] : a[0] < b[0] ? -1 : 1,
+        ),
+    };
+}
+
 export function run(corpus: Case[], router: Router): [CaseResult[], number] {
     const results: CaseResult[] = [];
     let misses = 0;
@@ -168,23 +336,76 @@ function _jsonDumps(obj: unknown): string {
 
 interface ParsedArgs {
     json: boolean;
+    /** `rule` keeps the pinned byte-identical contract; `skill` is additive. */
+    scope: 'rule' | 'skill';
 }
 
 function parse_args(argv: string[]): ParsedArgs {
-    const out: ParsedArgs = { json: false };
-    for (const a of argv) {
+    const out: ParsedArgs = { json: false, scope: 'rule' };
+    for (let i = 0; i < argv.length; i += 1) {
+        const a = argv[i]!;
         if (a === '--json') {
             out.json = true;
+        } else if (a === '--scope') {
+            const v = argv[i + 1];
+            if (v !== 'rule' && v !== 'skill') {
+                process.stderr.write(`error: --scope must be rule or skill, got: ${v ?? '(missing)'}\n`);
+                process.exit(2);
+            }
+            out.scope = v;
+            i += 1;
         } else if (a === '-h' || a === '--help') {
-            process.stdout.write('usage: trigger_coverage [-h] [--json]\n');
+            process.stdout.write('usage: trigger_coverage [-h] [--json] [--scope rule|skill]\n');
             process.exit(0);
         }
     }
     return out;
 }
 
+/**
+ * `--scope skill` — report only, never a gate.
+ *
+ * Exit 0 regardless of the count: no ceiling is committed here. The skill-side
+ * census is a NEW number (see the scope's header note on why the rule-side 433
+ * is structurally out of reach), and this package does not commit a threshold
+ * before it has a reading. A tranche is judged against a published baseline in
+ * its own change, not against a number invented on the day the instrument
+ * shipped.
+ */
+function runSkillScopeCli(json: boolean): number {
+    const report = runSkillScope(loadSkillTriggers(SKILLS_DIR), loadMatrixPrompts(MATRIX_DIR));
+    if (json) {
+        process.stdout.write(_jsonDumps(report) + '\n');
+        return 0;
+    }
+    process.stdout.write(
+        `skill trigger coverage over ${report.prompts} matrix prompts\n` +
+            `  skills declaring triggers: ${report.skills_with_triggers}\n` +
+            `  activations:               ${report.activations}\n` +
+            `  prompts activating ≥1:     ${report.prompts_activating}\n`,
+    );
+    if (report.skills_with_triggers === 0) {
+        process.stdout.write(
+            '\nNo skill declares a trigger yet. That is the schema landing empty of\n' +
+                'adopters, not a failure: the capability is declarable before anything\n' +
+                'declares it, so a seeded tranche is measurable before it is written.\n',
+        );
+        return 0;
+    }
+    process.stdout.write('\nper skill:\n');
+    for (const [id, n] of report.histogram) {
+        process.stdout.write(`  ${String(n).padStart(4)}  ${id}\n`);
+    }
+    process.stdout.write(
+        '\nReport only — no ceiling is enforced here. The rule-side 433 census is a\n' +
+            'different catalogue and cannot move from this scope.\n',
+    );
+    return 0;
+}
+
 export function main(argv: string[] | null = null): number {
     const args = parse_args(argv ?? process.argv.slice(2));
+    if (args.scope === 'skill') return runSkillScopeCli(args.json);
 
     let isFile = false;
     try {