@event4u/agent-config 16.2.0 → 16.4.0

package/src/scripts/skill_tools/score_skill_relevance.ts

@@ -39,11 +39,16 @@ import { fileURLToPath, pathToFileURL } from 'node:url';
 // a second copy (road-to-skill-delivery-over-mcp risk 6). Everything below is
 // the disk half — globbing, frontmatter, the CLI — which stays here.
 import {
-    scoreSkill,
+    buildTermStats,
+    compareRanked,
+    rawSkillScore,
+    roundHalfToEven,
     skillTerms as _sharedSkillTerms,
     tokenize as _sharedTokenize,
     triggerTextFromFlatLines,
+    widensTermSource,
     type RankOptions,
+    type TermStats,
 } from '../../shared/skillRanking.js';
 
 const _HERE = fileURLToPath(import.meta.url);
@@ -67,6 +72,20 @@ export const ROOT = path.resolve(path.dirname(_HERE), '..', '..', '..');
  * CLI turns that into a distinct exit code rather than an empty list.
  */
 import { DEFAULT_CATALOGUE_ROOTS, resolveSkillCatalogueRoots } from '../_lib/skill_catalogue.js';
+// Authorship + scan primitives only. Deliberately NOT `neighbour_census.js`,
+// whose module graph reaches the host-hook merger, the secret detector and the
+// install-ownership reader — this module is inlined into the shared hook bundle,
+// where every byte is paid by every concern on every dispatch.
+import {
+    fileDigest,
+    makeSkillOriginResolver,
+    qualifiedSkillName,
+    readScanCache,
+    scanCachePath,
+    scanVerdict,
+    type ScanCache,
+    type SkillOrigin,
+} from '../_lib/skill_origin.js';
 
 /**
  * Every readable catalogue root, in precedence order — possibly empty.
@@ -100,7 +119,17 @@ export interface Frontmatter {
 export function _parse_frontmatter(filePath: string): Frontmatter {
     // text.read_text(errors="replace") — Node `utf-8` decode replaces the same
     // way for the inputs these tools see (well-formed SKILL.md files).
-    const text = fs.readFileSync(filePath, 'utf-8');
+    return _parse_frontmatter_text(fs.readFileSync(filePath, 'utf-8'));
+}
+
+/**
+ * The same reader over text already in hand.
+ *
+ * The loader needs the body as well as the frontmatter, and reading each
+ * SKILL.md twice to get them would double the per-call I/O of a ranker that
+ * runs on a hook path.
+ */
+export function _parse_frontmatter_text(text: string): Frontmatter {
     if (!text.startsWith('---')) {
         return {};
     }
@@ -164,38 +193,204 @@ function _rstrip(s: string): string {
 
 export interface Skill {
     name: string;
+    /**
+     * Where the skill came from — `package` for ours, `project` / `home` for a
+     * neighbour another installer put in a root this ranker unions in.
+     */
+    origin: SkillOrigin;
+    /** What a route line prints: `name` for ours, `origin:name` otherwise. */
+    qualified: string;
+    /**
+     * Finding kind when a neighbour body did not pass the shape scan, else
+     * `null`. Such a skill is still ranked — by its NAME only; see
+     * {@link _load_skills}.
+     */
+    unscanned: string | null;
     description: string;
     personas: string[];
     terms: Set<string>;
     /** `triggers[].keyword` / `.phrase` prose. Indexed only under keyword-v2. */
     triggerText: string[];
+    /** The body's `## When to use` section. Indexed only behind its own flag. */
+    whenToUseText: string;
+    /** The body's `##` / `###` heading lines. Indexed only behind its own flag. */
+    headingText: string[];
+}
+
+/** Everything after the frontmatter block, or the whole text when there is none. */
+export function _body_of(text: string): string {
+    if (!text.startsWith('---')) return text;
+    const end = text.indexOf('\n---', 3);
+    if (end === -1) return text;
+    return text.slice(end + 4);
 }
 
 /**
- * Load one root, or several with the FIRST occurrence of a name winning.
+ * The `## When to use` section, and every `##` / `###` heading, out of a body.
+ *
+ * Line-oriented, and bounded in two ways that a first version got wrong in the
+ * same direction — it indexed something other than what this block describes,
+ * which makes a measurement of the FLAG a measurement of the PARSER:
+ *
+ *   - the section ends at the next heading of the SAME OR A HIGHER level, so a
+ *     `###` subsection of `## When to use` stays inside it. Ending at any
+ *     heading truncated the capture at the first subsection;
+ *   - a line inside a fenced code block is never a heading. These skills are
+ *     documentation and routinely fence markdown samples, so a `## …` line in a
+ *     fence was both harvested as a topic and used to terminate the section.
  *
- * Precedence by name, never by root: every root is read, and a collision
- * resolves to the earliest root that carries it — `resolveSkillCatalogueRoots`
- * owns what that order means. Reading only the first root is the defect this
- * replaced; silently ranking one name twice would be the next one.
+ * Without the first bound a skill whose whole file is one section could still
+ * not put its procedure into an index whose flag says "when to use", because a
+ * sibling `##` ends it.
  */
-function _load_skills_across(roots: readonly string[]): Skill[] {
+export function _body_signals(body: string): { whenToUse: string; headings: string[] } {
+    const headings: string[] = [];
+    const when: string[] = [];
+    let whenLevel = 0;
+    let fenceMark: string | null = null;
+    let fenceLen = 0;
+    for (const raw of body.split('\n')) {
+        const line = raw.trimEnd();
+        const f = /^\s*(`{3,}|~{3,})(.*)$/.exec(line);
+        if (f) {
+            const run = f[1] as string;
+            const mark = run[0] as string;
+            if (fenceMark === null) {
+                fenceMark = mark;
+                fenceLen = run.length;
+                // The DELIMITER and its info string are not prose. Pushing them
+                // put the language tag of every fenced sample (`bash`, `yaml`)
+                // into the index, which is the parser leaking into the signal in
+                // the opposite direction from the truncation this block fixed.
+                continue;
+            }
+            // A closing fence has to be at least as long as the opening one and
+            // use the same marker; otherwise a ``` inside a ```` block would
+            // close it early.
+            if (mark === fenceMark && run.length >= fenceLen && (f[2] as string).trim() === '') {
+                fenceMark = null;
+                fenceLen = 0;
+            }
+            continue;
+        }
+        if (fenceMark !== null) {
+            if (whenLevel > 0) when.push(raw);
+            continue;
+        }
+        // `#{1,6}`, not `#{2,6}`: a level-1 heading is strictly HIGHER than the
+        // section and must close it. Matching only `##`+ let a `# New Section`
+        // and everything after it leak into an open capture.
+        const h = /^(#{1,6})\s+(.*)$/.exec(line);
+        if (h) {
+            const level = (h[1] as string).length;
+            const title = (h[2] as string).trim();
+            if (level >= 2 && level <= 3) headings.push(title);
+            if (whenLevel > 0 && level <= whenLevel) {
+                whenLevel = 0;
+            } else if (whenLevel > 0) {
+                // A SUBSECTION TITLE inside `## When to use` is part of that
+                // section's text. Skipping it indexed the prose under a `### Do
+                // NOT use when` while dropping the words that say what it is.
+                when.push(title);
+            }
+            // Only OPEN a section when none is open. Assigning unconditionally
+            // let a `### When to use it on a monorepo` overwrite the outer
+            // level 2 with 3, after which the next sibling `###` satisfied
+            // `level <= whenLevel` and closed the section early — silently
+            // dropping everything from that sibling to the real end.
+            if (whenLevel === 0 && /^when to use\b/i.test(title)) whenLevel = level;
+            continue;
+        }
+        if (whenLevel > 0) when.push(raw);
+    }
+    return { whenToUse: when.join(' ').trim(), headings };
+}
+
+/**
+ * Everything a load needs to know about who owns a file and whether its body was
+ * scanned. Built once per `rank` call, never per file.
+ */
+export interface NeighbourContext {
+    origin: (skillMdPath: string) => SkillOrigin;
+    scans: ScanCache | null;
+}
+
+/**
+ * The context for a workspace: manifest-backed origins plus the scan cache.
+ *
+ * Built once per `rank` call rather than per file — one manifest read and one
+ * cache read for a catalogue of several hundred entries.
+ */
+export function defaultNeighbourContext(workspaceRoot: string = ROOT): NeighbourContext {
+    return {
+        origin: makeSkillOriginResolver(workspaceRoot, { packageRoot: ROOT }),
+        scans: readScanCache(scanCachePath(workspaceRoot)),
+    };
+}
+
+/**
+ * Load one root, or several, with the FIRST occurrence of a QUALIFIED name winning.
+ *
+ * **It used to dedupe on the bare name, and that was the defect.** Every root is
+ * read, so a second package's `design-system` in `~/.claude/skills` and ours both
+ * reached this loop — and whichever root came second was dropped, silently, with
+ * the surviving row printing one bare name. The reader could not tell which skill
+ * the route line meant, and in the direction where the neighbour came first, ours
+ * was simply gone.
+ *
+ * Deduping on the qualified name keeps the only collision that is genuinely a
+ * duplicate — the same PACKAGE skill reached through two roots, which is the
+ * authored tree beside its own projection — and lets a collision between two
+ * different owners rank twice, which is the thing a reader needs to see. A
+ * no-manifest tree labels everything `package` (`makeOriginResolver`), so there
+ * the behavior is byte-identical to the bare-name dedupe it replaced.
+ */
+function _load_skills_across(
+    roots: readonly string[],
+    opts: RankOptions = {},
+    ctx?: NeighbourContext,
+): Skill[] {
+    const context = ctx ?? defaultNeighbourContext();
     const out: Skill[] = [];
     const seen = new Set<string>();
     for (const root of roots) {
-        for (const s of _load_skills(root)) {
-            if (seen.has(s.name)) continue;
-            seen.add(s.name);
+        for (const s of _load_skills(root, opts, context)) {
+            if (seen.has(s.qualified)) continue;
+            seen.add(s.qualified);
             out.push(s);
         }
     }
     return out;
 }
 
-function _load_skills(skillsDir: string): Skill[] {
+/**
+ * One root's skills.
+ *
+ * A NEIGHBOUR whose body did not pass the shape scan is loaded with every term
+ * source but its name emptied — description, triggers, `## When to use`,
+ * headings and personas — so it ranks by name and nothing from the file can
+ * reach the index or, through it, the injected route line. It is NOT dropped:
+ * a skill hidden for a finding is a skill the reader cannot weigh, and the
+ * finding kind rides along on `unscanned` instead.
+ *
+ * "Did not pass" includes "was never scanned". `scanVerdict` refuses a missing
+ * census record and a stale digest as firmly as a real finding, which is what
+ * makes this a floor rather than a preference — a gate that passes on missing
+ * evidence is satisfied by deleting the evidence.
+ */
+function _load_skills(
+    skillsDir: string,
+    opts: RankOptions = {},
+    ctx: NeighbourContext = defaultNeighbourContext(),
+): Skill[] {
+    // The body is parsed only when a flag indexes it. Under keyword-v1 the
+    // loader's work is exactly what it always was, which is what keeps the
+    // default path's per-prompt cost comparable to the pre-flag reading.
+    const wantsBody = Boolean(opts.includeWhenToUse || opts.includeHeadings);
     const skills: Skill[] = [];
     for (const skillMd of _globSkillMd(skillsDir)) {
-        const fm = _parse_frontmatter(skillMd);
+        const text = fs.readFileSync(skillMd, 'utf-8');
+        const fm = _parse_frontmatter_text(text);
         const rawName = fm['name'];
         const name = _truthyStr(rawName) ? String(rawName) : path.basename(path.dirname(skillMd));
         const rawDesc = fm['description'];
@@ -209,12 +404,41 @@ function _load_skills(skillsDir: string): Skill[] {
         const triggerText = Array.isArray(rawTriggers)
             ? triggerTextFromFlatLines(rawTriggers as string[])
             : [];
+        const { whenToUse, headings } = wantsBody
+            ? _body_signals(_body_of(text))
+            : { whenToUse: '', headings: [] as string[] };
+        const origin = ctx.origin(skillMd);
+        const qualified = qualifiedSkillName(name, origin);
+        const verdict =
+            origin === 'package'
+                ? ({ ok: true } as const)
+                : scanVerdict(ctx.scans, qualified, fileDigest(skillMd));
+        if (!verdict.ok) {
+            skills.push({
+                name,
+                origin,
+                qualified,
+                unscanned: verdict.kind,
+                description: '',
+                personas: [],
+                terms: _tokenize(name),
+                triggerText: [],
+                whenToUseText: '',
+                headingText: [],
+            });
+            continue;
+        }
         skills.push({
             name,
+            origin,
+            qualified,
+            unscanned: null,
             description: desc,
             personas: personaList,
             terms: _tokenize(name + ' ' + desc),
             triggerText,
+            whenToUseText: whenToUse,
+            headingText: headings,
         });
     }
     return skills;
@@ -234,7 +458,7 @@ function _truthyStr(v: string | string[] | undefined): boolean {
  * Python sorts `Path` objects component-wise. Since every match is
  * `<dir>/SKILL.md`, sorting by the directory component reproduces that order.
  */
-function _globSkillMd(root: string): string[] {
+export function _globSkillMd(root: string): string[] {
     let names: string[];
     try {
         names = fs.readdirSync(root);
@@ -257,54 +481,104 @@ function _globSkillMd(root: string): string[] {
     return dirs.map((name) => path.join(root, name, 'SKILL.md'));
 }
 
-function _score(taskTerms: Set<string>, skill: Skill, opts: RankOptions = {}): number {
-    // Single-sourced in `src/shared/skillRanking.ts`. Under keyword-v1 (the
-    // default) `skill.terms` is already `tokenize(name + ' ' + description)`, so
-    // the precomputed set is reused unchanged; keyword-v2 re-derives the term
-    // set with the skill's trigger prose folded in.
-    const rankable = {
+function _rankable(skill: Skill): {
+    name: string;
+    description: string;
+    personas: string[];
+    triggerText: string[];
+    whenToUseText: string;
+    headingText: string[];
+} {
+    return {
         name: skill.name,
         description: skill.description,
         personas: skill.personas,
         triggerText: skill.triggerText,
+        whenToUseText: skill.whenToUseText,
+        headingText: skill.headingText,
     };
-    const terms = opts.includeTriggers ? _sharedSkillTerms(rankable, opts) : skill.terms;
-    return scoreSkill(taskTerms, rankable, terms);
+}
+
+/**
+ * The indexed term set for one skill.
+ *
+ * With no term-source flag set, `skill.terms` is already
+ * `tokenize(name + ' ' + description)` and the precomputed set is reused
+ * unchanged; a flag that widens the source re-derives it with that source
+ * folded in. Separated from `_score` so the caller can build each set ONCE and
+ * share it between the document-frequency pass and the scoring loop.
+ */
+function _terms(skill: Skill, opts: RankOptions): ReadonlySet<string> {
+    return widensTermSource(opts) ? _sharedSkillTerms(_rankable(skill), opts) : skill.terms;
+}
+
+function _rawScore(
+    taskTerms: Set<string>,
+    skill: Skill,
+    terms: ReadonlySet<string>,
+    opts: RankOptions = {},
+    stats?: TermStats,
+): number {
+    // Single-sourced in `src/shared/skillRanking.ts`; `scoreSkill` rounds this.
+    return rawSkillScore(taskTerms, _rankable(skill), terms, opts, stats);
 }
 
 export type RankRow = [string, number, string[]];
 
 /**
  * @param skillsDir one root, or several to rank across (see `_load_skills_across`).
+ * @param ctx       origin + scan context; omitted, it is built from {@link ROOT}.
+ *
+ * Row names are QUALIFIED (`project:design-system`) for a neighbour and bare for
+ * ours, so two same-named skills are two distinguishable rows rather than one
+ * ambiguous one. On a tree with no installed-tools manifest every skill resolves
+ * to `package` and every row is bare, which is what it was before.
  */
 export function rank(
     task: string,
     skillsDir: string | readonly string[],
     opts: RankOptions = {},
+    ctx?: NeighbourContext,
 ): RankRow[] {
+    return rankDetailed(task, skillsDir, opts, ctx).rows;
+}
+
+/**
+ * {@link rank}, plus each ranked skill's unrounded score. The tie metrics of
+ * `report_skill_ranker_confusion` need the second half to tell a tie the
+ * ordering key still cannot break from one the unrounded score already broke.
+ */
+export function rankDetailed(
+    task: string,
+    skillsDir: string | readonly string[],
+    opts: RankOptions = {},
+    ctx?: NeighbourContext,
+): { rows: RankRow[]; raw: ReadonlyMap<string, number> } {
     const taskTerms = _tokenize(task);
-    const skills = _load_skills_across(typeof skillsDir === 'string' ? [skillsDir] : skillsDir);
+    const skills = _load_skills_across(
+        typeof skillsDir === 'string' ? [skillsDir] : skillsDir,
+        opts,
+        ctx,
+    );
+    // ONE term set per skill, shared between the document-frequency pass and the
+    // scoring loop — the two used to tokenize the whole catalogue separately.
+    const termSets = skills.map((s) => _terms(s, opts));
+    const stats = opts.idfWeighting ? buildTermStats(skills.map(_rankable), termSets) : undefined;
     const rows: RankRow[] = [];
-    for (const s of skills) {
-        const score = _score(taskTerms, s, opts);
+    const raw = new Map<string, number>();
+    for (let i = 0; i < skills.length; i += 1) {
+        const s = skills[i] as Skill;
+        const unrounded = _rawScore(taskTerms, s, termSets[i] as ReadonlySet<string>, opts, stats);
+        const score = roundHalfToEven(unrounded);
         if (score > 0) {
-            rows.push([s.name, score, [...s.personas]]);
+            rows.push([s.qualified, score, [...s.personas]]);
+            raw.set(s.qualified, unrounded);
         }
     }
-    // rows.sort(key=lambda r: (-r[1], r[0])) — Python stable tuple sort.
-    rows.sort((a, b) => {
-        if (b[1] !== a[1]) {
-            return b[1] - a[1];
-        }
-        if (a[0] < b[0]) {
-            return -1;
-        }
-        if (a[0] > b[0]) {
-            return 1;
-        }
-        return 0;
-    });
-    return rows;
+    // rows.sort(key=lambda r: (-r[1], r[0])) — Python stable tuple sort, with
+    // the opt-in unrounded tie-break between the two keys.
+    rows.sort((a, b) => compareRanked(a[0], a[1], b[0], b[1], opts, raw));
+    return { rows, raw };
 }
 
 /** Mirror Python `f"{s:<{w}}"` (left-justify) over code-point width. */

package/src/scripts/sweep_skill_ranker_signals.ts

@@ -0,0 +1,129 @@
+#!/usr/bin/env tsx
+/**
+ * sweep_skill_ranker_signals — every candidate signal, alone, on both slices.
+ *
+ * `road-to-a-ranker-that-routes` 2.2 asks for top-1, top-3 and MRR with
+ * intervals for each candidate flag measured ALONE, and the flags are a table in
+ * `measure_skill_ranker_baseline.RANKER_LABELS`. This script is the thing that
+ * produces that table, and it exists as a committed script rather than a
+ * one-off because a number nobody can regenerate is a claim rather than a
+ * measurement.
+ *
+ * IT PRINTS BOTH SLICES, and the two are read for different purposes. The
+ * TUNING reading is what a signal may be CHOSEN on. The SEALED reading is the
+ * only one a lift may be CLAIMED on. Printing only the sealed column would
+ * invite choosing on it, which is the one thing the partition exists to stop;
+ * printing only the tuning column would leave the claim unsupported.
+ *
+ * Usage:
+ *     ./scripts-run src/scripts/sweep_skill_ranker_signals
+ *     ./scripts-run src/scripts/sweep_skill_ranker_signals keyword-v1 idf
+ *     ./scripts-run src/scripts/sweep_skill_ranker_signals --json
+ */
+
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { wilsonInterval } from './_lib/capture_rate.js';
+import {
+    RANKER_LABELS,
+    REPO,
+    SKILLS_DIR,
+    type SliceName,
+    isKnownRanker,
+    partitionBySlice,
+    readMatrixLabelledPrompts,
+} from './measure_skill_ranker_baseline.js';
+import { meanReciprocalRank, readRows } from './report_skill_ranker_confusion.js';
+
+export interface SweepRow {
+    label: string;
+    slice: SliceName;
+    n: number;
+    top1: number;
+    top1_ci95: { lower: number; upper: number };
+    top3: number;
+    top3_ci95: { lower: number; upper: number };
+    mrr: number;
+}
+
+function round3(n: number): number {
+    return Math.round(n * 1000) / 1000;
+}
+
+function ci(successes: number, trials: number): { lower: number; upper: number } {
+    const raw = wilsonInterval(successes, trials);
+    return { lower: round3(raw.lower), upper: round3(raw.upper) };
+}
+
+/**
+ * Every configuration on both slices, ONE ranking pass per prompt.
+ *
+ * `rank()` re-globs and re-reads the whole SKILL.md catalogue on every call, so
+ * it is the dominant cost by a wide margin and ranking each prompt twice doubles
+ * the whole sweep. The earlier shape took top-1/top-3 from `measureAccuracy` and
+ * MRR from a second pass in `readRows`, which also derived the row set from two
+ * independent corpus reads — so a future divergence between them would have been
+ * silent rather than loud. All three rates now come off the same readings of the
+ * same rows.
+ */
+export function sweep(labels: readonly string[], repo = REPO, skillsDir = SKILLS_DIR): SweepRow[] {
+    const whole = readMatrixLabelledPrompts(repo);
+    const out: SweepRow[] = [];
+    for (const label of labels) {
+        for (const slice of ['tuning', 'sealed'] as const) {
+            const rows = partitionBySlice(whole, slice);
+            const readings = readRows(rows, skillsDir, label);
+            const n = readings.length;
+            const t1 = readings.filter((r) => r.rankOfExpected === 1).length;
+            const t3 = readings.filter((r) => r.rankOfExpected >= 1 && r.rankOfExpected <= 3).length;
+            out.push({
+                label,
+                slice,
+                n,
+                top1: n ? round3(t1 / n) : 0,
+                top1_ci95: ci(t1, n),
+                top3: n ? round3(t3 / n) : 0,
+                top3_ci95: ci(t3, n),
+                mrr: round3(meanReciprocalRank(readings)),
+            });
+        }
+    }
+    return out;
+}
+
+export function renderTable(rows: readonly SweepRow[]): string {
+    const L = ['| configuration | slice | n | top-1 | 95 % CI | top-3 | 95 % CI | MRR |', '|---|---|---:|---:|---|---:|---|---:|'];
+    for (const r of rows) {
+        L.push(
+            `| \`${r.label}\` | ${r.slice} | ${String(r.n)} | ${r.top1.toFixed(3)} | ` +
+                `${r.top1_ci95.lower.toFixed(3)} – ${r.top1_ci95.upper.toFixed(3)} | ${r.top3.toFixed(3)} | ` +
+                `${r.top3_ci95.lower.toFixed(3)} – ${r.top3_ci95.upper.toFixed(3)} | ${r.mrr.toFixed(3)} |`,
+        );
+    }
+    return `${L.join('\n')}\n`;
+}
+
+export function main(argv: readonly string[]): number {
+    const json = argv.includes('--json');
+    const named = argv.filter((a) => !a.startsWith('--'));
+    const labels = named.length > 0 ? named : Object.keys(RANKER_LABELS);
+    // `isKnownRanker` (Object.hasOwn), not `in`: `in` walks the prototype chain,
+    // so `constructor` and `toString` would pass here while `Object.keys` below
+    // prints a set that does not contain them — two answers to one question.
+    const unknown = labels.filter((l) => !isKnownRanker(l));
+    if (unknown.length > 0) {
+        process.stderr.write(
+            `sweep_skill_ranker_signals: unknown configuration(s) ${unknown.join(', ')}; ` +
+                `known: ${Object.keys(RANKER_LABELS).join(', ')}\n`,
+        );
+        return 2;
+    }
+    const rows = sweep(labels);
+    process.stdout.write(json ? `${JSON.stringify(rows, null, 2)}\n` : renderTable(rows));
+    return 0;
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
+    process.exit(main(process.argv.slice(2)));
+}