@event4u/agent-config 16.2.0 → 16.3.0

package/src/scripts/schemas/rule.schema.json

@@ -212,6 +212,11 @@
       },
       "description": "What mechanically enforces this rule. `validator:<script>` and `test:<path>` must resolve on disk AND be wired into a taskfile, workflow, or the hook manifest — a script that exists but runs nowhere resolves to unwired, not covered. `hook:<name>` must be registered in hook_manifest.yaml with fail_closed: true; a hook that never blocks is `observer:`, not `validator:`. `instruction-only: <reason>` declares an honestly model-carried obligation and is counted as a recorded gap — the reason is MANDATORY in the pattern, because a bare marker is a claim with no triage record behind it and resolves to `missing`, which the --check ratchet reds on. The bare `none` spelling it replaces stays legal for exactly one reason: `non-destructive-by-default` still carries it and is a kernel rule that block_kernel_rule_writes denies agent writes to, so retiring the value outright would make a CI-green tree unreachable for any agent. Resolved, not trusted, by check_enforcement_coverage.ts."
     },
+    "consequence_class": {
+      "type": "string",
+      "enum": ["irreversible-external", "security-boundary", "authority-bypass"],
+      "description": "Which clause of the high-consequence criterion this rule meets, declared only by rules that meet one (road-to-rule-laws-that-can-stand Phase 2). THE CRITERION, and it is meant to be applied by reading the rule's own law section rather than its name: a rule is high-consequence when violating the obligation its law states would (a) `irreversible-external` \u2014 perform an action that leaves this repository and that this agent cannot undo: a deploy, a published egress, a paid render, a rewrite of published history, a destroyed record; or (b) `security-boundary` \u2014 cross a security or data-exposure boundary: authorization, secrets, private or personal data, confidentiality, or untrusted content reaching a privileged path; or (c) `authority-bypass` \u2014 take, or let the agent take, a decision reserved to the user, the owner, or an independent evaluator. A rule whose violation costs quality, tokens, consistency, consistency of style, or rework is NOT high-consequence however important it is \u2014 that reading is what makes the class a measurement rather than a list of favourites. Exactly one clause per rule: where two apply, the declared one is the clause the rule's own law text names. The class is what the criterion produces: every member, every near-miss considered and rejected with its reason, and every member whose law cannot yet stand in a stub are all recorded in src/config/rule-consequence-class.json and checked by tests/scripts/rule_consequence_class.test.ts. Consumed by project_thin_rules, which ships a class member as a stub plus its own law section, byte-copied and digest-checked."
+    },
     "obligation_frequency": {
       "type": "string",
       "enum": ["per-edit", "per-turn", "per-task", "per-session", "per-event", "per-commit", "per-file-write", "none"],

package/src/scripts/self_review_gate.ts

@@ -34,7 +34,13 @@ import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 
-import { AnthropicClient, load_anthropic_key } from './ai_council/clients.js';
+import { CLI_CONSUMER_SELF_REVIEW } from './ai_council/cli_call_budget.js';
+import {
+    AnthropicClient,
+    AnthropicCliClient,
+    type ExternalAIClient,
+    load_anthropic_key,
+} from './ai_council/clients.js';
 import { independenceFields } from './_lib/review_independence.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
@@ -1012,6 +1018,49 @@ function resolveKey(): string | null {
     }
 }
 
+/** The transport a reviewer was reached on, for the run's own report. */
+interface Reviewer {
+    client: ExternalAIClient;
+    transport: 'cli' | 'api';
+    how: string;
+}
+
+/**
+ * Resolve the review client CLI-FIRST — the subscription transport before the
+ * metered one, which is the ordering the rest of this suite already uses for
+ * coding work and which the council has used since it grew a `mode: cli`.
+ *
+ * The ordering is not a preference. An API key is a METERED resource with a
+ * balance that runs out, and when it does the gate reports NEUTRAL — a release
+ * then carries no review at all. That is not hypothetical: it is how 16.3.0
+ * stalled, with six chunks returning `credit balance is too low` and the
+ * release step refusing for want of a findings artefact. A vendor CLI backed by
+ * a subscription has no balance to exhaust, so trying it first removes the one
+ * failure mode that has actually produced an unreviewed release here.
+ *
+ * `AnthropicCliClient`'s constructor THROWS when `claude` is not on PATH, which
+ * is the probe: a hosted GitHub runner has no vendor CLI and falls straight
+ * through to the API transport, so this change costs CI nothing and buys every
+ * local and self-hosted run a free review. The catch is deliberately broad —
+ * anything that goes wrong reaching for the CLI resolves to the API path rather
+ * than to a NEUTRAL, because a reviewed release beats a diagnosed one.
+ *
+ * Calls book against `CLI_CONSUMER_SELF_REVIEW` so the daily CLI counter can
+ * tell a review apart from a council round; without the attribution the gate's
+ * six chunks would read as unexplained council spend.
+ */
+function resolveReviewer(): Reviewer | null {
+    try {
+        const client = new AnthropicCliClient({ consumer: CLI_CONSUMER_SELF_REVIEW });
+        return { client, transport: 'cli', how: `vendor CLI (${client.binary}), subscription` };
+    } catch {
+        // No `claude` on PATH, or the client refused to construct. Fall through.
+    }
+    const key = resolveKey();
+    if (!key) return null;
+    return { client: new AnthropicClient({ api_key: key }), transport: 'api', how: 'ANTHROPIC_API_KEY' };
+}
+
 // ── CLI ───────────────────────────────────────────────────────────────
 export function main(argv: string[]): 0 | 2 {
     const dryRun = argv.includes('--dry-run');
@@ -1087,22 +1136,28 @@ export function main(argv: string[]): 0 | 2 {
         return 0;
     }
 
-    const key = resolveKey();
-    if (!key) {
+    const reviewer = resolveReviewer();
+    if (!reviewer) {
         // Explicit NEUTRAL state — never a bare green that reads as "reviewed".
+        // BOTH transports are named, because naming only the API one is what
+        // made the CLI path invisible: an operator reading this warning set the
+        // secret and never learned a logged-in `claude` would also have served.
         process.stdout.write(
-            '::warning::self-review-gate NEUTRAL — no ANTHROPIC_API_KEY configured, NOTHING was reviewed. ' +
-                'Set the repo secret to enable the live dogfooded review.\n',
+            '::warning::self-review-gate NEUTRAL — no reviewer reachable (no `claude` CLI on PATH, ' +
+                'no ANTHROPIC_API_KEY), NOTHING was reviewed. Log in to the vendor CLI or set the ' +
+                'repo secret to enable the live dogfooded review.\n',
         );
         const summary = process.env.GITHUB_STEP_SUMMARY;
         if (summary) {
             appendFileSync(
                 summary,
-                '### Self-review gate: NEUTRAL\n\nNo `ANTHROPIC_API_KEY` secret — nothing was reviewed. This is not a pass.\n',
+                '### Self-review gate: NEUTRAL\n\nNo reviewer reachable — neither the `claude` CLI ' +
+                    '(subscription) nor `ANTHROPIC_API_KEY` (metered). Nothing was reviewed. This is not a pass.\n',
             );
         }
         return 0;
     }
+    process.stdout.write(`::notice::self-review-gate — transport: ${reviewer.transport} (${reviewer.how}).\n`);
 
     // Belt-and-suspenders: the WHOLE live path is wrapped so that ANYTHING going
     // wrong — no API credit / balance (HTTP 402), rate-limit (429), network error,
@@ -1111,7 +1166,7 @@ export function main(argv: string[]): 0 | 2 {
     // gate is advisory by design. (Only an --enforce run with real blocking
     // findings returns 2 — an error is not a finding.)
     try {
-        const client = new AnthropicClient({ api_key: key });
+        const client = reviewer.client;
         const systemPrompt = buildSystemPrompt(plan.release, baseRef);
         const partition = plan.partition;
 

package/src/scripts/skill_tools/score_skill_relevance.ts

@@ -39,11 +39,14 @@ import { fileURLToPath, pathToFileURL } from 'node:url';
 // a second copy (road-to-skill-delivery-over-mcp risk 6). Everything below is
 // the disk half — globbing, frontmatter, the CLI — which stays here.
 import {
+    buildTermStats,
     scoreSkill,
     skillTerms as _sharedSkillTerms,
     tokenize as _sharedTokenize,
     triggerTextFromFlatLines,
+    widensTermSource,
     type RankOptions,
+    type TermStats,
 } from '../../shared/skillRanking.js';
 
 const _HERE = fileURLToPath(import.meta.url);
@@ -67,6 +70,20 @@ export const ROOT = path.resolve(path.dirname(_HERE), '..', '..', '..');
  * CLI turns that into a distinct exit code rather than an empty list.
  */
 import { DEFAULT_CATALOGUE_ROOTS, resolveSkillCatalogueRoots } from '../_lib/skill_catalogue.js';
+// Authorship + scan primitives only. Deliberately NOT `neighbour_census.js`,
+// whose module graph reaches the host-hook merger, the secret detector and the
+// install-ownership reader — this module is inlined into the shared hook bundle,
+// where every byte is paid by every concern on every dispatch.
+import {
+    fileDigest,
+    makeSkillOriginResolver,
+    qualifiedSkillName,
+    readScanCache,
+    scanCachePath,
+    scanVerdict,
+    type ScanCache,
+    type SkillOrigin,
+} from '../_lib/skill_origin.js';
 
 /**
  * Every readable catalogue root, in precedence order — possibly empty.
@@ -100,7 +117,17 @@ export interface Frontmatter {
 export function _parse_frontmatter(filePath: string): Frontmatter {
     // text.read_text(errors="replace") — Node `utf-8` decode replaces the same
     // way for the inputs these tools see (well-formed SKILL.md files).
-    const text = fs.readFileSync(filePath, 'utf-8');
+    return _parse_frontmatter_text(fs.readFileSync(filePath, 'utf-8'));
+}
+
+/**
+ * The same reader over text already in hand.
+ *
+ * The loader needs the body as well as the frontmatter, and reading each
+ * SKILL.md twice to get them would double the per-call I/O of a ranker that
+ * runs on a hook path.
+ */
+export function _parse_frontmatter_text(text: string): Frontmatter {
     if (!text.startsWith('---')) {
         return {};
     }
@@ -164,38 +191,204 @@ function _rstrip(s: string): string {
 
 export interface Skill {
     name: string;
+    /**
+     * Where the skill came from — `package` for ours, `project` / `home` for a
+     * neighbour another installer put in a root this ranker unions in.
+     */
+    origin: SkillOrigin;
+    /** What a route line prints: `name` for ours, `origin:name` otherwise. */
+    qualified: string;
+    /**
+     * Finding kind when a neighbour body did not pass the shape scan, else
+     * `null`. Such a skill is still ranked — by its NAME only; see
+     * {@link _load_skills}.
+     */
+    unscanned: string | null;
     description: string;
     personas: string[];
     terms: Set<string>;
     /** `triggers[].keyword` / `.phrase` prose. Indexed only under keyword-v2. */
     triggerText: string[];
+    /** The body's `## When to use` section. Indexed only behind its own flag. */
+    whenToUseText: string;
+    /** The body's `##` / `###` heading lines. Indexed only behind its own flag. */
+    headingText: string[];
+}
+
+/** Everything after the frontmatter block, or the whole text when there is none. */
+export function _body_of(text: string): string {
+    if (!text.startsWith('---')) return text;
+    const end = text.indexOf('\n---', 3);
+    if (end === -1) return text;
+    return text.slice(end + 4);
 }
 
 /**
- * Load one root, or several with the FIRST occurrence of a name winning.
+ * The `## When to use` section, and every `##` / `###` heading, out of a body.
+ *
+ * Line-oriented, and bounded in two ways that a first version got wrong in the
+ * same direction — it indexed something other than what this block describes,
+ * which makes a measurement of the FLAG a measurement of the PARSER:
  *
- * Precedence by name, never by root: every root is read, and a collision
- * resolves to the earliest root that carries it — `resolveSkillCatalogueRoots`
- * owns what that order means. Reading only the first root is the defect this
- * replaced; silently ranking one name twice would be the next one.
+ *   - the section ends at the next heading of the SAME OR A HIGHER level, so a
+ *     `###` subsection of `## When to use` stays inside it. Ending at any
+ *     heading truncated the capture at the first subsection;
+ *   - a line inside a fenced code block is never a heading. These skills are
+ *     documentation and routinely fence markdown samples, so a `## …` line in a
+ *     fence was both harvested as a topic and used to terminate the section.
+ *
+ * Without the first bound a skill whose whole file is one section could still
+ * not put its procedure into an index whose flag says "when to use", because a
+ * sibling `##` ends it.
+ */
+export function _body_signals(body: string): { whenToUse: string; headings: string[] } {
+    const headings: string[] = [];
+    const when: string[] = [];
+    let whenLevel = 0;
+    let fenceMark: string | null = null;
+    let fenceLen = 0;
+    for (const raw of body.split('\n')) {
+        const line = raw.trimEnd();
+        const f = /^\s*(`{3,}|~{3,})(.*)$/.exec(line);
+        if (f) {
+            const run = f[1] as string;
+            const mark = run[0] as string;
+            if (fenceMark === null) {
+                fenceMark = mark;
+                fenceLen = run.length;
+                // The DELIMITER and its info string are not prose. Pushing them
+                // put the language tag of every fenced sample (`bash`, `yaml`)
+                // into the index, which is the parser leaking into the signal in
+                // the opposite direction from the truncation this block fixed.
+                continue;
+            }
+            // A closing fence has to be at least as long as the opening one and
+            // use the same marker; otherwise a ``` inside a ```` block would
+            // close it early.
+            if (mark === fenceMark && run.length >= fenceLen && (f[2] as string).trim() === '') {
+                fenceMark = null;
+                fenceLen = 0;
+            }
+            continue;
+        }
+        if (fenceMark !== null) {
+            if (whenLevel > 0) when.push(raw);
+            continue;
+        }
+        // `#{1,6}`, not `#{2,6}`: a level-1 heading is strictly HIGHER than the
+        // section and must close it. Matching only `##`+ let a `# New Section`
+        // and everything after it leak into an open capture.
+        const h = /^(#{1,6})\s+(.*)$/.exec(line);
+        if (h) {
+            const level = (h[1] as string).length;
+            const title = (h[2] as string).trim();
+            if (level >= 2 && level <= 3) headings.push(title);
+            if (whenLevel > 0 && level <= whenLevel) {
+                whenLevel = 0;
+            } else if (whenLevel > 0) {
+                // A SUBSECTION TITLE inside `## When to use` is part of that
+                // section's text. Skipping it indexed the prose under a `### Do
+                // NOT use when` while dropping the words that say what it is.
+                when.push(title);
+            }
+            // Only OPEN a section when none is open. Assigning unconditionally
+            // let a `### When to use it on a monorepo` overwrite the outer
+            // level 2 with 3, after which the next sibling `###` satisfied
+            // `level <= whenLevel` and closed the section early — silently
+            // dropping everything from that sibling to the real end.
+            if (whenLevel === 0 && /^when to use\b/i.test(title)) whenLevel = level;
+            continue;
+        }
+        if (whenLevel > 0) when.push(raw);
+    }
+    return { whenToUse: when.join(' ').trim(), headings };
+}
+
+/**
+ * Everything a load needs to know about who owns a file and whether its body was
+ * scanned. Built once per `rank` call, never per file.
  */
-function _load_skills_across(roots: readonly string[]): Skill[] {
+export interface NeighbourContext {
+    origin: (skillMdPath: string) => SkillOrigin;
+    scans: ScanCache | null;
+}
+
+/**
+ * The context for a workspace: manifest-backed origins plus the scan cache.
+ *
+ * Built once per `rank` call rather than per file — one manifest read and one
+ * cache read for a catalogue of several hundred entries.
+ */
+export function defaultNeighbourContext(workspaceRoot: string = ROOT): NeighbourContext {
+    return {
+        origin: makeSkillOriginResolver(workspaceRoot, { packageRoot: ROOT }),
+        scans: readScanCache(scanCachePath(workspaceRoot)),
+    };
+}
+
+/**
+ * Load one root, or several, with the FIRST occurrence of a QUALIFIED name winning.
+ *
+ * **It used to dedupe on the bare name, and that was the defect.** Every root is
+ * read, so a second package's `design-system` in `~/.claude/skills` and ours both
+ * reached this loop — and whichever root came second was dropped, silently, with
+ * the surviving row printing one bare name. The reader could not tell which skill
+ * the route line meant, and in the direction where the neighbour came first, ours
+ * was simply gone.
+ *
+ * Deduping on the qualified name keeps the only collision that is genuinely a
+ * duplicate — the same PACKAGE skill reached through two roots, which is the
+ * authored tree beside its own projection — and lets a collision between two
+ * different owners rank twice, which is the thing a reader needs to see. A
+ * no-manifest tree labels everything `package` (`makeOriginResolver`), so there
+ * the behavior is byte-identical to the bare-name dedupe it replaced.
+ */
+function _load_skills_across(
+    roots: readonly string[],
+    opts: RankOptions = {},
+    ctx?: NeighbourContext,
+): Skill[] {
+    const context = ctx ?? defaultNeighbourContext();
     const out: Skill[] = [];
     const seen = new Set<string>();
     for (const root of roots) {
-        for (const s of _load_skills(root)) {
-            if (seen.has(s.name)) continue;
-            seen.add(s.name);
+        for (const s of _load_skills(root, opts, context)) {
+            if (seen.has(s.qualified)) continue;
+            seen.add(s.qualified);
             out.push(s);
         }
     }
     return out;
 }
 
-function _load_skills(skillsDir: string): Skill[] {
+/**
+ * One root's skills.
+ *
+ * A NEIGHBOUR whose body did not pass the shape scan is loaded with every term
+ * source but its name emptied — description, triggers, `## When to use`,
+ * headings and personas — so it ranks by name and nothing from the file can
+ * reach the index or, through it, the injected route line. It is NOT dropped:
+ * a skill hidden for a finding is a skill the reader cannot weigh, and the
+ * finding kind rides along on `unscanned` instead.
+ *
+ * "Did not pass" includes "was never scanned". `scanVerdict` refuses a missing
+ * census record and a stale digest as firmly as a real finding, which is what
+ * makes this a floor rather than a preference — a gate that passes on missing
+ * evidence is satisfied by deleting the evidence.
+ */
+function _load_skills(
+    skillsDir: string,
+    opts: RankOptions = {},
+    ctx: NeighbourContext = defaultNeighbourContext(),
+): Skill[] {
+    // The body is parsed only when a flag indexes it. Under keyword-v1 the
+    // loader's work is exactly what it always was, which is what keeps the
+    // default path's per-prompt cost comparable to the pre-flag reading.
+    const wantsBody = Boolean(opts.includeWhenToUse || opts.includeHeadings);
     const skills: Skill[] = [];
     for (const skillMd of _globSkillMd(skillsDir)) {
-        const fm = _parse_frontmatter(skillMd);
+        const text = fs.readFileSync(skillMd, 'utf-8');
+        const fm = _parse_frontmatter_text(text);
         const rawName = fm['name'];
         const name = _truthyStr(rawName) ? String(rawName) : path.basename(path.dirname(skillMd));
         const rawDesc = fm['description'];
@@ -209,12 +402,41 @@ function _load_skills(skillsDir: string): Skill[] {
         const triggerText = Array.isArray(rawTriggers)
             ? triggerTextFromFlatLines(rawTriggers as string[])
             : [];
+        const { whenToUse, headings } = wantsBody
+            ? _body_signals(_body_of(text))
+            : { whenToUse: '', headings: [] as string[] };
+        const origin = ctx.origin(skillMd);
+        const qualified = qualifiedSkillName(name, origin);
+        const verdict =
+            origin === 'package'
+                ? ({ ok: true } as const)
+                : scanVerdict(ctx.scans, qualified, fileDigest(skillMd));
+        if (!verdict.ok) {
+            skills.push({
+                name,
+                origin,
+                qualified,
+                unscanned: verdict.kind,
+                description: '',
+                personas: [],
+                terms: _tokenize(name),
+                triggerText: [],
+                whenToUseText: '',
+                headingText: [],
+            });
+            continue;
+        }
         skills.push({
             name,
+            origin,
+            qualified,
+            unscanned: null,
             description: desc,
             personas: personaList,
             terms: _tokenize(name + ' ' + desc),
             triggerText,
+            whenToUseText: whenToUse,
+            headingText: headings,
         });
     }
     return skills;
@@ -234,7 +456,7 @@ function _truthyStr(v: string | string[] | undefined): boolean {
  * Python sorts `Path` objects component-wise. Since every match is
  * `<dir>/SKILL.md`, sorting by the directory component reproduces that order.
  */
-function _globSkillMd(root: string): string[] {
+export function _globSkillMd(root: string): string[] {
     let names: string[];
     try {
         names = fs.readdirSync(root);
@@ -257,38 +479,81 @@ function _globSkillMd(root: string): string[] {
     return dirs.map((name) => path.join(root, name, 'SKILL.md'));
 }
 
-function _score(taskTerms: Set<string>, skill: Skill, opts: RankOptions = {}): number {
-    // Single-sourced in `src/shared/skillRanking.ts`. Under keyword-v1 (the
-    // default) `skill.terms` is already `tokenize(name + ' ' + description)`, so
-    // the precomputed set is reused unchanged; keyword-v2 re-derives the term
-    // set with the skill's trigger prose folded in.
-    const rankable = {
+function _rankable(skill: Skill): {
+    name: string;
+    description: string;
+    personas: string[];
+    triggerText: string[];
+    whenToUseText: string;
+    headingText: string[];
+} {
+    return {
         name: skill.name,
         description: skill.description,
         personas: skill.personas,
         triggerText: skill.triggerText,
+        whenToUseText: skill.whenToUseText,
+        headingText: skill.headingText,
     };
-    const terms = opts.includeTriggers ? _sharedSkillTerms(rankable, opts) : skill.terms;
-    return scoreSkill(taskTerms, rankable, terms);
+}
+
+/**
+ * The indexed term set for one skill.
+ *
+ * With no term-source flag set, `skill.terms` is already
+ * `tokenize(name + ' ' + description)` and the precomputed set is reused
+ * unchanged; a flag that widens the source re-derives it with that source
+ * folded in. Separated from `_score` so the caller can build each set ONCE and
+ * share it between the document-frequency pass and the scoring loop.
+ */
+function _terms(skill: Skill, opts: RankOptions): ReadonlySet<string> {
+    return widensTermSource(opts) ? _sharedSkillTerms(_rankable(skill), opts) : skill.terms;
+}
+
+function _score(
+    taskTerms: Set<string>,
+    skill: Skill,
+    terms: ReadonlySet<string>,
+    opts: RankOptions = {},
+    stats?: TermStats,
+): number {
+    // Single-sourced in `src/shared/skillRanking.ts`.
+    return scoreSkill(taskTerms, _rankable(skill), terms, opts, stats);
 }
 
 export type RankRow = [string, number, string[]];
 
 /**
  * @param skillsDir one root, or several to rank across (see `_load_skills_across`).
+ * @param ctx       origin + scan context; omitted, it is built from {@link ROOT}.
+ *
+ * Row names are QUALIFIED (`project:design-system`) for a neighbour and bare for
+ * ours, so two same-named skills are two distinguishable rows rather than one
+ * ambiguous one. On a tree with no installed-tools manifest every skill resolves
+ * to `package` and every row is bare, which is what it was before.
  */
 export function rank(
     task: string,
     skillsDir: string | readonly string[],
     opts: RankOptions = {},
+    ctx?: NeighbourContext,
 ): RankRow[] {
     const taskTerms = _tokenize(task);
-    const skills = _load_skills_across(typeof skillsDir === 'string' ? [skillsDir] : skillsDir);
+    const skills = _load_skills_across(
+        typeof skillsDir === 'string' ? [skillsDir] : skillsDir,
+        opts,
+        ctx,
+    );
+    // ONE term set per skill, shared between the document-frequency pass and the
+    // scoring loop — the two used to tokenize the whole catalogue separately.
+    const termSets = skills.map((s) => _terms(s, opts));
+    const stats = opts.idfWeighting ? buildTermStats(skills.map(_rankable), termSets) : undefined;
     const rows: RankRow[] = [];
-    for (const s of skills) {
-        const score = _score(taskTerms, s, opts);
+    for (let i = 0; i < skills.length; i += 1) {
+        const s = skills[i] as Skill;
+        const score = _score(taskTerms, s, termSets[i] as ReadonlySet<string>, opts, stats);
         if (score > 0) {
-            rows.push([s.name, score, [...s.personas]]);
+            rows.push([s.qualified, score, [...s.personas]]);
         }
     }
     // rows.sort(key=lambda r: (-r[1], r[0])) — Python stable tuple sort.

package/src/scripts/sweep_skill_ranker_signals.ts

@@ -0,0 +1,129 @@
+#!/usr/bin/env tsx
+/**
+ * sweep_skill_ranker_signals — every candidate signal, alone, on both slices.
+ *
+ * `road-to-a-ranker-that-routes` 2.2 asks for top-1, top-3 and MRR with
+ * intervals for each candidate flag measured ALONE, and the flags are a table in
+ * `measure_skill_ranker_baseline.RANKER_LABELS`. This script is the thing that
+ * produces that table, and it exists as a committed script rather than a
+ * one-off because a number nobody can regenerate is a claim rather than a
+ * measurement.
+ *
+ * IT PRINTS BOTH SLICES, and the two are read for different purposes. The
+ * TUNING reading is what a signal may be CHOSEN on. The SEALED reading is the
+ * only one a lift may be CLAIMED on. Printing only the sealed column would
+ * invite choosing on it, which is the one thing the partition exists to stop;
+ * printing only the tuning column would leave the claim unsupported.
+ *
+ * Usage:
+ *     ./scripts-run src/scripts/sweep_skill_ranker_signals
+ *     ./scripts-run src/scripts/sweep_skill_ranker_signals keyword-v1 idf
+ *     ./scripts-run src/scripts/sweep_skill_ranker_signals --json
+ */
+
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { wilsonInterval } from './_lib/capture_rate.js';
+import {
+    RANKER_LABELS,
+    REPO,
+    SKILLS_DIR,
+    type SliceName,
+    isKnownRanker,
+    partitionBySlice,
+    readMatrixLabelledPrompts,
+} from './measure_skill_ranker_baseline.js';
+import { meanReciprocalRank, readRows } from './report_skill_ranker_confusion.js';
+
+export interface SweepRow {
+    label: string;
+    slice: SliceName;
+    n: number;
+    top1: number;
+    top1_ci95: { lower: number; upper: number };
+    top3: number;
+    top3_ci95: { lower: number; upper: number };
+    mrr: number;
+}
+
+function round3(n: number): number {
+    return Math.round(n * 1000) / 1000;
+}
+
+function ci(successes: number, trials: number): { lower: number; upper: number } {
+    const raw = wilsonInterval(successes, trials);
+    return { lower: round3(raw.lower), upper: round3(raw.upper) };
+}
+
+/**
+ * Every configuration on both slices, ONE ranking pass per prompt.
+ *
+ * `rank()` re-globs and re-reads the whole SKILL.md catalogue on every call, so
+ * it is the dominant cost by a wide margin and ranking each prompt twice doubles
+ * the whole sweep. The earlier shape took top-1/top-3 from `measureAccuracy` and
+ * MRR from a second pass in `readRows`, which also derived the row set from two
+ * independent corpus reads — so a future divergence between them would have been
+ * silent rather than loud. All three rates now come off the same readings of the
+ * same rows.
+ */
+export function sweep(labels: readonly string[], repo = REPO, skillsDir = SKILLS_DIR): SweepRow[] {
+    const whole = readMatrixLabelledPrompts(repo);
+    const out: SweepRow[] = [];
+    for (const label of labels) {
+        for (const slice of ['tuning', 'sealed'] as const) {
+            const rows = partitionBySlice(whole, slice);
+            const readings = readRows(rows, skillsDir, label);
+            const n = readings.length;
+            const t1 = readings.filter((r) => r.rankOfExpected === 1).length;
+            const t3 = readings.filter((r) => r.rankOfExpected >= 1 && r.rankOfExpected <= 3).length;
+            out.push({
+                label,
+                slice,
+                n,
+                top1: n ? round3(t1 / n) : 0,
+                top1_ci95: ci(t1, n),
+                top3: n ? round3(t3 / n) : 0,
+                top3_ci95: ci(t3, n),
+                mrr: round3(meanReciprocalRank(readings)),
+            });
+        }
+    }
+    return out;
+}
+
+export function renderTable(rows: readonly SweepRow[]): string {
+    const L = ['| configuration | slice | n | top-1 | 95 % CI | top-3 | 95 % CI | MRR |', '|---|---|---:|---:|---|---:|---|---:|'];
+    for (const r of rows) {
+        L.push(
+            `| \`${r.label}\` | ${r.slice} | ${String(r.n)} | ${r.top1.toFixed(3)} | ` +
+                `${r.top1_ci95.lower.toFixed(3)} – ${r.top1_ci95.upper.toFixed(3)} | ${r.top3.toFixed(3)} | ` +
+                `${r.top3_ci95.lower.toFixed(3)} – ${r.top3_ci95.upper.toFixed(3)} | ${r.mrr.toFixed(3)} |`,
+        );
+    }
+    return `${L.join('\n')}\n`;
+}
+
+export function main(argv: readonly string[]): number {
+    const json = argv.includes('--json');
+    const named = argv.filter((a) => !a.startsWith('--'));
+    const labels = named.length > 0 ? named : Object.keys(RANKER_LABELS);
+    // `isKnownRanker` (Object.hasOwn), not `in`: `in` walks the prototype chain,
+    // so `constructor` and `toString` would pass here while `Object.keys` below
+    // prints a set that does not contain them — two answers to one question.
+    const unknown = labels.filter((l) => !isKnownRanker(l));
+    if (unknown.length > 0) {
+        process.stderr.write(
+            `sweep_skill_ranker_signals: unknown configuration(s) ${unknown.join(', ')}; ` +
+                `known: ${Object.keys(RANKER_LABELS).join(', ')}\n`,
+        );
+        return 2;
+    }
+    const rows = sweep(labels);
+    process.stdout.write(json ? `${JSON.stringify(rows, null, 2)}\n` : renderTable(rows));
+    return 0;
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
+    process.exit(main(process.argv.slice(2)));
+}