@event4u/agent-config 16.2.0 → 16.3.0

package/src/scripts/measure_skill_ranker_baseline.ts

@@ -43,6 +43,7 @@
  *     ./scripts-run src/scripts/measure_skill_ranker_baseline --corpus routing-matrix
  *     ./scripts-run src/scripts/measure_skill_ranker_baseline --corpus all
  *     ./scripts-run src/scripts/measure_skill_ranker_baseline --ranker keyword-v2
+ *     ./scripts-run src/scripts/measure_skill_ranker_baseline --corpus routing-matrix --slice sealed
  */
 
 import * as fs from 'node:fs';
@@ -50,6 +51,7 @@ import * as path from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 import { wilsonInterval } from './_lib/capture_rate.js';
+import { fnv1a } from './_lib/fnv.js';
 import { rank } from './skill_tools/score_skill_relevance.js';
 import type { RankOptions } from '../shared/skillRanking.js';
 
@@ -204,6 +206,84 @@ export function promptsForCorpus(corpus: CorpusName, repo = REPO): LabelledPromp
     return [...readLabelledPrompts(repo), ...readMatrixLabelledPrompts(repo)];
 }
 
+/**
+ * Which half of the corpus a row belongs to.
+ *
+ * `road-to-a-ranker-that-routes` 1.2. Tuning and reading on one corpus of 390
+ * overfits it: a signal chosen because it lifted the number on the rows it was
+ * chosen from will lift the number on exactly those rows, and the interval
+ * printed beside it then describes the fitting rather than the ranker. Every
+ * lift claim in Phases 2 and 3 is read on `sealed` only.
+ */
+export type SliceName = 'all' | 'tuning' | 'sealed';
+export const SLICE_NAMES: readonly SliceName[] = ['all', 'tuning', 'sealed'];
+
+/**
+ * One row in five — nominally 20 %, measured 19.2 % (75 of 390) on the live
+ * corpus, because a hash is not a shuffle and the exact count is a property of
+ * the ids.
+ *
+ * Stated as a modulus rather than a fraction because the partition has to be a
+ * pure function of the id: a share applied to a shuffled list would move every
+ * row the moment a prompt is added, and the sealed slice would stop being the
+ * same slice between two runs of the same measurement.
+ */
+export const SEALED_MODULUS = 5;
+
+/**
+ * The slice one case id falls in — deterministic and order-independent.
+ *
+ * STABILITY UNDER CORPUS CHANGE IS PARTIAL, and an earlier version of this block
+ * claimed otherwise. The matrix id is `rule#section[ordinal]` and the ordinal is
+ * POSITIONAL — it counts `- prompt:` entries within a section. So:
+ *
+ *   - APPENDING a prompt to the end of a section is stable: every existing row
+ *     keeps its ordinal, its id, and its side of the seal. This is the workflow
+ *     the matrix README documents, which is why the partition survives the
+ *     release-to-release growth the `history` array is for.
+ *   - INSERTING or DELETING mid-section renumbers every later row in THAT
+ *     section, which re-hashes them and moves roughly one in five across the
+ *     seal.
+ *
+ * A content key (`rule|section|prompt`) would close the second case and is the
+ * right shape for the next corpus version. It is deliberately NOT applied here:
+ * the signals in Phase 2 were chosen by reading the TUNING rows of this
+ * partition, so re-drawing the boundary now would move ~20 % of those rows into
+ * the sealed slice and silently contaminate the very claims the seal exists to
+ * protect. Changing a partition after choosing on it is worse than a partially
+ * stable partition, so the key moves when the corpus does, not before.
+ */
+export function sliceForId(id: string): Exclude<SliceName, 'all'> {
+    return fnv1a(id) % SEALED_MODULUS === 0 ? 'sealed' : 'tuning';
+}
+
+/** The rows of `prompts` that fall in `slice`; `all` is the identity. */
+export function partitionBySlice<T extends { id: string }>(
+    prompts: readonly T[],
+    slice: SliceName,
+): T[] {
+    if (slice === 'all') return [...prompts];
+    return prompts.filter((p) => sliceForId(p.id) === slice);
+}
+
+/** How the corpus divides, printed on every run so a reader never assumes it. */
+export interface SliceSizes {
+    all: number;
+    tuning: number;
+    sealed: number;
+    sealed_modulus: number;
+}
+
+export function sliceSizes(prompts: readonly { id: string }[]): SliceSizes {
+    const sealed = prompts.filter((p) => sliceForId(p.id) === 'sealed').length;
+    return {
+        all: prompts.length,
+        tuning: prompts.length - sealed,
+        sealed,
+        sealed_modulus: SEALED_MODULUS,
+    };
+}
+
 /**
  * The packs a skill declares, read from its own `packs:` frontmatter block.
  *
@@ -278,6 +358,10 @@ export interface Interval {
 
 export interface AccuracyArm {
     corpus: CorpusName;
+    /** Which half of the corpus this arm read. `all` is the whole of it. */
+    slice: SliceName;
+    /** The division of the FULL corpus, printed whichever slice was read. */
+    slice_sizes: SliceSizes;
     corpus_prompts: number;
     top1: number;
     top1_ci95: Interval;
@@ -350,8 +434,11 @@ export function measureAccuracy(opts: {
     repo: string;
     skillsDir: string;
     rankOpts: RankOptions;
+    slice?: SliceName;
 }): AccuracyArm {
-    const labelled = promptsForCorpus(opts.corpus, opts.repo);
+    const slice = opts.slice ?? 'all';
+    const whole = promptsForCorpus(opts.corpus, opts.repo);
+    const labelled = partitionBySlice(whole, slice);
     let top1 = 0;
     let top3 = 0;
     const misses: string[] = [];
@@ -365,6 +452,8 @@ export function measureAccuracy(opts: {
     const census = packCensus(labelled, opts.skillsDir);
     return {
         corpus: opts.corpus,
+        slice,
+        slice_sizes: sliceSizes(whole),
         corpus_prompts: n,
         top1: n ? round3(top1 / n) : 0,
         top1_ci95: ci(top1, n),
@@ -381,20 +470,80 @@ export function measureAccuracy(opts: {
     };
 }
 
+/**
+ * The ranker label → the options it means.
+ *
+ * One table rather than a conditional at each call site, because a measurement
+ * harness that resolves `--ranker` differently from the report that quotes it
+ * is a comparison of two things under one name. Every entry past `keyword-v1`
+ * is a candidate configuration under `road-to-a-ranker-that-routes` Phase 2,
+ * each a single flag so it can be measured alone, plus the combinations the
+ * same phase names.
+ *
+ * AN UNKNOWN LABEL THROWS. It used to resolve to `keyword-v1` — the baseline —
+ * which is the exact failure this table exists to prevent: every consumer
+ * echoes the requested label back into its output, so a typo published a
+ * BASELINE number under another configuration's name, in the report's own title
+ * and in the regenerate command it prints beside it. The two sibling parsers in
+ * this file refuse an unknown value and so does the sweep; the lenient branch
+ * was the inconsistent one.
+ */
+export const RANKER_LABELS: Readonly<Record<string, RankOptions>> = {
+    'keyword-v1': {},
+    'keyword-v2': { includeTriggers: true },
+    'when-to-use': { includeWhenToUse: true },
+    headings: { includeHeadings: true },
+    idf: { idfWeighting: true },
+    'idf+when-to-use': { idfWeighting: true, includeWhenToUse: true },
+};
+
+/**
+ * The message carries NO tool name.
+ *
+ * Three scripts resolve `--ranker` through this module now, so a hardcoded
+ * prefix would print one tool's name when another was the failing command. The
+ * caller's own `process.stderr` line is where the command is identified.
+ */
+export class UnknownRankerLabel extends Error {
+    constructor(label: string) {
+        super(
+            `unknown --ranker ${label || '(nothing)'}; ` +
+                `known: ${Object.keys(RANKER_LABELS).join(' | ')}`,
+        );
+        this.name = 'UnknownRankerLabel';
+    }
+}
+
+/** Whether `label` is one of the table's OWN keys — never an inherited one. */
+export function isKnownRanker(label: string): boolean {
+    return Object.hasOwn(RANKER_LABELS, label);
+}
+
+export function rankOptionsFor(ranker: string): RankOptions {
+    // `Object.hasOwn`, not a truthiness check on the lookup: `RANKER_LABELS`
+    // is an object literal, so `constructor`, `toString` and `valueOf` resolve
+    // to inherited FUNCTIONS rather than to undefined. A plain `=== undefined`
+    // guard lets those three through, scores them as the baseline, and echoes
+    // the label into the report title — which is the whole defect this guard
+    // was added to close, surviving in its own fix.
+    if (!isKnownRanker(ranker)) throw new UnknownRankerLabel(ranker);
+    return RANKER_LABELS[ranker] as RankOptions;
+}
+
 export function measure(opts: {
     ranker: string;
     commit: string;
     corpus?: CorpusName;
+    slice?: SliceName;
     skillsDir?: string;
     repo?: string;
 }): RankerBaseline {
     const repo = opts.repo ?? REPO;
     const skillsDir = opts.skillsDir ?? SKILLS_DIR;
-    // `keyword-v2` is Phase 3.1: the same formula with `triggers:` prose folded
-    // into each skill's term source. Any other label measures v1.
-    const rankOpts: RankOptions = opts.ranker === 'keyword-v2' ? { includeTriggers: true } : {};
+    // Resolved through the one table, which throws on a label it does not know.
+    const rankOpts: RankOptions = rankOptionsFor(opts.ranker);
     const corpus = opts.corpus ?? 'labelled';
-    const accuracy = measureAccuracy({ corpus, repo, skillsDir, rankOpts });
+    const accuracy = measureAccuracy({ corpus, repo, skillsDir, rankOpts, slice: opts.slice ?? 'all' });
 
     const cases = readMatrixCases(repo);
     let withResult = 0;
@@ -446,17 +595,76 @@ function parseCorpus(argv: readonly string[]): CorpusName {
     return raw as CorpusName;
 }
 
+/**
+ * The `--ranker` value, or the baseline when the flag is absent.
+ *
+ * A PRESENT flag with no value after it is an error, not an omission — the
+ * sibling `--slice` has always refused that shape, and substituting the
+ * baseline for a malformed flag is the same silent-fallback defect one level
+ * up from the label table.
+ */
+export function parseRanker(argv: readonly string[]): string {
+    const i = argv.indexOf('--ranker');
+    if (i === -1) return 'keyword-v1';
+    const raw = argv[i + 1];
+    if (!raw || raw.startsWith('--')) {
+        throw new Error(
+            `--ranker expects one of ${Object.keys(RANKER_LABELS).join(' | ')}, got ${raw ?? '(nothing)'}`,
+        );
+    }
+    return raw;
+}
+
+/**
+ * A flag's value, or `fallback` when the flag is absent.
+ *
+ * A PRESENT flag followed by nothing — or by the next flag — is an error. Both
+ * `--commit` and `--date` are echoed verbatim into published artifacts (the
+ * baseline's provenance field, the confusion report's own header), so
+ * `--commit --ranker idf` silently publishing `--ranker` as the commit is the
+ * same silent-substitution class `parseRanker` was written to close, two flags
+ * over in the same two functions.
+ */
+export function parseFlagValue(argv: readonly string[], flag: string, fallback: string): string {
+    const i = argv.indexOf(flag);
+    if (i === -1) return fallback;
+    const raw = argv[i + 1];
+    if (!raw || raw.startsWith('--')) {
+        throw new Error(`${flag} expects a value, got ${raw ?? '(nothing)'}`);
+    }
+    return raw;
+}
+
+export function parseSlice(argv: readonly string[]): SliceName {
+    const i = argv.indexOf('--slice');
+    if (i === -1) return 'all';
+    const raw = argv[i + 1];
+    if (!raw || !SLICE_NAMES.includes(raw as SliceName)) {
+        throw new Error(
+            `measure_skill_ranker_baseline: --slice expects one of ${SLICE_NAMES.join(' | ')}, got ${raw ?? '(nothing)'}`,
+        );
+    }
+    return raw as SliceName;
+}
+
 export function main(argv: readonly string[]): number {
-    const ranker = argv.includes('--ranker') ? (argv[argv.indexOf('--ranker') + 1] ?? 'keyword-v1') : 'keyword-v1';
-    const commit = argv.includes('--commit') ? (argv[argv.indexOf('--commit') + 1] ?? 'unknown') : 'unknown';
     let corpus: CorpusName;
+    let slice: SliceName;
+    let ranker: string;
+    let commit: string;
+    let out: RankerBaseline;
     try {
         corpus = parseCorpus(argv);
+        slice = parseSlice(argv);
+        ranker = parseRanker(argv);
+        commit = parseFlagValue(argv, '--commit', 'unknown');
+        // Inside the same guard as the two parsers above: an unknown --ranker is
+        // the same class of mistake and now gets the same exit code.
+        out = measure({ ranker, commit, corpus, slice });
     } catch (err) {
         process.stderr.write(`${(err as Error).message}\n`);
         return 2;
     }
-    const out = measure({ ranker, commit, corpus });
     process.stdout.write(`${JSON.stringify(out, null, 2)}\n`);
     return 0;
 }

package/src/scripts/memory_learn_hook.ts

@@ -22,6 +22,11 @@
  *   PROMOTED lesson is surfaced as a one-line visibility marker pointing
  *   at the existing /memory:propose flow. Nothing auto-writes curated
  *   `agents/memory/*.yml`.
+ * - Measured: every run appends one line to the gitignored dogfood ledger
+ *   (`agents/runtime/state/learning-dogfood.jsonl`) BEFORE the early
+ *   returns, so the 30-day window the default flip is gated on can read a
+ *   zero rather than a silence. See `runLearn` for why that distinction is
+ *   the whole point.
  *
  * Exit codes (dispatcher contract): 0 always (allow).
  */
@@ -44,8 +49,9 @@ import type { SettingsClass } from '../shared/settingsClasses.js';
 import { consentVerdict, type ConsentVerdict } from '../shared/settingsConsent.js';
 
 import {
-    buildSidecar,
+    buildSidecarFromSignals,
     LESSONS_NAME,
+    readSignals,
     renderLessonsMd,
     SIDECAR_NAME,
 } from './learning_sidecar.js';
@@ -175,23 +181,102 @@ export function enabled(root: string): boolean {
     return learnConsent(root) === 'granted';
 }
 
-/** Run the aggregation. Returns the one-line marker to emit, or null. */
+/**
+ * The dogfood ledger: repo-relative POSIX path, one JSON object per line.
+ *
+ * Under `agents/runtime/` — gitignored wholesale — because it is one machine's
+ * timings and counts, not a shared artefact. Its continuity-surface row records
+ * it as `excluded`: nothing restores a session from it.
+ */
+export const DOGFOOD_LEDGER_POSIX = 'agents/runtime/state/learning-dogfood.jsonl';
+
+/** One session end, as the ledger records it. */
+export interface DogfoodLine {
+    /** The `now` the aggregation ran against. */
+    at: string;
+    /** Wall-clock ms for read + aggregate — the portion the 2 s budget governs. */
+    wall_ms: number;
+    /** Well-formed signal records read from the intake. */
+    signals_in: number;
+    /** Lessons the aggregation produced. */
+    lessons_out: number;
+    /** Of those, how many carry the `preferred` verdict. */
+    preferred: number;
+}
+
+/**
+ * Append one measurement line. Never throws: the ledger is a measurement, and
+ * a measurement that can break the thing it measures is worse than no ledger.
+ */
+export function appendDogfoodLine(root: string, line: DogfoodLine): void {
+    try {
+        const file = path.join(root, ...DOGFOOD_LEDGER_POSIX.split('/'));
+        fs.mkdirSync(path.dirname(file), { recursive: true });
+        fs.appendFileSync(file, `${JSON.stringify(line)}\n`, 'utf8');
+    } catch {
+        // fail-open, same contract as the aggregation itself.
+    }
+}
+
+/** Read the ledger back. Malformed lines are skipped, an absent file is empty. */
+export function readDogfoodLines(root: string): DogfoodLine[] {
+    let text: string;
+    try {
+        text = fs.readFileSync(path.join(root, ...DOGFOOD_LEDGER_POSIX.split('/')), 'utf8');
+    } catch {
+        return [];
+    }
+    const out: DogfoodLine[] = [];
+    for (const raw of text.split('\n')) {
+        const line = raw.trim();
+        if (!line) continue;
+        try {
+            out.push(JSON.parse(line) as DogfoodLine);
+        } catch {
+            continue;
+        }
+    }
+    return out;
+}
+
+/**
+ * Run the aggregation. Returns the one-line marker to emit, or null.
+ *
+ * MEASURE FIRST, THEN DECIDE. The flip condition this hook's default is gated
+ * on (`src/config/agent-settings.template.yml:1376-1378`) names two numbers —
+ * non-trivial signal AND session-end p95 < 2 s — and until the ledger below
+ * existed, neither was recorded: every early return here fired before any
+ * measurement point, so a checkout with an empty intake produced an unbroken
+ * silence. A silence and a measured zero read identically to whoever opens the
+ * window, which is the one failure a 30-day window cannot recover from. So the
+ * read, the aggregation and the ledger line all happen before the first return.
+ */
 export function runLearn(root: string, nowIso: string, budgetMs: number = BUDGET_MS): string | null {
     const started = Date.now();
     const intakeDir = path.join(root, 'agents', 'memory', 'intake');
     const outDir = path.join(root, 'agents', 'memory');
-    if (!fs.existsSync(intakeDir)) return null;
-    const sidecar = buildSidecar(intakeDir, nowIso);
+    // An absent intake directory is a zero-signal session, not an absent one.
+    const haveIntake = fs.existsSync(intakeDir);
+    const signals = haveIntake ? readSignals(intakeDir) : [];
+    const sidecar = buildSidecarFromSignals(signals, nowIso);
+    const preferred = sidecar.lessons.filter((l) => l.verdict === 'preferred').length;
+    appendDogfoodLine(root, {
+        at: nowIso,
+        wall_ms: Date.now() - started,
+        signals_in: signals.length,
+        lessons_out: sidecar.lessons.length,
+        preferred,
+    });
+    if (!haveIntake) return null;
     if (Date.now() - started > budgetMs) return null; // over budget — skip the write
     if (sidecar.lessons.length === 0) return null;
     fs.writeFileSync(path.join(outDir, SIDECAR_NAME), `${JSON.stringify(sidecar, null, 2)}\n`, 'utf8');
     fs.writeFileSync(path.join(outDir, LESSONS_NAME), renderLessonsMd(sidecar), 'utf8');
-    const promoted = sidecar.lessons.filter((l) => l.verdict === 'preferred').length;
     const deadEnds = sidecar.lessons.filter((l) => l.verdict === 'dead_end').length;
     // Memory-visibility contract shape: one line, ids/counters only, no bodies.
     return (
         `🧠 Memory: sidecar refreshed — ${sidecar.lessons.length} lesson(s) ` +
-        `(${promoted} preferred · ${deadEnds} dead-end); promote via /memory:propose`
+        `(${preferred} preferred · ${deadEnds} dead-end); promote via /memory:propose`
     );
 }
 

package/src/scripts/model_rule_injection.ts

@@ -65,6 +65,7 @@ import {
     triggerlessRuleIds,
     type Router,
 } from './_lib/rule_injection.js';
+import { ruleBody } from './_lib/rule_law_section.js';
 import {
     renderArmExperiment,
     runArmExperiment,
@@ -682,6 +683,34 @@ export function runEndpoints(corpusDir: string): EndpointResult[] {
     ]);
     const thinnedAndPathOnly = pathOnlyReach.filter((id) => !alwaysEager.has(id));
 
+    // PROMPT-LEVEL beside RULE-LEVEL — step 0.4 of
+    // `road-to-a-rule-carrier-that-works-outside-the-repo`.
+    //
+    // The two numbers above are per RULE: a rule counts as reachable when ANY
+    // of its labelled positives matches. That is the right unit for the bar —
+    // "no labelled rule left unreachable" — and it is the wrong unit for the
+    // question a reader actually asks next, which is how much of the corpus the
+    // shipped binding delivers. A rule reachable on one of twelve prompts and a
+    // rule reachable on all twelve are the same row in `byRule`.
+    //
+    // So the pair is reported in both units. The SUMMARY view has computed the
+    // prompt-level figures since the two-readings split landed; what was missing
+    // is that `--endpoints`, which is the view a claim cites, printed only the
+    // rule-level half. A reader comparing the two views saw 99/102 and 305/335
+    // and had no way to tell they measure different things.
+    const promptHonoured = positives.filter((c) => reach(c, true)).length;
+    const promptIgnored = positives.filter((c) => reach(c, false)).length;
+
+    // The SUMMARY view prints a LOWER pair over the same corpus, and the gap is
+    // not noise. `scoreExact` calls `matchTierRules(router, c.prompt, of)` with
+    // three arguments and drops the case's `command`, so a positive whose rule
+    // is reached by a `command:` trigger scores as a miss there and as a hit
+    // here. Recorded rather than repaired: this phase changes no behaviour, and
+    // the SUMMARY figures are cited in a roadmap and in the prereg. Both
+    // readings are printed so neither view has to be trusted over the other.
+    const summaryHonoured = scoreExact(router, cases, true);
+    const summaryIgnored = scoreExact(router, cases, false);
+
     // (c) A near-miss must never deliver its labelled rule.
     const exact = scoreExact(router, cases, true);
 
@@ -716,7 +745,15 @@ export function runEndpoints(corpusDir: string): EndpointResult[] {
                 `${byRule.size - pathOnlyReach.length}/${byRule.size}; reachable only via a ` +
                 `path trigger: ${pathOnlyReach.length === 0 ? 'none' : pathOnlyReach.join(', ')}` +
                 ` (of which thinned, i.e. at no scope on that route: ` +
-                `${thinnedAndPathOnly.length === 0 ? 'none' : thinnedAndPathOnly.join(', ')})`,
+                `${thinnedAndPathOnly.length === 0 ? 'none' : thinnedAndPathOnly.join(', ')})` +
+                ` | per-prompt reach: ${promptIgnored}/${positives.length} with open_files ` +
+                `IGNORED against ${promptHonoured}/${positives.length} honoured — the same ` +
+                `corpus in prompts rather than in rules, because a rule reachable on one of ` +
+                `twelve positives and one reachable on all twelve are one row above` +
+                ` | SUMMARY prints ${summaryIgnored.hits}/${summaryIgnored.positives} and ` +
+                `${summaryHonoured.hits}/${summaryHonoured.positives} for the same corpus: ` +
+                `scoreExact drops the case's command trigger, so a command-reached positive ` +
+                `misses there and hits here`,
             bar: 'unreachable == 0 (a rule with zero matched positives is a rule the mode removed)',
             passed: unreachable.length === 0,
         },
@@ -781,13 +818,23 @@ export function runSelftest(corpusDir: string): SelftestCase[] {
         });
     } else {
         const id = injection.rules[0] as string;
-        const projected = loadRuleBody(REPO_ROOT, id) ?? '';
-        const mutated = `${projected.trim().slice(0, -1)}~MUTATED~`;
+        // THE DELIVERED FORM, not the raw file. Since step 1.4 the concern emits
+        // `ruleBody` of the projection — frontmatter and HTML comments stripped
+        // by the same parser `project_thin_rules` uses — so comparing against
+        // the raw file asserts a substring the payload no longer contains. The
+        // endpoint's claim is unchanged: the payload carries the body verbatim
+        // and rejects a mutation of it. What "the body" means moved, and this
+        // moved with it.
+        const projected = ruleBody(loadRuleBody(REPO_ROOT, id) ?? '');
+        const mutated = `${projected.slice(0, -1)}~MUTATED~`;
         out.push({
             endpoint: 'a-delivery',
             name: 'one-byte mutation rejected',
-            passed: injection.body.includes(projected.trim()) && !injection.body.includes(mutated),
-            detail: `${id}: the concern's payload carries the projected body verbatim and rejects a mutation of it`,
+            passed:
+                projected !== ''
+                && injection.body.includes(projected)
+                && !injection.body.includes(mutated),
+            detail: `${id}: the concern's payload carries the delivered body verbatim and rejects a mutation of it`,
         });
     }