@event4u/agent-config 14.12.0 → 14.14.0

package/src/scripts/measure_routing_signal.ts

@@ -0,0 +1,307 @@
+#!/usr/bin/env tsx
+/**
+ * measure_routing_signal — does the skill BODY carry routing signal the
+ * description does not? (`road-to-governed-harness-evolution` step 5.1.)
+ *
+ * THE BAR WAS FIXED BEFORE THIS FILE EXISTED. Every threshold, the k, the
+ * corpus partition, the verdict function and a committed prediction live in
+ * `agents/evidence/analysis/routing-signal-preregistration-2026-08-31.md`,
+ * which is committed in an earlier commit that adds no measurement module. The
+ * constants below are transcribed from it; they are not chosen here.
+ *
+ * TWO GAPS, AND ONLY ONE OF THEM IS MEASURABLE HERE.
+ *
+ *   Gap A — proxy-to-real-session fidelity. Stated by the description-surface
+ *           route checker's own header: asking a model which units it would
+ *           load is not the host's selection procedure. Measuring it needs real
+ *           sessions, and step 5.2 parks the live harness for this whole
+ *           roadmap. So this run emits `null` with `status:
+ *           unmeasured-by-construction` and the reason. That field is not a
+ *           caveat — it bounds the external validity of the Gap B verdict, and
+ *           a consumer reading the verdict file gets both or neither.
+ *
+ *   Gap B — the description-vs-body question this run answers.
+ *
+ * WHY McNEMAR AND NOT A RERUN BAND. The estimator is deterministic: same tree,
+ * same ranking, every time. A rerun band would be identically zero and would
+ * measure nothing. The variance that exists is sampling variance over cases, so
+ * the paired discordant-pair test is the form that addresses it.
+ *
+ * ZERO MODEL CALLS AND ZERO SPEND, and that follows from the imports rather
+ * than from good behaviour: this module imports a file reader and a pure
+ * scorer, and nothing that can open a socket.
+ *
+ * Usage:
+ *   ./scripts-run src/scripts/measure_routing_signal            human table
+ *   ./scripts-run src/scripts/measure_routing_signal --json     machine record
+ *   ./scripts-run src/scripts/measure_routing_signal --write    persist verdict
+ *
+ * Exit 0 = the measurement ran · 2 = it could not run.
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+import {
+    ARMS,
+    type Arm,
+    type CatalogueEntry,
+    type CorpusCase,
+    legacyShaped,
+    loadCatalogue,
+    loadTrainCases,
+    termIndex,
+    topK,
+} from './_lib/routing_corpus.js';
+
+export const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
+
+/** Where the verdict lands. Step 6.5 derives its input set from this file. */
+export const VERDICT_REL = 'agents/evidence/analysis/routing-body-signal-verdict.json';
+export const PREREG_REL = 'agents/evidence/analysis/routing-signal-preregistration-2026-08-31.md';
+
+// --- the pre-registered constants, transcribed ------------------------------
+
+export const K = 5;
+export const RECALL_GAIN_BAR_PP = 5.0;
+export const FALSE_ACTIVATION_GUARD_PP = 2.0;
+export const POWER_FLOOR_DISCORDANT = 10;
+export const ALPHA = 0.05;
+
+export type Verdict = 'signal' | 'null' | 'harmful' | 'underpowered';
+
+export interface ArmCounts {
+    readonly positives: number;
+    readonly positiveHits: number;
+    readonly negatives: number;
+    readonly negativeHits: number;
+}
+
+export interface Discordance {
+    /** control miss → body hit. */
+    readonly gained: number;
+    /** control hit → body miss. */
+    readonly lost: number;
+}
+
+export interface Measurement {
+    readonly k: number;
+    readonly catalogueSize: number;
+    readonly trainCorpora: number;
+    readonly legacyShaped: readonly string[];
+    readonly cases: number;
+    readonly arms: Record<Arm, ArmCounts>;
+    readonly recallPp: Record<Arm, number>;
+    readonly falseActivationPp: Record<Arm, number>;
+    readonly deltaRecallPp: number;
+    readonly deltaFalseActivationPp: number;
+    readonly positiveDiscordance: Discordance;
+    readonly pValue: number;
+    readonly verdict: Verdict;
+    readonly verdictReason: string;
+}
+
+/** Exact two-sided McNemar: a binomial sign test over the discordant pairs. */
+export function mcnemarExactP(gained: number, lost: number): number {
+    const n = gained + lost;
+    if (n === 0) return 1;
+    const logFact: number[] = [0];
+    for (let i = 1; i <= n; i++) logFact.push((logFact[i - 1] as number) + Math.log(i));
+    const logChoose = (a: number, b: number): number =>
+        (logFact[a] as number) - (logFact[b] as number) - (logFact[a - b] as number);
+    const extreme = Math.min(gained, lost);
+    let tail = 0;
+    for (let i = 0; i <= extreme; i++) tail += Math.exp(logChoose(n, i) - n * Math.LN2);
+    return Math.min(1, 2 * tail);
+}
+
+const pp = (hits: number, total: number): number => (total === 0 ? 0 : (hits / total) * 100);
+
+export function measure(repo = REPO, k = K): Measurement {
+    const catalogue: CatalogueEntry[] = loadCatalogue(repo);
+    const cases: CorpusCase[] = loadTrainCases(repo);
+    if (catalogue.length === 0 || cases.length === 0) {
+        throw new Error('measure_routing_signal: empty catalogue or corpus — a run over nothing');
+    }
+    const indexes = new Map<Arm, ReturnType<typeof termIndex>>();
+    for (const arm of ARMS) indexes.set(arm, termIndex(catalogue, arm));
+
+    const counts: Record<string, { p: number; ph: number; n: number; nh: number }> = {};
+    for (const arm of ARMS) counts[arm] = { p: 0, ph: 0, n: 0, nh: 0 };
+    let gained = 0;
+    let lost = 0;
+
+    // One ranking pass per (case, arm). The prompt is re-tokenized per arm by
+    // `topK`; that is the price of keeping the control arm byte-identical to
+    // the shipped call rather than hand-rolling a shared fast path.
+    for (const c of cases) {
+        const hit: Record<string, boolean> = {};
+        for (const arm of ARMS) {
+            const ranked = topK(c.prompt, catalogue, indexes.get(arm) as Map<string, Set<string>>, k);
+            hit[arm] = ranked.some((r) => r.name === c.skill);
+            const bucket = counts[arm] as { p: number; ph: number; n: number; nh: number };
+            if (c.expect) {
+                bucket.p += 1;
+                if (hit[arm]) bucket.ph += 1;
+            } else {
+                bucket.n += 1;
+                if (hit[arm]) bucket.nh += 1;
+            }
+        }
+        if (c.expect) {
+            const control = hit['description'] === true;
+            const body = hit['description+body'] === true;
+            if (!control && body) gained += 1;
+            if (control && !body) lost += 1;
+        }
+    }
+
+    const arms = {} as Record<Arm, ArmCounts>;
+    const recallPp = {} as Record<Arm, number>;
+    const falsePp = {} as Record<Arm, number>;
+    for (const arm of ARMS) {
+        const b = counts[arm] as { p: number; ph: number; n: number; nh: number };
+        arms[arm] = { positives: b.p, positiveHits: b.ph, negatives: b.n, negativeHits: b.nh };
+        recallPp[arm] = pp(b.ph, b.p);
+        falsePp[arm] = pp(b.nh, b.n);
+    }
+    const deltaRecall = (recallPp['description+body'] as number) - (recallPp['description'] as number);
+    const deltaFalse =
+        (falsePp['description+body'] as number) - (falsePp['description'] as number);
+    const p = mcnemarExactP(gained, lost);
+
+    // The pre-registered verdict function, in its pre-registered order.
+    let verdict: Verdict;
+    let reason: string;
+    if (gained + lost < POWER_FLOOR_DISCORDANT) {
+        verdict = 'underpowered';
+        reason = `${gained + lost} discordant positive pairs < power floor ${POWER_FLOOR_DISCORDANT}`;
+    } else if (deltaFalse > FALSE_ACTIVATION_GUARD_PP) {
+        verdict = 'harmful';
+        reason = `false activation +${deltaFalse.toFixed(2)} pp breaches the +${FALSE_ACTIVATION_GUARD_PP.toFixed(1)} pp guard`;
+    } else if (deltaRecall >= RECALL_GAIN_BAR_PP && p < ALPHA) {
+        verdict = 'signal';
+        reason = `recall +${deltaRecall.toFixed(2)} pp at p=${p.toExponential(2)}`;
+    } else {
+        verdict = 'null';
+        reason = `recall +${deltaRecall.toFixed(2)} pp at p=${p.toExponential(2)} does not clear +${RECALL_GAIN_BAR_PP.toFixed(1)} pp and p<${ALPHA}`;
+    }
+
+    return {
+        k,
+        catalogueSize: catalogue.length,
+        trainCorpora: new Set(cases.map((c) => c.skill)).size,
+        legacyShaped: legacyShaped(repo),
+        cases: cases.length,
+        arms,
+        recallPp,
+        falseActivationPp: falsePp,
+        deltaRecallPp: deltaRecall,
+        deltaFalseActivationPp: deltaFalse,
+        positiveDiscordance: { gained, lost },
+        pValue: p,
+        verdict,
+        verdictReason: reason,
+    };
+}
+
+/**
+ * The verdict record step 6.5 reads.
+ *
+ * `proxy_to_real_fidelity` is a REQUIRED field carrying `null`. A consumer that
+ * reads `body_signal` without reading it has taken the conclusion without its
+ * bound, so the two ship in one object.
+ */
+export function verdictRecord(m: Measurement, measuredAt: string): Record<string, unknown> {
+    return {
+        schema: 1,
+        step: '5.1',
+        roadmap: 'road-to-governed-harness-evolution',
+        preregistration: PREREG_REL,
+        measured_at: measuredAt,
+        proxy_to_real_fidelity: {
+            value: null,
+            status: 'unmeasured-by-construction',
+            reason:
+                'Step 5.2 parks the live routing harness for this roadmap; a fidelity ' +
+                'figure requires real sessions. Every routing conclusion in this file is ' +
+                'bounded by this unmeasured quantity.',
+        },
+        body_signal: {
+            verdict: m.verdict,
+            reason: m.verdictReason,
+            k: m.k,
+            delta_recall_pp: Number(m.deltaRecallPp.toFixed(3)),
+            delta_false_activation_pp: Number(m.deltaFalseActivationPp.toFixed(3)),
+            p_value: Number(m.pValue.toPrecision(4)),
+            discordant_positive_pairs: m.positiveDiscordance,
+            bars: {
+                recall_gain_pp: RECALL_GAIN_BAR_PP,
+                false_activation_guard_pp: FALSE_ACTIVATION_GUARD_PP,
+                power_floor_discordant: POWER_FLOOR_DISCORDANT,
+                alpha: ALPHA,
+            },
+            arms: m.arms,
+            recall_pp: {
+                description: Number((m.recallPp['description'] as number).toFixed(3)),
+                'description+body': Number((m.recallPp['description+body'] as number).toFixed(3)),
+            },
+            false_activation_pp: {
+                description: Number((m.falseActivationPp['description'] as number).toFixed(3)),
+                'description+body': Number(
+                    (m.falseActivationPp['description+body'] as number).toFixed(3),
+                ),
+            },
+        },
+        corpus: {
+            partition: 'train',
+            holdout_sealed: true,
+            train_corpora: m.trainCorpora,
+            legacy_shaped_corpora: m.legacyShaped,
+            cases: m.cases,
+            catalogue_size: m.catalogueSize,
+        },
+    };
+}
+
+export function render(m: Measurement): string {
+    const rows = ARMS.map(
+        (arm) =>
+            `  ${arm.padEnd(18)} recall@${m.k} ${(m.recallPp[arm] as number).toFixed(2).padStart(6)} %` +
+            `   false-activation@${m.k} ${(m.falseActivationPp[arm] as number).toFixed(2).padStart(6)} %`,
+    );
+    return [
+        `measure_routing_signal — step 5.1, k=${m.k}`,
+        `  catalogue ${m.catalogueSize} skills · train corpora ${m.trainCorpora}` +
+            ` (${m.legacyShaped.length} legacy-shaped: ${m.legacyShaped.join(', ') || 'none'})` +
+            ` · cases ${m.cases}`,
+        ...rows,
+        `  delta recall ${m.deltaRecallPp >= 0 ? '+' : ''}${m.deltaRecallPp.toFixed(2)} pp` +
+            ` (bar +${RECALL_GAIN_BAR_PP.toFixed(1)} pp)`,
+        `  delta false activation ${m.deltaFalseActivationPp >= 0 ? '+' : ''}${m.deltaFalseActivationPp.toFixed(2)} pp` +
+            ` (guard +${FALSE_ACTIVATION_GUARD_PP.toFixed(1)} pp)`,
+        `  discordant positives gained ${m.positiveDiscordance.gained} lost ${m.positiveDiscordance.lost}` +
+            ` · McNemar exact p=${m.pValue.toExponential(3)}`,
+        `  proxy-to-real fidelity: null (unmeasured-by-construction — 5.2 park)`,
+        `  VERDICT: ${m.verdict} — ${m.verdictReason}`,
+    ].join('\n');
+}
+
+export function main(argv: readonly string[]): number {
+    const m = measure();
+    const record = verdictRecord(m, new Date().toISOString().slice(0, 10));
+    if (argv.includes('--write')) {
+        fs.writeFileSync(path.join(REPO, VERDICT_REL), `${JSON.stringify(record, null, 2)}\n`);
+    }
+    process.stdout.write(argv.includes('--json') ? `${JSON.stringify(record, null, 2)}\n` : `${render(m)}\n`);
+    return 0;
+}
+
+if (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) {
+    try {
+        process.exit(main(process.argv.slice(2)));
+    } catch (err) {
+        process.stderr.write(`measure_routing_signal: ${(err as Error).message}\n`);
+        process.exit(2);
+    }
+}

package/src/scripts/migration_status.ts

@@ -10,7 +10,7 @@
  * dashboard (default: `agents/evidence/migration-status.md`).
  *
  * Usage:
- *   npx tsx src/scripts/migration_status.ts [--out <path>]
+ *   npx tsx migration_status [--out <path>]
  *
  * Node builtins only — no third-party imports.
  */
@@ -240,7 +240,7 @@ function renderDashboard(
     lines.push('# Python → TypeScript Migration Status');
     lines.push('');
     lines.push(
-        `> Generated by \`src/scripts/migration_status.ts\` — do not edit by hand. Timestamp: ${timestamp}`,
+        `> Generated by \`migration_status\` — do not edit by hand. Timestamp: ${timestamp}`,
     );
     lines.push('');
     lines.push(
@@ -311,7 +311,7 @@ function main(): void {
     process.stdout.write(`Tracked .py after exclusions: ${pyFiles.length}\n`);
 }
 
-// Run the CLI only when executed directly (`npx tsx src/scripts/migration_status.ts`),
+// Run the CLI only when executed directly (`npx tsx migration_status`),
 // not when imported (e.g. by src/scripts/parity/phase_gate.ts for `categorize`).
 function _isCliEntry(): boolean {
     if (process.argv[1] === undefined) {

package/src/scripts/model_rule_injection.ts

@@ -52,6 +52,7 @@ import { parse as parseYaml } from 'yaml';
 
 import { LexicalIndex } from './_lib/lexical_index.js';
 import { gpt_tokens } from './_lib/token_count.js';
+import { makePromptShortlist } from './_lib/lexical_shortlist.js';
 import {
     allTierRules,
     kernelIds,
@@ -63,6 +64,12 @@ import {
     triggerlessRuleIds,
     type Router,
 } from './_lib/rule_injection.js';
+import {
+    renderArmExperiment,
+    runArmExperiment,
+    type ArmName,
+    type ArmRow,
+} from './_lib/delivery_arm_experiment.js';
 import { thin_entry } from './project_thin_rules.js';
 import { CAP_BYTES, buildInjection } from './hooks/rule_inject_hook.js';
 
@@ -802,11 +809,56 @@ export function runSelftest(corpusDir: string): SelftestCase[] {
     return out;
 }
 
+/**
+ * Step 6.1's three-arm run, assembled from parts that already exist.
+ *
+ * The standing token figures come from `standingCorpora` — the SAME function
+ * the price grid reads — so the delivery table and the price table can never
+ * disagree about what a shape costs to stand up. `capBytes` is a parameter
+ * rather than a constant so the run can be shown sensitive to the mechanism it
+ * reports on; the default is the concern's own `CAP_BYTES`.
+ */
+export function runThreeArm(
+    corpusDir: string,
+    capBytes: number = CAP_BYTES,
+    useShortlist = false,
+): { rows: ArmRow[]; lines: string[] } {
+    const router = loadRouter(REPO_ROOT);
+    const cases = loadCorpus(corpusDir);
+    const sc = standingCorpora(router);
+    const standing: Record<ArmName, number> = {
+        'eager-all': sc.eagerTokens,
+        thin: sc.thinTokens,
+        delivery: sc.thinTokens,
+    };
+    // STEP 6.3 — the shortlist is OPT-IN and OFF by default, so the figures 6.1
+    // recorded reproduce byte-for-byte without it. It reaches the delivery arm
+    // as a tie-break under the matcher's score and can move nothing but which
+    // matched bodies survive a binding cap.
+    const rows = runArmExperiment({
+        repoRoot: REPO_ROOT,
+        router,
+        cases,
+        capBytes,
+        standing,
+        tokensOf,
+        shortlist: useShortlist ? makePromptShortlist(REPO_ROOT, router) : null,
+    });
+    const lines = renderArmExperiment(rows, capBytes);
+    lines.push(
+        useShortlist
+            ? 'shortlist: ON — lexical BM25 tie-break under the matcher score (step 6.3)'
+            : 'shortlist: OFF — cap order is score then router order (step 6.1 baseline)',
+    );
+    return { rows, lines };
+}
+
 // ── CLI ──────────────────────────────────────────────────────────────────
 
 const USAGE = `usage: model_rule_injection [--corpus DIR] [--baseline-comparison]
                             [--honour-open-files]
                             [--selftest] [--endpoints] [--json]
+                            [--three-arm] [--cap-bytes N] [--shortlist]
 
 Offline recall + cost model for trigger-delivered rule bodies. No metered call
 on any path.
@@ -820,6 +872,9 @@ export function main(argv: string[] | null = null): number {
     let selftest = false;
     let endpoints = false;
     let json = false;
+    let threeArm = false;
+    let shortlist = false;
+    let capBytes = CAP_BYTES;
     for (let i = 0; i < args.length; i += 1) {
         const a = args[i] as string;
         if (a === '--corpus') {
@@ -835,6 +890,17 @@ export function main(argv: string[] | null = null): number {
             endpoints = true;
         } else if (a === '--json') {
             json = true;
+        } else if (a === '--three-arm') {
+            threeArm = true;
+        } else if (a === '--shortlist') {
+            shortlist = true;
+        } else if (a === '--cap-bytes') {
+            i += 1;
+            capBytes = Number.parseInt(args[i] as string, 10);
+            if (!Number.isFinite(capBytes) || capBytes < 1) {
+                process.stderr.write(`model_rule_injection: --cap-bytes needs a positive integer\n`);
+                return 2;
+            }
         } else if (a === '-h' || a === '--help') {
             process.stdout.write(USAGE);
             return 0;
@@ -844,6 +910,16 @@ export function main(argv: string[] | null = null): number {
         }
     }
 
+    if (threeArm) {
+        const res = runThreeArm(corpus, capBytes, shortlist);
+        if (json) {
+            process.stdout.write(`${JSON.stringify(res.rows, null, 2)}\n`);
+        } else {
+            process.stdout.write(`${res.lines.join('\n')}\n`);
+        }
+        return 0;
+    }
+
     if (selftest) {
         const results = runSelftest(corpus);
         const failed = results.filter((r) => !r.passed);

package/src/scripts/orchestration_record.ts

@@ -127,6 +127,20 @@ export function main(argv: string[] = process.argv.slice(2)): number {
         tier_source: str(flags, 'tier-source') as TierSource | undefined,
         task_class: str(flags, 'task-class'),
         dispatch_mode: str(flags, 'dispatch-mode') as never,
+        // `--skills-applied=a,b` records ids; `--skills-applied=` (empty)
+        // records the EMPTY array, which is a real observation ("none
+        // applied") and deliberately not the same as omitting the flag
+        // ("not recorded"). The empty-string check therefore cannot be folded
+        // into the `?.split()` chain above, which would map both onto
+        // undefined and destroy the distinction the field exists for.
+        expected_output: str(flags, 'expected-output') as never,
+        diff_lines: int(flags, 'diff-lines'),
+        empty_cycles: int(flags, 'empty-cycles'),
+        skills_applied: (() => {
+            const raw = str(flags, 'skills-applied');
+            if (raw === undefined) return undefined;
+            return raw.split(',').map((v) => v.trim()).filter(Boolean);
+        })(),
         task_size_estimate: int(flags, 'task-size-estimate'),
         wall_clock_ms: int(flags, 'wall-clock-ms'),
         dispatch_tokens: int(flags, 'dispatch-tokens'),