@event4u/agent-config 14.20.0 → 14.21.0

package/src/scripts/bench_humanizer_eval.ts

@@ -37,9 +37,38 @@ const ROOT = process.cwd();
 const FIXTURES = join(ROOT, "tests", "fixtures", "ai-tells");
 const REPORT_DIR = join(ROOT, "internal", "bench", "reports");
 const LENGTH_TOLERANCE = 0.25;
+const SPLITS_FILE = join(FIXTURES, "SPLITS.json");
+
+export type Split = "tune" | "holdout";
+
+/**
+ * Tune / holdout membership, declared in `tests/fixtures/ai-tells/SPLITS.json`.
+ *
+ * Before this existed `loadPairs()` returned one array, so a rule tuned against
+ * the twenty pairs was measured against the same twenty pairs and overfitting
+ * was excluded by construction — of the measurement, not of the rule. Every
+ * figure this bench prints now names the half it came from, and a number that
+ * mixes the halves is not produced at all.
+ */
+export function loadSplits(file: string = SPLITS_FILE): Record<Split, string[]> {
+  const raw = JSON.parse(readFileSync(file, "utf8")) as Record<string, unknown>;
+  const pick = (k: Split): string[] => {
+    const v = raw[k];
+    if (!Array.isArray(v)) throw new Error(`SPLITS.json: "${k}" is not an array`);
+    return v as string[];
+  };
+  return { tune: pick("tune"), holdout: pick("holdout") };
+}
 
 interface PairResult {
   name: string;
+  split: Split;
+  /**
+   * Per rule id, how many occurrences the humanize pass removed (before minus
+   * after, positive only). Without this the judge returns one bit per pair and
+   * a family landed in Phase 2 cannot be tied to any effect at all.
+   */
+  families_removed: Record<string, number>;
   language: "en" | "de";
   words_before: number;
   words_after: number;
@@ -50,6 +79,23 @@ interface PairResult {
   judge?: { preferred: "after" | "before"; order: "after-first" | "before-first" };
 }
 
+/** before minus after, per rule, keeping only what the pass actually removed. */
+export function familyDelta(before: TellReport, after: TellReport): Record<string, number> {
+  const out: Record<string, number> = {};
+  for (const [id, n] of Object.entries(before.per_pattern)) {
+    const removed = n - (after.per_pattern[id] ?? 0);
+    if (removed > 0) out[id] = removed;
+  }
+  return out;
+}
+
+interface JudgedAggregate {
+  judge_model?: string;
+  judged_pairs: number;
+  prefers_after: number;
+  prefers_before: number;
+}
+
 function slim(r: TellReport) {
   return {
     hard_total: r.hard_total,
@@ -63,8 +109,23 @@ function coin(seed: number): boolean {
   return ((seed * 1103515245 + 12345) >>> 16) % 2 === 0;
 }
 
-function loadPairs(): Array<{ name: string; language: "en" | "de"; before: string; after: string }> {
-  const out: Array<{ name: string; language: "en" | "de"; before: string; after: string }> = [];
+export interface LoadedPair {
+  name: string;
+  split: Split;
+  language: "en" | "de";
+  before: string;
+  after: string;
+}
+
+export function loadPairs(splits: Record<Split, string[]> = loadSplits()): LoadedPair[] {
+  const assigned = new Map<string, Split>();
+  for (const s of ["tune", "holdout"] as const) {
+    for (const key of splits[s]) {
+      if (assigned.has(key)) throw new Error(`SPLITS.json: ${key} is in both halves`);
+      assigned.set(key, s);
+    }
+  }
+  const out: LoadedPair[] = [];
   for (const language of ["en", "de"] as const) {
     const dir = join(FIXTURES, language);
     let names: string[] = [];
@@ -77,8 +138,15 @@ function loadPairs(): Array<{ name: string; language: "en" | "de"; before: strin
       continue;
     }
     for (const name of names) {
+      const split = assigned.get(`${language}/${name}`);
+      if (split === undefined) {
+        // Silence here would let a new pair join the tune half by default,
+        // which is the failure the split exists to prevent.
+        throw new Error(`SPLITS.json assigns no half to ${language}/${name}`);
+      }
       out.push({
         name,
+        split,
         language,
         before: readFileSync(join(dir, `${name}.before.md`), "utf8"),
         after: readFileSync(join(dir, `${name}.after.md`), "utf8"),
@@ -111,6 +179,18 @@ async function judgePair(
 async function main(): Promise<void> {
   const runJudge = process.argv.includes("--judge");
   const confirmSpend = process.argv.includes("--confirm-spend");
+  const excluded = new Set<string>();
+  for (let i = 0; i < process.argv.length; i += 1) {
+    if (process.argv[i] === "--disable-family") {
+      const id = process.argv[i + 1];
+      if (id === undefined || id.startsWith("--")) {
+        console.error("--disable-family needs a rule id");
+        process.exit(2);
+      }
+      excluded.add(id);
+    }
+  }
+  const opts = { exclude: excluded };
   const pairs = loadPairs();
   if (pairs.length < 20) {
     console.error(`corpus too small: ${pairs.length} pairs (< 20)`);
@@ -123,8 +203,8 @@ async function main(): Promise<void> {
   // never an interactive prompt. The objective-only default path is free.
   if (runJudge && !confirmSpend) {
     const controlledCount = pairs.filter((p) => {
-      const b = analyzeText(p.before, p.language).words;
-      const a = analyzeText(p.after, p.language).words;
+      const b = analyzeText(p.before, p.language, opts).words;
+      const a = analyzeText(p.after, p.language, opts).words;
       return Math.abs(1 - a / b) <= LENGTH_TOLERANCE;
     }).length;
     process.stderr.write(
@@ -138,12 +218,14 @@ async function main(): Promise<void> {
   const results: PairResult[] = [];
   for (let i = 0; i < pairs.length; i++) {
     const p = pairs[i]!;
-    const before = analyzeText(p.before, p.language);
-    const after = analyzeText(p.after, p.language);
+    const before = analyzeText(p.before, p.language, opts);
+    const after = analyzeText(p.after, p.language, opts);
     const ratio = after.words / before.words;
     const controlled = Math.abs(1 - ratio) <= LENGTH_TOLERANCE;
     const row: PairResult = {
       name: p.name,
+      split: p.split,
+      families_removed: familyDelta(before, after),
       language: p.language,
       words_before: before.words,
       words_after: after.words,
@@ -161,31 +243,116 @@ async function main(): Promise<void> {
 
   const mean = (xs: number[]) =>
     Math.round((xs.reduce((s, x) => s + x, 0) / Math.max(xs.length, 1)) * 100) / 100;
-  const agg = {
-    pairs: results.length,
-    length_controlled_pairs: results.filter((r) => r.length_controlled).length,
-    mean_hard_before: mean(results.map((r) => r.before.hard_total)),
-    mean_hard_after: mean(results.map((r) => r.after.hard_total)),
-    mean_cluster_before: mean(results.map((r) => r.before.cluster_score_per_500)),
-    mean_cluster_after: mean(results.map((r) => r.after.cluster_score_per_500)),
-    mean_dash_before: mean(results.map((r) => r.before.dash_density_per_500)),
-    mean_dash_after: mean(results.map((r) => r.after.dash_density_per_500)),
-    judge_model: undefined as string | undefined,
-    judged_pairs: 0,
-    prefers_after: 0,
-    prefers_before: 0,
+  /**
+   * A density is `null` when the text sits under the detector's word floor, so
+   * the rate was never computed. Averaging those as zero would report "measured
+   * clean" for text nobody measured; they are dropped and the surviving
+   * denominator is published beside the mean.
+   */
+  const meanDefined = (xs: Array<number | null>): { value: number | null; n: number } => {
+    const defined = xs.filter((x): x is number => x !== null);
+    return { value: defined.length === 0 ? null : mean(defined), n: defined.length };
+  };
+  const show = (m: { value: number | null; n: number }): string =>
+    m.value === null ? `not evaluated (n=0)` : `${m.value} (n=${m.n})`;
+  const aggregateOver = (rows: PairResult[]) => {
+    const judgedRows = rows.filter((r) => r.judge);
+    return {
+      pairs: rows.length,
+      length_controlled_pairs: rows.filter((r) => r.length_controlled).length,
+      mean_hard_before: mean(rows.map((r) => r.before.hard_total)),
+      mean_hard_after: mean(rows.map((r) => r.after.hard_total)),
+      mean_cluster_before: meanDefined(rows.map((r) => r.before.cluster_score_per_500)),
+      mean_cluster_after: meanDefined(rows.map((r) => r.after.cluster_score_per_500)),
+      mean_dash_before: meanDefined(rows.map((r) => r.before.dash_density_per_500)),
+      mean_dash_after: meanDefined(rows.map((r) => r.after.dash_density_per_500)),
+      judged_pairs: judgedRows.length,
+      prefers_after: judgedRows.filter((r) => r.judge!.preferred === "after").length,
+      prefers_before: judgedRows.filter((r) => r.judge!.preferred === "before").length,
+      judge_model: undefined as string | undefined,
+    };
+  };
+  /**
+   * The last recorded judged run, when this one is objective-only.
+   *
+   * An objective-only re-run used to overwrite the canonical report with a
+   * "Not run" placeholder, which deleted the only evidence backing
+   * `claim:humanizer-tell-reduction` — a free re-run silently unbacking a
+   * claim is a worse failure than a stale number, because nothing announces it.
+   * The carried-forward block is labelled with its own date and is never
+   * presented as a measurement of the current register.
+   */
+  const priorJudged = ((): { generated: string; agg: JudgedAggregate } | null => {
+    let best: { generated: string; agg: JudgedAggregate } | null = null;
+    let names: string[] = [];
+    try {
+      names = readdirSync(REPORT_DIR).filter((n) => n.endsWith("-humanizer-v1.json"));
+    } catch {
+      return null;
+    }
+    for (const n of names.sort()) {
+      try {
+        const doc = JSON.parse(readFileSync(join(REPORT_DIR, n), "utf8")) as {
+          generated?: string;
+          aggregate?: JudgedAggregate;
+        };
+        if ((doc.aggregate?.judged_pairs ?? 0) > 0 && doc.generated !== undefined) {
+          best = { generated: doc.generated, agg: doc.aggregate as JudgedAggregate };
+        }
+      } catch {
+        continue;
+      }
+    }
+    return best;
+  })();
+
+  const agg = aggregateOver(results);
+
+  /**
+   * Attribution — which families the preference could have tracked.
+   *
+   * `pairs_removed` is deterministic and free. `pairs_removed_and_preferred`
+   * is the crossing with the blind judge and exists only on a `--judge` run;
+   * it is a CO-OCCURRENCE, never an isolated effect, because the pass removes
+   * several families at once from the same pair. What it rules out is the
+   * opposite: a family that appears in no preferred pair did not carry the
+   * preference, and that is the half a binary verdict could not say at all.
+   */
+  const attribution = (() => {
+    const rows = new Map<string, { pairs_removed: number; pairs_removed_and_preferred: number }>();
+    for (const r of results) {
+      for (const id of Object.keys(r.families_removed)) {
+        const row = rows.get(id) ?? { pairs_removed: 0, pairs_removed_and_preferred: 0 };
+        row.pairs_removed += 1;
+        if (r.judge?.preferred === "after") row.pairs_removed_and_preferred += 1;
+        rows.set(id, row);
+      }
+    }
+    return [...rows.entries()]
+      .map(([id, v]) => ({ id, ...v }))
+      .sort((a, b) => b.pairs_removed - a.pairs_removed || a.id.localeCompare(b.id));
+  })();
+  const bySplit = {
+    tune: aggregateOver(results.filter((r) => r.split === "tune")),
+    holdout: aggregateOver(results.filter((r) => r.split === "holdout")),
   };
-  const judged = results.filter((r) => r.judge);
-  agg.judged_pairs = judged.length;
-  agg.prefers_after = judged.filter((r) => r.judge!.preferred === "after").length;
-  agg.prefers_before = judged.filter((r) => r.judge!.preferred === "before").length;
   if (runJudge) {
     const { DEFAULT_ANTHROPIC_MODEL } = await import("./ai_council/clients.js");
     agg.judge_model = DEFAULT_ANTHROPIC_MODEL as string;
+    bySplit.tune.judge_model = agg.judge_model;
+    bySplit.holdout.judge_model = agg.judge_model;
   }
 
   const iso = new Date().toISOString().replace(/\.\d+Z$/, "Z").replace(/:/g, "-");
-  const payload = { generated: iso, corpus: "tests/fixtures/ai-tells", aggregate: agg, results };
+  const payload = {
+    generated: iso,
+    corpus: "tests/fixtures/ai-tells",
+    aggregate: agg,
+    by_split: bySplit,
+    disabled_families: [...excluded].sort(),
+    attribution,
+    results,
+  };
 
   const md = [
     "# Humanizer paired eval — v1",
@@ -195,19 +362,59 @@ async function main(): Promise<void> {
     "",
     "## Objective — AI-tell reduction (deterministic)",
     "",
-    "| Metric (mean) | Before | After |",
-    "|---|---|---|",
-    `| Hard hits | ${agg.mean_hard_before} | ${agg.mean_hard_after} |`,
-    `| Cluster score /500w | ${agg.mean_cluster_before} | ${agg.mean_cluster_after} |`,
-    `| Dash density /500w | ${agg.mean_dash_before} | ${agg.mean_dash_after} |`,
+    "Every figure names the split it came from. `tune` is the half a rule may be",
+    "looked at while it is being written; `holdout` is scored and never read",
+    "during authoring, so a gain that appears only in `tune` is overfitting and",
+    "says so on its face. Split membership: `tests/fixtures/ai-tells/SPLITS.json`.",
+    "",
+    `| Metric (mean) | tune before (n=${bySplit.tune.pairs}) | tune after | holdout before (n=${bySplit.holdout.pairs}) | holdout after | both before (n=${agg.pairs}) | both after |`,
+    "|---|---|---|---|---|---|---|",
+    `| Hard hits | ${bySplit.tune.mean_hard_before} | ${bySplit.tune.mean_hard_after} | ` +
+      `${bySplit.holdout.mean_hard_before} | ${bySplit.holdout.mean_hard_after} | ` +
+      `${agg.mean_hard_before} | ${agg.mean_hard_after} |`,
+    `| Cluster score /500w | ${show(bySplit.tune.mean_cluster_before)} | ${show(bySplit.tune.mean_cluster_after)} | ` +
+      `${show(bySplit.holdout.mean_cluster_before)} | ${show(bySplit.holdout.mean_cluster_after)} | ` +
+      `${show(agg.mean_cluster_before)} | ${show(agg.mean_cluster_after)} |`,
+    `| Dash density /500w | ${show(bySplit.tune.mean_dash_before)} | ${show(bySplit.tune.mean_dash_after)} | ` +
+      `${show(bySplit.holdout.mean_dash_before)} | ${show(bySplit.holdout.mean_dash_after)} | ` +
+      `${show(agg.mean_dash_before)} | ${show(agg.mean_dash_after)} |`,
     "",
     "## Blind preference (length-controlled)",
     "",
     agg.judged_pairs > 0
       ? `Judge ${agg.judge_model}: prefers the humanized text in **${agg.prefers_after}/${agg.judged_pairs}** ` +
-        `pairs (randomized A/B order, deterministic seed). An honest null here keeps the detector as a ` +
+        `pairs across both halves — **${bySplit.tune.prefers_after}/${bySplit.tune.judged_pairs}** on tune, ` +
+        `**${bySplit.holdout.prefers_after}/${bySplit.holdout.judged_pairs}** on holdout — ` +
+        `(randomized A/B order, deterministic seed). An honest null here keeps the detector as a ` +
         `hygiene gate; the claim ledger only carries what this table shows.`
-      : "_Not run (objective-only invocation)._",
+      : priorJudged !== null
+        ? `_Not re-run — this invocation is objective-only._ Carried forward from the judged run of ` +
+          `**${priorJudged.generated}**: judge ${priorJudged.agg.judge_model ?? "unrecorded"} ` +
+          `prefers the humanized text in **${priorJudged.agg.prefers_after}/${priorJudged.agg.judged_pairs}** ` +
+          `length-controlled pairs (randomized A/B order, deterministic seed). That figure was measured ` +
+          `against the register and the length-controlled set as they stood on that date; the objective ` +
+          `table above is newer. It is reproduced so a free re-run cannot silently unback the claim it ` +
+          `supports, and it is NOT a measurement of the current register.`
+        : "_Not run (objective-only invocation, and no earlier judged run on record)._",
+    "",
+    "## Attribution — which families the preference tracked",
+    "",
+    excluded.size > 0
+      ? `Families disabled for this run: ${[...excluded].sort().map((x) => `\`${x}\``).join(", ")}. ` +
+        "They were excluded from the scan, not merely from this table, so every figure above moved with them."
+      : "No family disabled. Re-run with `--disable-family <rule-id>` to see what a family was carrying.",
+    "",
+    "| Family | Pairs it was removed from | …and the judge preferred the humanized text |",
+    "|---|---|---|",
+    ...attribution.map(
+      (a) => `| \`${a.id}\` | ${a.pairs_removed} | ${agg.judged_pairs > 0 ? a.pairs_removed_and_preferred : "not judged"} |`,
+    ),
+    "",
+    "The right-hand column is a **co-occurrence, never an isolated effect**: the",
+    "pass removes several families from the same pair, so a high number does not",
+    "attribute the preference to that family. What the table does establish is the",
+    "negative — a family removed in no preferred pair did not carry the preference —",
+    "and that is what a binary verdict could not say at all.",
     "",
     "## Scope note",
     "",
@@ -215,14 +422,20 @@ async function main(): Promise<void> {
     'It never measures third-party "AI detector" outcomes — that claim class is banned',
     "(unfalsifiable from our side; see roadmap non-goals).",
     "",
-    "## Open question — real-draft lift is unmeasured",
+    "## Open question — real-draft lift is unmeasured, and stays that way this round",
     "",
     "The `before` fixtures were **deliberately tell-seeded**, so a perfect score",
     "measures seeded-tell removal on a self-constructed corpus — NOT that real",
-    "ghostwriter drafts get better. Real-world lift stays **unmeasured** until",
-    "write-engine step 4b has processed real `/ghostwriter:write` drafts and those",
-    "have been paired-evaluated (the `road-to-humanizer-hardening` live-usage",
-    "blocker). The claim ledger is scoped to \"on the fixture corpus\" accordingly.",
+    "ghostwriter drafts get better. Real-world lift is **unmeasured**.",
+    "",
+    "Collecting it was declined for this round. Retaining real drafts, and",
+    "retaining metrics derived from real drafts, both create a new retention",
+    "practice beyond the fixture-only data-handling floor this package records,",
+    "and neither was authorized. That is a decision about this round: **future",
+    "authorization is neither granted nor refused**, and the owner-facing question",
+    "is unchanged. The claim ledger stays scoped to \"on the fixture corpus\"",
+    "accordingly, and widening it needs an authorization AND a measurement, not",
+    "either one alone.",
     "",
   ].join("\n");
 
@@ -234,7 +447,18 @@ async function main(): Promise<void> {
   process.stdout.write(md + "\n");
 }
 
-main().catch((err) => {
-  console.error(String(err));
-  process.exit(2);
-});
+/**
+ * Guarded, because this module is now IMPORTED — `loadPairs`, `loadSplits` and
+ * `familyDelta` are the split and attribution surface the suite tests. Without
+ * the guard an import ran the whole bench as a side effect and overwrote the
+ * canonical report, so the test suite silently wrote tracked files.
+ */
+const invokedDirectly =
+  process.argv[1] !== undefined &&
+  import.meta.url.endsWith(process.argv[1].split("/").pop() ?? "");
+if (invokedDirectly) {
+  main().catch((err) => {
+    console.error(String(err));
+    process.exit(2);
+  });
+}

package/src/scripts/bench_prose_tells_fp.ts

@@ -0,0 +1,198 @@
+#!/usr/bin/env node
+/**
+ * bench_prose_tells_fp — per-rule false-positive count for the prose-tell
+ * registry, measured over a clean corpus.
+ *
+ * The seeded corpus in `tests/fixtures/ai-tells/{en,de}/` can only answer
+ * "does the rule fire on text written to make it fire". This bench answers the
+ * other half: how often each rule in `ai_tells_rules.ts` fires on prose that
+ * was never written for it. Pre-registration:
+ * `internal/bench/corpora/prose-tells-fp-PREREG.md` — read it before reading a
+ * number from this tool.
+ *
+ * Counting is PER FILE, not per hit: a rule matching six lines of one file is
+ * one false positive, because the unit a writer experiences is "this rule
+ * flagged this file". Reported alongside, never instead: the count of rules
+ * with a non-zero M1, and how many clean files the shipped thresholds reject
+ * outright.
+ *
+ * Every run prints a corpus SHA-256 over the sorted `<path>:<sha256>` list. A
+ * number quoted without its corpus hash is not comparable to any other number.
+ *
+ * Usage:
+ *   npx tsx src/scripts/bench_prose_tells_fp.ts            # table + pin
+ *   npx tsx src/scripts/bench_prose_tells_fp.ts --json
+ *   npx tsx src/scripts/bench_prose_tells_fp.ts --gate tell-x tell-y
+ *
+ * `--gate <rule-id...>` is the Phase-2 promotion gate: exit 1 when any named
+ * rule has a non-zero clean-corpus count, naming the files that hit. Promotion
+ * of a family is refused while its count is above zero.
+ *
+ * Exit codes: 0 report-only or gate clean · 1 gate failed · 2 usage / no corpus.
+ */
+
+import { createHash } from "node:crypto";
+import { readFileSync, readdirSync } from "node:fs";
+import { join, relative } from "node:path";
+import { fileURLToPath } from "node:url";
+import { analyzeText, exceedsThresholds } from "./detect_ai_tells.js";
+import {
+  ALL_TELL_RULES,
+  DEFAULT_MAX_CLUSTER_SCORE,
+  DEFAULT_MAX_DASH_DENSITY,
+  DEFAULT_MAX_HARD,
+} from "./ai_tells_rules.js";
+
+const REPO = join(fileURLToPath(import.meta.url), "..", "..", "..");
+export const CLEAN_DIR = join(REPO, "tests", "fixtures", "ai-tells", "clean");
+
+const THRESHOLDS = {
+  maxHard: DEFAULT_MAX_HARD,
+  maxScore: DEFAULT_MAX_CLUSTER_SCORE,
+  maxDashDensity: DEFAULT_MAX_DASH_DENSITY,
+};
+
+export interface CleanFile {
+  path: string;
+  language: "en" | "de";
+  text: string;
+}
+
+/** Every file in the clean corpus, sorted, with its language from the folder. */
+export function loadCleanCorpus(dir: string = CLEAN_DIR): CleanFile[] {
+  const out: CleanFile[] = [];
+  for (const language of ["en", "de"] as const) {
+    let names: string[] = [];
+    try {
+      names = readdirSync(join(dir, language)).filter((f) => f.endsWith(".md")).sort();
+    } catch {
+      continue;
+    }
+    for (const name of names) {
+      const full = join(dir, language, name);
+      out.push({
+        path: relative(REPO, full),
+        language,
+        text: readFileSync(full, "utf8"),
+      });
+    }
+  }
+  return out;
+}
+
+/** SHA-256 over the sorted `<relative path>:<sha256 of contents>` list. */
+export function corpusPin(files: CleanFile[]): string {
+  const lines = files
+    .map((f) => `${f.path}:${createHash("sha256").update(f.text).digest("hex")}`)
+    .sort();
+  return createHash("sha256").update(lines.join("\n")).digest("hex");
+}
+
+export interface RuleFp {
+  id: string;
+  language: string;
+  severity: string;
+  /** M1 — number of clean files on which the rule produced at least one match. */
+  files: number;
+  /** Which files, so a non-zero count is actionable rather than a number. */
+  hits: string[];
+}
+
+export interface FpReport {
+  corpus_pin: string;
+  files: number;
+  per_rule: RuleFp[];
+  rules_with_findings: number;
+  /** Clean files the shipped thresholds would reject outright. */
+  rejected_files: Array<{ path: string; reasons: string[] }>;
+}
+
+export function measure(files: CleanFile[]): FpReport {
+  const byRule = new Map<string, string[]>();
+  const rejected: Array<{ path: string; reasons: string[] }> = [];
+
+  for (const f of files) {
+    const report = analyzeText(f.text, f.language);
+    for (const [id, count] of Object.entries(report.per_pattern)) {
+      if (count > 0) byRule.set(id, [...(byRule.get(id) ?? []), f.path]);
+    }
+    const reasons = exceedsThresholds(report, THRESHOLDS);
+    if (reasons.length > 0) rejected.push({ path: f.path, reasons });
+  }
+
+  const perRule: RuleFp[] = ALL_TELL_RULES.map((r) => ({
+    id: r.id,
+    language: r.language,
+    severity: r.severity,
+    files: (byRule.get(r.id) ?? []).length,
+    hits: byRule.get(r.id) ?? [],
+  })).sort((a, b) => b.files - a.files || a.id.localeCompare(b.id));
+
+  return {
+    corpus_pin: corpusPin(files),
+    files: files.length,
+    per_rule: perRule,
+    rules_with_findings: perRule.filter((r) => r.files > 0).length,
+    rejected_files: rejected,
+  };
+}
+
+function render(r: FpReport): string {
+  const lines: string[] = [];
+  lines.push(`prose-tell false-positive count — ${r.files} clean files`);
+  lines.push(`corpus pin: ${r.corpus_pin}`);
+  lines.push("");
+  lines.push("| Rule | Lang | Severity | M1 (files) |");
+  lines.push("|---|---|---|---|");
+  for (const rule of r.per_rule) {
+    lines.push(`| ${rule.id} | ${rule.language} | ${rule.severity} | ${rule.files} |`);
+  }
+  lines.push("");
+  lines.push(`rules with a non-zero M1: ${r.rules_with_findings} / ${r.per_rule.length}`);
+  lines.push(`clean files the shipped thresholds reject: ${r.rejected_files.length} / ${r.files}`);
+  for (const f of r.rejected_files) lines.push(`  ❌ ${f.path} — ${f.reasons.join("; ")}`);
+  for (const rule of r.per_rule.filter((x) => x.files > 0)) {
+    lines.push(`  · ${rule.id} hit: ${rule.hits.join(", ")}`);
+  }
+  return lines.join("\n");
+}
+
+function main(): void {
+  const args = process.argv.slice(2);
+  const json = args.includes("--json");
+  const gateAt = args.indexOf("--gate");
+  const gated = gateAt === -1 ? [] : args.slice(gateAt + 1).filter((a) => !a.startsWith("--"));
+
+  const files = loadCleanCorpus();
+  if (files.length === 0) {
+    console.error(`no clean corpus at ${CLEAN_DIR} — nothing to measure`);
+    process.exit(2);
+  }
+  const report = measure(files);
+  process.stdout.write((json ? JSON.stringify(report, null, 2) : render(report)) + "\n");
+
+  if (gated.length > 0) {
+    const known = new Set(ALL_TELL_RULES.map((r) => r.id));
+    const unknown = gated.filter((g) => !known.has(g));
+    if (unknown.length > 0) {
+      console.error(`--gate names no such rule: ${unknown.join(", ")}`);
+      process.exit(2);
+    }
+    const failing = report.per_rule.filter((r) => gated.includes(r.id) && r.files > 0);
+    if (failing.length > 0) {
+      for (const f of failing) {
+        console.error(
+          `promotion refused: ${f.id} has ${f.files} clean-corpus false positive(s) — ${f.hits.join(", ")}`,
+        );
+      }
+      process.exit(1);
+    }
+    process.stdout.write(`promotion gate clean for: ${gated.join(", ")}\n`);
+  }
+  process.exit(0);
+}
+
+const invokedDirectly =
+  process.argv[1] !== undefined &&
+  import.meta.url.endsWith(process.argv[1].split("/").pop() ?? "");
+if (invokedDirectly) main();