@event4u/agent-config 7.2.0 → 7.4.0

package/src/scripts/bench_rule_load_latency.ts

@@ -0,0 +1,178 @@
+#!/usr/bin/env tsx
+/**
+ * On-demand rule-load latency micro-benchmark (token-saving Phase 0).
+ *
+ * Under thin projection, a non-kernel rule body is a router-resolved pointer
+ * that the agent loads ON DEMAND when the rule fires. This measures the
+ * mechanical cost the suite controls: resolve the rule id → read its body file
+ * from `dist/agent-src/rules/<id>.md`. It reports the TAIL (p50/p95/p99), not
+ * just the mean — a fat tail on pointer resolution would erode the thin-
+ * projection win.
+ *
+ * Honest scope: this is the file-IO + locate cost, NOT the live-agent
+ * round-trip (model re-reading context). It bounds the part the package owns;
+ * it does not model host latency.
+ *
+ * Kernel rules are always-loaded (never on-demand), so they are excluded.
+ *
+ * CLI:
+ *   ./scripts-run src/scripts/bench_rule_load_latency
+ *   ./scripts-run src/scripts/bench_rule_load_latency --iterations 100 --json
+ *   ./scripts-run src/scripts/bench_rule_load_latency --write   # → internal/bench/reports/rule-load-latency.json
+ *
+ * Exit codes: 0 measured · 1 file error. Measurement only — never gates.
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { performance } from 'node:perf_hooks';
+import process from 'node:process';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const _HERE = fileURLToPath(import.meta.url);
+const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
+const RULES_SOURCE = path.join(REPO_ROOT, 'dist/agent-src/rules');
+const ROUTER = path.join(REPO_ROOT, 'dist/router.json');
+const REPORT = path.join(REPO_ROOT, 'internal/bench/reports/rule-load-latency.json');
+
+type Json = Record<string, unknown>;
+
+/** Non-kernel (on-demand) rule ids from the router: tier_1 + tier_2. */
+export function on_demand_rule_ids(router: Json): string[] {
+  const ids: string[] = [];
+  for (const key of ['tier_1', 'tier_2']) {
+    const arr = router[key];
+    if (!Array.isArray(arr)) continue;
+    for (const entry of arr) {
+      if (typeof entry === 'string') ids.push(entry);
+      else if (entry && typeof entry === 'object' && typeof (entry as Json).id === 'string') {
+        ids.push((entry as Json).id as string);
+      }
+    }
+  }
+  return ids;
+}
+
+export interface Summary {
+  count: number;
+  mean_ms: number;
+  p50_ms: number;
+  p95_ms: number;
+  p99_ms: number;
+  max_ms: number;
+  min_ms: number;
+}
+
+/** Nearest-rank percentile over an ascending-sorted sample (p in [0,100]). */
+export function percentile(sortedAsc: number[], p: number): number {
+  if (sortedAsc.length === 0) return 0;
+  const rank = Math.ceil((p / 100) * sortedAsc.length);
+  const idx = Math.min(sortedAsc.length - 1, Math.max(0, rank - 1));
+  return sortedAsc[idx] ?? 0;
+}
+
+export function summarize(samples: number[]): Summary {
+  if (samples.length === 0) {
+    return { count: 0, mean_ms: 0, p50_ms: 0, p95_ms: 0, p99_ms: 0, max_ms: 0, min_ms: 0 };
+  }
+  const sorted = [...samples].sort((a, b) => a - b);
+  const sum = sorted.reduce((a, b) => a + b, 0);
+  const round = (x: number) => Math.round(x * 1000) / 1000; // µs precision
+  return {
+    count: sorted.length,
+    mean_ms: round(sum / sorted.length),
+    p50_ms: round(percentile(sorted, 50)),
+    p95_ms: round(percentile(sorted, 95)),
+    p99_ms: round(percentile(sorted, 99)),
+    max_ms: round(sorted[sorted.length - 1] ?? 0),
+    min_ms: round(sorted[0] ?? 0),
+  };
+}
+
+function main(argv: string[]): number {
+  let iterations = 50;
+  let asJson = false;
+  let write = false;
+  for (let i = 0; i < argv.length; i += 1) {
+    const a = argv[i];
+    if (a === '--json') asJson = true;
+    else if (a === '--write') write = true;
+    else if (a === '--iterations') {
+      const v = Number(argv[(i += 1)]);
+      if (!Number.isInteger(v) || v < 1) {
+        process.stderr.write('error: --iterations must be a positive integer\n');
+        return 1;
+      }
+      iterations = v;
+    } else if (a === '-h' || a === '--help') {
+      process.stdout.write('usage: bench_rule_load_latency [--iterations N] [--json] [--write]\n');
+      return 0;
+    } else {
+      process.stderr.write(`error: unknown argument: ${a}\n`);
+      return 1;
+    }
+  }
+
+  let router: Json;
+  try {
+    router = JSON.parse(fs.readFileSync(ROUTER, 'utf-8')) as Json;
+  } catch {
+    process.stderr.write(`error: cannot read ${path.relative(REPO_ROOT, ROUTER)} — run \`task sync\` first\n`);
+    return 1;
+  }
+  const ids = on_demand_rule_ids(router);
+  const paths: string[] = [];
+  for (const id of ids) {
+    const p = path.join(RULES_SOURCE, `${id}.md`);
+    if (fs.existsSync(p)) paths.push(p);
+  }
+  if (paths.length === 0) {
+    process.stderr.write('error: no on-demand rule bodies found under dist/agent-src/rules\n');
+    return 1;
+  }
+
+  // Warm-up pass (page the files in) so we measure steady-state, not cold cache.
+  for (const p of paths) fs.readFileSync(p, 'utf-8');
+
+  const samples: number[] = [];
+  for (let it = 0; it < iterations; it += 1) {
+    for (const p of paths) {
+      const t0 = performance.now();
+      fs.readFileSync(p, 'utf-8'); // resolve + read = the on-demand load
+      samples.push(performance.now() - t0);
+    }
+  }
+
+  const summary = summarize(samples);
+  const payload = {
+    measured_at: new Date().toISOString(),
+    rules_measured: paths.length,
+    iterations,
+    note: 'mechanical resolve+read latency of on-demand (non-kernel) rule bodies; excludes live-agent round-trip',
+    ...summary,
+  };
+
+  if (write) {
+    fs.writeFileSync(REPORT, JSON.stringify(payload, null, 2) + '\n');
+  }
+  if (asJson) {
+    process.stdout.write(JSON.stringify(payload, null, 2) + '\n');
+  } else {
+    process.stdout.write(
+      `on-demand rule load (${paths.length} rules × ${iterations} iters, ${summary.count} samples):\n` +
+        `  p50 ${summary.p50_ms}ms · p95 ${summary.p95_ms}ms · p99 ${summary.p99_ms}ms · ` +
+        `mean ${summary.mean_ms}ms · max ${summary.max_ms}ms\n` +
+        (write ? `→ wrote ${path.relative(REPO_ROOT, REPORT)}\n` : ''),
+    );
+  }
+  return 0;
+}
+
+const _IS_MAIN =
+  process.argv[1] !== undefined &&
+  import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href;
+
+if (_IS_MAIN) {
+  process.exit(main(process.argv.slice(2)));
+}
+
+export { main };

package/src/scripts/check_quality_regression.ts

@@ -0,0 +1,244 @@
+#!/usr/bin/env tsx
+/**
+ * Length-controlled paired judge + quality-regression gate (token-saving Phase 0).
+ *
+ * Proves OUTPUT QUALITY is held constant when a token lever changes the
+ * projection (thin vs eager). Hand-rolled on the suite's existing Wilcoxon
+ * layer (`bench_ab_v2_stats.wilcoxon`) — no `promptfoo` dependency (council
+ * 2026-06-26: the bias controls are custom either way; the package is lean).
+ *
+ * Bias controls (the part that makes an LLM judge trustworthy):
+ *   - **Position**: every pair is judged in BOTH orders (thin-first AND
+ *     eager-first). A verdict that does not survive the swap is `inconsistent`
+ *     and excluded — this is the "reject the judge if it flips" control, and it
+ *     subsumes order-randomisation (we are position-balanced by construction).
+ *   - **Length / verbosity**: per decisive win we record whether the winner was
+ *     also the LONGER answer; the aggregate surfaces the length-confound rate so
+ *     a win driven by verbosity bias (measured +17.3% in LLM judges) is visible.
+ *
+ * Two layers:
+ *   LIBRARY (here, unit-tested with a mock judge): `evaluatePair` / `aggregate`
+ *   / `gateVerdict` — the deterministic harness an operator's live runner calls.
+ *   GATE (CLI): reads a completed run report and fails if thin's win-rate < the
+ *   threshold (default 0.48). The live run (generate thin+eager answers, call a
+ *   judge model) is the operator/cost step — until its report exists the gate is
+ *   INERT (exit 0), so CI stays green.
+ *
+ * quality-run.json (produced by the operator's live runner via this library):
+ *   { "threshold": 0.48, "judge_model": "...",
+ *     "results": [ { "id", "winner": thin|eager|tie|inconsistent,
+ *                    "length_delta": <int>, "winner_is_longer": bool|null } ] }
+ *
+ * CLI:
+ *   ./scripts-run src/scripts/check_quality_regression            # gate (inert w/o run data)
+ *   ./scripts-run src/scripts/check_quality_regression --json
+ *   ./scripts-run src/scripts/check_quality_regression --threshold 0.48
+ *
+ * Exit codes: 0 ok / inert · 1 file error · 2 quality regression (win-rate < threshold).
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import process from 'node:process';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+import { wilcoxon } from './bench_ab_v2_stats.js';
+
+const _HERE = fileURLToPath(import.meta.url);
+const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
+const RUN_REPORT = path.join(REPO_ROOT, 'internal/bench/reports/quality-run.json');
+const DEFAULT_THRESHOLD = 0.48;
+
+export type Arm = 'thin' | 'eager';
+/** A judge's preference between the two PRESENTED answers (order-relative). */
+export type JudgeVerdict = 'first' | 'second' | 'tie';
+/** The pluggable judge — a mock in tests, a model call in the live runner. */
+export type JudgeFn = (
+  ctx: { id: string; rubric: string },
+  first: string,
+  second: string,
+) => JudgeVerdict;
+
+export type PairWinner = 'thin' | 'eager' | 'tie' | 'inconsistent';
+
+export interface PairResult {
+  id: string;
+  winner: PairWinner;
+  length_delta: number; // chars(thin) - chars(eager)
+  winner_is_longer: boolean | null; // null when tie/inconsistent
+}
+
+/**
+ * Judge one (thin, eager) answer pair in BOTH orders and resolve the winner.
+ * A verdict that flips with order is `inconsistent` (position bias → excluded).
+ */
+export function evaluatePair(
+  task: { id: string; rubric: string },
+  answerThin: string,
+  answerEager: string,
+  judge: JudgeFn,
+): PairResult {
+  const v1 = judge(task, answerThin, answerEager); // first=thin, second=eager
+  const v2 = judge(task, answerEager, answerThin); // first=eager, second=thin
+  const w1: PairWinner = v1 === 'first' ? 'thin' : v1 === 'second' ? 'eager' : 'tie';
+  const w2: PairWinner = v2 === 'second' ? 'thin' : v2 === 'first' ? 'eager' : 'tie';
+
+  const length_delta = answerThin.length - answerEager.length;
+  let winner: PairWinner;
+  if (w1 === 'tie' && w2 === 'tie') {
+    winner = 'tie';
+  } else if (w1 === w2) {
+    winner = w1; // consistent decisive win (thin or eager)
+  } else {
+    winner = 'inconsistent'; // flipped or one-tie-one-decisive → not robust
+  }
+
+  let winner_is_longer: boolean | null = null;
+  if (winner === 'thin') winner_is_longer = length_delta > 0;
+  else if (winner === 'eager') winner_is_longer = length_delta < 0;
+
+  return { id: task.id, winner, length_delta, winner_is_longer };
+}
+
+export interface QualityAggregate {
+  total: number;
+  thin_wins: number;
+  eager_wins: number;
+  ties: number;
+  inconsistent: number;
+  decisive: number;
+  thin_win_rate: number | null; // thin_wins / decisive
+  inconsistency_rate: number | null; // inconsistent / total (judge-reliability health)
+  length_confound_rate: number | null; // share of decisive wins where winner was longer
+  wilcoxon_p: number | null;
+  verdict: 'ok' | 'regression' | 'no-data';
+}
+
+/** Aggregate per-pair results into a win-rate + bias diagnostics + Wilcoxon. */
+export function aggregate(results: PairResult[], threshold = DEFAULT_THRESHOLD): QualityAggregate {
+  const total = results.length;
+  let thin_wins = 0;
+  let eager_wins = 0;
+  let ties = 0;
+  let inconsistent = 0;
+  let longerWins = 0;
+  const diffs: number[] = [];
+  for (const r of results) {
+    if (r.winner === 'thin') {
+      thin_wins += 1;
+      diffs.push(1);
+      if (r.winner_is_longer) longerWins += 1;
+    } else if (r.winner === 'eager') {
+      eager_wins += 1;
+      diffs.push(-1);
+      if (r.winner_is_longer) longerWins += 1;
+    } else if (r.winner === 'tie') {
+      ties += 1;
+      diffs.push(0);
+    } else {
+      inconsistent += 1;
+      diffs.push(0);
+    }
+  }
+  const decisive = thin_wins + eager_wins;
+  const thin_win_rate = decisive > 0 ? thin_wins / decisive : null;
+  const wil = wilcoxon(diffs);
+  const verdict: QualityAggregate['verdict'] =
+    thin_win_rate === null ? 'no-data' : thin_win_rate < threshold ? 'regression' : 'ok';
+  return {
+    total,
+    thin_wins,
+    eager_wins,
+    ties,
+    inconsistent,
+    decisive,
+    thin_win_rate,
+    inconsistency_rate: total > 0 ? inconsistent / total : null,
+    length_confound_rate: decisive > 0 ? longerWins / decisive : null,
+    wilcoxon_p: wil.n > 0 ? wil.p : null,
+    verdict,
+  };
+}
+
+/** Gate exit code from an aggregate. */
+export function gateVerdict(agg: QualityAggregate): 0 | 2 {
+  return agg.verdict === 'regression' ? 2 : 0;
+}
+
+interface RunReport {
+  threshold?: number;
+  results?: PairResult[];
+}
+
+function main(argv: string[]): number {
+  let asJson = false;
+  let threshold = DEFAULT_THRESHOLD;
+  for (let i = 0; i < argv.length; i += 1) {
+    const a = argv[i];
+    if (a === '--json') asJson = true;
+    else if (a === '--threshold') {
+      const v = Number(argv[(i += 1)]);
+      if (!Number.isFinite(v) || v < 0 || v > 1) {
+        process.stderr.write('error: --threshold must be in [0,1]\n');
+        return 1;
+      }
+      threshold = v;
+    } else if (a === '-h' || a === '--help') {
+      process.stdout.write('usage: check_quality_regression [--json] [--threshold F]\n');
+      return 0;
+    } else {
+      process.stderr.write(`error: unknown argument: ${a}\n`);
+      return 1;
+    }
+  }
+
+  if (!fs.existsSync(RUN_REPORT)) {
+    const msg =
+      `⚠️  no quality-run report (${path.relative(REPO_ROOT, RUN_REPORT)}) — ` +
+      'gate inert. Produce it with a thin-vs-eager judge run (operator/cost gate).';
+    if (asJson) process.stdout.write(JSON.stringify({ verdict: 'no-data', inert: true }, null, 2) + '\n');
+    else process.stdout.write(msg + '\n');
+    return 0;
+  }
+
+  let report: RunReport;
+  try {
+    report = JSON.parse(fs.readFileSync(RUN_REPORT, 'utf-8')) as RunReport;
+  } catch (e) {
+    process.stderr.write(`error: cannot parse ${path.relative(REPO_ROOT, RUN_REPORT)}: ${(e as Error).message}\n`);
+    return 1;
+  }
+  const results = Array.isArray(report.results) ? report.results : [];
+  const thr = typeof report.threshold === 'number' ? report.threshold : threshold;
+  const agg = aggregate(results, thr);
+
+  if (asJson) {
+    process.stdout.write(JSON.stringify({ ...agg, threshold: thr }, null, 2) + '\n');
+  } else {
+    process.stdout.write(
+      `quality run: ${agg.total} pairs · thin ${agg.thin_wins} / eager ${agg.eager_wins} / ` +
+        `tie ${agg.ties} / inconsistent ${agg.inconsistent}\n` +
+        `  thin win-rate ${agg.thin_win_rate === null ? 'n/a' : (agg.thin_win_rate * 100).toFixed(1) + '%'} ` +
+        `(threshold ${(thr * 100).toFixed(0)}%) · ` +
+        `length-confound ${agg.length_confound_rate === null ? 'n/a' : (agg.length_confound_rate * 100).toFixed(0) + '%'} · ` +
+        `inconsistency ${agg.inconsistency_rate === null ? 'n/a' : (agg.inconsistency_rate * 100).toFixed(0) + '%'}\n`,
+    );
+    if (agg.verdict === 'regression') {
+      process.stdout.write(`❌  quality regression: thin win-rate below ${(thr * 100).toFixed(0)}%.\n`);
+    } else if (agg.verdict === 'no-data') {
+      process.stdout.write(`⚠️  no decisive pairs — inconclusive.\n`);
+    } else {
+      process.stdout.write(`✅  quality held: thin win-rate within tolerance.\n`);
+    }
+  }
+  return gateVerdict(agg);
+}
+
+const _IS_MAIN =
+  process.argv[1] !== undefined &&
+  import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href;
+
+if (_IS_MAIN) {
+  process.exit(main(process.argv.slice(2)));
+}
+
+export { main };

package/src/scripts/check_token_quality_golden.ts

@@ -0,0 +1,253 @@
+#!/usr/bin/env tsx
+/**
+ * Validator + coverage reporter for the token-quality golden set
+ * (token-saving Phase 0). Loads `internal/bench/corpora/token-quality-golden.yaml`,
+ * validates each task against the schema (TOKEN-QUALITY-GOLDEN-SCHEMA.md), and
+ * reports rule-coverage vs `dist/router.json` + label-status counts.
+ *
+ * The `expected` anchors are HAND-LABELLED operator work — so an unfilled
+ * (`stub`) task or an uncovered rule is REPORTED, not a failure, by default.
+ * Structural errors (unknown rule id, bad scenario, a `labelled` task with a
+ * TODO rubric / no anchors) always fail. `--require-complete` is the operator-
+ * completion gate (Phase 0 exit): it also fails on any remaining stub, any
+ * uncovered rule, or a missing required scenario.
+ *
+ * CLI:
+ *   ./scripts-run src/scripts/check_token_quality_golden
+ *   ./scripts-run src/scripts/check_token_quality_golden --json
+ *   ./scripts-run src/scripts/check_token_quality_golden --require-complete
+ *
+ * Exit codes: 0 valid (scaffold or complete) · 1 file error · 2 invalid / incomplete-when-required.
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import process from 'node:process';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+import yaml from 'js-yaml';
+
+const _HERE = fileURLToPath(import.meta.url);
+const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
+const CORPUS = path.join(REPO_ROOT, 'internal/bench/corpora/token-quality-golden.yaml');
+const ROUTER = path.join(REPO_ROOT, 'dist/router.json');
+
+const SCENARIOS = ['single', 'multi-turn', 'conflicting-rule', 'corner-case'];
+const REQUIRED_SCENARIOS = ['multi-turn', 'conflicting-rule', 'corner-case'];
+const LABEL_STATUS = ['stub', 'labelled'];
+
+type Json = Record<string, unknown>;
+
+/** Collect every rule id from the router (kernel strings + tier_* objects). */
+export function router_rule_ids(router: Json): Set<string> {
+  const ids = new Set<string>();
+  for (const key of ['kernel', 'tier_1', 'tier_2']) {
+    const arr = router[key];
+    if (!Array.isArray(arr)) continue;
+    for (const entry of arr) {
+      if (typeof entry === 'string') ids.add(entry);
+      else if (entry && typeof entry === 'object' && typeof (entry as Json).id === 'string') {
+        ids.add((entry as Json).id as string);
+      }
+    }
+  }
+  return ids;
+}
+
+export interface GoldenReport {
+  ok: boolean;
+  errors: string[];
+  task_count: number;
+  labelled: number;
+  stubs: number;
+  covered: number;
+  uncovered: string[];
+  scenarios: Record<string, number>;
+  missing_scenarios: string[];
+}
+
+function _isTodoRubric(rubric: unknown): boolean {
+  return (
+    typeof rubric !== 'string' ||
+    rubric.trim() === '' ||
+    /^todo\b/i.test(rubric.trim())
+  );
+}
+
+/** Pure validation + coverage. `ruleIds` is the authoritative rule universe. */
+export function validate(corpus: Json, ruleIds: Set<string>): GoldenReport {
+  const errors: string[] = [];
+  const tasks = Array.isArray(corpus.tasks) ? (corpus.tasks as Json[]) : null;
+  if (corpus.corpus_id !== 'token-quality-golden') {
+    errors.push(`corpus_id must be "token-quality-golden" (got ${JSON.stringify(corpus.corpus_id)})`);
+  }
+  if (tasks === null) {
+    errors.push('`tasks` must be a list');
+    return {
+      ok: false,
+      errors,
+      task_count: 0,
+      labelled: 0,
+      stubs: 0,
+      covered: 0,
+      uncovered: [...ruleIds].sort(),
+      scenarios: {},
+      missing_scenarios: [...REQUIRED_SCENARIOS],
+    };
+  }
+
+  const seenIds = new Set<string>();
+  const covered = new Set<string>();
+  const scenarios: Record<string, number> = {};
+  let labelled = 0;
+  let stubs = 0;
+
+  for (const [i, task] of tasks.entries()) {
+    const where = `task[${i}]${typeof task.id === 'string' ? ` (${task.id})` : ''}`;
+    if (typeof task.id !== 'string' || !/^tq-[a-z0-9-]+-\d+$/.test(task.id)) {
+      errors.push(`${where}: id must match tq-<area>-NN`);
+    } else if (seenIds.has(task.id)) {
+      errors.push(`${where}: duplicate id`);
+    } else {
+      seenIds.add(task.id);
+    }
+
+    const rules = Array.isArray(task.rules) ? (task.rules as unknown[]) : [];
+    if (rules.length === 0) errors.push(`${where}: rules must be a non-empty list`);
+    for (const r of rules) {
+      if (typeof r !== 'string' || !ruleIds.has(r)) {
+        errors.push(`${where}: unknown rule id ${JSON.stringify(r)}`);
+      } else {
+        covered.add(r);
+      }
+    }
+
+    if (typeof task.scenario !== 'string' || !SCENARIOS.includes(task.scenario)) {
+      errors.push(`${where}: scenario must be one of ${SCENARIOS.join(' | ')}`);
+    } else {
+      scenarios[task.scenario] = (scenarios[task.scenario] ?? 0) + 1;
+    }
+
+    if (typeof task.prompt !== 'string' || task.prompt.trim() === '') {
+      errors.push(`${where}: prompt must be a non-empty string`);
+    }
+
+    const status = task.label_status;
+    if (typeof status !== 'string' || !LABEL_STATUS.includes(status)) {
+      errors.push(`${where}: label_status must be one of ${LABEL_STATUS.join(' | ')}`);
+    } else if (status === 'labelled') {
+      labelled += 1;
+    } else {
+      stubs += 1;
+    }
+
+    const expected = (task.expected ?? {}) as Json;
+    const mustInclude = Array.isArray(expected.must_include) ? expected.must_include : [];
+    // A LABELLED task must carry a real rubric + ≥1 anchor; a stub need not.
+    if (status === 'labelled') {
+      if (_isTodoRubric(expected.rubric)) {
+        errors.push(`${where}: labelled task has a TODO/empty rubric`);
+      }
+      if (mustInclude.length === 0) {
+        errors.push(`${where}: labelled task needs ≥1 expected.must_include anchor`);
+      }
+    }
+  }
+
+  const uncovered = [...ruleIds].filter((r) => !covered.has(r)).sort();
+  const missing_scenarios = REQUIRED_SCENARIOS.filter((s) => (scenarios[s] ?? 0) === 0);
+
+  return {
+    ok: errors.length === 0,
+    errors,
+    task_count: tasks.length,
+    labelled,
+    stubs,
+    covered: covered.size,
+    uncovered,
+    scenarios,
+    missing_scenarios,
+  };
+}
+
+function _readJson(file: string): Json | null {
+  try {
+    return JSON.parse(fs.readFileSync(file, 'utf-8')) as Json;
+  } catch {
+    return null;
+  }
+}
+
+function main(argv: string[]): number {
+  const asJson = argv.includes('--json');
+  const requireComplete = argv.includes('--require-complete');
+  for (const a of argv) {
+    if (!['--json', '--require-complete', '-h', '--help'].includes(a)) {
+      process.stderr.write(`error: unknown argument: ${a}\n`);
+      return 1;
+    }
+    if (a === '-h' || a === '--help') {
+      process.stdout.write('usage: check_token_quality_golden [--json] [--require-complete]\n');
+      return 0;
+    }
+  }
+
+  let corpus: Json | null;
+  try {
+    corpus = yaml.load(fs.readFileSync(CORPUS, 'utf-8')) as Json;
+  } catch (e) {
+    process.stderr.write(`error: cannot read/parse ${path.relative(REPO_ROOT, CORPUS)}: ${(e as Error).message}\n`);
+    return 1;
+  }
+  const router = _readJson(ROUTER);
+  if (router === null) {
+    process.stderr.write(`error: cannot read ${path.relative(REPO_ROOT, ROUTER)} — run \`task sync\` first\n`);
+    return 1;
+  }
+  const ruleIds = router_rule_ids(router);
+  const report = validate(corpus, ruleIds);
+
+  const incomplete =
+    report.stubs > 0 || report.uncovered.length > 0 || report.missing_scenarios.length > 0;
+
+  if (asJson) {
+    process.stdout.write(JSON.stringify({ ...report, require_complete: requireComplete, incomplete }, null, 2) + '\n');
+  } else {
+    for (const e of report.errors) process.stdout.write(`❌  ${e}\n`);
+    process.stdout.write(
+      `golden set: ${report.task_count} task(s) · ${report.labelled} labelled · ${report.stubs} stub · ` +
+        `coverage ${report.covered}/${ruleIds.size} rules\n`,
+    );
+    if (report.uncovered.length > 0) {
+      process.stdout.write(`⚠️  ${report.uncovered.length} rule(s) uncovered (operator to add tasks)\n`);
+    }
+    if (report.missing_scenarios.length > 0) {
+      process.stdout.write(`⚠️  missing required scenario(s): ${report.missing_scenarios.join(', ')}\n`);
+    }
+    if (!report.ok) {
+      process.stdout.write(`❌  golden set has structural errors (see above).\n`);
+    } else if (incomplete) {
+      process.stdout.write(`✅  structurally valid — scaffold awaiting operator labels.\n`);
+    } else {
+      process.stdout.write(`✅  golden set complete.\n`);
+    }
+  }
+
+  if (!report.ok) return 2;
+  if (requireComplete && incomplete) {
+    if (!asJson) {
+      process.stdout.write(`❌  --require-complete: golden set is not yet complete.\n`);
+    }
+    return 2;
+  }
+  return 0;
+}
+
+const _IS_MAIN =
+  process.argv[1] !== undefined &&
+  import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href;
+
+if (_IS_MAIN) {
+  process.exit(main(process.argv.slice(2)));
+}
+
+export { main };