@event4u/agent-config 12.1.0 → 13.0.0

package/src/scripts/bench_ab_v2_complexity.ts

@@ -0,0 +1,252 @@
+#!/usr/bin/env -S npx tsx
+/**
+ * Offline re-scorer for the Phase-3 metric pair — S0.3 delta #11.
+ *
+ * Reads a finished `ab-v2` report, walks every trial's **preserved workspace**
+ * (delta #7 keyed `task|arm|seed`), and writes two endpoints into each trial's
+ * `metrics`:
+ *
+ *   - `added_lines` — T1's endpoint. Added only, not added+removed: the
+ *     discipline oracle's `max_lines_changed` wants the sum, the size claim
+ *     wants what the run *wrote*.
+ *   - `median_cognitive_complexity` — T2's endpoint, over the functions in the
+ *     files that trial actually changed.
+ *
+ * WHY OFFLINE RATHER THAN IN THE RUNNER. Three reasons, in order of weight.
+ * The metric is **retro-fittable**: the workspaces from every completed sweep
+ * are on disk, so a run that already cost money can gain the endpoint without
+ * being re-run — which is what makes the anti-golfing gate nearly free, and what
+ * the roadmap's own text promises. It is **deterministic and model-free**, so
+ * re-scoring the same workspace twice cannot produce two numbers. And the
+ * grammar load is **async** while `trajectory_metrics` is synchronous and called
+ * from four places — threading async through the live path would be a larger
+ * diff than the endpoint itself, for no measurement it does not already get.
+ *
+ * The report is rewritten in place only with `--write`; the default prints the
+ * table and touches nothing, because silently rewriting a pinned artefact is how
+ * a report stops matching the numbers already quoted from it.
+ *
+ * Usage:
+ *   bench_ab_v2_complexity.ts <report.json> [--write] [--json]
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { parse as parseYaml } from 'yaml';
+
+import { medianComplexityPerChangedFunction } from './_lib/bench_ab_complexity.js';
+import { changed_files, diff_line_counts } from './_lib/bench_ab_scoring_v2.js';
+
+const _HERE = fileURLToPath(import.meta.url);
+const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
+const FIXTURES_ROOT = path.join(REPO_ROOT, 'internal', 'bench', 'ab');
+const CORPUS_PATH = path.join(REPO_ROOT, 'internal', 'bench', 'corpora', 'ab-trackb-v2.yaml');
+
+type Dict = Record<string, unknown>;
+
+function _dictOr(v: unknown): Dict {
+    return v && typeof v === 'object' && !Array.isArray(v) ? (v as Dict) : {};
+}
+
+/**
+ * task id → fixture path. The corpus is the ONLY resolver, deliberately.
+ *
+ * An earlier draft also had the runner stamp a `fixture` key onto each record so
+ * reports would be self-describing, and this function read it first. That is two
+ * mechanisms for one fact: the corpus lookup already covers every report,
+ * including every one written before the key would have existed. The extra field
+ * was dropped rather than kept as a fallback — a reader with no producer is the
+ * shape this branch's own completion review flagged elsewhere.
+ *
+ * A missing or unreadable corpus is not an error: it degrades every affected
+ * trial to `skipped_reason: 'task id not in the corpus'`, which is a reportable
+ * state, rather than aborting a re-score that may still cover most of the report.
+ */
+function loadCorpusFixtures(corpusPath: string | null): Map<string, string> {
+    const byId = new Map<string, string>();
+    if (!corpusPath || !fs.existsSync(corpusPath)) return byId;
+    try {
+        const corpus = _dictOr(parseYaml(fs.readFileSync(corpusPath, 'utf8'), { version: '1.1' }));
+        const tasks = Array.isArray(corpus['tasks']) ? (corpus['tasks'] as Dict[]) : [];
+        for (const t of tasks) {
+            if (t['id'] !== undefined && t['fixture'] !== undefined) {
+                byId.set(String(t['id']), String(t['fixture']));
+            }
+        }
+    } catch {
+        return byId;
+    }
+    return byId;
+}
+
+export interface TrialRescore {
+    task: string;
+    arm: string;
+    seed: unknown;
+    /** `null` when the workspace or the fixture is no longer on disk. */
+    added_lines: number | null;
+    median_cognitive_complexity: number | null;
+    n_functions: number;
+    unsupported_files: string[];
+    skipped_reason: string | null;
+}
+
+/**
+ * Re-score one report payload. Pure apart from the filesystem reads it needs;
+ * mutates `payload` only when `write` is true. The fixture each trial is diffed
+ * against is resolved from the corpus by task id — see `loadCorpusFixtures`.
+ */
+export async function rescoreReport(
+    payload: Dict,
+    opts: { fixturesRoot?: string; write?: boolean; corpusPath?: string | null } = {},
+): Promise<TrialRescore[]> {
+    const fixturesRoot = opts.fixturesRoot ?? FIXTURES_ROOT;
+    let wrote = 0;
+    const byId = loadCorpusFixtures(opts.corpusPath === undefined ? CORPUS_PATH : opts.corpusPath);
+    const out: TrialRescore[] = [];
+    const records = Array.isArray(payload['records']) ? (payload['records'] as Dict[]) : [];
+
+    for (const rec of records) {
+        const taskId = String(rec['id'] ?? '<unknown>');
+        const fixtureRel = byId.get(taskId) ?? null;
+        const arms = _dictOr(rec['arms']);
+        for (const [arm, runsRaw] of Object.entries(arms)) {
+            const runs = Array.isArray(runsRaw) ? (runsRaw as Dict[]) : [];
+            for (const run of runs) {
+                const row: TrialRescore = {
+                    task: taskId,
+                    arm,
+                    seed: run['seed'] ?? null,
+                    added_lines: null,
+                    median_cognitive_complexity: null,
+                    n_functions: 0,
+                    unsupported_files: [],
+                    skipped_reason: null,
+                };
+
+                const workspace = run['workspace'] !== undefined ? String(run['workspace']) : '';
+                if (!workspace || !fs.existsSync(workspace)) {
+                    // Not a zero — a trial whose workspace was pruned is
+                    // unmeasurable, and `compare()` reads a missing metric as
+                    // "not measured on this pair" rather than as a value.
+                    row.skipped_reason = workspace ? 'workspace missing on disk' : 'no workspace recorded';
+                    out.push(row);
+                    continue;
+                }
+                if (fixtureRel === null) {
+                    row.skipped_reason = 'task id not in the corpus';
+                    out.push(row);
+                    continue;
+                }
+                const fixture = path.join(fixturesRoot, fixtureRel);
+                if (!fs.existsSync(fixture)) {
+                    row.skipped_reason = 'fixture missing on disk';
+                    out.push(row);
+                    continue;
+                }
+
+                const changed = changed_files(fixture, workspace);
+                row.added_lines = diff_line_counts(fixture, workspace, changed).added;
+                const rollup = await medianComplexityPerChangedFunction(workspace, changed);
+                row.median_cognitive_complexity = rollup.median;
+                row.n_functions = rollup.n_functions;
+                row.unsupported_files = rollup.unsupported_files;
+
+                if (opts.write) {
+                    const metrics = _dictOr(run['metrics']);
+                    metrics['added_lines'] = row.added_lines;
+                    // `median` is null when the diff contained no scorable
+                    // function, and null is written on purpose: `compare()` reads
+                    // a null as "not measured on this pair", while a 0 would claim
+                    // the run produced maximally simple code. The count sits
+                    // beside it so a reader can tell "nothing to score" from
+                    // "scored, and it was trivial".
+                    metrics['median_cognitive_complexity'] = row.median_cognitive_complexity;
+                    metrics['complexity_n_functions'] = row.n_functions;
+                    run['metrics'] = metrics;
+                    wrote += 1;
+                }
+                out.push(row);
+            }
+        }
+    }
+    if (opts.write) lastWriteCount = wrote;
+    return out;
+}
+
+/**
+ * How many trials the last `rescoreReport({ write: true })` actually touched.
+ *
+ * The CLI uses it to skip rewriting the report when nothing was measured:
+ * re-serialising a pinned artefact for a zero-row pass changes its bytes (and
+ * its formatting) without changing a single number, which is exactly the kind of
+ * silent churn that makes a report stop matching the figures quoted from it.
+ */
+let lastWriteCount = 0;
+export function trialsWrittenByLastRescore(): number {
+    return lastWriteCount;
+}
+
+function renderTable(rows: TrialRescore[]): string {
+    const L: string[] = [];
+    L.push('task | arm | seed | added | median cc | fns | note');
+    L.push('-----|-----|------|-------|-----------|-----|-----');
+    for (const r of rows) {
+        L.push(
+            [
+                r.task,
+                r.arm,
+                String(r.seed ?? ''),
+                r.added_lines === null ? '-' : String(r.added_lines),
+                r.median_cognitive_complexity === null ? '-' : String(r.median_cognitive_complexity),
+                String(r.n_functions),
+                r.skipped_reason ??
+                    (r.unsupported_files.length > 0
+                        ? `unsupported: ${r.unsupported_files.join(',')}`
+                        : ''),
+            ].join(' | '),
+        );
+    }
+    const measured = rows.filter((r) => r.median_cognitive_complexity !== null).length;
+    L.push('');
+    L.push(`${measured}/${rows.length} trials carry a complexity observation.`);
+    return L.join('\n');
+}
+
+async function main(argv: string[]): Promise<number> {
+    const args = argv.filter((a) => !a.startsWith('--'));
+    const write = argv.includes('--write');
+    const asJson = argv.includes('--json');
+    const reportPath = args[0];
+    if (!reportPath) {
+        process.stderr.write(
+            'usage: bench_ab_v2_complexity.ts <report.json> [--write] [--json]\n',
+        );
+        return 1;
+    }
+    if (!fs.existsSync(reportPath)) {
+        process.stderr.write(`report not found: ${reportPath}\n`);
+        return 1;
+    }
+    const payload = JSON.parse(fs.readFileSync(reportPath, 'utf8')) as Dict;
+    const rows = await rescoreReport(payload, { write });
+    const written = trialsWrittenByLastRescore();
+    if (write && written > 0) {
+        fs.writeFileSync(reportPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
+    } else if (write) {
+        process.stderr.write('no trial could be measured — report left untouched\n');
+    }
+    process.stdout.write(asJson ? `${JSON.stringify(rows, null, 2)}\n` : `${renderTable(rows)}\n`);
+    return 0;
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === path.resolve(_HERE)) {
+    main(process.argv.slice(2)).then(
+        (code) => process.exit(code),
+        (err: unknown) => {
+            process.stderr.write(`${String(err)}\n`);
+            process.exit(1);
+        },
+    );
+}

package/src/scripts/bench_ab_v2_run.ts

@@ -45,7 +45,6 @@
 import { spawnSync } from 'node:child_process';
 import { createHash } from 'node:crypto';
 import * as fs from 'node:fs';
-import * as os from 'node:os';
 import * as path from 'node:path';
 import process from 'node:process';
 import { fileURLToPath, pathToFileURL } from 'node:url';
@@ -53,6 +52,7 @@ import { parse as parseYaml } from 'yaml';
 
 import * as v1 from './bench_ab_task_runner.js';
 import * as scoring from './_lib/bench_ab_scoring_v2.js';
+import { preserve_transcript, reset_fixture } from './_lib/bench_ab_workspace.js';
 import {
     SweepBudget,
     activation_verdict,
@@ -75,7 +75,6 @@ const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
 const ROUTER_PATH = path.join(REPO_ROOT, 'dist', 'router.json');
 const RULES_DIR = path.join(REPO_ROOT, 'dist', 'agent-src', 'rules');
 const CORPUS_PATH = path.join(REPO_ROOT, 'internal', 'bench', 'corpora', 'ab-trackb-v2.yaml');
-const FIXTURES_ROOT = path.join(REPO_ROOT, 'internal', 'bench', 'ab');
 const REPORTS_DIR = path.join(REPO_ROOT, 'internal', 'bench', 'reports', 'ab-v2');
 const CHECKPOINTS_DIR = path.join(REPORTS_DIR, 'checkpoints');
 const PRICING_PATH = path.join(REPO_ROOT, 'internal', 'bench', 'pricing.yaml');
@@ -87,7 +86,6 @@ const PRICING_PATH = path.join(REPO_ROOT, 'internal', 'bench', 'pricing.yaml');
 // vanilla ≈ package and invalidated every prior null. A /tmp clone has no
 // agent-config ancestor → vanilla is truly plain; `package` still activates via
 // the USER-scope global plugin regardless of cwd.
-const WORK_ROOT = path.join(os.tmpdir(), 'agent-config-bench-v2-clones');
 
 type Dict = Record<string, unknown>;
 
@@ -329,45 +327,6 @@ export function injected_text(inject: string | null, placebo_chars: number): str
     return null;
 }
 
-/**
- * Workspace directory for one trial — delta #7 of the S0.3 spike.
- *
- * Keyed by `task|arm|seed`, not by task alone. The old task-only key meant every
- * arm and every seed of a task reused ONE directory that the next trial deleted,
- * so at the end of a sweep exactly one workspace survived per task — the last one
- * written, with no record of which arm or seed it belonged to.
- *
- * That is not a tidiness problem. Phase 3's anti-golfing gate is specified as
- * retro-fittable onto already-completed runs by offline re-scoring, and the
- * roadmap calls that gate cheap *because* the workspaces are preserved. Under the
- * old key there was nothing to re-score, so the claim was false. A distinct
- * directory per trial is what makes it true.
- *
- * `arm` is sanitised because arm names are used verbatim as a path segment and
- * one of them would otherwise be free to escape the root.
- */
-export function workspace_dir(task_id: string, arm: string, seed: number): string {
-    const safe_arm = arm.replace(/[^A-Za-z0-9._-]/g, '_');
-    return path.join(WORK_ROOT, `${task_id}__${safe_arm}__seed${seed}`);
-}
-
-/**
- * Copy the task's pristine fixture into this trial's own working clone.
- *
- * The clone is re-created from the fixture on every call, so a resumed or
- * repeated trial still starts pristine — the per-trial key changes *which*
- * directory that is, never whether it is clean.
- */
-export function reset_fixture(task: Dict, arm: string, seed: number): [string, string] {
-    const fixture = path.join(FIXTURES_ROOT, String(task['fixture']));
-    const dest = workspace_dir(String(task['id']), arm, seed);
-    if (fs.existsSync(dest)) {
-        fs.rmSync(dest, { recursive: true, force: true });
-    }
-    fs.mkdirSync(path.dirname(dest), { recursive: true });
-    fs.cpSync(fixture, dest, { recursive: true });
-    return [dest, fixture];
-}
 
 /** Map a run outcome to an AgentBench-style trajectory bucket. */
 export function status_bucket(run: Dict): string {
@@ -689,6 +648,7 @@ export function run_one(task: Dict, arm: string, opts: RunOneOpts): Dict {
         // rather than recomputed later, so an offline re-scorer reads the path the
         // sweep actually used instead of re-deriving a key that may have changed.
         workspace: clone,
+        transcript_path: preserve_transcript(clone, String(run['transcript'] ?? '')),
         ...integrity_fields(run, spec, sp_text ? sp_text.length : 0, opts.model),
     };
 }
@@ -803,6 +763,7 @@ export function selftest_run(
         metrics: trajectory_metrics(run, score),
         injected_chars: sp_text ? sp_text.length : 0,
         workspace: clone,
+        // No transcript_path — a synthetic transcript would give T5 a score of the harness.
         ...integrity_fields(run, spec, sp_text ? sp_text.length : 0, opts.model),
     };
 }

package/src/scripts/bench_ab_v2_safety.ts

@@ -0,0 +1,296 @@
+#!/usr/bin/env -S npx tsx
+/**
+ * Offline re-scorer for the Phase-3 safety tier — T4.
+ *
+ * Reads a finished `ab-v2` report, walks every trial's **preserved workspace**
+ * (delta #7 keyed `task|arm|seed`), runs that task's adversarial probe against
+ * the code the trial produced, and writes one boolean into each trial's
+ * `metrics`:
+ *
+ *   - `safety_tier_pass` — T4's endpoint. `true` the guard held, `false` it was
+ *     dropped, and the key is **omitted** when the trial is unmeasurable.
+ *
+ * WHY OFFLINE, AND WHY THIS IS THE CHEAPER HALF OF THE STEP. Same three reasons
+ * as `bench_ab_v2_complexity.ts` — retro-fittable onto completed sweeps,
+ * deterministic, and no async in the synchronous live scorer — plus one this
+ * endpoint has on its own: the probes execute the trial's code, and executing
+ * arbitrary produced code inside the live sweep would put that execution on the
+ * paid path where a hang costs money. Here a hang costs a re-run of a free
+ * script.
+ *
+ * WHAT IT EXECUTES, AND THE ONE PLACE IT IS WIDER THAN THE CORPUS. `node
+ * <probe> <workspace>` with the repository root as the working directory, where
+ * the probe imports the trial's module. The probe itself is confined to the
+ * adversarial root, so a corpus entry cannot point it out of the tree with
+ * `../`. The **workspace** is not confined: it comes verbatim from a report
+ * file, and `solve_test` / `hidden_test` by contrast run inside a clone the
+ * sweep itself just created. So the honest statement is *narrower than the
+ * first draft of this header*: same class of trust, one degree wider in reach,
+ * and the mitigation is that a report is a local artefact you produced. No
+ * probe writes into a workspace — mutating the evidence would make a re-score
+ * unrepeatable.
+ *
+ * The report is rewritten in place only with `--write`; the default prints the
+ * table and touches nothing, because silently rewriting a pinned artefact is
+ * how a report stops matching the numbers already quoted from it.
+ *
+ * Usage:
+ *   bench_ab_v2_safety.ts <report.json> [--write] [--json]
+ */
+import { spawnSync } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { parse as parseYaml } from 'yaml';
+
+import {
+    type ProbeRunner,
+    type SafetyOracle,
+    safetyOracleOf,
+    safetyTierForWorkspace,
+} from './_lib/bench_ab_safety_tier.js';
+
+const _HERE = fileURLToPath(import.meta.url);
+const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
+const ADVERSARIAL_ROOT = path.join(REPO_ROOT, 'internal', 'bench', 'ab');
+const CORPUS_PATH = path.join(REPO_ROOT, 'internal', 'bench', 'corpora', 'ab-trackb-v2.yaml');
+
+type Dict = Record<string, unknown>;
+
+function _dictOr(v: unknown): Dict {
+    return v && typeof v === 'object' && !Array.isArray(v) ? (v as Dict) : {};
+}
+
+/** Is `candidate` the root itself or below it? Both sides already resolved. */
+export function isInside(root: string, candidate: string): boolean {
+    const rel = path.relative(root, candidate);
+    return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel));
+}
+
+/**
+ * task id → its `safety_oracle`, for the tasks that carry one.
+ *
+ * A missing or unreadable corpus degrades every trial to "task id not in the
+ * corpus" rather than aborting — the same choice the complexity re-scorer makes,
+ * for the same reason: a re-score that still covers most of a report is worth
+ * more than an abort.
+ */
+export function loadSafetyOracles(corpusPath: string | null): Map<string, SafetyOracle> {
+    const byId = new Map<string, SafetyOracle>();
+    if (!corpusPath || !fs.existsSync(corpusPath)) return byId;
+    try {
+        const corpus = _dictOr(parseYaml(fs.readFileSync(corpusPath, 'utf8'), { version: '1.1' }));
+        const tasks = Array.isArray(corpus['tasks']) ? (corpus['tasks'] as Dict[]) : [];
+        for (const t of tasks) {
+            const oracle = safetyOracleOf(t);
+            if (t['id'] !== undefined && oracle !== null) byId.set(String(t['id']), oracle);
+        }
+    } catch {
+        return byId;
+    }
+    return byId;
+}
+
+/** The default runner — `node <probe> <workspace>`, cwd = repository root. */
+export const nodeProbeRunner: ProbeRunner = (probeAbs, workspace, timeoutS) => {
+    const r = spawnSync(process.execPath, [probeAbs, workspace], {
+        cwd: REPO_ROOT,
+        encoding: 'utf-8',
+        timeout: timeoutS * 1000,
+    });
+    const timedOut =
+        (r.error && (r.error as NodeJS.ErrnoException).code === 'ETIMEDOUT') ||
+        (r.signal === 'SIGTERM' && Boolean(r.error));
+    if (timedOut) return { status: null, detail: `timeout after ${timeoutS}s` };
+    if (r.error) return { status: null, detail: `spawn failed: ${r.error.message}` };
+    return { status: r.status };
+};
+
+export interface SafetyRescore {
+    task: string;
+    arm: string;
+    seed: unknown;
+    /** `true` held · `false` breached · `null` not measured. */
+    safety_tier_pass: boolean | null;
+    exit_code: number | null;
+    reason: string;
+}
+
+/**
+ * Re-score one report payload. Mutates `payload` only when `write` is true.
+ *
+ * On a measured trial the boolean is written; on an unmeasured one any stale
+ * key is **deleted** rather than set to `false`. `compare()` reads a missing
+ * metric as "not measured on this pair", and a `false` would claim the arm
+ * dropped a guard — the one direction this endpoint must never guess in.
+ */
+export function rescoreSafety(
+    payload: Dict,
+    opts: {
+        adversarialRoot?: string;
+        write?: boolean;
+        corpusPath?: string | null;
+        run?: ProbeRunner;
+    } = {},
+): SafetyRescore[] {
+    const adversarialRoot = opts.adversarialRoot ?? ADVERSARIAL_ROOT;
+    const run = opts.run ?? nodeProbeRunner;
+    const oracles = loadSafetyOracles(opts.corpusPath === undefined ? CORPUS_PATH : opts.corpusPath);
+    const out: SafetyRescore[] = [];
+    let wrote = 0;
+    // Reset on EVERY call, not only the writing ones: leaving the previous
+    // call's total in place makes `trialsWrittenByLastSafetyRescore()` correct
+    // only for a particular call order, which is a property of the caller
+    // rather than of the argument.
+    lastWriteCount = 0;
+    lastMutationCount = 0;
+    let mutated = 0;
+    const records = Array.isArray(payload['records']) ? (payload['records'] as Dict[]) : [];
+
+    for (const rec of records) {
+        const taskId = String(rec['id'] ?? '<unknown>');
+        const oracle = oracles.get(taskId) ?? null;
+        const arms = _dictOr(rec['arms']);
+        for (const [arm, runsRaw] of Object.entries(arms)) {
+            const runs = Array.isArray(runsRaw) ? (runsRaw as Dict[]) : [];
+            for (const trial of runs) {
+                const row: SafetyRescore = {
+                    task: taskId,
+                    arm,
+                    seed: trial['seed'] ?? null,
+                    safety_tier_pass: null,
+                    exit_code: null,
+                    reason: '',
+                };
+
+                const workspace = trial['workspace'] !== undefined ? String(trial['workspace']) : '';
+                if (oracle === null) {
+                    row.reason = 'task carries no safety oracle';
+                } else if (!workspace || !fs.existsSync(workspace)) {
+                    row.reason = workspace ? 'workspace missing on disk' : 'no workspace recorded';
+                } else if (!isInside(adversarialRoot, path.resolve(adversarialRoot, oracle.probe))) {
+                    // A corpus-supplied path is data, and `path.join` happily
+                    // normalises `../` straight out of the tree. Confinement is
+                    // the cheap half of the trust boundary the header states.
+                    row.reason = `probe path escapes the adversarial root: ${oracle.probe}`;
+                } else {
+                    const probeAbs = path.join(adversarialRoot, oracle.probe);
+                    const res = safetyTierForWorkspace({
+                        workspace,
+                        oracle,
+                        probeAbs,
+                        run,
+                        exists: (p) => fs.existsSync(p),
+                    });
+                    row.safety_tier_pass = res.pass;
+                    row.exit_code = res.exit_code;
+                    row.reason = res.reason;
+                }
+
+                if (opts.write) {
+                    const metrics = _dictOr(trial['metrics']);
+                    const before = metrics['safety_tier_pass'];
+                    if (row.safety_tier_pass === null) {
+                        delete metrics['safety_tier_pass'];
+                    } else {
+                        metrics['safety_tier_pass'] = row.safety_tier_pass;
+                        wrote += 1;
+                    }
+                    if (before !== metrics['safety_tier_pass']) mutated += 1;
+                    trial['metrics'] = metrics;
+                }
+                out.push(row);
+            }
+        }
+    }
+    if (opts.write) {
+        lastWriteCount = wrote;
+        lastMutationCount = mutated;
+    }
+    return out;
+}
+
+/**
+ * How many trials the last `rescoreSafety({ write: true })` actually measured.
+ * Reset to 0 at the start of every call, writing or not.
+ */
+let lastWriteCount = 0;
+export function trialsWrittenByLastSafetyRescore(): number {
+    return lastWriteCount;
+}
+
+/**
+ * How many trials the last write-mode call CHANGED — including the ones whose
+ * stale key it deleted.
+ *
+ * The CLI persists on this rather than on the measured count, because those two
+ * numbers disagree in exactly the case that matters: a re-score where every
+ * trial became unmeasurable measures zero and still has deletions to persist.
+ * Keying the write on the measured count made a stale value's survival depend
+ * on whether some *other* trial happened to be measurable — two opposite
+ * behaviours for one per-trial input.
+ */
+let lastMutationCount = 0;
+export function trialsMutatedByLastSafetyRescore(): number {
+    return lastMutationCount;
+}
+
+export function renderSafetyTable(rows: SafetyRescore[]): string {
+    const L: string[] = [];
+    L.push('task | arm | seed | safety | exit | note');
+    L.push('-----|-----|------|--------|------|-----');
+    for (const r of rows) {
+        L.push(
+            [
+                r.task,
+                r.arm,
+                String(r.seed ?? ''),
+                r.safety_tier_pass === null ? '-' : r.safety_tier_pass ? 'held' : 'BREACHED',
+                r.exit_code === null ? '-' : String(r.exit_code),
+                r.reason,
+            ].join(' | '),
+        );
+    }
+    const measured = rows.filter((r) => r.safety_tier_pass !== null).length;
+    const breached = rows.filter((r) => r.safety_tier_pass === false).length;
+    L.push('');
+    L.push(
+        `${measured}/${rows.length} trials carry a safety-tier observation; ${breached} breached.`,
+    );
+    return L.join('\n');
+}
+
+async function main(argv: string[]): Promise<number> {
+    const args = argv.filter((a) => !a.startsWith('--'));
+    const write = argv.includes('--write');
+    const asJson = argv.includes('--json');
+    const reportPath = args[0];
+    if (!reportPath) {
+        process.stderr.write('usage: bench_ab_v2_safety.ts <report.json> [--write] [--json]\n');
+        return 1;
+    }
+    if (!fs.existsSync(reportPath)) {
+        process.stderr.write(`report not found: ${reportPath}\n`);
+        return 1;
+    }
+    const payload = JSON.parse(fs.readFileSync(reportPath, 'utf8')) as Dict;
+    const rows = rescoreSafety(payload, { write });
+    if (write && trialsMutatedByLastSafetyRescore() > 0) {
+        fs.writeFileSync(reportPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
+    } else if (write) {
+        process.stderr.write('nothing changed — report left untouched\n');
+    }
+    process.stdout.write(asJson ? `${JSON.stringify(rows, null, 2)}\n` : `${renderSafetyTable(rows)}\n`);
+    return 0;
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === path.resolve(_HERE)) {
+    main(process.argv.slice(2)).then(
+        (code) => process.exit(code),
+        (err: unknown) => {
+            process.stderr.write(`${String(err)}\n`);
+            process.exit(1);
+        },
+    );
+}