@event4u/agent-config 9.9.0 → 9.10.0

package/src/scripts/anchor_scoring_run.ts

@@ -0,0 +1,239 @@
+#!/usr/bin/env tsx
+/**
+ * Anchor-scoring runner — constrained anchor evaluation with frozen verdicts
+ * (ADR-202, amended; attempt 2).
+ *
+ * Two phases, deliberately separate commands, because the freeze is the thing
+ * that makes the run reproducible:
+ *
+ *   --generate DIR   answer both arms once, freeze transcripts.json
+ *   --score DIR      evaluate every anchor with both evaluators, freeze
+ *                    verdicts.json, then score — κ, δ, thresholds, verdict
+ *
+ * Re-scoring a frozen directory is repeatable by construction: the scorer is a
+ * pure function over the frozen verdicts. Re-GENERATING is a new experiment with
+ * its own directory and its own δ, never a re-run of this one.
+ *
+ * Exit codes: 0 ok · 1 usage/IO error · 2 verdict FAIL or instrument FAIL.
+ */
+import { createRequire } from 'node:module';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import process from 'node:process';
+import { fileURLToPath } from 'node:url';
+
+import yaml from 'js-yaml';
+
+import { build_thin, RULES_SOURCE } from './project_thin_rules.js';
+import { cohensKappa } from './check_quality_regression.js';
+import {
+    delta_from_spread,
+    eval_prompt,
+    eval_with_retry,
+    key,
+    resolve,
+    verdict,
+    type Arm,
+    type TaskAnchors,
+} from './_lib/anchor_eval.js';
+
+const _HERE = fileURLToPath(import.meta.url);
+const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
+const CORPUS = path.join(REPO_ROOT, 'internal/bench/corpora/token-quality-golden.yaml');
+const REPORT = path.join(REPO_ROOT, 'internal/bench/reports/quality-run.json');
+
+const ANSWER_MODEL = 'claude-sonnet-4-5';
+const EVAL_ANTHROPIC = 'claude-sonnet-4-5';
+const EVAL_OPENAI = process.env['ANCHOR_EVAL_OPENAI_MODEL'] ?? 'gpt-5';
+const ANSWER_SYSTEM =
+    'You are an AI coding agent governed by the following always-loaded package rules. '
+    + 'Answer the user request as that agent would, honouring these rules:\n\n';
+const EVAL_SYSTEM = 'You are a strict checklist grader. You output only the requested JSON.';
+
+interface RawTask {
+    id: string;
+    rules: string[];
+    prompt: string;
+    expected: { must_include?: string[]; must_not?: string[] };
+}
+
+export function load_tasks(limit: number | null): Array<RawTask & TaskAnchors> {
+    const doc = yaml.load(fs.readFileSync(CORPUS, 'utf-8')) as { tasks: RawTask[] };
+    const all = doc.tasks.map((t) => ({
+        ...t,
+        must_include: t.expected.must_include ?? [],
+        must_not: t.expected.must_not ?? [],
+    }));
+    return limit === null ? all : all.slice(0, limit);
+}
+
+function contexts(): Record<Arm, string> {
+    // Mirror `project_thin_rules.measure()` exactly — same file set, same join —
+    // so the arms this run bills for are the arms the token measurement reports.
+    // `build_thin`'s first parameter is the rules DIRECTORY, not a scope; passing
+    // a scope string there silently yields an empty map and a rule-free "thin"
+    // arm, which would measure nothing.
+    const names = fs.readdirSync(RULES_SOURCE).filter((f) => f.endsWith('.md')).sort();
+    const eager = names.map((f) => fs.readFileSync(path.join(RULES_SOURCE, f), 'utf-8')).join('');
+    const thin = [...build_thin(RULES_SOURCE, null).values()].join('');
+    if (thin.length === 0) throw new Error('thin projection is empty — refusing to run a rule-free arm');
+    return { eager, thin };
+}
+
+function anthropic(model: string): (s: string, u: string) => string {
+    const require = createRequire(import.meta.url);
+    const c = require('./ai_council/clients.js') as {
+        AnthropicClient: new (o: { model: string; api_key: string }) => { ask(s: string, u: string): { text: string } };
+        load_anthropic_key: () => string;
+    };
+    const cl = new c.AnthropicClient({ model, api_key: c.load_anthropic_key() });
+    return (s, u) => cl.ask(s, u).text;
+}
+
+function openai(model: string): (s: string, u: string) => string {
+    const require = createRequire(import.meta.url);
+    const c = require('./ai_council/clients.js') as {
+        OpenAIClient: new (o: { model: string; api_key: string }) => { ask(s: string, u: string): { text: string } };
+        load_openai_key: () => string;
+    };
+    const cl = new c.OpenAIClient({ model, api_key: c.load_openai_key() });
+    return (s, u) => cl.ask(s, u).text;
+}
+
+// ── generate ────────────────────────────────────────────────────────────────
+function generate(dir: string, limit: number | null): number {
+    const tasks = load_tasks(limit);
+    const ctx = contexts();
+    const ask = anthropic(ANSWER_MODEL);
+    fs.mkdirSync(dir, { recursive: true });
+    const out: Record<string, Record<Arm, string>> = {};
+    let i = 0;
+    for (const t of tasks) {
+        i += 1;
+        process.stdout.write(`  [${i}/${tasks.length}] ${t.id}\n`);
+        out[t.id] = {
+            thin: ask(ANSWER_SYSTEM + ctx.thin, t.prompt),
+            eager: ask(ANSWER_SYSTEM + ctx.eager, t.prompt),
+        };
+    }
+    fs.writeFileSync(
+        path.join(dir, 'transcripts.json'),
+        JSON.stringify({ answer_model: ANSWER_MODEL, task_count: tasks.length, limit, transcripts: out }, null, 2) + '\n',
+    );
+    process.stdout.write(`\n❄️  frozen: ${path.relative(REPO_ROOT, dir)}/transcripts.json (${tasks.length} tasks × 2 arms)\n`);
+    return 0;
+}
+
+// ── score ───────────────────────────────────────────────────────────────────
+function score(dir: string): number {
+    const frozen = JSON.parse(fs.readFileSync(path.join(dir, 'transcripts.json'), 'utf-8')) as {
+        answer_model: string;
+        transcripts: Record<string, Record<Arm, string>>;
+    };
+    const ids = new Set(Object.keys(frozen.transcripts));
+    const tasks = load_tasks(null).filter((t) => ids.has(t.id));
+    const evalA = anthropic(EVAL_ANTHROPIC);
+    const evalB = openai(EVAL_OPENAI);
+
+    const labelsA: string[] = [];
+    const labelsB: string[] = [];
+    const resolved = new Map<string, boolean>();
+    const disagreements: Array<{ task_id: string; arm: Arm; kind: string; anchor: string; a: string; b: string; resolved_to: boolean }> = [];
+    let retries = 0;
+    let n = 0;
+
+    for (const t of tasks) {
+        for (const arm of ['thin', 'eager'] as const) {
+            n += 1;
+            process.stdout.write(`  [${n}/${tasks.length * 2}] ${t.id} · ${arm}\n`);
+            const p = eval_prompt(t.prompt, frozen.transcripts[t.id]![arm], t.must_include, t.must_not);
+            const ra = eval_with_retry((u) => evalA(EVAL_SYSTEM, u), p, t.must_include.length, t.must_not.length);
+            const rb = eval_with_retry((u) => evalB(EVAL_SYSTEM, u), p, t.must_include.length, t.must_not.length);
+            if (ra.retried) retries += 1;
+            if (rb.retried) retries += 1;
+
+            const record = (kind: 'must_include' | 'must_not', idx: number, anchor: string, a: boolean | null, b: boolean | null): void => {
+                labelsA.push(String(a));
+                labelsB.push(String(b));
+                const r = resolve(kind, a, b);
+                resolved.set(key(t.id, arm, kind, idx), r);
+                if (a !== b) {
+                    disagreements.push({ task_id: t.id, arm, kind, anchor, a: String(a), b: String(b), resolved_to: r });
+                }
+            };
+            t.must_include.forEach((anchor, idx) => record('must_include', idx, anchor, ra.include[idx]!, rb.include[idx]!));
+            t.must_not.forEach((anchor, idx) => record('must_not', idx, anchor, ra.not[idx]!, rb.not[idx]!));
+        }
+    }
+
+    const kappa = cohensKappa(labelsA, labelsB);
+    const input = { tasks, resolved };
+    const v = verdict(input, { kappa });
+    const delta = delta_from_spread(input);
+
+    fs.writeFileSync(
+        path.join(dir, 'verdicts.json'),
+        JSON.stringify({ evaluators: [EVAL_ANTHROPIC, EVAL_OPENAI], retries, labelsA, labelsB, disagreements }, null, 2) + '\n',
+    );
+
+    // quality-run.json — the schema check_quality_regression reads. Per-pair
+    // winners are emitted for compatibility; the DECISION is the two-leg
+    // aggregate below, and saying so here is the split ADR-202 required be explicit.
+    const results = tasks.map((t) => {
+        let thin = 0;
+        let eager = 0;
+        t.must_include.forEach((_, i) => {
+            if (resolved.get(key(t.id, 'thin', 'must_include', i)) === true) thin += 1;
+            if (resolved.get(key(t.id, 'eager', 'must_include', i)) === true) eager += 1;
+        });
+        return {
+            id: t.id,
+            winner: thin > eager ? 'thin' : eager > thin ? 'eager' : 'tie',
+            length_delta: 0,
+            winner_is_longer: null,
+        };
+    });
+    fs.mkdirSync(path.dirname(REPORT), { recursive: true });
+    fs.writeFileSync(
+        REPORT,
+        JSON.stringify(
+            {
+                threshold: 0.48,
+                judge_model: `anchor-scoring:${EVAL_ANTHROPIC}+${EVAL_OPENAI}`,
+                note: 'Per-pair winners are schema compatibility only. The decision is the two-leg aggregate in `anchor_verdict`.',
+                anchor_verdict: { ...v, delta_from_spread_pp: delta, retries, disagreement_count: disagreements.length },
+                results,
+            },
+            null,
+            2,
+        ) + '\n',
+    );
+
+    const line = (ok: boolean, s: string): void => {
+        process.stdout.write(`  ${ok ? '✅' : '❌'} ${s}\n`);
+    };
+    process.stdout.write('\n── verdict ──\n');
+    line(v.instrument_ok, `κ = ${kappa.toFixed(3)} (floor ${v.kappa_floor})`);
+    line(v.delta_registered, `δ = ${delta.toFixed(2)} pp (ceiling ${v.delta_ceiling_pp} pp)`);
+    line(v.must_not_ok, `must_not: ${v.thin.must_not_violations.length} thin / ${v.eager.must_not_violations.length} eager violations`);
+    line(v.non_inferiority_ok, `must_include: thin ${(v.thin.rate * 100).toFixed(1)}% vs eager ${(v.eager.rate * 100).toFixed(1)}% (gap ${v.rate_gap_pp.toFixed(2)} pp)`);
+    line(v.per_rule_floor_ok, `per-rule floor: ${v.per_rule_floor_breaches.length} breach(es)`);
+    process.stdout.write(`\n  ${v.pass ? '✅ PASS' : '❌ FAIL'} · disagreements ${disagreements.length} · retries ${retries}\n`);
+    return v.pass ? 0 : 2;
+}
+
+function main(argv: string[]): number {
+    const gi = argv.indexOf('--generate');
+    const si = argv.indexOf('--score');
+    const li = argv.indexOf('--limit');
+    const limit = li >= 0 ? Number(argv[li + 1]) : null;
+    if (gi >= 0) return generate(path.resolve(argv[gi + 1] ?? ''), limit);
+    if (si >= 0) return score(path.resolve(argv[si + 1] ?? ''));
+    process.stderr.write('usage: anchor_scoring_run --generate DIR [--limit N] | --score DIR\n');
+    return 1;
+}
+
+if (process.argv[1] !== undefined && process.argv[1].endsWith('anchor_scoring_run.ts')) {
+    process.exit(main(process.argv.slice(2)));
+}
+export { generate, score };

package/src/scripts/annotate_discovery.ts

@@ -6,23 +6,19 @@
  * Python→TS migration, Phase 8 / Wave 8g). Mirrors the Python CLI contract
  * EXACTLY — the `--pack` choice flag + positional `paths`, exit code (0),
  * the stdout/stderr split, byte-identical messages, and byte-identical
- * frontmatter rewrites of the source, the dist/agent-src mirror, and the
- * condensation-hash file.
+ * frontmatter rewrites of the source and the dist/agent-src mirror.
  *
  * Walks a list of artefacts in `.agent-src.uncondensed/` and:
  *   1. Inserts the 5 ADR-013 frontmatter keys (workspaces, packs, lifecycle,
  *      trust, install) before the closing `---`, deterministically.
  *   2. Mirrors the new keys into the matching `dist/agent-src/` counterpart so
  *      the condensed projection stays consistent (body preserved).
- *   3. Refreshes `internal/.condensation-hashes.json` for each touched source
- *      path so `task check-condensation` stays green.
  *
  * Idempotent: re-runs leave already-annotated files untouched.
  *
  * Mapping table (`PACK_DEFAULTS`) is the council-locked authority. No
  * behaviour changes — historical quirks preserved (consumers pin the exact behaviour).
  */
-import * as crypto from 'node:crypto';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
@@ -32,7 +28,6 @@ const _HERE = fileURLToPath(import.meta.url);
 export const ROOT = path.resolve(path.dirname(_HERE), '..', '..');
 export const SRC = path.join(ROOT, '.agent-src.uncondensed');
 export const DST = path.join(ROOT, 'dist/agent-src');
-export const HASH_FILE = path.join(ROOT, 'internal', '.condensation-hashes.json');
 
 // Pack → [workspace_id, trust_level, default_install, removable, lifecycle].
 export const PACK_DEFAULTS: Record<string, [string, string, boolean, boolean, string]> = {
@@ -131,13 +126,6 @@ function _mirrorToCondensed(rel: string, pack: string): void {
     fs.writeFileSync(dst, newText, 'utf-8');
 }
 
-function _refreshHash(rel: string, hashes: Record<string, string>): void {
-    const src = path.join(SRC, rel);
-    // rel.as_posix() — POSIX separators for the JSON key, regardless of OS.
-    const key = rel.split(path.sep).join('/');
-    hashes[key] = crypto.createHash('sha256').update(fs.readFileSync(src)).digest('hex');
-}
-
 function _isFile(p: string): boolean {
     try {
         return fs.statSync(p).isFile();
@@ -202,9 +190,6 @@ export function parseArgs(argv: string[]): Args {
 export function main(argv: string[] | null = null): number {
     const args = parseArgs(argv ?? process.argv.slice(2));
 
-    const hashes: Record<string, string> = _isFile(HASH_FILE)
-        ? (JSON.parse(fs.readFileSync(HASH_FILE, 'utf-8')) as Record<string, string>)
-        : {};
     let changed = 0;
     for (const raw of args.paths) {
         let rel = raw;
@@ -226,60 +211,11 @@ export function main(argv: string[] | null = null): number {
             changed += 1;
         }
         _mirrorToCondensed(inner, args.pack);
-        _refreshHash(inner, hashes);
     }
-    // json.dumps(hashes, indent=2, sort_keys=True) + "\n"
-    fs.mkdirSync(path.dirname(HASH_FILE), { recursive: true });
-    fs.writeFileSync(HASH_FILE, _dumpsSorted(hashes) + '\n', 'utf-8');
     process.stdout.write(`annotated ${changed} files with pack=${args.pack}\n`);
     return 0;
 }
 
-// --- json.dumps(indent=2, sort_keys=True) for a flat string→string map -------
-
-function _pyJsonStr(s: string): string {
-    let out = '"';
-    for (const ch of s) {
-        const code = ch.codePointAt(0) as number;
-        if (ch === '"') {
-            out += '\\"';
-        } else if (ch === '\\') {
-            out += '\\\\';
-        } else if (ch === '\n') {
-            out += '\\n';
-        } else if (ch === '\r') {
-            out += '\\r';
-        } else if (ch === '\t') {
-            out += '\\t';
-        } else if (ch === '\b') {
-            out += '\\b';
-        } else if (ch === '\f') {
-            out += '\\f';
-        } else if (code < 0x20) {
-            out += `\\u${code.toString(16).padStart(4, '0')}`;
-        } else if (code < 0x7f) {
-            out += ch;
-        } else if (code <= 0xffff) {
-            out += `\\u${code.toString(16).padStart(4, '0')}`;
-        } else {
-            const c = code - 0x10000;
-            const hi = 0xd800 + (c >> 10);
-            const lo = 0xdc00 + (c & 0x3ff);
-            out += `\\u${hi.toString(16).padStart(4, '0')}\\u${lo.toString(16).padStart(4, '0')}`;
-        }
-    }
-    return out + '"';
-}
-
-function _dumpsSorted(obj: Record<string, string>): string {
-    const keys = Object.keys(obj).sort();
-    if (keys.length === 0) {
-        return '{}';
-    }
-    const parts = keys.map((k) => `  ${_pyJsonStr(k)}: ${_pyJsonStr(obj[k] as string)}`);
-    return `{\n${parts.join(',\n')}\n}`;
-}
-
 function _isCliEntry(): boolean {
     if (process.argv[1] === undefined) {
         return false;

package/src/scripts/bench_quality_run.ts

@@ -126,6 +126,116 @@ export function run_golden_judge(
   });
 }
 
+// ── Export mode (--dump-answers): blinded pairs for an external instrument ──
+//
+// Needed because LLM-paired judging on thin-vs-eager is closed-by-diagnosis
+// (`docs/benchmark.md` § Length-neutral judge RERUN). Any admissible re-open
+// needs the raw answer pairs, whether the instrument is deterministic
+// anchor-scoring against `must_include`/`must_not` or a human judge. Export mode
+// makes NO judge calls, so it costs ~half a full judged run (2 arms, 0 judges).
+
+/** Deterministic per-task coin flip — FNV-1a over `seed:taskId`, low bit.
+ * Reproducible blinding without Math.random, so re-running the export yields
+ * the same A/B assignment and a partially-filled verdicts.csv stays valid. */
+export function blind_flip(taskId: string, seed: number): boolean {
+  const s = `${seed}:${taskId}`;
+  let h = 0x811c9dc5;
+  for (let i = 0; i < s.length; i += 1) {
+    h ^= s.charCodeAt(i);
+    h = Math.imul(h, 0x01000193) >>> 0;
+  }
+  return (h & 1) === 1;
+}
+
+export interface BlindPair {
+  id: string;
+  /** Which arm was presented as ANSWER A. Belongs in _mapping.json ONLY. */
+  a_arm: 'thin' | 'eager';
+  answer_a: string;
+  answer_b: string;
+}
+
+export function build_blind_pair(
+  task: GoldenTask,
+  thinAnswer: string,
+  eagerAnswer: string,
+  seed: number,
+): BlindPair {
+  const thinIsA = blind_flip(task.id, seed);
+  return {
+    id: task.id,
+    a_arm: thinIsA ? 'thin' : 'eager',
+    answer_a: thinIsA ? thinAnswer : eagerAnswer,
+    answer_b: thinIsA ? eagerAnswer : thinAnswer,
+  };
+}
+
+/** Render the operator-facing pair file. MUST NOT leak the arm mapping —
+ * `tests/scripts/bench_quality_run.test.ts` asserts the blinding invariant. */
+export function render_pair_file(task: GoldenTask, pair: BlindPair): string {
+  const lines: string[] = [`# ${task.id}`, '', '## Request', '', task.prompt, '', '## Rubric', '', task.rubric, ''];
+  if (task.must_include.length) lines.push('## Must satisfy', '', ...task.must_include.map((s) => `- ${s}`), '');
+  if (task.must_not.length) lines.push('## Must avoid', '', ...task.must_not.map((s) => `- ${s}`), '');
+  lines.push(
+    '## Judging rules',
+    '',
+    '- Judge ONLY which answer better satisfies the rubric above.',
+    '- IGNORE length and verbosity. A longer answer is NOT a better answer.',
+    '- `tie` is a valid verdict — do not force a winner.',
+    '- Do NOT open `_mapping.json` until every verdict is recorded.',
+    '',
+    '## ANSWER A',
+    '',
+    pair.answer_a,
+    '',
+    '## ANSWER B',
+    '',
+    pair.answer_b,
+    '',
+    '## Verdict',
+    '',
+    `Append to verdicts.csv — one of: \`${task.id},A\` · \`${task.id},B\` · \`${task.id},tie\``,
+    '',
+  );
+  return lines.join('\n');
+}
+
+/** Write the blinded export: one file per task + the withheld mapping + a CSV stub. */
+export function write_export(
+  dir: string,
+  tasks: GoldenTask[],
+  pairs: BlindPair[],
+  meta: { seed: number; model: string; dry_run: boolean },
+): void {
+  fs.mkdirSync(dir, { recursive: true });
+  const byId = new Map(tasks.map((t) => [t.id, t]));
+  for (const pair of pairs) {
+    const task = byId.get(pair.id);
+    if (task === undefined) continue;
+    fs.writeFileSync(path.join(dir, `${pair.id}.md`), render_pair_file(task, pair));
+  }
+  fs.writeFileSync(
+    path.join(dir, '_mapping.json'),
+    JSON.stringify(
+      {
+        generated_by: 'bench_quality_run --dump-answers',
+        warning: 'DO NOT READ until every verdict is recorded in verdicts.csv.',
+        seed: meta.seed,
+        answer_model: meta.model,
+        dry_run: meta.dry_run,
+        threshold: DEFAULT_THRESHOLD,
+        mapping: pairs.map((p) => ({ id: p.id, a_arm: p.a_arm })),
+      },
+      null,
+      2,
+    ) + '\n',
+  );
+  const csv = path.join(dir, 'verdicts.csv');
+  if (!fs.existsSync(csv)) {
+    fs.writeFileSync(csv, `task_id,verdict\n${pairs.map((p) => `${p.id},`).join('\n')}\n`);
+  }
+}
+
 // ── Mock model (--dry-run): deterministic, no API ───────────────────────────
 function mock_answer(_ctx: string, task: GoldenTask): string {
   return `Answer to ${task.id}: ${(task.must_include[0] ?? 'addresses the request')}.`;
@@ -137,16 +247,41 @@ interface Args {
   model: string;
   limit: number | null;
   output: string;
+  dumpAnswers: string | null;
+  seed: number;
 }
 
+const DEFAULT_SEED = 20260729;
+
 function parse_args(argv: string[]): Args | number {
-  const a: Args = { dryRun: false, model: 'claude-sonnet-4-5', limit: null, output: OUT };
+  const a: Args = {
+    dryRun: false,
+    model: 'claude-sonnet-4-5',
+    limit: null,
+    output: OUT,
+    dumpAnswers: null,
+    seed: DEFAULT_SEED,
+  };
   for (let i = 0; i < argv.length; i += 1) {
     const arg = argv[i];
     if (arg === '--dry-run') a.dryRun = true;
     else if (arg === '--model') a.model = String(argv[(i += 1)] ?? a.model);
     else if (arg === '--output') a.output = String(argv[(i += 1)] ?? a.output);
-    else if (arg === '--limit') {
+    else if (arg === '--dump-answers') {
+      const v = argv[(i += 1)];
+      if (v === undefined || v.startsWith('--')) {
+        process.stderr.write('error: --dump-answers requires a directory path\n');
+        return 1;
+      }
+      a.dumpAnswers = v;
+    } else if (arg === '--seed') {
+      const v = Number(argv[(i += 1)]);
+      if (!Number.isInteger(v)) {
+        process.stderr.write('error: --seed must be an integer\n');
+        return 1;
+      }
+      a.seed = v;
+    } else if (arg === '--limit') {
       const v = Number(argv[(i += 1)]);
       if (!Number.isInteger(v) || v < 1) {
         process.stderr.write('error: --limit must be a positive integer\n');
@@ -154,7 +289,13 @@ function parse_args(argv: string[]): Args | number {
       }
       a.limit = v;
     } else if (arg === '-h' || arg === '--help') {
-      process.stdout.write('usage: bench_quality_run [--dry-run] [--model M] [--limit N] [--output PATH]\n');
+      process.stdout.write(
+        'usage: bench_quality_run [--dry-run] [--model M] [--limit N] [--output PATH]\n' +
+          '                        [--dump-answers DIR] [--seed N]\n\n' +
+          '  --dump-answers DIR  export blinded pairs for an external instrument; makes NO\n' +
+          '                      judge calls (~half the cost of a judged run)\n' +
+          `  --seed N            deterministic A/B blinding seed (default ${String(DEFAULT_SEED)})\n`,
+      );
       return 0;
     } else {
       process.stderr.write(`error: unknown argument: ${arg}\n`);
@@ -214,8 +355,31 @@ function main(argv: string[]): number {
       );
     modelLabel = args.model;
     process.stdout.write(
-      `Live run: ${tasks.length} tasks × 2 arms + 2 judge calls each via ${args.model}. This costs API $.\n`,
+      args.dumpAnswers === null
+        ? `Live run: ${tasks.length} tasks × 2 arms + 2 judge calls each via ${args.model}. This costs API $.\n`
+        : `Live export: ${tasks.length} tasks × 2 arms via ${args.model}, NO judge calls. This costs API $.\n`,
+    );
+  }
+
+  // Export mode: generate both arms, blind them, write the pairs, judge nothing.
+  if (args.dumpAnswers !== null) {
+    let pairs: BlindPair[];
+    try {
+      pairs = tasks.map((task) =>
+        build_blind_pair(task, answer(contexts.thin, task), answer(contexts.eager, task), args.seed),
+      );
+    } catch (e) {
+      process.stderr.write(`error: answer generation failed: ${(e as Error).message}\n`);
+      return 1;
+    }
+    const dir = path.resolve(args.dumpAnswers);
+    write_export(dir, tasks, pairs, { seed: args.seed, model: modelLabel, dry_run: args.dryRun });
+    process.stdout.write(
+      `→ wrote ${pairs.length} blinded pairs to ${dir}\n` +
+        `   Arm mapping withheld in _mapping.json — do not open it until verdicts.csv is complete.\n` +
+        `   Judged nothing: LLM-paired judging is closed-by-diagnosis for this question.\n`,
     );
+    return 0;
   }
 
   let results: PairResult[];