@bongos/core 1.21.175 → 1.21.176

package/scripts/bongos/grade-correlation-audit.js

@@ -0,0 +1,310 @@
+#!/usr/bin/env node
+'use strict';
+
+// scripts/bongos/grade-correlation-audit.js — do the grader's panel workers fail
+// DIFFERENTLY, or are we paying four times for two votes? (task 1002843,
+// ADR 0089 §5 — the structural answer to "Nine Judges, Two Effective Votes").
+//
+// WHY. Aggregation cannot manufacture independence a panel does not have. ADR
+// 0089 therefore makes panel composition a measured choice, and nothing measured
+// it: /grade-audit and /grader-health are qualitative sweeps with no inter-judge
+// statistics. This reads PERSISTED grades — never re-grades, never spawns a
+// model, never writes anything — and reports:
+//   (a) pairwise agreement between the panel workers on the same task: raw
+//       agreement, Cohen's kappa (agreement corrected for chance) and the phi
+//       correlation of their objections;
+//   (b) each worker's Cohen's kappa against the OWNER'S decisions;
+//   (c) which workers look redundant with another and which carry independent
+//       signal, plus the panel's effective number of independent votes.
+//
+// A VOTE is binary: a worker OBJECTS (verdict concerns or fail) or CLEARS (pass).
+// A worker that errored without a finding produced no signal and casts no vote
+// on that task — the same missing-signal rule the gate applies (#473). The
+// deterministic pseudo-judges and the non-gating reader are not voters.
+//
+// THE OWNER'S DECISIONS — the only labels used, per the task:
+//   accept   an approved override request, or a manual confirm of work whose
+//            stored grade did not pass (task confirmed/shipped anyway);
+//   reject   a denied override request.
+// A passing grade that auto-landed is NOT a label: that is the grader agreeing
+// with itself, and counting it would inflate every worker's kappa. The labelled
+// set is therefore a selected one — tasks the panel did not pass, plus the ones
+// someone argued with — and the report says so.
+//
+// THE MINIMUM-N GUARD. Every statistic is REFUSED, with the reason, below
+// MIN_N observations or when either side has fewer than MIN_PER_CLASS of a
+// class (a kappa over a constant rater is undefined, and one over five labels is
+// noise). --min-n may raise the bar, never lower it. Report-only: no gate reads
+// this, and nothing changes status (ADR 0162).
+//
+// Usage:
+//   node scripts/bongos/grade-correlation-audit.js [--recent N] [--min-n N] [--json]
+//        pull the N most recently finished tasks' grades (default 400) + the
+//        decided override requests (Metic+ read) through the API
+//   ... --save <file>   also write the pulled dataset, to re-run offline
+//   ... --file <file>   audit a saved dataset instead of the live API
+
+const fs = require('node:fs');
+const { arg, hasFlag, cliClient } = require('./cli-lib');
+const { fetchRecentGrades, trimRecordedGrade } = require('./grade-replay');
+
+const PANEL = ['narc', 'quality', 'hacker', 'efficiency'];
+const MIN_N = 30;
+const MIN_PER_CLASS = 5;
+const REDUNDANT_KAPPA = 0.8;
+const INDEPENDENT_KAPPA = 0.4;
+
+// A worker's vote on one grade: 1 objects, 0 clears, null no signal.
+function voteOf(w) {
+  if (!w || typeof w !== 'object') return null;
+  // A stored attempt keeps severities only (task 1004292); a full grade keeps issues.
+  const severities = Array.isArray(w.severities)
+    ? w.severities
+    : (Array.isArray(w.issues) ? w.issues : []).map((i) => i && i.severity);
+  const substantive = severities.some((sv) => sv === 'blocker' || sv === 'major');
+  if (w.error && !substantive) return null;
+  if (w.verdict === 'pass') return 0;
+  if (w.verdict === 'concerns' || w.verdict === 'fail') return 1;
+  return null;
+}
+
+// { kind -> vote } for the panel workers of one grade.
+function panelVotes(workers) {
+  const out = {};
+  for (const w of Array.isArray(workers) ? workers : []) {
+    if (!w || !PANEL.includes(w.kind)) continue;
+    const v = voteOf(w);
+    if (v != null) out[w.kind] = v;
+  }
+  return out;
+}
+
+// Why a binary series cannot carry a statistic, or null when it can.
+function refusal(pairs, minN) {
+  if (pairs.length < minN) return `n=${pairs.length} is below the minimum of ${minN}`;
+  for (const side of [0, 1]) {
+    const ones = pairs.filter((p) => p[side] === 1).length;
+    const zeros = pairs.length - ones;
+    if (Math.min(ones, zeros) < MIN_PER_CLASS) {
+      return `${side === 0 ? 'the first' : 'the second'} rater has only ${Math.min(ones, zeros)} of one class (need ${MIN_PER_CLASS}+ of each)`;
+    }
+  }
+  return null;
+}
+
+// Cohen's kappa + raw agreement + phi for [[a, b], ...] binary pairs. Pure; the
+// caller has already applied the minimum-N guard.
+function agreementStats(pairs) {
+  const n = pairs.length;
+  let n11 = 0; let n10 = 0; let n01 = 0; let n00 = 0;
+  for (const [a, b] of pairs) {
+    if (a === 1 && b === 1) n11++; else if (a === 1) n10++; else if (b === 1) n01++; else n00++;
+  }
+  const po = (n11 + n00) / n;
+  const pa = (n11 + n10) / n;
+  const pb = (n11 + n01) / n;
+  const pe = pa * pb + (1 - pa) * (1 - pb);
+  const kappa = pe === 1 ? null : (po - pe) / (1 - pe);
+  const den = Math.sqrt((n11 + n10) * (n01 + n00) * (n11 + n01) * (n10 + n00));
+  const phi = den === 0 ? null : (n11 * n00 - n10 * n01) / den;
+  return { n, agreement: po, kappa, phi, table: { both_object: n11, only_first: n10, only_second: n01, both_clear: n00 } };
+}
+
+function guardedStats(pairs, minN) {
+  const why = refusal(pairs, minN);
+  return why ? { n: pairs.length, refused: why } : agreementStats(pairs);
+}
+
+// Effective number of independent votes for k raters whose mean pairwise
+// correlation is rho (the design-effect formula). A negative mean is clamped to
+// 0 — this is a ceiling on independence, not a claim of anti-correlation.
+function effectiveVotes(k, rho) {
+  const r = Math.max(0, rho);
+  return k / (1 + (k - 1) * r);
+}
+
+// The owner's decision per task id, from the override ledger and the tasks'
+// fates. Returns Map(task_id -> { label: 'accept'|'reject', source }). An
+// approval outranks a denial on the same task: it was the later decision.
+function ownerLabels({ grades, overrides }) {
+  const labels = new Map();
+  for (const o of overrides || []) {
+    if (o.status === 'approved') labels.set(String(o.task_id), { label: 'accept', source: 'override_approved' });
+    else if (o.status === 'denied' && !labels.has(String(o.task_id))) labels.set(String(o.task_id), { label: 'reject', source: 'override_denied' });
+  }
+  for (const g of grades || []) {
+    const id = String(g.task_id);
+    if (labels.has(id)) continue;
+    const landed = g.task_status === 'confirmed' || g.task_status === 'shipped';
+    if (landed && g.passed === false) labels.set(id, { label: 'accept', source: 'manual_confirm_after_fail' });
+  }
+  return labels;
+}
+
+function classify(kappa) {
+  if (kappa == null) return 'undetermined';
+  if (kappa >= REDUNDANT_KAPPA) return 'redundant';
+  if (kappa >= INDEPENDENT_KAPPA) return 'overlapping';
+  return 'independent';
+}
+
+// One stored grade → the slim row the audit reads (review prose hashed, as in
+// the replay corpus): { task_id, task_status, passed, workers }, or null when
+// it is not a panel grade.
+//
+// task 1004292: grade.attempts (every round that kept its verdicts, oldest first,
+// served beside the grade) becomes `rounds`; a task with none falls back to its
+// final grade alone, exactly as before.
+function toAuditRow({ task_id, task_status, grade }) {
+  const attempts = grade && Array.isArray(grade.attempts) ? grade.attempts : [];
+  const rounds = attempts.map((a) => (a && Array.isArray(a.workers) ? a.workers : null)).filter((w) => w && w.length);
+  const row = trimRecordedGrade({ task_id, grade });
+  if (!row && rounds.length === 0) return null;
+  return {
+    task_id: String(task_id),
+    task_status: task_status || null,
+    passed: !!(grade && grade.passed === true),
+    workers: row ? row.workers : rounds[rounds.length - 1],
+    ...(rounds.length ? { rounds } : {}),
+  };
+}
+
+// The panels one dataset row contributes: every recorded round when there are
+// any (the LAST one is the final grade, the one the owner's decision is about),
+// else the final grade alone.
+function panelsOf(g) {
+  const sets = Array.isArray(g.rounds) && g.rounds.length ? g.rounds : [g.workers];
+  return sets
+    .map((workers, i) => ({ task_id: String(g.task_id), final: i === sets.length - 1, votes: panelVotes(workers) }))
+    .filter((p) => Object.keys(p.votes).length > 0);
+}
+
+// The whole report, pure over a dataset { grades: [toAuditRow…], overrides: [{task_id, status}] }.
+function audit(dataset, { minN = MIN_N } = {}) {
+  const floor = Math.max(MIN_N, Number(minN) || MIN_N);
+  // Agreement between workers reads EVERY round; the owner's decision is about
+  // the work as it landed, so the owner-facing statistics read final rounds only.
+  const panels = (dataset.grades || []).flatMap(panelsOf);
+  const finals = panels.filter((p) => p.final);
+  const labels = ownerLabels(dataset);
+  // 1 when the worker's vote disagrees with the owner (objected to accepted
+  // work, or cleared rejected work); only on labelled tasks it voted on.
+  const errorOf = (p, kind) => (!p.final ? null : (labels.has(p.task_id) && kind in p.votes
+    ? Number(p.votes[kind] !== (labels.get(p.task_id).label === 'reject' ? 1 : 0))
+    : null));
+
+  const pairwise = [];
+  for (let i = 0; i < PANEL.length; i++) {
+    for (let j = i + 1; j < PANEL.length; j++) {
+      const a = PANEL[i]; const b = PANEL[j];
+      const pairs = panels.filter((p) => a in p.votes && b in p.votes).map((p) => [p.votes[a], p.votes[b]]);
+      const stats = guardedStats(pairs, floor);
+      // Error correlation: do the two get the SAME tasks wrong against the owner?
+      const errPairs = panels.map((p) => [errorOf(p, a), errorOf(p, b)]).filter(([x, y]) => x != null && y != null);
+      pairwise.push({ a, b, ...stats, reading: stats.refused ? 'undetermined' : classify(stats.kappa), errors: guardedStats(errPairs, floor) });
+    }
+  }
+  // The panel's effective vote count is a claim about ALL of it, so it needs
+  // every pair — one computable pair extrapolated to four workers would be the
+  // over-reading this guard exists to stop.
+  const phis = pairwise.filter((p) => !p.refused && p.phi != null).map((p) => p.phi);
+  const meanPhi = phis.length === pairwise.length ? phis.reduce((s, x) => s + x, 0) / phis.length : null;
+
+  const byLabel = { accept: 0, reject: 0 };
+  for (const v of labels.values()) byLabel[v.label]++;
+  const perWorker = PANEL.map((kind) => {
+    const pairs = finals
+      .filter((p) => labels.has(p.task_id) && kind in p.votes)
+      .map((p) => [p.votes[kind], labels.get(p.task_id).label === 'reject' ? 1 : 0]);
+    return { kind, ...guardedStats(pairs, floor) };
+  });
+
+  return {
+    min_n: floor,
+    panels: panels.length,
+    tasks: finals.length,
+    pairwise,
+    mean_pairwise_phi: meanPhi,
+    effective_votes: meanPhi == null ? null : effectiveVotes(PANEL.length, meanPhi),
+    labels: { total: labels.size, ...byLabel },
+    vs_owner: perWorker,
+  };
+}
+
+// ---------- the CLI ----------
+
+async function fetchDataset(recent) {
+  const client = await cliClient();
+  const grades = (await fetchRecentGrades({ recent, client })).map(toAuditRow).filter(Boolean);
+  const overrides = [];
+  for (const status of ['approved', 'denied']) {
+    const r = await client.overrideRequests.getOverrideRequests({ query: { status } });
+    if (!r.ok) throw new Error(`GET /override-requests?status=${status} → HTTP ${r.status} (a Metic+ read)`);
+    for (const o of (r.data && r.data.requests) || []) overrides.push({ task_id: String(o.task_id), status: o.status, decided_at: o.decided_at || null });
+  }
+  return { pulled_at: new Date().toISOString(), grades, overrides };
+}
+
+const pct = (x) => `${(x * 100).toFixed(1)}%`;
+const num = (x) => (x == null ? 'n/a' : x.toFixed(2));
+const statLine = (s) => (s.refused ? `REFUSED — ${s.refused}` : `n=${s.n}  agree ${pct(s.agreement)}  kappa ${num(s.kappa)}  phi ${num(s.phi)}`);
+
+function printReport(r) {
+  const out = [];
+  out.push(`grade correlation audit — ${r.panels} recorded panel round(s) over ${r.tasks} task(s), minimum n = ${r.min_n}`);
+  out.push('A vote is OBJECTS (concerns/fail) or CLEARS (pass); an errored worker with no finding casts none.');
+  out.push('');
+  out.push('(a) Do the workers fail differently? — agreement between each pair on the same task');
+  for (const p of r.pairwise) {
+    out.push(`  ${`${p.a} × ${p.b}`.padEnd(22)} ${statLine(p)}${p.refused ? '' : `  → ${p.reading}`}`);
+    out.push(`  ${''.padEnd(22)} same tasks wrong vs the owner: ${statLine(p.errors)}`);
+  }
+  const cleared = r.pairwise.filter((p) => !p.refused).length;
+  out.push(r.effective_votes == null
+    ? `  effective independent votes: not computable — needs all ${r.pairwise.length} pairs, ${cleared} cleared the minimum`
+    : `  mean pairwise phi ${num(r.mean_pairwise_phi)} → about ${r.effective_votes.toFixed(1)} effective independent vote(s) out of 4`);
+  out.push('');
+  out.push(`(b) Each worker against the owner's decisions — ${r.labels.total} labelled task(s): ${r.labels.accept} accepted, ${r.labels.reject} rejected`);
+  for (const w of r.vs_owner) out.push(`  ${w.kind.padEnd(22)} ${statLine(w)}`);
+  out.push('');
+  const redundant = r.pairwise.filter((p) => p.reading === 'redundant').map((p) => `${p.a}+${p.b}`);
+  const independent = r.pairwise.filter((p) => p.reading === 'independent').map((p) => `${p.a}+${p.b}`);
+  out.push('(c) Reading');
+  out.push(`  redundant pairs (kappa >= 0.8): ${redundant.join(', ') || 'none'}`);
+  out.push(`  independent pairs (kappa < 0.4): ${independent.join(', ') || 'none'}`);
+  out.push('');
+  out.push('Caveats: labels exist only where a person decided — override requests and manual confirms of work the');
+  out.push('panel did not pass — so they are a selected set, not a sample. A grade that auto-landed on a pass is');
+  out.push('never a label (that is the grader agreeing with itself). Agreement (a) reads every round that kept its');
+  out.push('per-worker verdicts (grade_attempts.worker_verdicts); rounds graded before that column existed are lost,');
+  out.push('and such a task counts once, as its final grade. Owner comparisons (b) and error correlation read final rounds only.');
+  console.log(out.join('\n'));
+}
+
+async function main() {
+  const file = arg('--file');
+  const dataset = file
+    ? JSON.parse(fs.readFileSync(file, 'utf8'))
+    : await fetchDataset(Number(arg('--recent')) || 400);
+  const save = arg('--save');
+  if (save && !file) fs.writeFileSync(save, `${JSON.stringify(dataset)}\n`);
+  const report = audit(dataset, { minN: Number(arg('--min-n')) || MIN_N });
+  if (hasFlag('--json')) console.log(JSON.stringify(report, null, 2));
+  else printReport(report);
+}
+
+module.exports = {
+  MIN_N,
+  agreementStats,
+  audit,
+  effectiveVotes,
+  ownerLabels,
+  panelVotes,
+  panelsOf,
+  toAuditRow,
+  voteOf,
+};
+
+if (require.main === module) {
+  main().catch((err) => { console.error(`grade-correlation-audit: ${err.message}`); process.exit(2); });
+}

package/scripts/bongos/grade-replay.js

@@ -0,0 +1,237 @@
+#!/usr/bin/env node
+'use strict';
+
+// scripts/bongos/grade-replay.js — the differential replay over RECORDED grades
+// (task 1002839, ADR 0089 §1). Read-only: it never grades, never spawns a model
+// and never writes a row.
+//
+// WHY. How a grade is aggregated and gated IS the live ship gate, so a change to
+// either is only behaviour-preserving if real recorded grades still come out the
+// way production recorded them. Every persisted panel grade carries its judges'
+// verdicts (signals.workers, the deterministic pseudo-judges included), so the
+// grade engine's aggregate + gate stages can be re-run over them at zero cost and
+// compared field by field with what the server stored.
+//
+// WHAT IS COMPARED — the decision, not the prose: panel_outcome, the four verdict
+// counts, the worker count, the panel's own cost, the forwarded confidence (skipped
+// when the server-side guards downgraded it — that number is theirs, not the
+// panel's), and the five rubric axes the server re-records.
+//
+// Usage:
+//   node scripts/bongos/grade-replay.js                    replay the committed corpus
+//   node scripts/bongos/grade-replay.js --file <corpus>    replay another corpus file
+//   node scripts/bongos/grade-replay.js --live [--recent N]
+//        pull the N most recently finished tasks' grades from the API (default 150)
+//   ... --write-fixture <path>   write the AGREEING replays, trimmed (issue text and
+//        stderr hashed), one per panel shape — how the test corpus is made
+//   ... --json                   machine-readable summary
+// Exit 0 when every replayed grade agrees, 1 when any does not. A report, not a
+// gate: nothing reads the exit code but the person who ran it.
+
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+const { arg, hasFlag, cliClient } = require('./cli-lib');
+// Through the module facade: requiring grader.js is what registers the code panel
+// as the engine's default work-type.
+const engine = require('../../modules/grading/grader').gradeEngine;
+
+const DEFAULT_CORPUS = path.join(__dirname, '..', '..', 'tests', 'fixtures', 'grade-replay', 'recorded-panel-grades.json');
+const COUNT_FIELDS = ['pass_count', 'concerns_count', 'fail_count', 'error_count', 'worker_count'];
+const RUBRIC_AXES = ['functional_fidelity', 'code_quality', 'economy', 'security_score', 'efficiency_score'];
+const PER_SHAPE = 1;
+
+const num = (v) => (v == null || v === '' ? null : Number(v));
+
+// The decision a stored grade row records. The rubric axes are the server's
+// re-derivation (validateAndScore) of the panel's medians — equal to them for
+// every grade since the null-axis fixes (tasks 1002653 / 1003318).
+function recordedDecision(grade) {
+  const s = (grade && grade.signals) || {};
+  const guard = s.server_guard || {};
+  const out = { panel_outcome: s.panel_outcome ?? null };
+  for (const k of COUNT_FIELDS) out[k] = s[k] ?? null;
+  out.panel_cost_usd = num(s.panel_cost_usd);
+  out.confidence = guard.downgraded_to ? undefined : num(s.confidence);
+  out.rubric = {};
+  for (const a of RUBRIC_AXES) out.rubric[a] = num(grade && grade.rubric_scores ? grade.rubric_scores[a] : null);
+  return out;
+}
+
+// The same decision, read off a replayed grade-result.
+function replayedDecision(result) {
+  const s = (result && result.signals) || {};
+  const out = { panel_outcome: s.panel_outcome ?? null };
+  for (const k of COUNT_FIELDS) out[k] = s[k] ?? null;
+  out.panel_cost_usd = num(s.panel_cost_usd);
+  out.confidence = num(s.confidence);
+  out.rubric = {};
+  for (const a of RUBRIC_AXES) out.rubric[a] = num(result && result.rubric_scores ? result.rubric_scores[a] : null);
+  return out;
+}
+
+const same = (a, b) => a === b || (typeof a === 'number' && typeof b === 'number' && Math.abs(a - b) < 1e-9);
+
+// Every field on which a replay disagrees with the record, as
+// { field, recorded, replayed }. An `undefined` recorded value is not compared.
+//
+// One pairing is NOT a disagreement: a rubric axis the panel left null (no judge
+// reported it) that the row stores as 0. That 0 is the server's, not the panel's —
+// validateAndScore coerced a missing axis with Number(null) === 0 until tasks
+// 1002653 / 1003318 made it record null. A real 0/10 median replays as 0, so the
+// pairing can only mean that coercion; any other value against a null still counts.
+function compareDecision(recorded, replayed) {
+  const out = [];
+  for (const k of ['panel_outcome', ...COUNT_FIELDS, 'panel_cost_usd', 'confidence']) {
+    if (recorded[k] === undefined) continue;
+    if (!same(recorded[k], replayed[k])) out.push({ field: k, recorded: recorded[k], replayed: replayed[k] });
+  }
+  for (const a of RUBRIC_AXES) {
+    const r = (recorded.rubric || {})[a];
+    const p = (replayed.rubric || {})[a];
+    if (r === undefined || same(r, p) || (r === 0 && p === null)) continue;
+    out.push({ field: `rubric.${a}`, recorded: r, replayed: p });
+  }
+  return out;
+}
+
+const shortHash = (s) => `h:${crypto.createHash('sha256').update(String(s)).digest('hex').slice(0, 10)}`;
+
+// A worker, reduced to what aggregate + gate read. Issue summaries, their file
+// paths and stderr are hashed: the gate reads a summary and a file only to
+// de-duplicate (the hash of the trimmed text preserves exactly that), and a
+// fixture should not carry verbatim review prose.
+function trimWorker(w) {
+  const t = {
+    kind: w.kind,
+    verdict: w.verdict,
+    error: w.error ?? null,
+    scores: w.scores || {},
+    confidence: w.confidence ?? null,
+    model: w.model ?? null,
+    cost_usd: w.cost_usd ?? null,
+    retried: !!w.retried,
+    issues: (Array.isArray(w.issues) ? w.issues : []).filter((i) => i && typeof i === 'object').map((i) => ({
+      severity: i.severity ?? null,
+      file: i.file == null ? null : shortHash(i.file),
+      summary: shortHash(String(i.summary ?? '').trim()),
+    })),
+  };
+  if (w.exit_code != null) t.exit_code = w.exit_code;
+  if (w.stderr_tail) t.stderr_tail = shortHash(w.stderr_tail);
+  return t;
+}
+
+// One recorded grade → a corpus row, or null when it is not a panel grade.
+function trimRecordedGrade({ task_id, graded_at = null, grade }) {
+  const s = (grade && grade.signals) || {};
+  if (!Array.isArray(s.workers) || s.workers.length === 0) return null;
+  return { task_id: String(task_id), graded_at, recorded: recordedDecision(grade), workers: s.workers.map(trimWorker) };
+}
+
+// The panel's shape — outcome plus each judge's kind/verdict/error/retry — so
+// the fixture keeps variety instead of 200 copies of an all-pass panel.
+function shapeKey(row) {
+  return [row.recorded.panel_outcome, ...row.workers.map((w) => `${w.kind}:${w.verdict}${w.error ? `!${w.error}` : ''}${w.retried ? '~r' : ''}`)].join('|');
+}
+
+// Replay one corpus row (or one raw recorded grade) through the engine.
+function replayRow(row) {
+  const result = engine.replayRecorded({ signals: { workers: row.workers } });
+  return { result, mismatches: compareDecision(row.recorded, replayedDecision(result)) };
+}
+
+// The most recently finished tasks' STORED grades, raw and newest first:
+// [{ task_id, task_status, at, grade }]. Read-only GETs. Shared with the
+// correlation audit (scripts/bongos/grade-correlation-audit.js), which needs the
+// task's fate beside its grade. The per-task reads run a few at a time — one
+// after another made a 400-task pull take minutes for no reason.
+const FETCH_CONCURRENCY = 6;
+async function fetchRecentGrades({ recent = 150, statuses = ['shipped', 'completed', 'confirmed'], client = null } = {}) {
+  const api = client || await cliClient();
+  const tasks = [];
+  for (const status of statuses) {
+    const r = await api.tasks.getTasks({ query: { status, limit: 1000 } });
+    if (!r.ok) throw new Error(`GET /tasks?status=${status} → HTTP ${r.status}`);
+    for (const t of r.data.tasks || []) tasks.push({ id: String(t.id), status: t.status || status, at: t.shipped_at || t.updated_at || '' });
+  }
+  tasks.sort((a, b) => String(b.at).localeCompare(String(a.at)));
+  const picked = tasks.slice(0, recent);
+  const out = new Array(picked.length).fill(null);
+  for (let i = 0; i < picked.length; i += FETCH_CONCURRENCY) {
+    await Promise.all(picked.slice(i, i + FETCH_CONCURRENCY).map(async (t, j) => {
+      const r = await api.tasks.getTasksId({ id: t.id, query: { include: 'grade,grade_attempts' } });
+      if (r.ok && r.data && r.data.grade) out[i + j] = { task_id: t.id, task_status: t.status, at: t.at, grade: r.data.grade };
+    }));
+  }
+  return out.filter(Boolean);
+}
+
+async function fetchLive(recent) {
+  const rows = [];
+  for (const g of await fetchRecentGrades({ recent })) {
+    const row = trimRecordedGrade({ task_id: g.task_id, graded_at: g.at, grade: g.grade });
+    if (row) rows.push(row);
+  }
+  return rows;
+}
+
+function readCorpus(file) {
+  const doc = JSON.parse(fs.readFileSync(file, 'utf8'));
+  return Array.isArray(doc) ? doc : doc.grades;
+}
+
+async function main() {
+  const live = hasFlag('--live');
+  const rows = live
+    ? await fetchLive(Number(arg('--recent')) || 150)
+    : readCorpus(arg('--file') || DEFAULT_CORPUS);
+  const report = rows.map((row) => ({ row, ...replayRow(row) }));
+  const disagree = report.filter((r) => r.mismatches.length);
+
+  const fixturePath = arg('--write-fixture');
+  if (fixturePath) {
+    const perShape = new Map();
+    const keep = [];
+    for (const r of report) {
+      if (r.mismatches.length) continue;
+      const k = shapeKey(r.row);
+      const n = perShape.get(k) || 0;
+      if (n >= PER_SHAPE) continue;
+      perShape.set(k, n + 1);
+      keep.push(r.row);
+    }
+    const note = 'Recorded panel grades for the differential replay (tests/grade_engine_replay.mjs). Generated by scripts/bongos/grade-replay.js --live --write-fixture: only grades whose replay agreed with the stored row, one per panel shape. Issue summaries, their files and stderr are hashed.';
+    fs.mkdirSync(path.dirname(fixturePath), { recursive: true });
+    // One grade per line: small in a diff, and a regenerated corpus diffs by row.
+    fs.writeFileSync(fixturePath, `{"note":${JSON.stringify(note)},\n"grades":[\n${keep.map((g) => JSON.stringify(g)).join(',\n')}\n]}\n`);
+    console.error(`wrote ${keep.length} grades (${perShape.size} panel shapes) → ${fixturePath}`);
+  }
+
+  if (hasFlag('--json')) {
+    console.log(JSON.stringify({
+      replayed: report.length,
+      agree: report.length - disagree.length,
+      disagree: disagree.map((r) => ({ task_id: r.row.task_id, graded_at: r.row.graded_at, mismatches: r.mismatches })),
+    }, null, 2));
+  } else {
+    console.log(`grade replay: ${report.length} recorded panel grade(s), ${report.length - disagree.length} agree, ${disagree.length} disagree`);
+    for (const r of disagree) {
+      const what = r.mismatches.map((m) => `${m.field} recorded=${JSON.stringify(m.recorded)} replayed=${JSON.stringify(m.replayed)}`).join('; ');
+      console.log(`  task ${r.row.task_id}${r.row.graded_at ? ` (${r.row.graded_at})` : ''}: ${what}`);
+    }
+  }
+  process.exitCode = disagree.length ? 1 : 0;
+}
+
+module.exports = {
+  compareDecision,
+  fetchRecentGrades,
+  replayRow,
+  shapeKey,
+  trimRecordedGrade,
+};
+
+if (require.main === module) {
+  main().catch((err) => { console.error(`grade-replay: ${err.message}`); process.exit(2); });
+}