@bglocation/tune-context 2.4.0 → 3.0.0

package/eval/verdict.mjs

@@ -0,0 +1,963 @@
+#!/usr/bin/env node
+// verdict — the step analyze.mjs deliberately stops before: may this run say
+// "one arm is cheaper"? See tasks/TASK-090.
+//
+// Pure predicates over the TASK-089 aggregate. No disk, no clock, no stdout —
+// an aggregate in, a decision out. TASK-091 renders it as HTML; formatVerdict()
+// renders it as text for a terminal. The CLI section at the bottom of this file
+// (TASK-096) is the ONLY place in it that reads a file, stdin or the process,
+// or writes anything: everything above it stays a function of its arguments.
+//
+// The failure this file exists to prevent is not a wrong number; it is a right
+// number about the wrong thing. An aggregate that adds up perfectly still
+// supports "the doctrine is 30% cheaper" when the two arms ran on different
+// models, when the cheaper arm hid two subagent sessions from the transcript, or
+// when the whole difference is the operator already knowing the answer the
+// second time. None of those show up in a chart. So the gates are their own
+// module with their own tests, and the only proof they work is a test where the
+// verdict is WITHHELD on data that looks encouraging.
+//
+// Three properties hold everywhere below:
+//
+//   • **A held gate is never a bare `false`.** It carries the reason, the value
+//     measured, the threshold it was measured against, and what is missing for a
+//     verdict to become possible ("one more complete pair in arm off"). A
+//     refusal without those is indistinguishable from a crash.
+//   • **Every gate is evaluated, always.** The reader has to learn how many
+//     things are wrong, not just the first one — a run that fixes the sample
+//     size and then trips on model homogeneity has cost two runs to learn what
+//     one report could have said.
+//   • **The thresholds are data, frozen by the PO, in one table apart from the
+//     logic** — the same anti-bias rule as MIN_SESSIONS_FOR_VERDICT in
+//     eval/adoption-report.mjs and the same shape as the price table in
+//     eval/pricing.mjs. The model does not get to pick the n at which its own
+//     tool comes out ahead. Nothing below re-states a threshold as a literal:
+//     every gate reads this table, so changing a number here changes the
+//     outcome, which is what makes the freeze meaningful rather than decorative.
+//
+// The verdict is built on the PAIRED pool only. Natural work holds nothing
+// constant — it is background for asking whether the paired result points the
+// same way, and it is never evidence (analyze.mjs keeps the two un-addable by
+// shape; this file simply never reaches for the second one).
+import { readFileSync } from 'node:fs';
+import { parseArgs } from 'node:util';
+import { endQuietlyOnClosedPipe, isEntryPoint, readStdin } from '../cli/entry-point.mjs';
+import { meterStamp, stampProblem } from '../cli/version.mjs';
+import { ARMS } from './analyze.mjs';
+import { formatUsd } from './pricing.mjs';
+
+/**
+ * An input this module refuses: an aggregate that is not one, a run journal of
+ * the wrong shape, and — in the CLI — a file that is missing or is not JSON.
+ * One class for all of them, so a CLI can answer exactly these with a message
+ * and exit 1, and let anything else go up as the bug it is.
+ *
+ * A TypeError, because the pure functions threw TypeError at this boundary
+ * before the class existed, and a caller matching on that keeps working.
+ */
+export class VerdictInputError extends TypeError {
+  constructor(message) {
+    super(message);
+    this.name = 'VerdictInputError';
+  }
+}
+
+/**
+ * Frozen with the PO on 2026-07-30 (COST_EVALUATION_PROCEDURE §10, §11); the
+ * seventh gate, meter_mixed, frozen by the PO on 2026-10-15 (EPIC-010).
+ *
+ * Each entry carries its own `frozen` date next to its value, not just the
+ * table: a later change to one threshold must be visible as a change to THAT
+ * line. A threshold moved after seeing the data is a threshold fitted to the
+ * result — if it has to happen, it lands in `amended` with a new date and the
+ * admission that it happened after the run. A gate ADDED before the next run is
+ * not that, which is why meter_mixed carries its own date and `amended` stays
+ * empty; the top-level `frozen` stays the date the table was first frozen.
+ */
+export const VERDICT_THRESHOLDS = {
+  schema: 'tune-context/verdict-thresholds@2',
+  frozen: '2026-07-30',
+  frozen_by: 'PO',
+  source: 'COST_EVALUATION_PROCEDURE.md §10 and §11 (tasks/TASK-090)',
+  // Empty by construction. Every entry here is a threshold that moved AFTER a
+  // run — the record of that, not a place to quietly rewrite the table.
+  amended: [],
+  gates: {
+    sample: {
+      min_pairs: 2,
+      frozen: '2026-07-30',
+      note:
+        'Deliberately the minimum: the first run asks whether anything happens at all, it does not rule.' +
+        ' At n = 2 the report says "observation", never "proof". The same number is the floor for sessions' +
+        ' per arm — §2a runs one scripted session per arm per pair, so an arm with fewer sessions than pairs' +
+        ' measured two tasks in one session and is a smaller sample than the pair count suggests. That is the' +
+        ' same frozen number read twice, not a second threshold chosen here.',
+    },
+    homogeneity: {
+      requires: 'identical',
+      frozen: '2026-07-30',
+      note:
+        'Zero tolerance, so no number: measured opus/sonnet input price differs by 1.7x, larger than any effect' +
+        ' being looked for (error class 4). A run whose arms differ in model or effort is withheld and repeated,' +
+        ' NEVER averaged across models.',
+    },
+    delegation: {
+      max_diff: 2,
+      directional: true,
+      frozen: '2026-07-30',
+      note:
+        'Withhold when the CHEAPER arm made more than 2 more Agent calls than the other. One delegation is a whole' +
+        ' subagent session (~62k tokens measured) whose turns never reach the transcript, so the cheaper arm is' +
+        ' cheaper by an unknown remainder (§11a). Directional on purpose: the reverse difference makes the verdict' +
+        ' cautious rather than inflated, and a gate that fires on it would refuse a result that was already conservative.',
+    },
+    order_effect: {
+      min_arm_over_order: 2,
+      frozen: '2026-07-30',
+      note:
+        'A verdict requires |arm effect| >= 2 x |order effect| (§11b). The second implementation of a task is cheaper' +
+        ' because the solution is already known, and that runs in the same direction as the effect being measured.' +
+        ' At 1x a tie would pass as a result. Below the threshold the report prints both numbers and rules on nothing.',
+    },
+    unpriced: {
+      max_rows: 0,
+      frozen: '2026-07-30',
+      note:
+        'Not a PO judgment call — it follows from precondition §1: a run does not start with UNPRICED rows. A cost' +
+        ' computed over some of the rows is not the cost of the pool.',
+    },
+    rejected: {
+      max_share: 0.1,
+      frozen: '2026-07-30',
+      note:
+        'Share of experiment rows thrown away. 2 pairs = 4 sessions, and 10% of 4 is less than one — so a single' +
+        ' rejected run exceeds it. That is the point of the threshold, not a flaw in it: at four sessions there is' +
+        ' nothing to spare. A broken run is repeated (§7), not absorbed.',
+    },
+    meter_mixed: {
+      requires: 'single identifiable stamp across the chain',
+      frozen: '2026-10-15',
+      note:
+        'Chosen by the PO (EPIC-010 decision 2), not by the model — the TASK-062 anti-bias rule. The meter is the whole' +
+        ' chain: rows (prompt-metrics), the aggregate (analyze) and this verdict each carry the stamp of the code that' +
+        ' computed them, and in the EPIC-009 series the measured tasks changed analyze and verdict themselves. So' +
+        ' exactly one stamp, and one that names code: `.dirty` was computed from uncommitted changes and `+unknown`' +
+        ' from a checkout git could not read — neither can be rebuilt, so neither can be checked. EPIC-009 was voided' +
+        ' because its rows came from a meter nobody could name.',
+    },
+  },
+};
+
+/**
+ * §7 rejections live in the run journal, not in the rows.
+ *
+ * A run the OPERATOR threw away — a skipped prompt, a prompt typed out of order,
+ * a prompt that differed between the arms — leaves nothing analyze.mjs can see.
+ * The procedure drops that session from the pool BEFORE the rows are aggregated
+ * (§8), by session id, because the alternative — deleting the transcript — is
+ * the failure class this epic exists to prevent. The aggregate's own counters
+ * therefore read zero, and gate 6 passed a series that had thrown away a third
+ * of its sessions. That happened, in the run this file was built by.
+ *
+ * Hence this input. It carries what the journal recorded, in the unit the gate
+ * already counts: rows, i.e. scripted prompts, because a rejected run takes all
+ * four of its own.
+ *
+ * `undefined` and `[]` are DIFFERENT answers and must never collapse:
+ *
+ *   []         the journal was read and says nothing was thrown away.
+ *   undefined  nobody asked the journal.
+ *
+ * The second may not produce a ruling. A share whose denominator is unknown is
+ * the arithmetic form of a quietly dropped sample — the same error as a
+ * fabricated verdict, wearing a passing gate.
+ */
+function normalizeJournalRejects(input) {
+  if (input === undefined || input === null) return { reported: false, rows: 0, runs: [] };
+  const problem = journalShapeProblem(input);
+  if (problem !== null) throw new VerdictInputError(`verdict: ${problem}`);
+  const runs = input.map((run) => ({
+    session: run.session ?? null,
+    task: run.task ?? null,
+    arm: run.arm ?? null,
+    prompts: run.prompts,
+    reason: run.reason ?? null,
+  }));
+  return { reported: true, rows: runs.reduce((n, r) => n + r.prompts, 0), runs };
+}
+
+/**
+ * What is wrong with a supplied journal, or null when nothing is. A message
+ * rather than a throw, because two layers report it in their own words — the
+ * pure function above, and the CLI reading a `--rejected-runs` file — and the
+ * check itself must exist once.
+ */
+function journalShapeProblem(input) {
+  if (!Array.isArray(input)) return 'rejectedRuns must be an array of journal entries, or omitted to say nobody asked';
+  const i = input.findIndex((run) => !Number.isInteger(run?.prompts) || run.prompts < 0);
+  if (i === -1) return null;
+  return (
+    `rejectedRuns[${i}].prompts must be a non-negative integer — this gate counts rows, and a` +
+    ' rejected run whose prompt count is a guess cannot be counted into a threshold'
+  );
+}
+
+/**
+ * Reject reasons that mean "a run that was TRYING to be part of the paired
+ * measurement failed" — the §7 invalidations, counted in rows because that is
+ * the unit analyze.mjs counts rejects in. A rejected session takes all four of
+ * its scripted prompts with it, so the row share reproduces the session
+ * arithmetic §10 does (one lost run of a two-pair series = 20% of the rows).
+ */
+const EXPERIMENT_REJECTS = new Set([
+  'branch-changed',
+  'branch-annotation-mismatch',
+  'off-convention-branch',
+  'unresolved-arm',
+  'mixed-arm-session',
+]);
+
+/**
+ * Reject reasons that are NOT the paired measurement failing: ordinary daily
+ * work whose arm nobody recorded, on a machine that also does ordinary daily
+ * work. prompt-metrics reads every transcript under ~/.claude/projects (§8), so
+ * these outnumber the scripted rows by orders of magnitude on any real run.
+ * Counting them in gate 6 would make it a gate nobody can ever pass — as
+ * useless as no gate at all — while saying nothing about the experiment. They
+ * are reported inside the gate's measurement instead of being silently dropped.
+ */
+function isBackgroundReject(reason) {
+  return reason === 'no-ledger' || reason === 'no-arm-in-force' || reason.startsWith('unusable-arm:');
+}
+
+function isExperimentReject(reason) {
+  return EXPERIMENT_REJECTS.has(reason) || reason.startsWith('conflict:');
+}
+
+/** Micro-dollar resolution, the same rounding rule as pricing.mjs and analyze.mjs. */
+function round6(n) {
+  return Math.round(n * 1e6) / 1e6;
+}
+
+function mean(values) {
+  if (values.length === 0) return null;
+  return round6(values.reduce((a, b) => a + b, 0) / values.length);
+}
+
+/** The other arm of the pair. Derived from ARMS rather than written out, so the
+ *  two-arm assumption lives in exactly one place. */
+function other(arm) {
+  return ARMS.find((a) => a !== arm);
+}
+
+/**
+ * The arithmetic of §11b, per pair, over the four numbers a two-pair run
+ * produces:
+ *
+ *   arm effect   = mean over pairs of (cost off - cost on)   > 0 means "on" cheaper
+ *   order effect = mean over pairs of (cost pass1 - cost pass2)  > 0 means the second pass cheaper
+ *
+ * Both are means over PAIRS, not over pool totals: the rotation (§9) is what
+ * lets the two be separated at all, and it only exists inside the pairs.
+ *
+ * A pair contributes only when all four inputs are real. Anything else is
+ * skipped WITH its reason rather than defaulted — a pair whose costs are
+ * incomplete, or whose two arms carry the same `pass_order`, cannot tell arm
+ * from order, and a zero standing in for it would quietly pull both effects
+ * toward the middle. The skipped list is what makes gate 4's refusal legible:
+ * "0 usable pairs" plus why beats a null.
+ */
+function pairEffects(pairs) {
+  const used = [];
+  const skipped = [];
+  for (const pair of pairs) {
+    const skip = (why) => skipped.push({ task: pair.task, why });
+    if (!pair.complete) {
+      skip('incomplete-pair');
+      continue;
+    }
+    const [off, on] = [pair.arms.off, pair.arms.on];
+    // One check, not two: `cost_complete` and a finite total are the same fact
+    // read from two fields (summarizeCost cannot produce one without the other),
+    // and a second skip reason for a state that cannot occur would be a branch
+    // no test can reach.
+    if (![off, on].every((a) => a.cost_complete && Number.isFinite(a.list_cost_usd))) {
+      skip('cost-incomplete');
+      continue;
+    }
+    // Exactly one of the arms ran first. Two arms sharing a pass_order (or
+    // carrying none) is a pair the rotation did not actually rotate, and the
+    // order effect computed from it would be an assumption dressed as a
+    // measurement.
+    const orders = [off.pass_order, on.pass_order];
+    if (!(orders.includes(1) && orders.includes(2))) {
+      skip('pass-order-unusable');
+      continue;
+    }
+    const first = off.pass_order === 1 ? off : on;
+    const second = off.pass_order === 1 ? on : off;
+    used.push({
+      task: pair.task,
+      off_usd: off.list_cost_usd,
+      on_usd: on.list_cost_usd,
+      arm_delta_usd: round6(off.list_cost_usd - on.list_cost_usd),
+      order_delta_usd: round6(first.list_cost_usd - second.list_cost_usd),
+    });
+  }
+
+  const arm = mean(used.map((p) => p.arm_delta_usd));
+  const order = mean(used.map((p) => p.order_delta_usd));
+  return {
+    // Named `*_usd` and left as list price, the same qualifier pricing.mjs
+    // attaches: this is what the work would have cost through the API, never
+    // money spent.
+    arm_effect_usd: arm,
+    order_effect_usd: order,
+    // Which arm the arithmetic points at — always reported, even when a gate
+    // refuses to let it be published. Null on an exact tie and when there is
+    // nothing to compute from; a tie is not a cheaper arm.
+    cheaper_arm: arm === null || arm === 0 ? null : arm > 0 ? 'on' : 'off',
+    pairs_used: used,
+    pairs_skipped: skipped,
+  };
+}
+
+function gate(id, question, passed, measured, threshold, { reason = null, missing = null } = {}) {
+  return { id, question, passed, measured, threshold, reason, missing };
+}
+
+// ─── the gates ──────────────────────────────────────────────────────────────
+//
+// Each one holds the verdict on its own and for its own reason. They are
+// written as independent functions over the same context rather than one
+// chain of ifs, because a gate that can only fire when the previous one passed
+// cannot report a run's second problem — and then the run gets repeated to
+// discover it.
+
+/** 1. Too small a sample — in pairs and in sessions per arm (§10). */
+function sampleGate({ arms, paired, t }) {
+  const min = t.sample.min_pairs;
+  const q = 'Is the sample big enough to rule on?';
+  const sessions = Object.fromEntries(ARMS.map((arm) => [arm, arms[arm].n_sessions]));
+  const measured = { n_pairs_complete: paired.n_pairs_complete, sessions_per_arm: sessions };
+  const short = ARMS.filter((arm) => sessions[arm] < min);
+  const pairsShort = paired.n_pairs_complete < min;
+  if (!pairsShort && short.length === 0) return gate('sample', q, true, measured, t.sample);
+
+  const reasons = [];
+  const missing = [];
+  if (pairsShort) {
+    reasons.push(`${paired.n_pairs_complete} complete pair(s), the frozen floor is ${min}`);
+    missing.push(`${min - paired.n_pairs_complete} more complete pair(s)`);
+  }
+  for (const arm of short) {
+    reasons.push(`arm ${arm} has ${sessions[arm]} session(s), the frozen floor is ${min}`);
+    missing.push(`${min - sessions[arm]} more session(s) in arm ${arm}`);
+  }
+  return gate('sample', q, false, measured, t.sample, {
+    reason: `sample below the PO floor: ${reasons.join('; ')}`,
+    missing: missing.join(', '),
+  });
+}
+
+/**
+ * 2. Model or effort differs between the arms (§10, error class 4).
+ *
+ * Read as a set union across both arms: one distinct value in the pool or the
+ * pair is not a pair. An arm with no rows contributes nothing — its emptiness is
+ * gate 1's business, and failing here as well would report one problem twice
+ * under two names.
+ */
+function homogeneityGate({ arms, t }) {
+  const seen = (field) => [...new Set(ARMS.flatMap((arm) => Object.keys(arms[arm][field])))].sort();
+  const models = seen('models');
+  const efforts = seen('efforts');
+  const measured = {
+    models_by_arm: Object.fromEntries(ARMS.map((arm) => [arm, arms[arm].models])),
+    efforts_by_arm: Object.fromEntries(ARMS.map((arm) => [arm, arms[arm].efforts])),
+    distinct_models: models,
+    distinct_efforts: efforts,
+  };
+  const q = 'Did both arms run on one model and one effort?';
+  const broken = [];
+  if (models.length > 1) broken.push(`model: ${models.join(', ')}`);
+  if (efforts.length > 1) broken.push(`effort: ${efforts.join(', ')}`);
+  if (broken.length === 0) return gate('homogeneity', q, true, measured, t.homogeneity);
+
+  return gate('homogeneity', q, false, measured, t.homogeneity, {
+    reason:
+      `the arms are not comparable — ${broken.join('; ')}; the threshold is "identical", with no tolerance` +
+      ' (measured opus/sonnet input price differs by 1.7x, more than any effect being looked for)',
+    // Deliberately not "collect more data": more rows from a heterogeneous run
+    // make a bigger heterogeneous run.
+    missing: 'nothing collectable — the run is invalid under §7 and must be repeated with one model and one effort in both arms',
+  });
+}
+
+/**
+ * 3. The cheaper arm delegated more (§11a).
+ *
+ * Counts, not costs, and that is the whole point: the cost of a delegation is
+ * known only from below (subagent turns never reach the transcript), so the
+ * number that is uncertain is exactly the one this gate must not depend on.
+ * `spawns` rather than `measured` for the same reason — a delegation with no
+ * usable record is the worst case, not an absent one.
+ */
+function delegationGate({ arms, effects, t }) {
+  const spawns = Object.fromEntries(ARMS.map((arm) => [arm, arms[arm].delegations.spawns]));
+  const cheaper = effects.cheaper_arm;
+  // One shape on every path, `direction_known` included: a field present in one
+  // branch and absent in another makes `undefined` and `false` mean the same
+  // thing to the reader downstream, and they do not.
+  const measured = { spawns_by_arm: spawns, cheaper_arm: cheaper, direction_known: cheaper !== null, diff: null };
+  const q = 'Did the cheaper arm win by hiding work in subagents?';
+  const max = t.delegation.max_diff;
+
+  if (cheaper === null) {
+    // No cheaper arm means no dangerous direction to fire in, and §11a is
+    // explicit that this gate is not symmetric. Holding here would report the
+    // order gate's refusal a second time under a different name. The counts are
+    // published either way.
+    return gate('delegation', q, true, measured, t.delegation);
+  }
+
+  const diff = spawns[cheaper] - spawns[other(cheaper)];
+  measured.diff = diff;
+  if (diff <= max) {
+    return gate('delegation', q, true, measured, t.delegation);
+  }
+  return gate('delegation', q, false, measured, t.delegation, {
+    reason:
+      `the cheaper arm (${cheaper}) made ${diff} more Agent call(s) than ${other(cheaper)}` +
+      ` (${spawns[cheaper]} vs ${spawns[other(cheaper)]}), above the frozen maximum of ${max};` +
+      ' a subagent\'s turns never reach the transcript, so that arm is cheaper by an unknown remainder',
+    missing: `a repeat run in which arm ${cheaper} delegates no more than ${max} times above ${other(cheaper)}`,
+  });
+}
+
+/**
+ * 4. The order effect swamps the arm effect (§11b).
+ *
+ * The one gate whose held state is the likeliest real outcome of a two-pair run,
+ * and the one whose refusal is most tempting to talk past: both numbers are
+ * reported so the report can print them side by side and rule on neither.
+ */
+function orderGate({ effects, t }) {
+  const ratio = t.order_effect.min_arm_over_order;
+  const arm = effects.arm_effect_usd;
+  const order = effects.order_effect_usd;
+  const q = 'Is the arm effect large enough next to the known order effect?';
+  const measured = {
+    arm_effect_usd: arm,
+    order_effect_usd: order,
+    // The observed ratio, for a report that wants to say how far off it was.
+    // Null when it cannot be formed (no pairs, or an order effect of exactly
+    // zero next to which every arm effect is infinitely large).
+    ratio: arm === null || order === null || order === 0 ? null : round6(Math.abs(arm) / Math.abs(order)),
+    pairs_used: effects.pairs_used.length,
+    pairs_skipped: effects.pairs_skipped,
+  };
+  if (arm === null || order === null) {
+    return gate('order_effect', q, false, measured, t.order_effect, {
+      reason:
+        'neither effect can be computed: 0 usable pair(s)' +
+        (effects.pairs_skipped.length > 0
+          ? ` (skipped: ${effects.pairs_skipped.map((p) => `${p.task} ${p.why}`).join(', ')})`
+          : ''),
+      missing: 'at least one pair with both arms priced and one arm at pass_order 1, the other at 2',
+    });
+  }
+  if (Math.abs(arm) >= ratio * Math.abs(order)) {
+    return gate('order_effect', q, true, measured, t.order_effect);
+  }
+  // Past both returns above, `ratio` is always a number: an order effect of
+  // exactly zero passes (every arm effect clears 2 x 0), so the only paths that
+  // reach here have a divisor.
+  return gate('order_effect', q, false, measured, t.order_effect, {
+    reason:
+      `the arm effect ${formatUsd(arm)} does not reach ${ratio}x the order effect ${formatUsd(order)}` +
+      ` (measured ratio ${measured.ratio}x);` +
+      ' the second implementation of a task is cheaper because the solution is already known, and that runs in the' +
+      ' same direction as the effect being measured',
+    missing:
+      `pairs in which |arm effect| reaches ${ratio}x |order effect| — with the rotation of §9 kept, since without it` +
+      ' the two effects cannot be separated at all',
+  });
+}
+
+/** 5. Part of the pool has no price (§1 precondition). */
+function unpricedGate({ arms, t }) {
+  const rows = Object.fromEntries(ARMS.map((arm) => [arm, arms[arm].cost.unpriced.rows]));
+  const total = ARMS.reduce((n, arm) => n + rows[arm], 0);
+  const measured = {
+    unpriced_rows: total,
+    unpriced_rows_by_arm: rows,
+    reasons_by_arm: Object.fromEntries(ARMS.map((arm) => [arm, arms[arm].cost.unpriced.reasons])),
+    models_by_arm: Object.fromEntries(ARMS.map((arm) => [arm, arms[arm].cost.unpriced.models])),
+  };
+  const q = 'Does the cost cover the whole pool?';
+  if (total <= t.unpriced.max_rows) return gate('unpriced', q, true, measured, t.unpriced);
+
+  const models = [...new Set(ARMS.flatMap((arm) => Object.keys(arms[arm].cost.unpriced.models)))].sort();
+  return gate('unpriced', q, false, measured, t.unpriced, {
+    reason:
+      `${total} row(s) carry no price (off ${rows.off}, on ${rows.on}) against a frozen maximum of` +
+      ` ${t.unpriced.max_rows}; a cost computed over part of the rows is not the cost of the pool`,
+    missing:
+      models.length > 0
+        ? `prices for ${models.join(', ')} in eval/pricing.mjs, then a recount (§8)`
+        : 'the price table completed for the models named in the unpriced reasons, then a recount (§8)',
+  });
+}
+
+/**
+ * 6. More of the measurement was thrown away than the threshold allows (§10).
+ *
+ * Rows, not sessions, because rows are what the aggregate counts rejects in —
+ * and because a rejected run loses all of its scripted prompts, the two units
+ * give the same answer at the sizes this runs at.
+ *
+ * One deliberate over-count remains: a `mixed-arm-session` in ordinary daily
+ * work (a session left open across a ledger `set`) is indistinguishable, in the
+ * aggregate, from a scripted session that switched arms — the counter carries no
+ * pool. It is counted, and the direction of that choice is the one to be wrong
+ * in: a false hold names the session ids and asks for a look, while the opposite
+ * error drops a genuinely switched experiment session out of sight, which is the
+ * failure class this epic exists to prevent.
+ */
+function rejectedGate({ agg, arms, t, journal }) {
+  const byReason = agg.counters.rejected_by_reason;
+  let rejected = 0;
+  let background = 0;
+  const reasons = {};
+  // Rows this gate cannot place: a reject reason analyze.mjs learned to emit
+  // after this file was written. Neither counted nor discarded — a discard that
+  // does not show up is the failure class the epic is built around, and a new
+  // reason silently missing from the share is exactly that. It surfaces in the
+  // measurement so the next reader classifies it deliberately.
+  const unclassified = {};
+  for (const [reason, n] of Object.entries(byReason)) {
+    if (isBackgroundReject(reason)) {
+      background += n;
+      continue;
+    }
+    if (!isExperimentReject(reason)) {
+      unclassified[reason] = n;
+      continue;
+    }
+    rejected += n;
+    reasons[reason] = n;
+  }
+  const used = ARMS.reduce((n, arm) => n + arms[arm].n_prompts, 0);
+  // The journal's rejects sit on the same side of the ratio as the parser's:
+  // both are measurement that was thrown away, and §10 freezes ONE share over
+  // the two. Keeping them in separate counters would let a series pass the
+  // threshold twice on half the evidence each time.
+  rejected += journal.rows;
+  const pool = used + rejected;
+  const share = pool === 0 ? 0 : round6(rejected / pool);
+  const measured = {
+    rejected_rows: rejected,
+    rows_in_pool: pool,
+    share,
+    rejected_by_reason: reasons,
+    // 'NOT REPORTED' rather than 0, for the same reason analyze.mjs prints it
+    // for unattributed entries: a zero the caller never supplied is a claim
+    // nobody made.
+    operator_rejects: journal.reported ? journal.rows : 'NOT REPORTED',
+    operator_reject_runs: journal.runs,
+    mixed_arm_sessions: agg.counters.mixed_arm_sessions,
+    mixed_arm_session_ids: agg.counters.mixed_arm_session_ids,
+    // Reported, never counted above — see isBackgroundReject().
+    background_rows_without_arm: background,
+    unclassified_rejects: unclassified,
+  };
+  const q = 'Did enough of the measurement survive to be about the measurement?';
+  // Not told, and there is a scripted pool to be told about. The share computed
+  // above is a floor, not a share, so it cannot clear a maximum — and passing on
+  // a floor is how this gate reported 0% for a series that had discarded 33%.
+  // A pool with no paired rows has no journal to read, so it is not held.
+  if (!journal.reported && used > 0) {
+    return gate('rejected', q, false, measured, t.rejected, {
+      reason:
+        'the journal count of §7 rejections was not supplied, so the share above counts only what the parser could' +
+        ` see (${rejected} of ${pool} row(s)); an operator-rejected run leaves no trace in the rows by design`,
+      missing:
+        'rejectedRuns from the run journal — pass [] to state that the journal was read and nothing was thrown away',
+    });
+  }
+  if (share <= t.rejected.max_share) return gate('rejected', q, true, measured, t.rejected);
+
+  return gate('rejected', q, false, measured, t.rejected, {
+    reason:
+      `${rejected} of ${pool} experiment row(s) rejected — ${(share * 100).toFixed(1)}% against a frozen maximum of` +
+      ` ${(t.rejected.max_share * 100).toFixed(1)}% (${Object.entries(reasons).map(([r, n]) => `${r} ${n}`).join(', ')});` +
+      ' a broken run is repeated, not absorbed',
+    missing: 'the rejected run(s) repeated under §7 — at four sessions there is nothing to spare',
+  });
+}
+
+/**
+ * 7. More than one meter computed this chain, or one that names no code
+ *    (EPIC-010, PO decision 2, frozen 2026-10-15).
+ *
+ * The stamps read are the paired pool's rows (`paired.meter_versions`), the
+ * aggregate's `analyzed_by` and this verdict's own `judged_by` — the whole
+ * chain, because in the EPIC-009 series the measured tasks changed analyze and
+ * verdict themselves, and rows from one version judged by another is the same
+ * error one floor up. Natural work never reaches the verdict, so its stamps are
+ * not read.
+ *
+ * analyze does not refuse mixed stamps (decision 2): the per-pair costs must
+ * stay visible to plan a repeat. So the refusal lives here, and it says which
+ * step carried which stamp — "two meters" alone does not say what to recompute.
+ */
+function meterGate({ agg, paired, judgedBy, t }) {
+  const q = 'Did one identifiable meter compute the whole chain, from the rows to this verdict?';
+  const carriers = new Map();
+  const carry = (stamp, what) => {
+    if (!carriers.has(stamp)) carriers.set(stamp, []);
+    carriers.get(stamp).push(what);
+  };
+  const rows = paired.meter_versions ?? {};
+  for (const [stamp, n] of Object.entries(rows)) carry(stamp, `rows ×${n}`);
+  // A step with no stamp is not a stamp of its own: it is a step nobody can
+  // name, and it holds the gate under that description.
+  const unstamped = [];
+  if (agg.analyzed_by) carry(agg.analyzed_by, 'analyze');
+  else unstamped.push('analyze');
+  if (judgedBy) carry(judgedBy, 'verdict');
+  else unstamped.push('verdict');
+
+  const stamps = [...carriers.keys()].sort();
+  const unidentifiable = Object.fromEntries(
+    stamps.map((stamp) => [stamp, stampProblem(stamp)]).filter(([, problem]) => problem !== null),
+  );
+  const measured = {
+    stamps,
+    carried_by: Object.fromEntries(stamps.map((stamp) => [stamp, carriers.get(stamp)])),
+    rows_by_stamp: { ...rows },
+    analyzed_by: agg.analyzed_by ?? null,
+    judged_by: judgedBy ?? null,
+    unidentifiable,
+    unstamped_steps: unstamped,
+  };
+  if (stamps.length === 1 && Object.keys(unidentifiable).length === 0 && unstamped.length === 0) {
+    return gate('meter_mixed', q, true, measured, t.meter_mixed);
+  }
+
+  const where = (stamp) => `${stamp} (${carriers.get(stamp).join(', ')})`;
+  const reasons = [];
+  if (stamps.length > 1) reasons.push(`the chain was computed by ${stamps.length} meters: ${stamps.map(where).join('; ')}`);
+  for (const [stamp, problem] of Object.entries(unidentifiable)) reasons.push(`${where(stamp)} names no code: ${problem}`);
+  if (unstamped.length > 0) reasons.push(`${unstamped.join(' and ')} left no stamp`);
+  return gate('meter_mixed', q, false, measured, t.meter_mixed, {
+    reason: reasons.join('; '),
+    missing:
+      'the whole chain from one meter that names its code — recompute the rows from the transcripts or from an' +
+      ' archive (`tune-context experiment archive`) with the current version, or run every step with the version' +
+      ' that computed the rows (`npx -p @bglocation/tune-context@<version> tune-context eval …`); a series runs' +
+      ' from a release or a clean commit, never from a .dirty tree',
+  });
+}
+
+const GATES = [sampleGate, homogeneityGate, delegationGate, orderGate, unpricedGate, rejectedGate, meterGate];
+
+/**
+ * Aggregate in, decision out.
+ *
+ * @param agg         the object analyze() returned (schema tune-context/analyze@2).
+ * @param thresholds    the frozen table. Injectable so a test can prove a gate
+ *                      READS it rather than repeating the number in code — which
+ *                      is the only way to tell a frozen threshold from a
+ *                      coincidence. Callers in production pass nothing.
+ * @param rejectedRuns  the §7 rejections the run journal recorded, as
+ *                      `{ session, task, arm, prompts, reason }`. Omitting it is
+ *                      not the same as passing `[]` — see
+ *                      normalizeJournalRejects(). A paired pool cannot be ruled
+ *                      on while it is omitted.
+ * @param judgedBy      the stamp of the code running this function, from the
+ *                      caller — the same arrangement as report's `generatedAt`,
+ *                      so this stays free of ambient state. Omitted, the
+ *                      meter_mixed gate holds: a verdict nobody can trace to code
+ *                      is the thing EPIC-010 exists to stop.
+ *
+ * Throws on an input that is not an analyze aggregate: the same layer-boundary
+ * rule classifyRow() applies to unannotated rows. A shape this does not
+ * recognise would otherwise read as a run with no pairs, no cost and no
+ * rejects — i.e. as a withheld verdict, which is the one wrong answer nobody
+ * would look at twice.
+ */
+export function verdict(agg, { thresholds = VERDICT_THRESHOLDS, rejectedRuns, judgedBy = null } = {}) {
+  if (!agg || typeof agg !== 'object' || agg.schema !== 'tune-context/analyze@2') {
+    // An @1 aggregate is refused, not upgraded: its rows carry no stamp, and the
+    // only way back to that data is through the transcripts (EPIC-010 decision 4).
+    const got = agg && typeof agg === 'object' && typeof agg.schema === 'string' ? ` — got ${agg.schema}` : '';
+    throw new VerdictInputError(
+      `verdict: expected an aggregate from analyze() (schema tune-context/analyze@2)${got}; recompute it with` +
+        ' `tune-context eval analyze` from stamped rows (EPIC-010: unstamped data is not used)',
+    );
+  }
+  const paired = agg.paired;
+  const arms = paired.arms;
+  const t = thresholds.gates;
+  const effects = pairEffects(paired.pairs);
+  const journal = normalizeJournalRejects(rejectedRuns);
+
+  const ctx = { agg, paired, arms, effects, t, journal, judgedBy };
+  const gates = GATES.map((fn) => fn(ctx));
+  const holds = gates.filter((g) => !g.passed);
+  // Three outcomes, named, because two of them are not "cheaper arm" and
+  // collapsing either into a bare `false` produces a refusal with no reason
+  // attached — the one thing this file may never emit.
+  //
+  //   withheld      a gate held; `holds` says which and what is missing.
+  //   no-difference every gate passed and the arms cost the same on average.
+  //                 That is a finding, not a failure: at this sample size "no
+  //                 difference measured" is exactly what the run may publish,
+  //                 and reporting it as a withheld verdict would hide a clean
+  //                 measurement behind a word that means "something is wrong".
+  //   cheaper-arm   every gate passed and the arithmetic points at one arm.
+  const finding = holds.length > 0 ? 'withheld' : effects.cheaper_arm === null ? 'no-difference' : 'cheaper-arm';
+  const ruled = finding !== 'withheld';
+
+  return {
+    schema: 'tune-context/verdict@2',
+    pool: 'paired',
+    // Which code judged — read by the meter_mixed gate above, and kept here so
+    // a verdict read a year later still names it.
+    judged_by: judgedBy,
+    // The table travels with the decision: a verdict read six months later has
+    // to show which thresholds it was judged against, without anyone trusting
+    // that the file still says the same thing.
+    thresholds,
+    effects,
+    gates,
+    // The same objects as above, filtered — not a second copy that could
+    // disagree. All of them, because a reader who fixes one problem and reruns
+    // has paid a run to learn about the second.
+    holds,
+    finding,
+    // True for both `cheaper-arm` and `no-difference`: the gates licensed a
+    // statement, and which statement it is, is `finding`. A consumer that reads
+    // `ruled` alone and reaches for `cheaper_arm` gets null rather than a name
+    // it could print.
+    ruled,
+    // Null unless every gate passed. `effects.cheaper_arm` above stays whatever
+    // the arithmetic said: the arithmetic is a measurement, this is a licence to
+    // publish it, and collapsing the two is exactly how a withheld verdict gets
+    // printed as a result anyway.
+    cheaper_arm: ruled ? effects.cheaper_arm : null,
+    n_pairs_used: effects.pairs_used.length,
+    // "Observation", never "proof" (§10) — and unconditionally so. The PO froze
+    // a MINIMUM and nothing above it, so there is no sample size at which this
+    // may flip to "proof". A comparison computed here (n > min => proof) would
+    // be the model choosing the point at which its own tool stops being
+    // qualified — the same anti-bias rule that keeps the thresholds in the
+    // table. Promoting a run past "observation" takes a PO decision with its
+    // own frozen date, which lands in the table, not in this expression.
+    observation_only: true,
+  };
+}
+
+// ─── text ───────────────────────────────────────────────────────────────────
+
+const pctText = (x) => (Number.isFinite(x) ? `${(x * 100).toFixed(1)}%` : '—');
+
+/**
+ * Per gate: what was measured, against what. One entry per gate in GATES — a
+ * test holds the two lists equal, so a new gate cannot reach the terminal as a
+ * bare id. Kept here rather than in the gates, because a gate's job is to
+ * decide and this is only how a decision reads at 80 columns.
+ */
+const GATE_SUMMARIES = {
+  sample: (m, t) =>
+    `${m.n_pairs_complete} complete pair(s), sessions off ${m.sessions_per_arm.off} / on ${m.sessions_per_arm.on}; floor ${t.min_pairs}`,
+  homogeneity: (m) => `model ${m.distinct_models.join(', ') || '—'}, effort ${m.distinct_efforts.join(', ') || '—'}; must be identical`,
+  delegation: (m, t) =>
+    `spawns off ${m.spawns_by_arm.off} / on ${m.spawns_by_arm.on}` +
+    (m.direction_known ? `, cheaper arm ${m.cheaper_arm} +${m.diff}; max +${t.max_diff}` : ', no cheaper arm'),
+  order_effect: (m, t) =>
+    m.arm_effect_usd === null || m.order_effect_usd === null
+      ? `no usable pair; needs |arm| >= ${t.min_arm_over_order}x |order|`
+      : `arm ${formatUsd(m.arm_effect_usd)}, order ${formatUsd(m.order_effect_usd)}, ratio ${m.ratio ?? '—'}x; needs >= ${t.min_arm_over_order}x`,
+  unpriced: (m, t) => `${m.unpriced_rows} unpriced row(s); max ${t.max_rows}`,
+  rejected: (m, t) =>
+    `${pctText(m.share)} rejected (${m.rejected_rows} of ${m.rows_in_pool} row(s)), journal ` +
+    `${m.operator_rejects === 'NOT REPORTED' ? 'NOT REPORTED' : `${m.operator_rejects} row(s)`}; max ${pctText(t.max_share)}`,
+  meter_mixed: (m) =>
+    `${m.stamps.map((stamp) => `${stamp} (${m.carried_by[stamp].join(', ')})`).join('; ') || 'no stamp'}` +
+    `${m.unstamped_steps.length > 0 ? `, unstamped: ${m.unstamped_steps.join(', ')}` : ''}; one identifiable stamp`,
+};
+
+/**
+ * The verdict as text, for stderr: one line per gate — passed or held, what it
+ * measured against its threshold, and why it holds — then the finding in one
+ * word. Every gate, always, in the order verdict() evaluated them, for the same
+ * reason verdict() evaluates them all: a reader who fixes one problem and
+ * reruns has paid a run to learn about the second.
+ *
+ * Pure: a verdict in, a string out.
+ */
+export function formatVerdict(v) {
+  const L = [`verdict: paired pool, ${v.n_pairs_used} usable pair(s) — an observation, never proof`];
+  for (const g of v.gates) {
+    const summary = GATE_SUMMARIES[g.id]?.(g.measured, g.threshold) ?? '';
+    L.push(`  ${g.passed ? '✓' : '✗'} ${g.id.padEnd(12)} ${summary}${g.passed ? '' : ` — ${g.reason}`}`);
+  }
+  // Said on its own line, because it is the one hold the operator causes by
+  // omission: the flag was not passed, so nobody asked the journal.
+  const journal = v.gates.find((g) => g.id === 'rejected');
+  if (journal && !journal.passed && journal.measured.operator_rejects === 'NOT REPORTED') {
+    L.push(
+      '  ! no --rejected-runs: nobody asked the run journal, so gate `rejected` holds by construction —' +
+        ' pass its §7 rejections, or a file containing [] once the journal has been read',
+    );
+  }
+  L.push(
+    `finding: ${
+      v.finding === 'withheld'
+        ? `withheld: ${v.holds.length} gate(s) hold — ${v.holds.map((g) => g.id).join(', ')}`
+        : v.finding === 'no-difference'
+          ? 'no-difference'
+          : `cheaper-arm: ${v.cheaper_arm}`
+    }`,
+  );
+  return L.join('\n');
+}
+
+// ─── CLI ────────────────────────────────────────────────────────────────────
+//
+// The only I/O in this file (see the header). `eval verdict` and `eval report`
+// both judge, so both read their input through readVerdictInput() below.
+
+const USAGE = `
+verdict — may this run say "one arm is cheaper"? The gates over an analyze aggregate (TASK-090).
+
+Usage:
+  verdict [--aggregate <path>] [--rejected-runs <path>]
+
+Options:
+  --aggregate <path>      JSON written by analyze (default: read stdin)
+  --rejected-runs <path>  JSON array of the run journal's §7 rejections,
+                          [{ session, task, arm, prompts, reason }]. OMITTING it
+                          is not the same as passing a file containing []: the
+                          first says nobody asked the journal, and gate 6 holds
+                          the verdict for it. Same flag, same meaning as report.
+  -h, --help              Show this help
+
+Writes the verdict as JSON to stdout — the same object report embeds in its
+HTML — and one line per gate to stderr. Exit 0 for every verdict, a withheld one
+included: withholding is an answer, not a failure. Exit 1 only when the input
+cannot be read.
+`.trim();
+
+/**
+ * A file the CLI was pointed at, as text — or the VerdictInputError naming the
+ * flag: missing, a directory, unreadable. Asking existsSync first let a
+ * directory through to readFileSync, and up as an EISDIR stacktrace.
+ */
+function readInputFile(path, flag) {
+  try {
+    return readFileSync(path, 'utf8');
+  } catch (e) {
+    throw new VerdictInputError(e.code === 'ENOENT' ? `no file at ${path} (${flag})` : `cannot read ${path} (${flag}): ${e.message}`);
+  }
+}
+
+function parseJson(text, what) {
+  try {
+    return JSON.parse(text);
+  } catch (e) {
+    throw new VerdictInputError(`${what} is not JSON: ${e.message}`);
+  }
+}
+
+/**
+ * The aggregate and the run journal, read and checked in ONE place for both
+ * steps that judge. Two copies would drift, and the drift nobody notices is the
+ * report accepting an input the verdict refuses.
+ *
+ * @param aggregate     a path, or undefined to read stdin
+ * @param rejectedRuns  a path, or undefined — and undefined STAYS undefined. It
+ *                      means nobody asked the journal, which gate 6 holds on;
+ *                      defaulting it to [] here would pass that gate on a share
+ *                      whose denominator nobody supplied.
+ * @returns {Promise<{agg: object, rejectedRuns: object[]|undefined}>}
+ * @throws {VerdictInputError} a file that is missing, a directory or
+ *         unreadable, a file that is not JSON, or a journal of the wrong shape —
+ *         the message names the flag and the path.
+ *         Whether the aggregate IS an aggregate is verdict()'s check, the same
+ *         one a module caller gets.
+ */
+export async function readVerdictInput({ aggregate, rejectedRuns }) {
+  // Both files before stdin: a mistyped --rejected-runs is reported at once,
+  // not after the aggregate has been piped in.
+  const aggText = aggregate === undefined ? undefined : readInputFile(aggregate, '--aggregate');
+  const runsText = rejectedRuns === undefined ? undefined : readInputFile(rejectedRuns, '--rejected-runs');
+  const agg =
+    aggText === undefined ? parseJson(await readStdin(), 'the aggregate on stdin') : parseJson(aggText, `--aggregate ${aggregate}`);
+  if (runsText === undefined) return { agg, rejectedRuns: undefined };
+  const runs = parseJson(runsText, `--rejected-runs ${rejectedRuns}`);
+  const problem = journalShapeProblem(runs);
+  if (problem !== null) throw new VerdictInputError(`--rejected-runs ${rejectedRuns}: ${problem}`);
+  return { agg, rejectedRuns: runs };
+}
+
+/**
+ * The line an input error prints: the step's name, then the message — unless
+ * the message already names this step (verdict()'s own errors start with
+ * `verdict:`), so `eval verdict` never stutters while `eval report` still says
+ * which layer refused.
+ */
+export function inputErrorLine(step, error) {
+  return error.message.startsWith(`${step}: `) ? error.message : `${step}: ${error.message}`;
+}
+
+/**
+ * The CLI. `args` is everything after the script name, or after
+ * `tune-context eval verdict` — one parser behind both doors (EPIC-010).
+ */
+export async function main(args = process.argv.slice(2)) {
+  endQuietlyOnClosedPipe();
+  let values;
+  try {
+    ({ values } = parseArgs({
+      args,
+      options: {
+        aggregate: { type: 'string' },
+        'rejected-runs': { type: 'string' },
+        help: { type: 'boolean', short: 'h', default: false },
+      },
+      strict: true,
+    }));
+  } catch (e) {
+    process.stderr.write(`verdict: ${e.message}\n`);
+    process.exitCode = 1;
+    return;
+  }
+  if (values.help) {
+    process.stdout.write(`${USAGE}\n`);
+    return;
+  }
+
+  let v;
+  try {
+    const { agg, rejectedRuns } = await readVerdictInput({ aggregate: values.aggregate, rejectedRuns: values['rejected-runs'] });
+    v = verdict(agg, { rejectedRuns, judgedBy: meterStamp() });
+  } catch (e) {
+    // Exactly the input boundary. A TypeError from inside a gate is a bug and
+    // keeps its stacktrace.
+    if (!(e instanceof VerdictInputError)) throw e;
+    process.stderr.write(`${inputErrorLine('verdict', e)}\n`);
+    process.exitCode = 1;
+    return;
+  }
+
+  process.stdout.write(`${JSON.stringify(v, null, 2)}\n`);
+  process.stderr.write(`${formatVerdict(v)}\n`);
+}
+
+if (isEntryPoint(import.meta.url)) {
+  await main();
+}