@bglocation/tune-context 2.4.0 → 2.5.0

package/eval/analyze.mjs

@@ -0,0 +1,689 @@
+#!/usr/bin/env node
+// analyze — per-prompt rows (TASK-085/086, arm-annotated by TASK-088) → one
+// aggregate per ARM. See tasks/TASK-089.
+//
+// This file walks the distance between "rows exist" and "this arm cost less",
+// and stops one step short of the second: it aggregates and it discards, it
+// never rules. The verdict, and every threshold behind one, is TASK-090.
+//
+// Two things it must make impossible rather than merely discourage:
+//
+//   • **Adding the two populations together.** The paired measurement (four
+//     scripted sessions, §2a) and ordinary daily work are not two samples of
+//     one thing — one holds model, effort, task and prompt text constant, the
+//     other holds nothing constant. So there is no collection of pools to fold
+//     over here, and no cost field above them: `paired` and `natural` are two
+//     named objects with different shapes (only `paired` has `pairs`), and a
+//     reader who wants a combined number has to write out both names and mean
+//     it. Criterion 3.
+//   • **A discard that does not show up.** Every row that goes in comes out
+//     either inside an arm or inside a counter, and `rows_in === rows_used +
+//     rows_rejected` is asserted by a test rather than by hope. The failure
+//     this guards against is the one earlier epics kept finding: a sample that
+//     quietly shrank to the rows that agreed with the conclusion.
+//
+// The arm is a property of the SESSION, not of the prompt: MCP servers and
+// settings.json are read once at startup, so a session whose rows disagree
+// about the arm was never in one arm at all. It is rejected whole (error class
+// 6), never split down the middle.
+import { existsSync, readFileSync } from 'node:fs';
+import { parseArgs } from 'node:util';
+import { claudeBaseDir } from '../hooks/config-dir.mjs';
+import { experimentPaths, readLedger } from '../cli/experiment.mjs';
+import { ledgerArmAt, parseArmBranch, rowTimestamp } from './arm.mjs';
+import { TOKEN_CLASS_FIELDS, formatUsd, summarizeCost } from './pricing.mjs';
+
+/** The two arms, always both present in the output — see armsOf(). */
+export const ARMS = ['off', 'on'];
+
+const USAGE = `
+analyze — per-arm aggregate from prompt-metrics rows (TASK-089).
+
+Usage:
+  analyze [--rows <path>]...
+
+Options:
+  --rows <path>  A JSONL file written by prompt-metrics. Repeatable.
+                 (default: read from stdin)
+  -h, --help     Show this help
+
+Writes the aggregate as JSON to stdout and a short summary to stderr. It does
+NOT rule on which arm is cheaper — that is TASK-090.
+`.trim();
+
+// ─── numbers ────────────────────────────────────────────────────────────────
+
+/** Micro-dollar resolution, the same as pricing.mjs — one rounding rule. */
+function round6(n) {
+  return Math.round(n * 1e6) / 1e6;
+}
+
+/** Milliseconds, for the two quantities measured in seconds. */
+function round3(n) {
+  return Math.round(n * 1000) / 1000;
+}
+
+/**
+ * Quantile by linear interpolation between closest ranks (the R-7 definition
+ * numpy and most spreadsheets use). Chosen over nearest-rank because these
+ * pools hold single-digit prompt counts, where nearest-rank makes p25 and the
+ * minimum the same number and the spread looks narrower than it is.
+ */
+function quantile(sorted, q) {
+  if (sorted.length === 1) return sorted[0];
+  const pos = (sorted.length - 1) * q;
+  const lo = Math.floor(pos);
+  const hi = Math.ceil(pos);
+  return sorted[lo] + (sorted[hi] - sorted[lo]) * (pos - lo);
+}
+
+/**
+ * Distribution of a per-prompt quantity: **median and spread, never the mean
+ * alone** (criterion 7).
+ *
+ * The mean rides along rather than being dropped, because the gap between the
+ * two is itself the finding — one 44-hour turn (measured: `work_s` 158999) or
+ * one prompt that read a 200k-token file moves a mean of six and leaves a
+ * median untouched. A single figure would have to pick which of those two
+ * stories to tell.
+ *
+ * Non-finite values are excluded and counted: `work_s` is null whenever a turn
+ * has no assistant reply, and a null coerced to 0 is a prompt that took no
+ * time.
+ */
+function stats(values, { round = round6 } = {}) {
+  const xs = values.filter((v) => Number.isFinite(v)).sort((a, b) => a - b);
+  const missing = values.length - xs.length;
+  if (xs.length === 0) {
+    return { n: 0, missing, total: null, mean: null, median: null, p25: null, p75: null, iqr: null, min: null, max: null };
+  }
+  const total = xs.reduce((a, b) => a + b, 0);
+  const p25 = quantile(xs, 0.25);
+  const p75 = quantile(xs, 0.75);
+  return {
+    n: xs.length,
+    missing,
+    total: round(total),
+    mean: round(total / xs.length),
+    median: round(quantile(xs, 0.5)),
+    p25: round(p25),
+    p75: round(p75),
+    iqr: round(p75 - p25),
+    min: round(xs[0]),
+    max: round(xs[xs.length - 1]),
+  };
+}
+
+/**
+ * Tokens for a set of rows: a total that is never published without the five
+ * classes it is made of (criterion 4).
+ *
+ * `cache_write_unsplit` is reported apart from the five rather than folded into
+ * them — the 5m and 1h write multipliers differ by 60%, so a class we cannot
+ * place is a number we cannot price, and it is exactly why the rows carrying it
+ * come back unpriced.
+ */
+function tokenTotals(rows) {
+  const by_class = Object.fromEntries(TOKEN_CLASS_FIELDS.map(([name]) => [name, 0]));
+  let unsplit = 0;
+  for (const row of rows) {
+    for (const [name, field] of TOKEN_CLASS_FIELDS) by_class[name] += row[field] || 0;
+    unsplit += row.cache_write_unsplit || 0;
+  }
+  return {
+    total: Object.values(by_class).reduce((a, b) => a + b, 0) + unsplit,
+    by_class,
+    cache_write_unsplit: unsplit,
+  };
+}
+
+/** Counts of a categorical field, e.g. `{ 'claude-opus-5': 7, mixed: 1 }`. */
+function distribution(rows, field) {
+  // Null-prototype: the keys are model ids and effort strings from a
+  // transcript, so `constructor` is a value this can legitimately be handed.
+  const out = Object.create(null);
+  for (const row of rows) {
+    const key = row[field] === null || row[field] === undefined ? '(none)' : String(row[field]);
+    out[key] = (out[key] ?? 0) + 1;
+  }
+  return out;
+}
+
+function sumField(rows, field) {
+  return rows.reduce((n, row) => n + (row[field] || 0), 0);
+}
+
+function distinct(rows, field) {
+  return new Set(rows.map((row) => row[field]).filter((v) => typeof v === 'string' && v !== ''));
+}
+
+// ─── classification ─────────────────────────────────────────────────────────
+
+/**
+ * Is this branch name TRYING to be an experiment branch and failing?
+ *
+ * The distinction the aggregate has to draw is not "valid or invalid" but
+ * "which population": `main` is not a broken `exp/task-089/on`, it is ordinary
+ * work and belongs to the natural pool by way of the ledger. `exp/task-089`,
+ * `exp/` and `exp/task-089/on/extra` are different — they claim the convention
+ * and miss it, so they go to a counter and to NO arm (criterion 1). Guessing
+ * which arm was meant is the silent mis-assignment TASK-088 exists to prevent.
+ */
+function isOffConventionBranch(branch) {
+  if (typeof branch !== 'string' || branch === '') return false;
+  const b = branch.trim();
+  return (b === 'exp' || b.startsWith('exp/')) && parseArmBranch(b) === null;
+}
+
+/**
+ * One row → the pool and arm it belongs to, or the reason it belongs to
+ * neither. Never throws and never returns "probably".
+ *
+ * Order matters and is the order of decreasing certainty:
+ *
+ *  1. `gitBranchChanged` — the arm marker moved INSIDE one prompt, so it
+ *     describes two states and therefore neither. Same family as a mixed
+ *     session, one level down.
+ *  2. An `exp/<task>/<arm>` branch → the paired pool, with the arm annotateRows
+ *     already reconciled against the ledger. An unresolved arm there carries
+ *     its own reason (`conflict:ledger`, `conflict:no-ledger-entry`, …) and
+ *     that reason is what gets counted — "12 rejected" says nothing, "12
+ *     rejected: conflict:ledger" says which run to repeat.
+ *  3. A branch that IS the convention on a row annotated as ordinary work, or a
+ *     branch that merely mimics the convention → counter only. Two different
+ *     disagreements, two different reasons, neither of them an arm.
+ *  4. Everything else is ordinary work, and its arm comes from the ledger,
+ *     which is the only record of what was switched on at the time.
+ *
+ * "Never throws" has one exception, and it is not about data: a row that never
+ * went through annotateRows is a wiring mistake in the caller, and the reading
+ * that would let it pass silently is the one this whole file exists to prevent.
+ */
+export function classifyRow(row, { ledger = [], ledgerAvailable = false } = {}) {
+  if (row.exp_arm_source === undefined) {
+    // The layer boundary arm.mjs warns about. An unannotated row has `undefined`
+    // where the contract promises `''`, and `undefined !== ''` would let it pass
+    // as ordinary work with a straight face. Fail loudly instead: this is a
+    // wiring mistake in the caller, not a property of the data.
+    throw new TypeError('analyze: rows must be annotated with annotateRows() from eval/arm.mjs first');
+  }
+  if (row.gitBranchChanged === true) return { pool: null, reject: 'branch-changed' };
+  if (row.exp_task) {
+    if (!row.exp_arm) return { pool: null, reject: row.exp_arm_source || 'unresolved-arm' };
+    return { pool: 'paired', task: row.exp_task, arm: row.exp_arm };
+  }
+  // The branch says "experiment" and the annotation says "ordinary work". One of
+  // the two is wrong and there is no way to tell which, so the row goes to
+  // neither pool. Unreachable through annotateRows today — it is here because
+  // the alternative reading is the silent one: without this the row would be
+  // attributed from the LEDGER as if it were daily work, which is how an
+  // experimental prompt ends up in the background pool with a straight face.
+  if (parseArmBranch(row.gitBranch) !== null) return { pool: null, reject: 'branch-annotation-mismatch' };
+  if (isOffConventionBranch(row.gitBranch)) return { pool: null, reject: 'off-convention-branch' };
+  // No ledger on this machine is an ABSENT control, not a negative one: the row
+  // is ordinary work whose arm nobody recorded, which is not the same as work
+  // done with the doctrine off. Refused either way, but under its own name.
+  if (!ledgerAvailable) return { pool: null, reject: 'no-ledger' };
+  const arm = ledgerArmAt(ledger, rowTimestamp(row));
+  if (arm === null) return { pool: null, reject: 'no-arm-in-force' };
+  // `mixed` is half the doctrine and half the RAG — a state the comparison has
+  // no name for, so it is refused rather than rounded up to "on".
+  if (!ARMS.includes(arm)) return { pool: null, reject: `unusable-arm:${arm}` };
+  return { pool: 'natural', arm };
+}
+
+/**
+ * Sessions whose rows do not agree on one arm, and are therefore thrown away
+ * whole (criterion 2).
+ *
+ * The key is pool AND arm, so a session that started on an `exp/` branch and
+ * finished on `main` is caught as well as one that flipped `off` to `on`: both
+ * mean the session's rows were produced under two different setups, and the
+ * setup is what the measurement is about.
+ *
+ * **A row that resolved to nothing counts as its own key**, not as an absence.
+ * Without that, a session half of whose rows resolved and half did not stayed in
+ * the sample with its resolved half — and that is the commonest form of the
+ * switch, not an exotic one: a ledger window closed by `restore` mid-session
+ * leaves the earlier prompts with an arm and the later ones with none. Half a
+ * session is not a smaller sample, it is a sample whose setup we cannot state.
+ *
+ * A session with a null id is skipped rather than lumped in with every other
+ * unnamed session; it is counted on its own so the omission is on the record.
+ */
+function mixedArmSessions(classified) {
+  const keysBySession = new Map();
+  for (const { row, verdict } of classified) {
+    const session = row.session;
+    if (typeof session !== 'string' || session === '') continue;
+    if (!keysBySession.has(session)) keysBySession.set(session, new Set());
+    keysBySession.get(session).add(verdict.pool === null ? 'unresolved' : `${verdict.pool}:${verdict.arm}`);
+  }
+  return new Set([...keysBySession.entries()].filter(([, keys]) => keys.size > 1).map(([session]) => session));
+}
+
+// ─── delegations ────────────────────────────────────────────────────────────
+
+/**
+ * The delegation records of a set of rows, **deduplicated by `agentId`**
+ * (criterion 8a).
+ *
+ * Three passes guard one number, at three different scopes, because they see
+ * different copies: the collector drops entries repeating a uuid within a file,
+ * then deduplicates by `agentId` within a prompt, and this pass closes the case
+ * neither can see — the same delegation landing on two ROWS, which is what
+ * happens when the copies carry different promptIds (measured) or come from two
+ * transcript files (the uuid index is per file, deliberately). Without the last
+ * one the only figure we have for subagent cost can still come out doubled, and
+ * it is a figure the verdict gate in §11a reads directly.
+ */
+function dedupeDelegations(rows) {
+  const byId = new Map();
+  let duplicates = 0;
+  for (const row of rows) {
+    for (const d of row.delegations ?? []) {
+      if (byId.has(d.agent_id)) {
+        duplicates++;
+        continue;
+      }
+      byId.set(d.agent_id, d);
+    }
+  }
+  return { list: [...byId.values()], duplicates };
+}
+
+/**
+ * What the subagents of one arm cost, **from below**.
+ *
+ * Three properties this shape enforces, all from COST_EVALUATION_PROCEDURE
+ * §11a:
+ *
+ *   • It is its own field. It is never added to the arm's cost, because a lower
+ *     bound folded into a total silently becomes the total.
+ *   • `bound: 'lower'` travels WITH the number. The evidence is that a
+ *     subagent's `cache_read` grows linearly with its tool-call count, which is
+ *     one turn re-reading its context, not a sum over turns — so the true cost
+ *     is this figure times an unknown factor ≥ 1.
+ *   • `spawns` and `measured` are both reported. Their difference is the part
+ *     of the delegation cost for which there is no figure at all; a report that
+ *     showed only `measured` would look complete.
+ */
+function delegationSummary(rows) {
+  const { list, duplicates } = dedupeDelegations(rows);
+  const spawns = sumField(rows, 'agent_spawns');
+  return {
+    bound: 'lower',
+    spawns,
+    measured: list.length,
+    unmeasured_spawns: Math.max(0, spawns - list.length),
+    duplicates_dropped: duplicates + sumField(rows, 'delegation_duplicates_dropped'),
+    unidentified_dropped: sumField(rows, 'delegations_unidentified'),
+    cost: summarizeCost(list),
+    tokens: tokenTotals(list),
+    by_agent_type: distribution(list, 'agent_type'),
+  };
+}
+
+// ─── the aggregate ──────────────────────────────────────────────────────────
+
+/**
+ * Everything the comparison needs about one arm — and nothing that decides it.
+ *
+ * `cost` comes from summarizeCost() rather than a sum written here, which is
+ * what keeps criterion 5 true by construction: an unpriced row is outside the
+ * total and inside `unpriced.rows`, with the model that caused it named. A pool
+ * whose cost is unknown for some of its rows says so through `cost_complete`
+ * instead of averaging over the rest.
+ */
+function armAggregate(arm, rows, { n_pairs = 0 } = {}) {
+  const cost = summarizeCost(rows);
+  return {
+    arm,
+    n_prompts: rows.length,
+    n_sessions: distinct(rows, 'session').size,
+    // Passed in rather than patched in afterwards: only the pool level knows
+    // what a pair is, and a field with two writers is a field whose value
+    // depends on the order they ran in. Zero in the natural pool, where the
+    // concept does not apply, but present so the two arm objects stay readable
+    // by one consumer.
+    n_pairs,
+    cost,
+    // Per-prompt spread alongside the arm's total: two arms can share a total
+    // and differ entirely in how it was spent (criterion 7).
+    cost_per_prompt: stats(rows.map((r) => r.list_cost_usd)),
+    cost_complete: cost.total_rows > 0 && cost.unpriced.rows === 0,
+    // work_s, not a duration measured from submission: the wait between sending
+    // a prompt and the first assistant token is queue time, which belongs to the
+    // user and to the machine's load, not to the arm.
+    work_s: stats(rows.map((r) => r.work_s), { round: round3 }),
+    // The one queue-time figure kept, because §8 lists it as a thing to check
+    // before believing a difference: a large gap inside a turn is a tool call
+    // that sat waiting for the operator to approve it.
+    max_gap_s: stats(rows.map((r) => r.max_gap_s), { round: round3 }),
+    tokens: tokenTotals(rows),
+    tools: {
+      calls_total: sumField(rows, 'tool_calls_total'),
+      agent_spawns: sumField(rows, 'agent_spawns'),
+      by_name: rows.reduce((acc, row) => {
+        for (const [name, n] of Object.entries(row.tools ?? {})) acc[name] = (acc[name] ?? 0) + n;
+        return acc;
+      }, Object.create(null)),
+    },
+    delegations: delegationSummary(rows),
+    // Held-constant variables, reported rather than assumed (§2). A pair whose
+    // arms ran on different models or efforts is not a pair; TASK-090 reads
+    // these two distributions to say so.
+    models: distribution(rows, 'model'),
+    efforts: distribution(rows, 'effort'),
+    // The honesty counters that belong to an arm rather than to the input as a
+    // whole. `unpriced` sits in `cost` above, where the reason and the model
+    // that caused it live too.
+    counters: {
+      synthetic_turns: sumField(rows, 'synthetic_turns'),
+      understated_turns: sumField(rows, 'understated_turns'),
+      multi_attempt_turns: sumField(rows, 'multi_attempt_turns'),
+      sidechain_turns: sumField(rows, 'sidechain_turns'),
+      api_turns: sumField(rows, 'api_turns'),
+    },
+  };
+}
+
+/**
+ * Both arms, always — an arm with no rows is emitted as zeros rather than
+ * omitted, so `agg.paired.arms.on` is never undefined and a missing arm reads
+ * as `n_prompts: 0` instead of crashing three layers up (criterion 6).
+ */
+function armsOf(rows, pairsPerArm = {}) {
+  const out = {};
+  for (const arm of ARMS) {
+    out[arm] = armAggregate(arm, rows.filter((r) => r.arm === arm), { n_pairs: pairsPerArm[arm] ?? 0 });
+  }
+  return out;
+}
+
+/**
+ * One task measured twice. This is the unit §11b does arithmetic on: the arm
+ * effect and the order effect are both means over pairs, and neither can be
+ * computed from pool totals alone — which is why the per-pair, per-arm cost is
+ * carried here rather than left to be re-derived.
+ *
+ * `complete` is false for a task with only one arm. Such a task is NOT dropped:
+ * a half-finished pair is a fact about the run (§7 says to repeat it), and a
+ * pair that vanishes from the aggregate cannot be repeated because nobody sees
+ * it is missing.
+ */
+function pairsOf(rows) {
+  const tasks = [...new Set(rows.map((r) => r.task))].sort();
+  return tasks.map((task) => {
+    const arms = {};
+    for (const arm of ARMS) {
+      const armRows = rows.filter((r) => r.task === task && r.arm === arm);
+      if (armRows.length === 0) continue;
+      const cost = summarizeCost(armRows);
+      const passOrders = [...new Set(armRows.map((r) => r.pass_order).filter((p) => p !== null && p !== undefined))];
+      arms[arm] = {
+        // One value or none. Two different pass_orders for one (task, arm) means
+        // the rows came from two different runs of the same arm, and picking one
+        // would be inventing which run this is.
+        pass_order: passOrders.length === 1 ? passOrders[0] : null,
+        n_prompts: armRows.length,
+        n_sessions: distinct(armRows, 'session').size,
+        list_cost_usd: cost.list_cost_usd,
+        cost_complete: cost.total_rows > 0 && cost.unpriced.rows === 0,
+        work_s_total: stats(armRows.map((r) => r.work_s)).total,
+        agent_spawns: sumField(armRows, 'agent_spawns'),
+        delegations_measured: dedupeDelegations(armRows).list.length,
+      };
+    }
+    return {
+      task,
+      complete: ARMS.every((arm) => arms[arm] !== undefined),
+      arms,
+    };
+  });
+}
+
+/**
+ * The paired measurement: four scripted sessions per §2a, everything held
+ * constant except the arm.
+ *
+ * The only pool with `pairs`, and that asymmetry is deliberate — it is what
+ * makes the two pools un-addable by shape rather than by comment (criterion 3).
+ */
+function pairedPool(rows) {
+  const pairs = pairsOf(rows);
+  const complete = pairs.filter((p) => p.complete);
+  const pairsPerArm = Object.fromEntries(
+    ARMS.map((arm) => [arm, complete.filter((p) => p.arms[arm] !== undefined).length]),
+  );
+  return {
+    pool: 'paired',
+    source: 'branch exp/<task>/<arm> (TASK-088), reconciled against the TASK-087 ledger',
+    arms: armsOf(rows, pairsPerArm),
+    pairs,
+    n_pairs_complete: complete.length,
+  };
+}
+
+/**
+ * Ordinary work, attributed by the ledger alone.
+ *
+ * Nothing is held constant here: different tasks, different days, different
+ * prompt text. It is background — useful for asking whether the paired result
+ * points the same way, useless as evidence on its own, and dangerous the moment
+ * its prompts are counted alongside the scripted ones.
+ */
+function naturalPool(rows) {
+  return {
+    pool: 'natural',
+    source: 'TASK-087 ledger, by timestamp',
+    arms: armsOf(rows),
+  };
+}
+
+/**
+ * Rows in, aggregate out. Pure: no disk, no clock, no ambient state.
+ *
+ * @param rows  prompt-metrics rows that have been through annotateRows().
+ * @param ledger / ledgerAvailable  the TASK-087 ledger, for natural work only.
+ *        Paired rows carry their arm already; ordinary work has no marker but
+ *        this one.
+ * @param unattributed  the collector's count of transcript entries that landed
+ *        in no row at all. It is not derivable from rows — that is the point of
+ *        it — so it is passed through, and stays **null** when the caller did
+ *        not supply it. Null means "not reported", never zero.
+ */
+export function analyze(rows, { ledger = [], ledgerAvailable = false, unattributed = null } = {}) {
+  const classified = rows.map((row) => ({ row, verdict: classifyRow(row, { ledger, ledgerAvailable }) }));
+  const mixed = mixedArmSessions(classified);
+
+  const pools = { paired: [], natural: [] };
+  const rejected = Object.create(null);
+  const offConventionNames = new Set();
+  const noArmSessions = new Set();
+  let rowsWithoutSession = 0;
+
+  const reject = (reason) => {
+    rejected[reason] = (rejected[reason] ?? 0) + 1;
+  };
+
+  for (const { row, verdict } of classified) {
+    if (typeof row.session !== 'string' || row.session === '') rowsWithoutSession++;
+    if (verdict.reject === 'off-convention-branch') offConventionNames.add(row.gitBranch);
+    if (verdict.reject === 'no-arm-in-force' || verdict.reject === 'no-ledger') {
+      if (typeof row.session === 'string' && row.session !== '') noArmSessions.add(row.session);
+    }
+    if (verdict.pool === null) {
+      reject(verdict.reject);
+      continue;
+    }
+    // The session-level refusal runs LAST and overrides a row that looked fine
+    // on its own: the arm is the session's property, so a well-formed row from a
+    // session that switched arms mid-flight is well-formed about the wrong
+    // thing (criterion 2).
+    if (mixed.has(row.session)) {
+      reject('mixed-arm-session');
+      continue;
+    }
+    // A copy of the row carrying its resolved `arm` and `task`, so the
+    // aggregators below read one flat object and never re-derive the
+    // classification. Neither key exists on a prompt-metrics row (it spells them
+    // `exp_arm`/`exp_task`), so nothing is being overwritten here.
+    pools[verdict.pool].push({ ...row, arm: verdict.arm, task: verdict.task ?? '' });
+  }
+
+  const used = pools.paired.length + pools.natural.length;
+  return {
+    schema: 'tune-context/analyze@1',
+    // No cost field at this level, on purpose. There is nothing here to add the
+    // pools into, and no array of pools to fold over — see the file header.
+    paired: pairedPool(pools.paired),
+    natural: naturalPool(pools.natural),
+    counters: {
+      rows_in: rows.length,
+      rows_used: used,
+      // rows_in === rows_used + rows_rejected, asserted in the tests. Every row
+      // is in exactly one of the two.
+      rows_rejected: rows.length - used,
+      rejected_by_reason: rejected,
+      mixed_arm_sessions: mixed.size,
+      mixed_arm_session_ids: [...mixed].sort(),
+      sessions_without_arm: noArmSessions.size,
+      off_convention_branches: rejected['off-convention-branch'] ?? 0,
+      off_convention_branch_names: [...offConventionNames].sort(),
+      rows_without_session: rowsWithoutSession,
+      unattributed,
+    },
+  };
+}
+
+// ─── reporting ──────────────────────────────────────────────────────────────
+
+function armLine(a) {
+  // An arm with no rows says so instead of printing $0.0000: a zero next to an
+  // arm that was never measured is the one number this whole file exists to
+  // keep off the page.
+  if (a.n_prompts === 0) return `    ${a.arm.padEnd(3)} no rows`;
+  const cost = a.cost_complete ? formatUsd(a.cost.list_cost_usd) : `${formatUsd(a.cost.list_cost_usd)} INCOMPLETE`;
+  return (
+    `    ${a.arm.padEnd(3)} ${a.n_prompts} prompt(s), ${a.n_sessions} session(s), ${a.n_pairs} pair(s)` +
+    ` | ${cost} (median/prompt ${formatUsd(a.cost_per_prompt.median)}, IQR ${formatUsd(a.cost_per_prompt.iqr)})` +
+    ` | work ${a.work_s.median ?? '—'}s median` +
+    ` | ${a.tools.agent_spawns} spawn(s), delegation floor ${formatUsd(a.delegations.cost.list_cost_usd)}`
+  );
+}
+
+/**
+ * A summary for stderr. Prints what was thrown away next to what was kept, and
+ * says outright that it is not a verdict — a per-arm table with no such line
+ * reads as one whatever the surrounding prose claims.
+ */
+export function formatAnalysis(agg) {
+  const L = [`analyze: ${agg.counters.rows_in} row(s) in, ${agg.counters.rows_used} used, ${agg.counters.rows_rejected} rejected`];
+  L.push('  paired measurement (TASK-088 — the only pool a verdict may be built on):');
+  for (const arm of ARMS) L.push(armLine(agg.paired.arms[arm]));
+  for (const pair of agg.paired.pairs) {
+    const orders = ARMS.filter((a) => pair.arms[a]).map((a) => `${a}=pass${pair.arms[a].pass_order ?? '?'}`);
+    L.push(`      ${pair.task}: ${orders.join(' ')}${pair.complete ? '' : ' — INCOMPLETE PAIR'}`);
+  }
+  L.push('  natural work (background — never added to the pool above):');
+  for (const arm of ARMS) L.push(armLine(agg.natural.arms[arm]));
+
+  const reasons = Object.entries(agg.counters.rejected_by_reason).sort((a, b) => b[1] - a[1]);
+  if (reasons.length > 0) {
+    L.push(`  rejected: ${reasons.map(([r, n]) => `${r} ${n}`).join(', ')}`);
+  }
+  if (agg.counters.mixed_arm_sessions > 0) {
+    L.push(
+      `  ${agg.counters.mixed_arm_sessions} session(s) changed arm mid-session and were dropped WHOLE` +
+        ' — the arm is a property of the session (EPIC-009 class 6), so that run is repeated, not absorbed',
+    );
+  }
+  if (agg.counters.off_convention_branch_names.length > 0) {
+    L.push(`  branches outside exp/<task>/<arm>: ${agg.counters.off_convention_branch_names.join(', ')} — counted, never assigned`);
+  }
+  if (agg.counters.unattributed === null) {
+    L.push('  unattributed transcript entries: NOT REPORTED (the caller did not pass them) — this is not a zero');
+  } else {
+    L.push(`  unattributed transcript entries: ${agg.counters.unattributed.entries ?? agg.counters.unattributed}`);
+  }
+  L.push('  this is an aggregate, NOT a verdict — thresholds and gates are TASK-090.');
+  return L.join('\n');
+}
+
+// ─── CLI ────────────────────────────────────────────────────────────────────
+
+async function readStdin() {
+  const chunks = [];
+  for await (const chunk of process.stdin) chunks.push(chunk);
+  return Buffer.concat(chunks).toString('utf8');
+}
+
+/** JSONL → rows, malformed lines counted rather than fatal — the same rule the
+ *  collector applies to transcripts, for the same reason: the file may be being
+ *  appended to while we read it. */
+export function parseRows(text) {
+  const rows = [];
+  let malformed = 0;
+  for (const line of text.split('\n')) {
+    if (line.trim() === '') continue;
+    try {
+      const row = JSON.parse(line);
+      if (row && typeof row === 'object' && !Array.isArray(row)) rows.push(row);
+      else malformed++;
+    } catch {
+      malformed++;
+    }
+  }
+  return { rows, malformed };
+}
+
+async function main() {
+  let text;
+  try {
+    const { values } = parseArgs({
+      args: process.argv.slice(2),
+      options: {
+        rows: { type: 'string', multiple: true },
+        help: { type: 'boolean', short: 'h', default: false },
+      },
+      strict: true,
+    });
+    if (values.help) {
+      process.stdout.write(`${USAGE}\n`);
+      return;
+    }
+    const paths = values.rows ?? [];
+    const missing = paths.filter((p) => !existsSync(p));
+    if (missing.length > 0) {
+      for (const p of missing) process.stderr.write(`analyze: no rows file at ${p}\n`);
+      process.exitCode = 1;
+      return;
+    }
+    text = paths.length > 0 ? paths.map((p) => readFileSync(p, 'utf8')).join('\n') : await readStdin();
+  } catch (e) {
+    process.stderr.write(`analyze: ${e.message}\n`);
+    process.exitCode = 1;
+    return;
+  }
+
+  const { rows, malformed } = parseRows(text);
+  if (malformed > 0) process.stderr.write(`analyze: ${malformed} malformed line(s) skipped\n`);
+
+  // The ledger is read here rather than inside analyze() for the same reason
+  // prompt-metrics reads it in its CLI: it is machine state, and a pure function
+  // that reaches for the disk cannot be tested against someone else's run.
+  const ledgerPath = experimentPaths(claudeBaseDir()).ledgerPath;
+  const ledgerAvailable = existsSync(ledgerPath);
+  const agg = analyze(rows, {
+    ledger: ledgerAvailable ? readLedger(claudeBaseDir()) : [],
+    ledgerAvailable,
+  });
+
+  process.stdout.write(`${JSON.stringify(agg, null, 2)}\n`);
+  process.stderr.write(`${formatAnalysis(agg)}\n`);
+}
+
+if (process.argv[1]?.endsWith('analyze.mjs')) {
+  await main();
+}