@bglocation/tune-context 2.3.1 → 2.5.0

package/eval/arm.mjs

@@ -0,0 +1,246 @@
+// arm — assigns a metrics row to an experiment arm (TASK-088).
+//
+// Two independent descriptions of the same fact, and the rule for when they
+// disagree:
+//
+//   • The BRANCH (`exp/<task>/<arm>`) carries the arm the operator INTENDED. It
+//     is on every transcript entry, works retroactively, and does not pollute
+//     the prompt text — which is why TASK-088 chose it as the marker.
+//   • The LEDGER (TASK-087) carries the arm actually SET before the session.
+//
+// These are not two competing truths; they are a measurement and its control
+// (COST_EVALUATION_PROCEDURE §8). A disagreement means the arm is UNKNOWN: the
+// row gets `exp_arm: ''` plus the reason in `exp_arm_source`, and the run goes
+// to §7. Nothing here guesses and nothing defaults to "on" — a silently
+// assigned arm is precisely the false saving TASK-088 exists to prevent.
+//
+// The module is pure: it never touches the disk. The caller (the CLI) loads the
+// ledger, so the same rules can be tested against in-memory entries and applied
+// to rows collected on another machine.
+
+/** The arms the procedure knows. Anything else is not an arm. */
+const ARMS = new Set(['on', 'off']);
+
+/**
+ * `exp/<task>/<arm>` → `{task, arm}`; **null** for everything else.
+ *
+ * The strictness is a function, not fussiness: TASK-088 criterion 1 says a
+ * branch outside the convention must not fall SILENTLY into an arm. So names
+ * close to valid ones are rejected too — `exp/task-089/off-by-one`,
+ * `feat/exp/task-089/on`, `exp/task-089/OFF` — rather than being stretched onto
+ * the nearest matching arm.
+ *
+ * The `<task>` segment stays free (non-empty, no whitespace): §9 of the
+ * procedure decides which tasks count; the parser only has to avoid lying about
+ * the shape of the name.
+ */
+export function parseArmBranch(branch) {
+  if (typeof branch !== 'string') return null;
+  const parts = branch.trim().split('/');
+  if (parts.length !== 3) return null;
+  const [prefix, task, arm] = parts;
+  if (prefix !== 'exp') return null;
+  if (task === '' || /\s/.test(task)) return null;
+  if (!ARMS.has(arm)) return null;
+  return { task, arm };
+}
+
+/** `{tc, rag}` → one label. Half and half is `mixed`, never rounded up to "on". */
+function armLabel(tc, rag) {
+  if (tc === 'on' && rag === 'on') return 'on';
+  if (tc === 'off' && rag === 'off') return 'off';
+  return 'mixed';
+}
+
+function msOf(value) {
+  if (typeof value !== 'string' || value === '') return null;
+  const ms = Date.parse(value);
+  return Number.isNaN(ms) ? null : ms;
+}
+
+/**
+ * The arm in force at `tsIso` according to the ledger: `'on'`, `'off'`,
+ * `'mixed'`, or **null**.
+ *
+ * Null means "no arm was in force" and covers two cases that are the same thing
+ * to the caller: there was no `set` entry before the session, or the covering
+ * entry is a `restore` — the measurement window was closed. That is exactly why
+ * TASK-087 appends `restore` to the ledger: without it the last arm would apply
+ * forever and every later piece of ordinary work would fall into the sample.
+ *
+ * Entries without a readable `ts` are skipped — they cannot be placed in time,
+ * and guessing their order would be inventing data.
+ */
+export function ledgerArmAt(entries, tsIso) {
+  const at = msOf(tsIso);
+  if (at === null || !Array.isArray(entries)) return null;
+  let covering = null;
+  let coveringMs = -Infinity;
+  for (const entry of entries) {
+    if (!entry || typeof entry !== 'object') continue;
+    const ms = msOf(entry.ts);
+    if (ms === null || ms > at) continue;
+    // `>=`, not `>`: on equal timestamps the later line in the file wins,
+    // because the ledger is append-only and write order is event order.
+    if (ms >= coveringMs) {
+      covering = entry;
+      coveringMs = ms;
+    }
+  }
+  if (covering === null) return null;
+  if (covering.event === 'restore') return null;
+  return armLabel(covering.tc, covering.rag);
+}
+
+/**
+ * The row's timestamp: when the prompt was submitted, or the first **readable**
+ * one of the remaining fields.
+ *
+ * One definition for every reader — the ledger window, `pass_order`, and the
+ * natural-work attribution in analyze.mjs (TASK-089), which asks the same ledger
+ * the same question about a row that is NOT on an `exp/` branch. Exported for
+ * that reason: a second "when did this row happen" would be a second answer.
+ *
+ * Two of these readers already drifted apart once: `??` only falls through on null, so it
+ * stopped at a `ts_submit` that was a string but unparseable — and a row the
+ * ledger DID cover came out as `conflict:no-ledger-entry`, i.e. a run to be
+ * repeated (§7) because of a cosmetic defect in the data.
+ */
+export function rowTimestamp(row) {
+  for (const value of [row.ts_submit, row.ts_first_assistant, row.ts_end]) {
+    if (msOf(value) !== null) return value;
+  }
+  return null;
+}
+
+function rowMs(row) {
+  return msOf(rowTimestamp(row));
+}
+
+function armFields(row, { ledger, ledgerAvailable }) {
+  const none = { exp_task: '', exp_arm: '', exp_arm_source: 'none', exp_ledger_arm: '' };
+  const parsed = parseArmBranch(row.gitBranch);
+  if (parsed === null) return none;
+
+  // The branch changed inside a single prompt — the arm marker then describes
+  // two states at once and therefore describes neither. That is a §8 signal, not
+  // a detail to average away.
+  if (row.gitBranchChanged === true) {
+    return { exp_task: parsed.task, exp_arm: '', exp_arm_source: 'conflict:branch-changed', exp_ledger_arm: '' };
+  }
+
+  // No ledger on this machine: the control is ABSENT, not negative. The row
+  // keeps the arm from the branch and says outright that nobody confirmed it —
+  // the same rule as `unknown` in `verify` (TASK-070): a thing that cannot be
+  // observed is neither verified nor broken.
+  if (!ledgerAvailable) {
+    return { exp_task: parsed.task, exp_arm: parsed.arm, exp_arm_source: 'branch', exp_ledger_arm: '' };
+  }
+
+  const fromLedger = ledgerArmAt(ledger, rowTimestamp(row));
+  if (fromLedger === null) {
+    return { exp_task: parsed.task, exp_arm: '', exp_arm_source: 'conflict:no-ledger-entry', exp_ledger_arm: '' };
+  }
+  if (fromLedger !== parsed.arm) {
+    return { exp_task: parsed.task, exp_arm: '', exp_arm_source: 'conflict:ledger', exp_ledger_arm: fromLedger };
+  }
+  return { exp_task: parsed.task, exp_arm: parsed.arm, exp_arm_source: 'branch+ledger', exp_ledger_arm: fromLedger };
+}
+
+/**
+ * `pass_order` — 1 for the arm that ran first within a task, 2 for the second.
+ * Derived from the start time, because the ledger does not know which task an
+ * entry belongs to (`--note` is prose for a human, not a field).
+ *
+ * The order is a covariate, not decoration: the second implementation of the
+ * same task is cheaper for a reason that has nothing to do with the arm, and
+ * that effect runs in the same direction as the one being measured (§11b).
+ * Without `pass_order` on the row the two cannot be separated.
+ *
+ * A row with no resolved arm gets `null` — ranking an unknown arm would assign
+ * it a place in an order we do not know.
+ */
+function assignPassOrder(rows) {
+  // Nested maps rather than a joined key: one structure instead of a
+  // build-key/split-key pair, so there is no separator to drift and nothing to
+  // misread in a diff.
+  const firstMs = new Map(); // task -> Map(arm -> earliest start)
+  for (const row of rows) {
+    if (!row.exp_task || !row.exp_arm) continue;
+    const ms = rowMs(row);
+    if (ms === null) continue;
+    if (!firstMs.has(row.exp_task)) firstMs.set(row.exp_task, new Map());
+    const arms = firstMs.get(row.exp_task);
+    const seen = arms.get(row.exp_arm);
+    if (seen === undefined || ms < seen) arms.set(row.exp_arm, ms);
+  }
+
+  const order = new Map(); // task -> Map(arm -> 1 | 2 | ...)
+  for (const [task, arms] of firstMs) {
+    // Ties broken by arm name, so two runs of the same analyzer cannot produce
+    // two different answers.
+    const ranked = [...arms.entries()].sort((a, b) => a[1] - b[1] || a[0].localeCompare(b[0]));
+    order.set(task, new Map(ranked.map(([arm], i) => [arm, i + 1])));
+  }
+
+  return rows.map((row) => ({
+    ...row,
+    pass_order: order.get(row.exp_task)?.get(row.exp_arm) ?? null,
+  }));
+}
+
+/**
+ * Adds five fields to every row: `exp_task`, `exp_arm`, `exp_arm_source`,
+ * `exp_ledger_arm`, `pass_order`. The fields are **always** present, empty ones
+ * included — a row that omits them is indistinguishable from a row the parser
+ * failed on (the same rule as `gitBranch: ''` in TASK-085).
+ *
+ * **This is a layer boundary, not an optional step.** A row straight from the
+ * collector (`aggregate`, `collectFiles(...).result()`) does NOT carry these
+ * fields: the collector never reads the ledger and has no business knowing about
+ * the experiment. A consumer reading rows through the module rather than through
+ * the CLI's JSONL (e.g. `analyze.mjs` in TASK-089) must call this itself —
+ * otherwise it gets `undefined` where the contract promises `''`, and then
+ * `undefined !== ''` lets a non-experiment row pass as an experimental one.
+ */
+export function annotateRows(rows, { ledger = [], ledgerAvailable = false } = {}) {
+  return assignPassOrder(rows.map((row) => ({ ...row, ...armFields(row, { ledger, ledgerAvailable }) })));
+}
+
+/**
+ * Arm summary for stderr — the §8 check ("are both arms in the data at all")
+ * computed by the tool instead of by a one-liner pasted next to it. Returns null
+ * when the data holds no row from an `exp/*` branch: an ordinary run has no
+ * reason to read a section about the experiment.
+ */
+export function formatArmSummary(rows) {
+  // Truthiness, not `!== ''`: a row that was never annotated has `undefined`
+  // here, and `undefined !== ''` would let ordinary work into the experiment
+  // section. Costs the same and does not depend on call order.
+  const experimental = rows.filter((row) => row.exp_task);
+  if (experimental.length === 0) return null;
+
+  const counts = new Map();
+  for (const row of experimental) {
+    const label = row.exp_arm === '' ? `${row.exp_task}/? (${row.exp_arm_source})` : `${row.exp_task}/${row.exp_arm}`;
+    const bucket = counts.get(label) ?? { rows: 0, passOrders: new Set() };
+    bucket.rows++;
+    if (row.pass_order !== null) bucket.passOrders.add(row.pass_order);
+    counts.set(label, bucket);
+  }
+
+  const lines = ['arms (branch × ledger):'];
+  for (const [label, bucket] of [...counts.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
+    const pass = bucket.passOrders.size ? ` pass_order=${[...bucket.passOrders].sort().join(',')}` : '';
+    lines.push(`  ${label}: ${bucket.rows} row(s)${pass}`);
+  }
+
+  const unresolved = experimental.filter((row) => row.exp_arm === '').length;
+  if (unresolved > 0) {
+    lines.push(
+      `  ${unresolved} row(s) on an exp/ branch could NOT be assigned to an arm — that run is invalid,`,
+      '  not averageable (COST_EVALUATION_PROCEDURE §7).',
+    );
+  }
+  return lines.join('\n');
+}