@bglocation/tune-context 2.3.1 → 2.4.0

package/cli/managed-block.mjs

@@ -24,3 +24,27 @@ export function upsertManagedBlock(existing, innerBlock) {
   const next = `${existing}${sep}\n${block}\n`;
   return { content: next, changed: true };
 }
+
+/**
+ * The inverse of upsertManagedBlock: drop the block AND its markers, keep every
+ * byte outside them. Used only by the experiment switch (TASK-087) to reach the
+ * `tc off` arm — init never removes the block, it replaces it.
+ *
+ * Deliberately conservative about what it deletes: only the marker span, plus
+ * the blank lines the removal would otherwise leave stacked at the seam. What
+ * the file's owner wrote below the closing marker is what an off arm has to
+ * keep — losing it would be the TASK-082 defect with a different trigger.
+ *
+ * @param {string|null} existing - current file content, or null if there is no file
+ * @returns {{content: string|null, changed: boolean}} content unchanged when there is no block
+ */
+export function removeManagedBlock(existing) {
+  if (existing == null) return { content: null, changed: false };
+  const startIdx = existing.indexOf(START_MARKER);
+  const endIdx = existing.indexOf(END_MARKER);
+  if (startIdx === -1 || endIdx === -1 || endIdx < startIdx) return { content: existing, changed: false };
+  const head = existing.slice(0, startIdx);
+  const tail = existing.slice(endIdx + END_MARKER.length).replace(/^\n+/, '');
+  const next = head.trim() === '' ? tail : `${head.trimEnd()}\n\n${tail}`;
+  return { content: next, changed: next !== existing };
+}

package/cli/verify.mjs

@@ -158,10 +158,19 @@ function freshnessChecks(claudeDir, pkgRoot) {
 
 /**
  * @param {string} [pkgRoot] - installed package root, for the staleness check
+ * @param {{tc: 'on'|'off', rag: 'on'|'off'}} [arm] - the experiment arm currently set
+ *   through `tune-context experiment` (TASK-087), read from the ledger state — NOT
+ *   detected here. Under `tc: 'off'` the doctrine is *supposed* to be absent, so the
+ *   expectations below are inverted rather than dropped: a missing registration is a
+ *   tick, and one that is still there is a RED, because an off arm that is not off
+ *   invalidates the measurement just as surely as a broken install invalidates a
+ *   session. Reporting "broken install" for a correctly disabled doctrine would be
+ *   the EPIC-006 false-red regression, only pointing the other way.
  * @returns {{ok: boolean, checks: {name: string, ok: boolean, detail?: string, unknown?: boolean}[]}}
  */
-export function verify({ claudeDir, userSettingsPath, expectedHookCommands, projectSettingsPath, expectedPermissions, pkgRoot }) {
+export function verify({ claudeDir, userSettingsPath, expectedHookCommands, projectSettingsPath, expectedPermissions, pkgRoot, arm }) {
   const checks = [];
+  const tcOff = arm?.tc === 'off';
 
   // Cross-check, not two independent presence checks. The old code asked
   // "does some command contain the script name?" and, separately, "does the
@@ -175,6 +184,19 @@ export function verify({ claudeDir, userSettingsPath, expectedHookCommands, proj
     const allCommands = entries.flatMap((g) => (g.hooks || []).map((h) => h.command)).filter(Boolean);
     const installedPath = resolve(join(claudeDir, 'hooks', scriptName));
 
+    if (tcOff) {
+      // Same looseness as the switch's own removal rule: any command naming the
+      // script counts, whatever dir it points at. A stricter test here would let
+      // a stray registration keep firing the hook while verify called the arm clean.
+      const still = allCommands.filter((c) => c.includes(scriptName));
+      checks.push({
+        name: `hooks.${event}: ${scriptName} unregistered (experiment arm tc=off)`,
+        ok: still.length === 0,
+        detail: still.length ? `still registered — the off arm is NOT in effect: ${still.join(', ')}` : undefined,
+      });
+      continue;
+    }
+
     // Classify by DESTINATION, not by spelling: a command that resolves to
     // the very file we installed is the CLI registration, whatever its path
     // happens to look like. Without this, PLUGIN_PATH_HINT's loose
@@ -254,7 +276,24 @@ export function verify({ claudeDir, userSettingsPath, expectedHookCommands, proj
     const scriptPath = join(claudeDir, 'hooks', scriptName);
     const exists = existsSync(scriptPath);
     const executable = exists && Boolean(statSync(scriptPath).mode & 0o111);
-    checks.push({ name: `${scriptName} installed + executable`, ok: exists && executable, detail: !exists ? 'missing' : !executable ? 'not executable' : undefined });
+    // Under an off arm a missing script is not a broken install: the switch
+    // parks REGISTRATIONS, never the scripts, so their absence just means the
+    // doctrine was never installed on this machine — which is what the arm asks
+    // for anyway. Reported as `?`, so it neither passes nor turns the arm red
+    // (epic criterion 4). Present-but-not-executable stays red in both arms: the
+    // arm did not cause it, and it breaks the moment the doctrine goes back on.
+    checks.push({
+      name: `${scriptName} installed + executable`,
+      ok: exists && executable,
+      unknown: tcOff && !exists,
+      detail: !exists
+        ? tcOff
+          ? 'not installed — this arm parks hook registrations, not the scripts themselves'
+          : 'missing'
+        : !executable
+          ? 'not executable'
+          : undefined,
+    });
   }
 
   if (projectSettingsPath && expectedPermissions?.length) {
@@ -268,8 +307,16 @@ export function verify({ claudeDir, userSettingsPath, expectedHookCommands, proj
 
   const skillsDir = join(claudeDir, 'skills');
   for (const skill of ['token-efficiency', 'caveman', 'cdd', 'phase-workflow']) {
-    const ok = existsSync(join(skillsDir, skill, 'SKILL.md'));
-    checks.push({ name: `skill ${skill} installed`, ok, detail: ok ? undefined : `missing ${join(skillsDir, skill, 'SKILL.md')}` });
+    const present = existsSync(join(skillsDir, skill, 'SKILL.md'));
+    if (tcOff) {
+      checks.push({
+        name: `skill ${skill} parked (experiment arm tc=off)`,
+        ok: !present,
+        detail: present ? `still installed at ${join(skillsDir, skill)} — the off arm is NOT in effect` : undefined,
+      });
+      continue;
+    }
+    checks.push({ name: `skill ${skill} installed`, ok: present, detail: present ? undefined : `missing ${join(skillsDir, skill, 'SKILL.md')}` });
   }
 
   checks.push(...freshnessChecks(claudeDir, pkgRoot));

package/eval/adoption-report.mjs

@@ -6,9 +6,10 @@
 // levers show adoption, which don't. See tasks/TASK-062.
 //
 // Honest scope (printed in the report too):
-//   • Cost is NOT here. Claude Code exposes token cost only via OTEL
-//     (`claude_code.cost.usage`, needs a collector) — never a hook. This report
-//     measures *behavior* (a proxy), not dollars. OTEL is the opt-in complement.
+//   • Cost is NOT here. This report reads the hook log, which carries no
+//     tokens — it measures *behavior* (a proxy), not dollars. For dollars see
+//     eval/prompt-metrics.mjs + eval/pricing.mjs (TASK-085/086), which price the
+//     transcript. OTEL stays the independent route to Anthropic's own figure.
 //   • caveman can't be measured: it's an output style, not a tool call.
 //   • RAG availability (the `~` line in the verdict) rests on a project's
 //     `.mcp.json` declaring a semantic-search server that predates the sessions
@@ -75,7 +76,7 @@ Options:
   -h, --help     Show this help
 
 The log is written by the adoption-log.mjs PostToolUse hook (installed by
-tune-context). Cost in dollars is NOT here — that needs OTEL (see report footer).
+tune-context). Cost in dollars is NOT here — see eval/prompt-metrics.mjs.
 `.trim();
 
 export function parseLog(text) {
@@ -494,8 +495,9 @@ export function formatReport(a) {
     }
   }
   L.push('');
-  L.push('Cost (dollars) is not measurable from hooks. For hard cost, enable OTEL:');
-  L.push('  CLAUDE_CODE_ENABLE_TELEMETRY=1 + an OTLP collector →');
+  L.push('Cost (dollars) is not in the hook log. Two independent routes to it:');
+  L.push('  eval/prompt-metrics.mjs — prices the transcript, works with hooks OFF too.');
+  L.push('  OTEL: CLAUDE_CODE_ENABLE_TELEMETRY=1 + an OTLP collector →');
   L.push('  metric `claude_code.cost.usage` (direct) / `claude_code.token.usage`.');
   L.push('  caveman (output style) leaves no tool call — not measured here.');
 

package/eval/arm.mjs

@@ -0,0 +1,242 @@
+// arm — assigns a metrics row to an experiment arm (TASK-088).
+//
+// Two independent descriptions of the same fact, and the rule for when they
+// disagree:
+//
+//   • The BRANCH (`exp/<task>/<arm>`) carries the arm the operator INTENDED. It
+//     is on every transcript entry, works retroactively, and does not pollute
+//     the prompt text — which is why TASK-088 chose it as the marker.
+//   • The LEDGER (TASK-087) carries the arm actually SET before the session.
+//
+// These are not two competing truths; they are a measurement and its control
+// (COST_EVALUATION_PROCEDURE §8). A disagreement means the arm is UNKNOWN: the
+// row gets `exp_arm: ''` plus the reason in `exp_arm_source`, and the run goes
+// to §7. Nothing here guesses and nothing defaults to "on" — a silently
+// assigned arm is precisely the false saving TASK-088 exists to prevent.
+//
+// The module is pure: it never touches the disk. The caller (the CLI) loads the
+// ledger, so the same rules can be tested against in-memory entries and applied
+// to rows collected on another machine.
+
+/** The arms the procedure knows. Anything else is not an arm. */
+const ARMS = new Set(['on', 'off']);
+
+/**
+ * `exp/<task>/<arm>` → `{task, arm}`; **null** for everything else.
+ *
+ * The strictness is a function, not fussiness: TASK-088 criterion 1 says a
+ * branch outside the convention must not fall SILENTLY into an arm. So names
+ * close to valid ones are rejected too — `exp/task-089/off-by-one`,
+ * `feat/exp/task-089/on`, `exp/task-089/OFF` — rather than being stretched onto
+ * the nearest matching arm.
+ *
+ * The `<task>` segment stays free (non-empty, no whitespace): §9 of the
+ * procedure decides which tasks count; the parser only has to avoid lying about
+ * the shape of the name.
+ */
+export function parseArmBranch(branch) {
+  if (typeof branch !== 'string') return null;
+  const parts = branch.trim().split('/');
+  if (parts.length !== 3) return null;
+  const [prefix, task, arm] = parts;
+  if (prefix !== 'exp') return null;
+  if (task === '' || /\s/.test(task)) return null;
+  if (!ARMS.has(arm)) return null;
+  return { task, arm };
+}
+
+/** `{tc, rag}` → one label. Half and half is `mixed`, never rounded up to "on". */
+function armLabel(tc, rag) {
+  if (tc === 'on' && rag === 'on') return 'on';
+  if (tc === 'off' && rag === 'off') return 'off';
+  return 'mixed';
+}
+
+function msOf(value) {
+  if (typeof value !== 'string' || value === '') return null;
+  const ms = Date.parse(value);
+  return Number.isNaN(ms) ? null : ms;
+}
+
+/**
+ * The arm in force at `tsIso` according to the ledger: `'on'`, `'off'`,
+ * `'mixed'`, or **null**.
+ *
+ * Null means "no arm was in force" and covers two cases that are the same thing
+ * to the caller: there was no `set` entry before the session, or the covering
+ * entry is a `restore` — the measurement window was closed. That is exactly why
+ * TASK-087 appends `restore` to the ledger: without it the last arm would apply
+ * forever and every later piece of ordinary work would fall into the sample.
+ *
+ * Entries without a readable `ts` are skipped — they cannot be placed in time,
+ * and guessing their order would be inventing data.
+ */
+export function ledgerArmAt(entries, tsIso) {
+  const at = msOf(tsIso);
+  if (at === null || !Array.isArray(entries)) return null;
+  let covering = null;
+  let coveringMs = -Infinity;
+  for (const entry of entries) {
+    if (!entry || typeof entry !== 'object') continue;
+    const ms = msOf(entry.ts);
+    if (ms === null || ms > at) continue;
+    // `>=`, not `>`: on equal timestamps the later line in the file wins,
+    // because the ledger is append-only and write order is event order.
+    if (ms >= coveringMs) {
+      covering = entry;
+      coveringMs = ms;
+    }
+  }
+  if (covering === null) return null;
+  if (covering.event === 'restore') return null;
+  return armLabel(covering.tc, covering.rag);
+}
+
+/**
+ * The row's timestamp: when the prompt was submitted, or the first **readable**
+ * one of the remaining fields.
+ *
+ * One definition for both readers (the ledger window and `pass_order`), because
+ * two of them already drifted apart once: `??` only falls through on null, so it
+ * stopped at a `ts_submit` that was a string but unparseable — and a row the
+ * ledger DID cover came out as `conflict:no-ledger-entry`, i.e. a run to be
+ * repeated (§7) because of a cosmetic defect in the data.
+ */
+function rowTs(row) {
+  for (const value of [row.ts_submit, row.ts_first_assistant, row.ts_end]) {
+    if (msOf(value) !== null) return value;
+  }
+  return null;
+}
+
+function rowMs(row) {
+  return msOf(rowTs(row));
+}
+
+function armFields(row, { ledger, ledgerAvailable }) {
+  const none = { exp_task: '', exp_arm: '', exp_arm_source: 'none', exp_ledger_arm: '' };
+  const parsed = parseArmBranch(row.gitBranch);
+  if (parsed === null) return none;
+
+  // The branch changed inside a single prompt — the arm marker then describes
+  // two states at once and therefore describes neither. That is a §8 signal, not
+  // a detail to average away.
+  if (row.gitBranchChanged === true) {
+    return { exp_task: parsed.task, exp_arm: '', exp_arm_source: 'conflict:branch-changed', exp_ledger_arm: '' };
+  }
+
+  // No ledger on this machine: the control is ABSENT, not negative. The row
+  // keeps the arm from the branch and says outright that nobody confirmed it —
+  // the same rule as `unknown` in `verify` (TASK-070): a thing that cannot be
+  // observed is neither verified nor broken.
+  if (!ledgerAvailable) {
+    return { exp_task: parsed.task, exp_arm: parsed.arm, exp_arm_source: 'branch', exp_ledger_arm: '' };
+  }
+
+  const fromLedger = ledgerArmAt(ledger, rowTs(row));
+  if (fromLedger === null) {
+    return { exp_task: parsed.task, exp_arm: '', exp_arm_source: 'conflict:no-ledger-entry', exp_ledger_arm: '' };
+  }
+  if (fromLedger !== parsed.arm) {
+    return { exp_task: parsed.task, exp_arm: '', exp_arm_source: 'conflict:ledger', exp_ledger_arm: fromLedger };
+  }
+  return { exp_task: parsed.task, exp_arm: parsed.arm, exp_arm_source: 'branch+ledger', exp_ledger_arm: fromLedger };
+}
+
+/**
+ * `pass_order` — 1 for the arm that ran first within a task, 2 for the second.
+ * Derived from the start time, because the ledger does not know which task an
+ * entry belongs to (`--note` is prose for a human, not a field).
+ *
+ * The order is a covariate, not decoration: the second implementation of the
+ * same task is cheaper for a reason that has nothing to do with the arm, and
+ * that effect runs in the same direction as the one being measured (§11b).
+ * Without `pass_order` on the row the two cannot be separated.
+ *
+ * A row with no resolved arm gets `null` — ranking an unknown arm would assign
+ * it a place in an order we do not know.
+ */
+function assignPassOrder(rows) {
+  // Nested maps rather than a joined key: one structure instead of a
+  // build-key/split-key pair, so there is no separator to drift and nothing to
+  // misread in a diff.
+  const firstMs = new Map(); // task -> Map(arm -> earliest start)
+  for (const row of rows) {
+    if (!row.exp_task || !row.exp_arm) continue;
+    const ms = rowMs(row);
+    if (ms === null) continue;
+    if (!firstMs.has(row.exp_task)) firstMs.set(row.exp_task, new Map());
+    const arms = firstMs.get(row.exp_task);
+    const seen = arms.get(row.exp_arm);
+    if (seen === undefined || ms < seen) arms.set(row.exp_arm, ms);
+  }
+
+  const order = new Map(); // task -> Map(arm -> 1 | 2 | ...)
+  for (const [task, arms] of firstMs) {
+    // Ties broken by arm name, so two runs of the same analyzer cannot produce
+    // two different answers.
+    const ranked = [...arms.entries()].sort((a, b) => a[1] - b[1] || a[0].localeCompare(b[0]));
+    order.set(task, new Map(ranked.map(([arm], i) => [arm, i + 1])));
+  }
+
+  return rows.map((row) => ({
+    ...row,
+    pass_order: order.get(row.exp_task)?.get(row.exp_arm) ?? null,
+  }));
+}
+
+/**
+ * Adds five fields to every row: `exp_task`, `exp_arm`, `exp_arm_source`,
+ * `exp_ledger_arm`, `pass_order`. The fields are **always** present, empty ones
+ * included — a row that omits them is indistinguishable from a row the parser
+ * failed on (the same rule as `gitBranch: ''` in TASK-085).
+ *
+ * **This is a layer boundary, not an optional step.** A row straight from the
+ * collector (`aggregate`, `collectFiles(...).result()`) does NOT carry these
+ * fields: the collector never reads the ledger and has no business knowing about
+ * the experiment. A consumer reading rows through the module rather than through
+ * the CLI's JSONL (e.g. `analyze.mjs` in TASK-089) must call this itself —
+ * otherwise it gets `undefined` where the contract promises `''`, and then
+ * `undefined !== ''` lets a non-experiment row pass as an experimental one.
+ */
+export function annotateRows(rows, { ledger = [], ledgerAvailable = false } = {}) {
+  return assignPassOrder(rows.map((row) => ({ ...row, ...armFields(row, { ledger, ledgerAvailable }) })));
+}
+
+/**
+ * Arm summary for stderr — the §8 check ("are both arms in the data at all")
+ * computed by the tool instead of by a one-liner pasted next to it. Returns null
+ * when the data holds no row from an `exp/*` branch: an ordinary run has no
+ * reason to read a section about the experiment.
+ */
+export function formatArmSummary(rows) {
+  // Truthiness, not `!== ''`: a row that was never annotated has `undefined`
+  // here, and `undefined !== ''` would let ordinary work into the experiment
+  // section. Costs the same and does not depend on call order.
+  const experimental = rows.filter((row) => row.exp_task);
+  if (experimental.length === 0) return null;
+
+  const counts = new Map();
+  for (const row of experimental) {
+    const label = row.exp_arm === '' ? `${row.exp_task}/? (${row.exp_arm_source})` : `${row.exp_task}/${row.exp_arm}`;
+    const bucket = counts.get(label) ?? { rows: 0, passOrders: new Set() };
+    bucket.rows++;
+    if (row.pass_order !== null) bucket.passOrders.add(row.pass_order);
+    counts.set(label, bucket);
+  }
+
+  const lines = ['arms (branch × ledger):'];
+  for (const [label, bucket] of [...counts.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
+    const pass = bucket.passOrders.size ? ` pass_order=${[...bucket.passOrders].sort().join(',')}` : '';
+    lines.push(`  ${label}: ${bucket.rows} row(s)${pass}`);
+  }
+
+  const unresolved = experimental.filter((row) => row.exp_arm === '').length;
+  if (unresolved > 0) {
+    lines.push(
+      `  ${unresolved} row(s) on an exp/ branch could NOT be assigned to an arm — that run is invalid,`,
+      '  not averageable (COST_EVALUATION_PROCEDURE §7).',
+    );
+  }
+  return lines.join('\n');
+}