@bglocation/tune-context 2.5.0 → 3.0.0

package/eval/prompt-metrics.mjs

@@ -43,7 +43,9 @@ import { createInterface } from 'node:readline';
 import { join } from 'node:path';
 import { parseArgs } from 'node:util';
 import { claudeBaseDir } from '../hooks/config-dir.mjs';
-import { experimentPaths, readLedger } from '../cli/experiment.mjs';
+import { endQuietlyOnClosedPipe, isEntryPoint } from '../cli/entry-point.mjs';
+import { readStepLedger } from '../cli/experiment.mjs';
+import { meterStamp } from '../cli/version.mjs';
 import { annotateRows, formatArmSummary } from './arm.mjs';
 import { formatCostLines, formatUsd, priceRow, summarizeCost, webSearchCost } from './pricing.mjs';
 
@@ -61,7 +63,7 @@ const USAGE = `
 prompt-metrics — per-prompt time/token/tool metrics from Claude Code transcripts.
 
 Usage:
-  prompt-metrics [--transcript <path>]... [--dir <path>]
+  prompt-metrics [--transcript <path>]... [--dir <path>] [--ledger <path>]
 
 Options:
   --transcript <path>  A single .jsonl transcript. Repeatable.
@@ -70,6 +72,12 @@ Options:
                        (default when neither is given: <config dir>/projects,
                        where <config dir> is $CLAUDE_CONFIG_DIR if set,
                        else ~/.claude)
+  --ledger <path>      The experiment ledger to assign arms by (default:
+                       <config dir>/tune-context/experiment.jsonl). For
+                       recomputing an archive — \`tune-context experiment
+                       archive\` keeps the ledger next to the transcripts. A
+                       path that does not exist, or is not a file, is an
+                       error.
   -h, --help           Show this help
 
 Writes JSONL to stdout, one row per prompt. A run summary — including how many
@@ -90,6 +98,12 @@ These five fields are added here, in the CLI. Rows obtained through the module
 (aggregate, collectFiles) carry the measurement only — call annotateRows from
 eval/arm.mjs to add the arm.
 
+Every row also carries meter_version: the code that computed it — the bare
+release number for an npm install, number+<sha> for a checkout, .dirty for
+uncommitted changes in the meter's paths (EPIC-010). The first line of the
+summary on stderr is that stamp, as \`meter <stamp>\`: it goes into the run
+journal before the first pair. analyze refuses rows without it.
+
 Each row also carries delegations[]: one record per Agent call that returned its
 own usage, priced with the same table. That figure is a LOWER BOUND on what the
 subagents cost and is never added to the row's own cost.
@@ -973,14 +987,21 @@ function defaultDir() {
   return join(claudeBaseDir(), 'projects');
 }
 
-async function main() {
+/**
+ * The CLI. `args` is everything after the script name, or after
+ * `tune-context eval prompt-metrics` — one parser behind both doors (EPIC-010).
+ */
+export async function main(args = process.argv.slice(2)) {
+  endQuietlyOnClosedPipe();
   let paths;
+  let ledger;
   try {
     const { values } = parseArgs({
-      args: process.argv.slice(2),
+      args,
       options: {
         transcript: { type: 'string', multiple: true },
         dir: { type: 'string' },
+        ledger: { type: 'string' },
         help: { type: 'boolean', short: 'h', default: false },
       },
       strict: true,
@@ -989,6 +1010,9 @@ async function main() {
       process.stdout.write(`${USAGE}\n`);
       return;
     }
+    // Read now, before the transcripts: a --ledger that is not a file stops the
+    // run here, with the same message analyze gives (TASK-098).
+    ledger = readStepLedger(values.ledger, claudeBaseDir());
     // Both flags together are a UNION, deduplicated, not a precedence rule:
     // silently ignoring one of two flags the user passed is how a run measures
     // something other than what was asked for.
@@ -1016,11 +1040,6 @@ async function main() {
     return;
   }
 
-  // Piping into `head` closes stdout early; that is a normal end, not a crash.
-  process.stdout.on('error', (e) => {
-    if (e.code !== 'EPIPE') throw e;
-  });
-
   const collector = await collectFiles(paths);
   const result = collector.result();
 
@@ -1028,19 +1047,21 @@ async function main() {
   // transcript into rows and has no business knowing about the experiment, and
   // the ledger is machine state rather than transcript content. The split also
   // lets the same rules run over rows collected elsewhere (TASK-088).
-  const ledgerPath = experimentPaths(claudeBaseDir()).ledgerPath;
-  const ledgerAvailable = existsSync(ledgerPath);
-  const rows = annotateRows(result.rows, {
-    ledger: ledgerAvailable ? readLedger(claudeBaseDir()) : [],
-    ledgerAvailable,
-  });
+  const rows = annotateRows(result.rows, ledger);
 
-  for (const row of rows) process.stdout.write(`${formatRow(row)}\n`);
-  process.stderr.write(`${formatSummary(result)}\n`);
+  // Which code computed the rows rides on every row — added here, with the arm,
+  // because it is a property of the process that ran, not of the transcript
+  // (TASK-097). Last on the row, so nothing that already read a row by position
+  // in a diff moves.
+  const stamp = meterStamp();
+  for (const row of rows) process.stdout.write(`${formatRow({ ...row, meter_version: stamp })}\n`);
+  // First line of the summary, on its own: this is what the operator copies
+  // into the run journal before the first pair.
+  process.stderr.write(`meter ${stamp}\n${formatSummary(result)}\n`);
   const arms = formatArmSummary(rows);
   if (arms !== null) process.stderr.write(`${arms}\n`);
 }
 
-if (process.argv[1]?.endsWith('prompt-metrics.mjs')) {
+if (isEntryPoint(import.meta.url)) {
   await main();
 }

package/eval/report.mjs

@@ -38,11 +38,13 @@
 // (light: contrast WARN on aqua/yellow/magenta, so both charts ship direct
 // labels AND a table twin — the documented relief), zero-baseline axes,
 // hairline chrome, 2px surface gaps instead of borders.
-import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { writeFileSync } from 'node:fs';
 import { parseArgs } from 'node:util';
+import { endQuietlyOnClosedPipe, isEntryPoint } from '../cli/entry-point.mjs';
+import { meterStamp } from '../cli/version.mjs';
 import { ARMS } from './analyze.mjs';
 import { TOKEN_CLASS_FIELDS, formatUsd } from './pricing.mjs';
-import { VERDICT_THRESHOLDS, verdict as computeVerdict } from './verdict.mjs';
+import { VERDICT_THRESHOLDS, VerdictInputError, verdict as computeVerdict, inputErrorLine, readVerdictInput } from './verdict.mjs';
 
 const USAGE = `
 report — one self-contained HTML file from an analyze aggregate (TASK-091).
@@ -51,7 +53,7 @@ Usage:
   report [--aggregate <path>] [--rejected-runs <path>] [--out <path>]
 
 Options:
-  --aggregate <path>      JSON written by analyze.mjs (default: read stdin)
+  --aggregate <path>      JSON written by analyze (default: read stdin)
   --rejected-runs <path>  JSON array of the run journal's §7 rejections,
                           [{ session, task, arm, prompts, reason }]. OMITTING it
                           is not the same as passing a file containing []: the
@@ -438,7 +440,7 @@ function verdictSection(v) {
       <p class="dim">Pool: paired measurement only (${num(v.n_pairs_used)} usable pair(s)). Natural work is never evidence.
       ${v.observation_only ? 'This report says "observation", never "proof" — the PO froze a minimum sample, not a level at which it becomes proof.' : ''}</p>
     </div>
-    <h2>The six gates</h2>
+    <h2>The ${v.gates.length} gates</h2>
     <ul class="gates">${gates}</ul>
   </section>`;
 }
@@ -602,6 +604,22 @@ function thresholdsSection(t) {
   </section>`;
 }
 
+/**
+ * Which code computed this page, step by step — rows, aggregate, verdict
+ * (EPIC-010). In the header rather than the footer, because EPIC-009's verdict
+ * was voided over exactly this question, asked of files that could not answer
+ * it. Whether the stamps agree is the meter_mixed gate's business; this line
+ * only names them, all of them.
+ */
+function meterLine(agg, v) {
+  const rows = Object.entries(agg.meter_versions ?? {});
+  const stamp = (s) => (s ? `<code>${esc(s)}</code>` : '<span class="flag">no stamp</span>');
+  return (
+    `<p class="src">meter — rows: ${rows.length === 0 ? '—' : rows.map(([s, n]) => `${stamp(s)} ×${num(n)}`).join(', ')}` +
+    ` · analyze: ${stamp(agg.analyzed_by)} · verdict: ${stamp(v.judged_by)}</p>`
+  );
+}
+
 // ─── the page ───────────────────────────────────────────────────────────────
 
 /** Palette and chrome from the `dataviz` reference instance, as CSS custom
@@ -711,18 +729,18 @@ footer { max-width:980px; margin:0 auto; color:var(--muted); font-size:12px; }
  * verdict() follow, and the reason a test can render a fabricated run without a
  * fixture directory.
  *
- * @param agg  an aggregate from analyze() (schema tune-context/analyze@1)
+ * @param agg  an aggregate from analyze() (schema tune-context/analyze@2)
  * @param verdict  the decision from verdict(). Computed by the caller rather
  *        than here, because the journal of §7 rejections is an input this layer
  *        has no way to read — and a verdict computed without it holds gate 6,
  *        which is the correct answer, not a bug to route around.
  */
 export function renderReport(agg, { verdict: v, generatedAt = null } = {}) {
-  if (!agg || typeof agg !== 'object' || agg.schema !== 'tune-context/analyze@1') {
-    throw new TypeError('report: expected an aggregate from analyze() (schema tune-context/analyze@1)');
+  if (!agg || typeof agg !== 'object' || agg.schema !== 'tune-context/analyze@2') {
+    throw new TypeError('report: expected an aggregate from analyze() (schema tune-context/analyze@2)');
   }
-  if (v?.schema !== 'tune-context/verdict@1') {
-    throw new TypeError('report: expected a verdict from verdict() (schema tune-context/verdict@1)');
+  if (v?.schema !== 'tune-context/verdict@2') {
+    throw new TypeError('report: expected a verdict from verdict() (schema tune-context/verdict@2)');
   }
 
   // Pool-level composition: the two arms of ONE pool added together. The pools
@@ -760,6 +778,7 @@ export function renderReport(agg, { verdict: v, generatedAt = null } = {}) {
   once with tune-context and RAG on, once with both parked. Generated${generatedAt ? ` ${esc(generatedAt)}` : ''} from
   an aggregate of ${num(agg.counters.rows_in)} prompt row(s). Costs are <b>list price through the API</b> — what the
   work would have cost, never money spent.</p>
+  ${meterLine(agg, v)}
 </header>
 ${verdictSection(v)}
 ${pairsSection(agg, v)}
@@ -794,17 +813,16 @@ ${thresholdsSection(v.thresholds)}
 
 // ─── CLI ────────────────────────────────────────────────────────────────────
 
-async function readStdin() {
-  const chunks = [];
-  for await (const chunk of process.stdin) chunks.push(chunk);
-  return Buffer.concat(chunks).toString('utf8');
-}
-
-async function main() {
+/**
+ * The CLI. `args` is everything after the script name, or after
+ * `tune-context eval report` — one parser behind both doors (EPIC-010).
+ */
+export async function main(args = process.argv.slice(2)) {
+  endQuietlyOnClosedPipe();
   let values;
   try {
     ({ values } = parseArgs({
-      args: process.argv.slice(2),
+      args,
       options: {
         aggregate: { type: 'string' },
         'rejected-runs': { type: 'string' },
@@ -823,48 +841,41 @@ async function main() {
     return;
   }
 
-  for (const [flag, path] of [['--aggregate', values.aggregate], ['--rejected-runs', values['rejected-runs']]]) {
-    if (path !== undefined && !existsSync(path)) {
-      process.stderr.write(`report: no file at ${path} (${flag})\n`);
-      process.exitCode = 1;
-      return;
-    }
-  }
-
-  let agg;
-  let rejectedRuns;
-  try {
-    agg = JSON.parse(values.aggregate ? readFileSync(values.aggregate, 'utf8') : await readStdin());
-    // Left `undefined` when the flag is absent, and that is the whole point: it
-    // is what makes gate 6 hold instead of passing on a share whose denominator
-    // nobody supplied.
-    if (values['rejected-runs'] !== undefined) {
-      rejectedRuns = JSON.parse(readFileSync(values['rejected-runs'], 'utf8'));
-    }
-  } catch (e) {
-    process.stderr.write(`report: ${e.message}\n`);
-    process.exitCode = 1;
-    return;
-  }
-
+  // The same reader and the same boundary as `eval verdict` (TASK-096): one
+  // place decides what a readable aggregate and journal are, so this page can
+  // never accept an input the verdict refuses — and only those refusals become
+  // a message. A TypeError from inside a gate or the page is a bug and keeps its
+  // stacktrace, here as there. `rejectedRuns` stays undefined when the flag is
+  // absent — that is what makes gate 6 hold instead of passing on a share whose
+  // denominator nobody supplied.
   let html;
   try {
-    const v = computeVerdict(agg, { rejectedRuns });
+    const { agg, rejectedRuns } = await readVerdictInput({ aggregate: values.aggregate, rejectedRuns: values['rejected-runs'] });
+    const v = computeVerdict(agg, { rejectedRuns, judgedBy: meterStamp() });
     html = renderReport(agg, { verdict: v, generatedAt: new Date().toISOString().slice(0, 10) });
   } catch (e) {
-    process.stderr.write(`report: ${e.message}\n`);
+    if (!(e instanceof VerdictInputError)) throw e;
+    process.stderr.write(`${inputErrorLine('report', e)}\n`);
     process.exitCode = 1;
     return;
   }
 
   if (values.out) {
-    writeFileSync(values.out, html);
+    try {
+      writeFileSync(values.out, html);
+    } catch (e) {
+      // Where to write is the operator's input too: a missing directory is a
+      // message, not a stacktrace.
+      process.stderr.write(`report: cannot write ${values.out}: ${e.message}\n`);
+      process.exitCode = 1;
+      return;
+    }
     process.stderr.write(`report: wrote ${values.out}\n`);
   } else {
     process.stdout.write(html);
   }
 }
 
-if (process.argv[1]?.endsWith('report.mjs')) {
+if (isEntryPoint(import.meta.url)) {
   await main();
 }