driftproof 0.4.0 → 0.6.0

package/lib/provider.js

@@ -7,6 +7,9 @@ const path = require('path');
 const { spawn } = require('child_process');
 const { withRetry, withTimeout } = require('./json');
 const { stubComplete, stubEnabled } = require('./stub');
+const {
+  parseClaudeCliJson, parseCodexJsonl, parseAnthropicApiUsage, parseOpenaiApiUsage, normalizeUsage,
+} = require('./usage');
 
 // Provider abstraction: one `complete()` call over a TWO-AXIS lane model —
 // provider (anthropic | openai) × surface (api | cli). Four concrete lanes:
@@ -109,9 +112,18 @@ const CODEX_OVERHEAD_NOTE =
   + 'control; the model id is set by us via -m (it is not echoed in the JSONL stream). '
   + 'Approval prompting is off by default on `codex exec` (no -a flag is passed).';
 
-// Send a single-turn prompt and return { text, usage, surface, provider }.
-// usage is { input_tokens, output_tokens } when the surface reports it (api
-// surfaces), else null (cli surfaces do not expose token counts to us).
+// Send a single-turn prompt and return { text, usage, wall_ms, surface, provider }.
+//
+// usage is the normalized v0.4 record { input_tokens, output_tokens, cached_tokens,
+// wall_ms } on EVERY lane — including the two CLI lanes, which report it in their
+// structured output (`claude -p --output-format json`; the `codex exec --json`
+// JSONL stream). See lib/usage.js for the per-surface shapes and the input-token
+// normalization. Fields the surface does not report stay null, never 0.
+//
+// wall_ms is measured HERE, around the successful attempt, so it means the same
+// thing on all four lanes (retries are excluded — a retried call's latency would
+// describe our backoff, not the model).
+//
 // `temperature` is honoured ONLY on api surfaces; cli surfaces control sampling
 // themselves, so temperature is ignored there and the receipt records that fact.
 async function complete({ system, prompt, model, maxTokens = 1024, timeoutMs = null, temperature = undefined }) {
@@ -120,7 +132,10 @@ async function complete({ system, prompt, model, maxTokens = 1024, timeoutMs = n
   // Offline stub surface: canned completion, zero model calls. The receipt still
   // records the real surface/provider so a stub run is not mistaken for a genuine
   // one at read time — only the generation/judge TEXT is canned.
-  if (stubEnabled()) return { ...stubComplete({ system, prompt }), surface, provider, attempts: 1 };
+  if (stubEnabled()) {
+    const s = stubComplete({ system, prompt });
+    return { ...s, usage: normalizeUsage(s.usage), wall_ms: (s.usage && s.usage.wall_ms) || 0, surface, provider, attempts: 1 };
+  }
 
   // Per-surface tolerance (see retryPolicyForSurface): a cli/subscription surface
   // spawns a first-party CLI subprocess whose cold-start is slow and occasionally
@@ -144,10 +159,20 @@ async function complete({ system, prompt, model, maxTokens = 1024, timeoutMs = n
   // throttling/cold-starts so one blip doesn't abort a multi-hour grind. `attempts`
   // counts every try (retries included) so the budget can charge for them.
   let attempts = 0;
-  const runner = () => { attempts += 1; return withTimeout(laneRunner, effTimeout, `provider(${surface})`); };
+  // Wall-clock of the attempt that SUCCEEDED (each attempt overwrites, so a
+  // retried call reports the latency of the call that actually produced the text,
+  // not the accumulated backoff).
+  let wallMs = null;
+  const runner = () => {
+    attempts += 1;
+    const t0 = Date.now();
+    return withTimeout(laneRunner, effTimeout, `provider(${surface})`)
+      .then((r) => { wallMs = Date.now() - t0; return r; });
+  };
   try {
     const out = await withRetry(runner, { tries: policy.tries, baseDelayMs });
-    return { ...out, surface, provider, attempts };
+    const usage = normalizeUsage({ ...(out.usage || {}), wall_ms: wallMs });
+    return { ...out, usage, wall_ms: wallMs, surface, provider, attempts };
   } catch (e) {
     // Surface the attempt count so a persistently-failing call can be charged for
     // (and, for a timeout, marked failed_timeout by the runner instead of fatal).
@@ -172,13 +197,7 @@ async function completeAnthropicApi({ system, prompt, model, maxTokens, temperat
   if (temperature !== undefined) params.temperature = temperature;
   const resp = await client.messages.create(params);
   const text = (resp.content || []).filter((b) => b.type === 'text').map((b) => b.text).join('');
-  return {
-    text,
-    usage: {
-      input_tokens: (resp.usage && resp.usage.input_tokens) || 0,
-      output_tokens: (resp.usage && resp.usage.output_tokens) || 0,
-    },
-  };
+  return { text, usage: parseAnthropicApiUsage(resp.usage) };
 }
 
 // ── anthropic/cli (claude -p) ──────────────────────────────────────────────────
@@ -189,7 +208,13 @@ function completeClaudeCli({ system, prompt, model, timeoutMs }) {
     const env = { ...process.env };
     delete env.ANTHROPIC_API_KEY;
 
-    const args = ['-p', '--model', resolveModel(model)];
+    // v0.4: `--output-format json` returns ONE JSON object carrying both the
+    // final text (`result`) and the token usage (`usage`) — the default text
+    // output carries no usage at all, which is why usage was previously null on
+    // this lane. The text is read from the parsed object; if the CLI ever emits
+    // something unparseable we fall back to the raw stdout so a run degrades to
+    // the old behaviour (text, no usage) rather than failing.
+    const args = ['-p', '--output-format', 'json', '--model', resolveModel(model)];
     if (system) args.push('--append-system-prompt', system);
 
     const child = spawn('claude', args, { env, stdio: ['pipe', 'pipe', 'pipe'] });
@@ -206,7 +231,10 @@ function completeClaudeCli({ system, prompt, model, timeoutMs }) {
     child.on('close', (code) => {
       clearTimeout(killer);
       if (code !== 0) return reject(new Error(`claude CLI exited ${code}: ${err.slice(0, 400)}`));
-      resolve({ text: out.trim(), usage: null });
+      const parsed = parseClaudeCliJson(out);
+      if (!parsed) return resolve({ text: out.trim(), usage: null });
+      if (parsed.isError) return reject(new Error(`claude CLI reported is_error: ${String(parsed.text || '').slice(0, 300)}`));
+      resolve({ text: String(parsed.text || '').trim(), usage: parsed.usage });
     });
     child.stdin.write(prompt);
     child.stdin.end();
@@ -250,14 +278,7 @@ async function completeOpenaiApi({ system, prompt, model, maxTokens, temperature
   let parsed;
   try { parsed = JSON.parse(raw); } catch (_e) { throw new Error(`openai/api: unparseable response: ${raw.slice(0, 200)}`); }
   const text = ((parsed.choices && parsed.choices[0] && parsed.choices[0].message && parsed.choices[0].message.content) || '');
-  const usage = parsed.usage || {};
-  return {
-    text: String(text).trim(),
-    usage: {
-      input_tokens: usage.prompt_tokens || 0,
-      output_tokens: usage.completion_tokens || 0,
-    },
-  };
+  return { text: String(text).trim(), usage: parseOpenaiApiUsage(parsed.usage) };
 }
 
 // Read the OpenAI provider config (base_url + api-key env) from the registry,
@@ -325,9 +346,15 @@ function completeCodexCli({ system, prompt, model, timeoutMs }) {
     // stdin (per its --help), which is content-agnostic.
     const args = codexFinalArgs({ model, outFile });
 
-    const child = spawn('codex', args, { env: { ...process.env }, stdio: ['pipe', 'ignore', 'pipe'] });
+    // v0.4: stdout is CAPTURED (it was 'ignore'). `--json` streams JSONL events
+    // there, and the terminal `turn.completed` event carries this call's token
+    // usage — the only place codex reports it. The final message still comes from
+    // the -o file (cleaner than scraping the stream); the JSONL is read for usage.
+    const child = spawn('codex', args, { env: { ...process.env }, stdio: ['pipe', 'pipe', 'pipe'] });
     let err = '';
+    let jsonl = '';
     const killer = setTimeout(() => { child.kill('SIGKILL'); }, timeoutMs + 5000);
+    child.stdout.on('data', (d) => { jsonl += d; });
     child.stderr.on('data', (d) => { err += d; });
     child.on('error', (e) => {
       clearTimeout(killer);
@@ -340,7 +367,10 @@ function completeCodexCli({ system, prompt, model, timeoutMs }) {
       try { text = fs.readFileSync(outFile, 'utf8'); } catch (_e) { /* fall through */ }
       try { fs.unlinkSync(outFile); } catch (_e) { /* best effort */ }
       if (code !== 0) return reject(new Error(`codex exec exited ${code}: ${String(err).slice(0, 400)}`));
-      resolve({ text: String(text).trim(), usage: null });
+      const ev = parseCodexJsonl(jsonl);
+      // Prefer the -o file; fall back to the stream's agent_message if it is empty.
+      const finalText = String(text || ev.text || '').trim();
+      resolve({ text: finalText, usage: ev.usage });
     });
     child.stdin.on('error', () => { /* ignore EPIPE if codex exits before reading all input */ });
     child.stdin.write(fullPrompt);

package/lib/receipt.js

@@ -14,7 +14,8 @@ const SCHEMA_FILES = {
   '0.1': 'receipt.v0.1.schema.json',
   '0.2': 'receipt.v0.2.schema.json',
   '0.3': 'receipt.v0.3.schema.json',
-  '0.3.1': 'receipt.schema.json',
+  '0.3.1': 'receipt.v0.3.1.schema.json',
+  '0.4': 'receipt.schema.json',
 };
 
 const _validators = {};
@@ -80,7 +81,7 @@ function aggregate(caseResults) {
 //   run:     { model_id, model_release_date, surface, runner_version, date_utc, judge, registry, transcripts }
 //   cases:   [ { id, mode, outcome, score, mean, stddev, samples, generation_hash, judge_sample_hashes, threshold, reason, judge } ]
 //   editorialReviews: optional [ { url, source, date } ]
-function buildReceipt({ skill, suite, run, cases, verificationLevel = 'TESTED', editorialReviews = null }) {
+function buildReceipt({ skill, suite, run, cases, economics = null, verificationLevel = 'TESTED', editorialReviews = null }) {
   // v0.3.1: cases marked failed_timeout are recorded in results.cases but EXCLUDED
   // from aggregates — a band is never fabricated from a case that did not complete.
   const okCases = cases.filter((c) => c.case_status !== 'failed_timeout');
@@ -138,6 +139,11 @@ function buildReceipt({ skill, suite, run, cases, verificationLevel = 'TESTED',
   //   skill.tokens              — estimated SKILL.md token size (value-per-token axis).
   if (run.surface_overhead_note) receipt.run.surface_overhead_note = run.surface_overhead_note;
   if (skill.tokens != null) receipt.skill.tokens = skill.tokens;
+  // v0.4 economics (additive-optional): the frozen prices this receipt's derived
+  // dollar figures were computed from, and the derived block itself. A receipt
+  // from a surface that reports no usage simply omits both.
+  if (run.pricing_snapshot) receipt.run.pricing_snapshot = run.pricing_snapshot;
+  if (economics) receipt.economics = economics;
   // v0.3.1: mark the receipt incomplete when any case failed (excluded above).
   if (failedCount > 0) {
     receipt.run.status = 'incomplete';

package/lib/revision.js

@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+const { bandVerdict, round } = require('./stats');
+const { EFFECT_FLOOR } = require('../config');
+
+// Revision drift — the fifth report type (spec 009, Report #006).
+//
+// Every other report type holds the skill text fixed and moves something
+// underneath it: the model release (#001, #003), the vendor surface (#002), the
+// capability tier (#004), the axes and the price (#005). This one inverts the
+// design. The substrate is held still — same model, same provider, same surface,
+// same suite, same fixed judge, same sampling — and the SKILL'S OWN TEXT moves,
+// from the revision a report pinned to the revision upstream ships today.
+//
+// This module holds the report type's LANGUAGE as code rather than as
+// hand-written page copy. That is deliberate: the fairness rule below is the one
+// a measurement project is most tempted to apply in one direction only, and a
+// rule that lives in prose cannot be gated before the run that would tempt it.
+
+// ── the cell headline ────────────────────────────────────────────────────────
+// A summary of the per-case band-overlap verdicts, worded about the REVISION.
+// The release-drift headline says "the skill is measurably weaker", which is a
+// sentence about a skill under a moving model. Here the model is the control.
+function revisionHeadline(perCase) {
+  const reg = perCase.filter((r) => r.verdict === 'regression').length;
+  const imp = perCase.filter((r) => r.verdict === 'improvement').length;
+  const s = (n) => (n === 1 ? '' : 's');
+  if (reg && imp) {
+    return `MIXED — the revision improved ${imp} case${s(imp)} and regressed ${reg} on non-overlapping bands.`;
+  }
+  if (reg) {
+    return `REVISION REGRESSED — ${reg} case${s(reg)} scored lower under the current upstream text (bands do not overlap).`;
+  }
+  if (imp) {
+    return `REVISION IMPROVED — ${imp} case${s(imp)} scored higher under the current upstream text (bands do not overlap); none regressed.`;
+  }
+  return 'WITHIN NOISE — the revision moved no case beyond its confidence band; the pinned text and the current text measure the same.';
+}
+
+// Classification word for a cell, from its headline. Kept separate so a caller
+// can branch on the class without parsing prose.
+function revisionClass(perCase) {
+  const h = revisionHeadline(perCase);
+  return h.split(' —')[0];
+}
+
+// ── the fairness sentence ────────────────────────────────────────────────────
+// Spec 009 § Fairness, clauses 1 and 4. Where a revision IMPROVED a skill, the
+// published #005 figure understates the pack a reader can install today; where it
+// REGRESSED one, #005 overstates it. Both sentences are generated by the same
+// function, from the same template, so the disclosure cannot quietly become a
+// one-directional courtesy — the symmetry is a property of the code, and the
+// gate asserts it.
+//
+// A cell within noise gets NO sentence. #005's figure stands unamended, because
+// nothing was measured that would amend it, and manufacturing a hedge for a null
+// result is how a report launders noise into a finding.
+function fairnessSentence({ slug, classification, report005Delta, measuredDelta }) {
+  const cls = String(classification || '');
+  if (cls !== 'REVISION IMPROVED' && cls !== 'REVISION REGRESSED') return null;
+  const improved = cls === 'REVISION IMPROVED';
+  const direction = improved ? 'understates' : 'overstates';
+  const d = (n) => (n == null ? 'n/a' : (n >= 0 ? '+' : '') + Number(n).toFixed(3));
+  return `Report #005 measured ${slug} at ${d(report005Delta)} on the text it had pinned. `
+    + `Report #006 measures the current upstream revision at ${d(measuredDelta)} on the same substrate and the same suite. `
+    + `#005's published figure therefore ${direction} the pack upstream ships today for this skill, `
+    + `and is amended by this report rather than corrected in place.`;
+}
+
+// ── per-cell scoping disclosures ─────────────────────────────────────────────
+// Spec 009 AC-10. One cell in Report #006 measures something other than what its
+// upstream author changed it to do, and the reader looking at that row is the
+// reader who needs to be told.
+const SCOPING_NOTES = {
+  'git-workflow-and-versioning':
+    'This revision changes the frontmatter `description:` line. In a skill runtime a description is a '
+    + 'routing trigger: it decides whether the skill loads, and never reaches the model as guidance. '
+    + 'Driftproof makes no routing decision — it always injects the skill, and passes the whole file, '
+    + 'frontmatter included, as the system prompt. This cell therefore measures the revision as added '
+    + 'context and cannot measure it as a trigger.',
+};
+function scopingNote(slug) {
+  return SCOPING_NOTES[slug] || null;
+}
+
+// ── the baseline-reproduction control ────────────────────────────────────────
+// Spec 009 AC-6, and the thing that makes the free pinned arm honest.
+//
+// Report #006 reuses #005's receipts as the pinned-text arm. That is valid only
+// if the substrate has not moved, and `run.model_release_date` is null on every
+// #005 receipt, so id equality is the only version evidence a receipt carries. A
+// provider that re-points a concrete id at a new snapshot is invisible to it.
+//
+// It does not have to be. Every fresh run emits a BASELINE arm: the same cases,
+// the same substrate, and no skill text at all. The revision cannot touch it by
+// construction, so comparing the fresh baseline against the reused receipt's
+// baseline re-measures exactly the thing id equality could not prove — at no
+// extra cost, because that arm is already paid for.
+//
+// A cell whose baselines do not reproduce is NOT MEASURED. The reuse is a tested
+// prediction, not an assumption the report asks the reader to grant.
+function baselineBands(receipt) {
+  const out = {};
+  for (const c of receipt.results.cases) {
+    if (c.mode !== 'baseline') continue;
+    out[c.id] = { mean: c.mean != null ? c.mean : c.score, stddev: c.stddev || 0 };
+  }
+  return out;
+}
+
+function baselineControl(reused, fresh) {
+  const A = baselineBands(reused);
+  const B = baselineBands(fresh);
+  const ids = [...new Set([...Object.keys(A), ...Object.keys(B)])];
+
+  const perCase = ids.map((id) => {
+    const before = A[id] || null;
+    const after = B[id] || null;
+    if (!before || !after) return { id, before, after, delta: null, moved: false, missing: true };
+    const delta = round(after.mean - before.mean);
+    // The same rule the study uses everywhere else: band separation AND the
+    // effect floor. A baseline that wobbles inside its band has not moved.
+    const raw = bandVerdict(before.mean, before.stddev, after.mean, after.stddev);
+    const separated = raw === 'regression' || raw === 'improvement';
+    return { id, before, after, delta, moved: separated && Math.abs(delta) >= EFFECT_FLOOR, missing: false };
+  });
+
+  const movedCases = perCase.filter((r) => r.moved);
+  const missing = perCase.filter((r) => r.missing);
+  const reproduced = movedCases.length === 0 && missing.length === 0;
+  const aggDelta = round(
+    (fresh.comparison && fresh.comparison.baseline_score != null ? fresh.comparison.baseline_score : 0)
+    - (reused.comparison && reused.comparison.baseline_score != null ? reused.comparison.baseline_score : 0),
+  );
+
+  return {
+    reproduced,
+    blocked: !reproduced,
+    verdict: reproduced ? 'MEASURED' : 'NOT MEASURED',
+    moved_cases: movedCases.map((r) => r.id),
+    missing_cases: missing.map((r) => r.id),
+    aggregate_baseline_delta: aggDelta,
+    floor: EFFECT_FLOOR,
+    // THE CONTROL PROVES NON-REPRODUCTION. IT CANNOT SAY WHY. These strings used
+    // to read 'the substrate moved' and 'the substrate held still' — a cause,
+    // asserted by a comparison that measures two baseline arms and nothing else.
+    // A 120-call stability probe then found generation-level sampling noise large
+    // enough to account for every gap this control saw, with no substrate movement
+    // required, and the report page retracted the claim while three committed
+    // control records still carried it (approval finding F-009-N). Reason strings
+    // only: no score, sample, hash or verdict changed with this edit.
+    reason: reproduced
+      ? 'the fresh baseline reproduces the reused receipt\'s baseline within the band and the floor, so the pinned-arm reuse stands for this cell'
+      : `the fresh baseline does not reproduce the reused receipt's baseline (${movedCases.length} case(s) moved beyond the band and the ${EFFECT_FLOOR} floor${missing.length ? `, ${missing.length} case(s) absent on one side` : ''}) — the reused pinned arm is not comparable to the fresh arm, so revision drift cannot be separated from whatever else changed in this cell; the control establishes non-reproduction and does not identify a cause`,
+    perCase,
+  };
+}
+
+module.exports = {
+  revisionHeadline, revisionClass, fairnessSentence, scopingNote, baselineControl,
+};

package/lib/run.js

@@ -1,14 +1,16 @@
 // SPDX-License-Identifier: Apache-2.0
 'use strict';
 
-const { complete, resolveModel, surfaceForModel, CODEX_OVERHEAD_NOTE } = require('./provider');
+const { complete, resolveModel, surfaceForModel, isMeteredSurface, CODEX_OVERHEAD_NOTE } = require('./provider');
 const { gradeSamples, judgeSettings } = require('./judge');
 const { buildReceipt } = require('./receipt');
 const { sha256 } = require('./canonical');
-const { registryStatus, providerForModel } = require('./models');
+const { registryStatus, providerForModel, priceForModel } = require('./models');
 const { perCallCostUSD } = require('./cost');
 const { runChecks } = require('./checks');
 const { estimateTokens } = require('./skillCost');
+const { hasUsage, normalizeUsage } = require('./usage');
+const { buildPricingSnapshot, computeEconomics } = require('./value');
 const { RUNNER_VERSION, DEFAULT_JUDGE_SAMPLES } = require('../config');
 
 // Known model release dates (best-effort; null when unknown). Recorded into the
@@ -51,8 +53,8 @@ async function runCase({ skillMd, caseObj, model, withSkill, timeoutMs }) {
     throw e;
   }
   const system = withSkill ? skillMd : undefined;
-  const { text, usage, attempts } = await complete({ system, prompt: caseObj.prompt, model, maxTokens: 1024, timeoutMs });
-  return { text, usage, attempts };
+  const { text, usage, wall_ms, attempts } = await complete({ system, prompt: caseObj.prompt, model, maxTokens: 1024, timeoutMs });
+  return { text, usage, wall_ms, attempts };
 }
 
 // Determine a case outcome from its sampled band and threshold.
@@ -91,6 +93,8 @@ async function judgeCase({ caseObj, response, generationHash, judgeModel, mode,
   // v0.3.1 deterministic post-checks (supplementary; NOT folded into `outcome`).
   const checks = runChecks(response, caseObj.checks);
   if (checks.length) caseResult.checks = checks;
+  // v0.4: grading overhead for this case row, kept OUT of the skill-value math.
+  if (hasUsage(g.usage)) caseResult.judge_usage = g.usage;
   return { caseResult, sampleTexts: g.sample_texts, attempts: g.attempts };
 }
 
@@ -170,6 +174,9 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
       const generationHash = sha256(String(gen.text || ''));
       onProgress({ case: c.id, mode, phase: 'judge', samples });
       const jr = await judgeCase({ caseObj: c, response: gen.text, generationHash, judgeModel, mode, timeoutMs: ct, samples });
+      // v0.4: the GENERATION call's usage is the skill-value measurement (the
+      // judge's own usage rides separately on judge_usage, above).
+      if (hasUsage(gen.usage)) jr.caseResult.usage = normalizeUsage({ ...gen.usage, wall_ms: gen.wall_ms });
       calls += samples;
       // Live budget: count all judge calls (retries included) for this (case, mode).
       if (budget) budget.add((jr.attempts || samples) * perCallCostUSD(judgeModel, 'judge'));
@@ -198,6 +205,24 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
   const transcripts = keepTranscripts ? pairs.map((p) => p.transcript) : null;
 
   const surface = surfaceForModel(modelId);
+  const nowIso = opts.nowIso || new Date().toISOString();
+  // v0.4 economics. The pricing snapshot is frozen HERE, at run time, from the
+  // registry; every derived dollar figure below is computed from the snapshot and
+  // never from the live registry, so this receipt keeps its meaning when prices
+  // later change.
+  const pricingSnapshot = buildPricingSnapshot({
+    models: [modelId, judgeModel],
+    lookup: priceForModel,
+    nowIso,
+  });
+  const economics = computeEconomics({
+    cases: caseResults,
+    modelId,
+    judgeModelId: judgeModel,
+    pricingSnapshot,
+    surface,
+    meteredSurface: isMeteredSurface(surface),
+  });
   const receipt = buildReceipt({
     skill: {
       name: skill.name, version: skill.version, contentHash: skill.contentHash,
@@ -213,12 +238,14 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
       // v0.3.1: on the openai/cli (codex) surface, record the fixed harness preamble.
       surface_overhead_note: surface === 'openai-cli' ? CODEX_OVERHEAD_NOTE : undefined,
       runner_version: RUNNER_VERSION,
-      date_utc: opts.nowIso || new Date().toISOString(),
+      date_utc: nowIso,
       registry: registryStatus(modelId),
       transcripts: keepTranscripts ? 'retained-local' : 'hashes-only',
       judge: judgeSettings(samples, judgeModel),
+      pricing_snapshot: pricingSnapshot,
     },
     cases: caseResults,
+    economics,
     verificationLevel: 'TESTED',
   });
 

package/lib/stub.js

@@ -39,16 +39,37 @@ function stubComplete({ system, prompt }) {
       pass: score >= 0.7,
       reason: `stub judge (DRIFTPROOF_STUB): ${helped ? 'with-skill marker present' : 'baseline'}`,
     };
-    return { text: JSON.stringify(body), usage: null };
+    return { text: JSON.stringify(body), usage: stubUsage({ kind: 'judge', prompt }) };
   }
   // Generation call: canned, marked by mode.
   const mark = system ? MARK_SKILL : MARK_BASE;
   const text = `${mark}\nfeat(stub): canned offline generation for CI (no model was called)`;
-  return { text, usage: null };
+  return { text, usage: stubUsage({ kind: 'gen', system, prompt }) };
+}
+
+// v0.4: deterministic synthetic usage, so a zero-model-call stub run still
+// exercises the whole economics path (usage → pricing snapshot → derived cost/
+// latency fields → the value report's three axes). The numbers are SYNTHETIC and
+// deliberately shaped like the real surfaces: a large fixed harness preamble that
+// is identical in both arms, plus the with-skill arm's extra SKILL.md input and
+// its slightly longer, slower output. No randomness — same input, same usage.
+const STUB_HARNESS_PREAMBLE_TOKENS = 20000;   // the fixed, arm-identical overhead
+function stubUsage({ kind, system, prompt }) {
+  const promptTokens = Math.ceil(String(prompt || '').length / 4);
+  if (kind === 'judge') {
+    return { input_tokens: 1200 + promptTokens, output_tokens: 60, cached_tokens: 800, wall_ms: 900 };
+  }
+  const skillTokens = system ? Math.ceil(String(system).length / 4) : 0;
+  return {
+    input_tokens: STUB_HARNESS_PREAMBLE_TOKENS + promptTokens + skillTokens,
+    output_tokens: system ? 420 : 350,
+    cached_tokens: STUB_HARNESS_PREAMBLE_TOKENS,
+    wall_ms: system ? 4200 : 3600,
+  };
 }
 
 function stubEnabled() {
   return process.env.DRIFTPROOF_STUB === '1';
 }
 
-module.exports = { stubComplete, stubEnabled, MARK_SKILL, MARK_BASE };
+module.exports = { stubComplete, stubEnabled, stubUsage, MARK_SKILL, MARK_BASE, STUB_HARNESS_PREAMBLE_TOKENS };

package/lib/usage.js

@@ -0,0 +1,168 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+// Usage capture — normalizing what each surface reports about a single call.
+//
+// Every lane reports token usage in its own shape, and two of them (the CLI
+// lanes) were previously discarding it entirely: the `claude -p` default text
+// output carries no usage block at all, and `codex exec --json` streams usage on
+// stdout that we were routing to /dev/null. Spec v0.4 captures it.
+//
+// ONE normalized shape, so three substrates are comparable:
+//
+//   { input_tokens, output_tokens, cached_tokens, wall_ms }
+//
+//   input_tokens   TOTAL input presented to the model for this call, INCLUDING
+//                  any portion served from cache. This is the field that most
+//                  needs normalizing: the surfaces disagree about it (below).
+//   cached_tokens  the portion of input_tokens served from cache (null when the
+//                  surface does not surface it — never guessed, never 0-filled).
+//   output_tokens  tokens generated, INCLUDING reasoning/thinking tokens where
+//                  the surface bundles them (both CLI lanes do; the split is
+//                  reported by the surface but deliberately not modelled here —
+//                  see the disclosure in REPORT-STYLE.md).
+//   wall_ms        measured BY US around the call (see lib/provider), not taken
+//                  from the surface. It is the only field whose definition is
+//                  identical across lanes, which is exactly why latency is
+//                  reported as an observed, surface-disclosed number.
+//
+// THE INPUT-TOKEN DISAGREEMENT (verified against real output, 2026-08-18):
+//   - `claude -p --output-format json` reports input_tokens EXCLUDING cache:
+//     total input = input_tokens + cache_creation_input_tokens + cache_read_input_tokens.
+//     (Observed: 10 + 6,974 + 18,134 = 25,118 for a two-word prompt.)
+//   - `codex exec --json` reports input_tokens INCLUDING cache, with
+//     cached_input_tokens as a SUBSET of it. (Observed: 10,807 of which 4,480 cached.)
+//   Normalizing to "total including cache" makes the two comparable; the raw
+//   fixtures both parsers are tested against are checked in under tests/fixtures/.
+//
+// Both CLI surfaces prepend a large fixed harness preamble that we do not
+// control (~25k tokens on claude-cli, ~11k on codex). It is IDENTICAL in the
+// with-skill and baseline arms, so it cancels in the incremental (Δ) figures —
+// which is why Δcost is the honest axis and absolute per-call cost is not.
+
+function n(x) { return Number.isFinite(Number(x)) ? Number(x) : 0; }
+
+// The empty/unknown usage record. Deliberately null (not zeros) so "the surface
+// did not tell us" never reads as "the call cost nothing".
+function emptyUsage() {
+  return { input_tokens: null, output_tokens: null, cached_tokens: null, wall_ms: null };
+}
+
+function hasUsage(u) {
+  return !!(u && (u.input_tokens != null || u.output_tokens != null));
+}
+
+// ── anthropic/cli — `claude -p --output-format json` ──────────────────────────
+// The whole call is one JSON object; `result` carries the final text and `usage`
+// the token counts. input_tokens EXCLUDES cache, so both cache fields are added
+// back to reach the total actually presented to the model.
+function parseClaudeCliJson(stdout) {
+  let j;
+  try { j = JSON.parse(String(stdout || '')); } catch (_e) { return null; }
+  if (!j || typeof j !== 'object') return null;
+  const u = j.usage || {};
+  const cacheRead = n(u.cache_read_input_tokens);
+  const cacheCreate = n(u.cache_creation_input_tokens);
+  const usage = {
+    input_tokens: n(u.input_tokens) + cacheRead + cacheCreate,
+    output_tokens: n(u.output_tokens),
+    // Cache READ is the portion genuinely served from cache. Cache CREATION was
+    // processed fresh this call (and billed at a premium), so it is not cached.
+    cached_tokens: cacheRead,
+    wall_ms: null,
+  };
+  return {
+    text: typeof j.result === 'string' ? j.result : '',
+    usage,
+    isError: j.is_error === true,
+    // The CLI reports its own dollar figure. Recorded here for completeness but
+    // NOT used: costs are computed uniformly from the frozen pricing snapshot so
+    // three substrates are on one basis (see lib/value.js).
+    surfaceReportedCostUsd: Number.isFinite(Number(j.total_cost_usd)) ? Number(j.total_cost_usd) : null,
+  };
+}
+
+// ── openai/cli — `codex exec --json` JSONL event stream ───────────────────────
+// One JSON object per line. Usage rides the terminal `turn.completed` event;
+// input_tokens already INCLUDES cached_input_tokens. Unparseable lines are
+// skipped (the stream also carries progress events we do not model).
+function parseCodexJsonl(stdout) {
+  const lines = String(stdout || '').split('\n').map((l) => l.trim()).filter(Boolean);
+  let usage = null;
+  let text = '';
+  for (const line of lines) {
+    let ev;
+    try { ev = JSON.parse(line); } catch (_e) { continue; }
+    if (!ev || typeof ev !== 'object') continue;
+    if (ev.type === 'turn.completed' && ev.usage) {
+      const u = ev.usage;
+      usage = {
+        input_tokens: n(u.input_tokens),          // already total (cache included)
+        output_tokens: n(u.output_tokens),        // includes reasoning_output_tokens
+        cached_tokens: u.cached_input_tokens == null ? null : n(u.cached_input_tokens),
+        wall_ms: null,
+      };
+    }
+    // The final message is normally read from the -o file; this is a fallback.
+    if (ev.type === 'item.completed' && ev.item && ev.item.type === 'agent_message' && typeof ev.item.text === 'string') {
+      text = ev.item.text;
+    }
+  }
+  return { usage, text };
+}
+
+// ── anthropic/api ─────────────────────────────────────────────────────────────
+// The Messages API reports input_tokens EXCLUDING cache, same as the CLI.
+function parseAnthropicApiUsage(u) {
+  if (!u) return emptyUsage();
+  const cacheRead = n(u.cache_read_input_tokens);
+  const cacheCreate = n(u.cache_creation_input_tokens);
+  return {
+    input_tokens: n(u.input_tokens) + cacheRead + cacheCreate,
+    output_tokens: n(u.output_tokens),
+    cached_tokens: u.cache_read_input_tokens == null ? null : cacheRead,
+    wall_ms: null,
+  };
+}
+
+// ── openai/api (Chat Completions-compatible) ──────────────────────────────────
+// prompt_tokens is the total; the cached portion, when present, is nested under
+// prompt_tokens_details.cached_tokens.
+function parseOpenaiApiUsage(u) {
+  if (!u) return emptyUsage();
+  const details = u.prompt_tokens_details || {};
+  return {
+    input_tokens: n(u.prompt_tokens),
+    output_tokens: n(u.completion_tokens),
+    cached_tokens: details.cached_tokens == null ? null : n(details.cached_tokens),
+    wall_ms: null,
+  };
+}
+
+// Sum a list of usage records into one (used for the N judge samples of a case).
+// Null-safe: unknown fields stay null unless at least one record reported them.
+function sumUsage(list) {
+  const records = (list || []).filter(Boolean);
+  if (!records.length) return emptyUsage();
+  const out = { input_tokens: null, output_tokens: null, cached_tokens: null, wall_ms: null };
+  for (const key of ['input_tokens', 'output_tokens', 'cached_tokens', 'wall_ms']) {
+    const present = records.filter((r) => r[key] != null);
+    if (present.length) out[key] = present.reduce((a, r) => a + n(r[key]), 0);
+  }
+  return out;
+}
+
+// Round a usage record's fields to integers (tokens and ms are whole units).
+function normalizeUsage(u) {
+  if (!u) return emptyUsage();
+  const out = emptyUsage();
+  for (const key of ['input_tokens', 'output_tokens', 'cached_tokens', 'wall_ms']) {
+    if (u[key] != null && Number.isFinite(Number(u[key]))) out[key] = Math.round(Number(u[key]));
+  }
+  return out;
+}
+
+module.exports = {
+  emptyUsage, hasUsage, sumUsage, normalizeUsage,
+  parseClaudeCliJson, parseCodexJsonl, parseAnthropicApiUsage, parseOpenaiApiUsage,
+};