driftproof 0.4.0 → 0.5.0

package/README.md

@@ -162,7 +162,7 @@ A receipt is the unit of evidence — one JSON document conforming to
 
 ```jsonc
 {
-  "schema_version": "0.3.1",
+  "schema_version": "0.4",
   "skill":  { "name": "commit-message-conventions", "version": "0.2.0",
               "content_hash": "…sha256 over SKILL.md + bundled files…" },
   "suite":  { "format": "agentskills.io/evals", "suite_hash": "…", "case_count": 10 },
@@ -171,7 +171,7 @@ A receipt is the unit of evidence — one JSON document conforming to
     "model_release_date": "2025-10-01",
     "provider": "anthropic",
     "surface": "claude-cli",
-    "runner_version": "0.4.0",
+    "runner_version": "0.5.0",
     "date_utc": "2026-07-27T…Z",
     "registry": "registered",
     "transcripts": "hashes-only",
@@ -233,7 +233,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: driftproofhq/driftproof@v0.4.0
+      - uses: driftproofhq/driftproof@v0.5.0
         with:
           skill-dir: skills/my-skill
           models: claude-haiku-4-5

package/config.js

@@ -9,7 +9,7 @@ const PROJECT_NAME = 'driftproof';
 // Bumped whenever the runner's behaviour or receipt-generation semantics change
 // in a way that could affect results. Recorded into every receipt as
 // run.runner_version so a receipt is reproducible against a known engine.
-const RUNNER_VERSION = '0.4.0';
+const RUNNER_VERSION = '0.5.0';
 
 // The eval format we CONSUME (we deliberately do not invent our own).
 const SUITE_FORMAT = 'agentskills.io/evals';
@@ -22,7 +22,14 @@ const SUITE_FORMAT = 'agentskills.io/evals';
 //        surface enums (openai-api/openai-cli), optional run.surface_overhead_note,
 //        optional per-case checks[] (deterministic post-checks), and optional
 //        skill.tokens (value-per-token axis). v0.1/v0.2/v0.3 receipts still load.
-const RECEIPT_SCHEMA_VERSION = '0.3.1';
+// v0.4   (additive over v0.3.1) adds the ECONOMICS axis: per-case, per-arm
+//        generation `usage` (input/output/cached tokens + measured wall_ms), a
+//        separate per-case `judge_usage` (measurement overhead, excluded from
+//        every skill-value figure), run.pricing_snapshot (registry prices frozen
+//        at run time so derived dollars stay reproducible), and the derived
+//        `economics` block. v0.3.1 is frozen as receipt.v0.3.1.schema.json;
+//        v0.1/v0.2/v0.3/v0.3.1 receipts all still load.
+const RECEIPT_SCHEMA_VERSION = '0.4';
 
 // Hard USD budget defaults per entry point (Week 4). --max-usd overrides any of
 // these. The projection is refused before any call if it exceeds the cap, on
@@ -73,10 +80,19 @@ const REPORT_004_BASE_MODEL = 'claude-opus-5';     // flagship tier
 const REPORT_004_FRONTIER_MODEL = 'claude-fable-5'; // frontier tier (full id — no alias)
 const REPORT_004_JUDGE_MODEL = 'claude-haiku-4-5';
 
+// Report #005 is a VALUE report — the fourth report type. It asks what a skill
+// COSTS to run alongside whether it helps, over three substrates, and shows the
+// three axes (accuracy lift / Δcost / Δlatency) side by side and never combined.
+// Same suites, same fixed judge; the substrate list spans two providers so the
+// economics are read across surfaces, not within one vendor's pricing.
+const REPORT_005_MODELS = ['claude-sonnet-5', 'claude-fable-5', 'gpt-5.6-sol'];
+const REPORT_005_JUDGE_MODEL = 'claude-haiku-4-5';
+
 module.exports = {
   PROJECT_NAME, RUNNER_VERSION, SUITE_FORMAT, RECEIPT_SCHEMA_VERSION, DEFAULT_JUDGE_SAMPLES,
   EFFECT_FLOOR, DEV_MAX_USD, REPORT_MAX_USD, TRIGGER_MAX_USD,
   REPORT_002_CLAUDE_MODEL, REPORT_002_GPT_MODEL, REPORT_002_JUDGE_MODEL,
   REPORT_003_NEW_MODEL, REPORT_003_OLD_MODEL, REPORT_003_JUDGE_MODEL,
   REPORT_004_BASE_MODEL, REPORT_004_FRONTIER_MODEL, REPORT_004_JUDGE_MODEL,
+  REPORT_005_MODELS, REPORT_005_JUDGE_MODEL,
 };

package/lib/judge.js

@@ -5,6 +5,7 @@ const { complete, surfaceForModel } = require('./provider');
 const { extractJsonObject } = require('./json');
 const { sha256 } = require('./canonical');
 const { mean, stddev } = require('./stats');
+const { sumUsage } = require('./usage');
 
 // Rubric-based LLM judge.
 //
@@ -82,16 +83,16 @@ function judgeSettings(samples, judgeModel) {
 // transcript auditability, and optionally retained under --keep-transcripts).
 async function gradeOnce({ task, response, rubric, model, timeoutMs, temperature }) {
   const prompt = buildJudgePrompt({ task, response, rubric });
-  const { text, attempts } = await complete({ system: JUDGE_SYSTEM, prompt, model, maxTokens: 400, timeoutMs, temperature });
+  const { text, attempts, usage } = await complete({ system: JUDGE_SYSTEM, prompt, model, maxTokens: 400, timeoutMs, temperature });
   let parsed;
   try {
     parsed = extractJsonObject(text);
   } catch (_e) {
     // Unsalvageable judge output → conservative 0 (a judge that can't be parsed
     // must never silently "pass"), tagged so the caller can see it happened.
-    return { score: 0, reason: 'judge output unparseable', unparsed: true, raw: String(text || ''), attempts: attempts || 1 };
+    return { score: 0, reason: 'judge output unparseable', unparsed: true, raw: String(text || ''), attempts: attempts || 1, usage };
   }
-  return { score: clamp01(parsed.score), reason: String(parsed.reason || '').slice(0, 300), raw: String(text || ''), attempts: attempts || 1 };
+  return { score: clamp01(parsed.score), reason: String(parsed.reason || '').slice(0, 300), raw: String(text || ''), attempts: attempts || 1, usage };
 }
 
 // Grade a response N times and return the sampled distribution:
@@ -103,6 +104,7 @@ async function gradeSamples({ task, response, rubric, model, samples = 5, timeou
   const scores = [];
   const reasons = [];
   const rawTexts = [];
+  const usages = [];
   let attemptsTotal = 0;
   for (let i = 0; i < samples; i++) {
     let r;
@@ -116,6 +118,7 @@ async function gradeSamples({ task, response, rubric, model, samples = 5, timeou
       throw e;
     }
     attemptsTotal += r.attempts || 1;
+    usages.push(r.usage || null);
     scores.push(r.score);
     reasons.push(r.reason);
     rawTexts.push(r.raw || '');
@@ -134,6 +137,11 @@ async function gradeSamples({ task, response, rubric, model, samples = 5, timeou
     model_id: model,
     rubric_hash: rubricHash(rubric),
     attempts: attemptsTotal,
+    // v0.4: the measurement overhead of grading this one case — the SUM over all
+    // N judge calls. Recorded in the receipt as the case's `judge_usage` and
+    // EXCLUDED from every skill-value figure (lib/value.js): it is a cost we
+    // impose to measure, not a cost of running the skill.
+    usage: sumUsage(usages),
   };
 }
 

package/lib/provider.js

@@ -7,6 +7,9 @@ const path = require('path');
 const { spawn } = require('child_process');
 const { withRetry, withTimeout } = require('./json');
 const { stubComplete, stubEnabled } = require('./stub');
+const {
+  parseClaudeCliJson, parseCodexJsonl, parseAnthropicApiUsage, parseOpenaiApiUsage, normalizeUsage,
+} = require('./usage');
 
 // Provider abstraction: one `complete()` call over a TWO-AXIS lane model —
 // provider (anthropic | openai) × surface (api | cli). Four concrete lanes:
@@ -109,9 +112,18 @@ const CODEX_OVERHEAD_NOTE =
   + 'control; the model id is set by us via -m (it is not echoed in the JSONL stream). '
   + 'Approval prompting is off by default on `codex exec` (no -a flag is passed).';
 
-// Send a single-turn prompt and return { text, usage, surface, provider }.
-// usage is { input_tokens, output_tokens } when the surface reports it (api
-// surfaces), else null (cli surfaces do not expose token counts to us).
+// Send a single-turn prompt and return { text, usage, wall_ms, surface, provider }.
+//
+// usage is the normalized v0.4 record { input_tokens, output_tokens, cached_tokens,
+// wall_ms } on EVERY lane — including the two CLI lanes, which report it in their
+// structured output (`claude -p --output-format json`; the `codex exec --json`
+// JSONL stream). See lib/usage.js for the per-surface shapes and the input-token
+// normalization. Fields the surface does not report stay null, never 0.
+//
+// wall_ms is measured HERE, around the successful attempt, so it means the same
+// thing on all four lanes (retries are excluded — a retried call's latency would
+// describe our backoff, not the model).
+//
 // `temperature` is honoured ONLY on api surfaces; cli surfaces control sampling
 // themselves, so temperature is ignored there and the receipt records that fact.
 async function complete({ system, prompt, model, maxTokens = 1024, timeoutMs = null, temperature = undefined }) {
@@ -120,7 +132,10 @@ async function complete({ system, prompt, model, maxTokens = 1024, timeoutMs = n
   // Offline stub surface: canned completion, zero model calls. The receipt still
   // records the real surface/provider so a stub run is not mistaken for a genuine
   // one at read time — only the generation/judge TEXT is canned.
-  if (stubEnabled()) return { ...stubComplete({ system, prompt }), surface, provider, attempts: 1 };
+  if (stubEnabled()) {
+    const s = stubComplete({ system, prompt });
+    return { ...s, usage: normalizeUsage(s.usage), wall_ms: (s.usage && s.usage.wall_ms) || 0, surface, provider, attempts: 1 };
+  }
 
   // Per-surface tolerance (see retryPolicyForSurface): a cli/subscription surface
   // spawns a first-party CLI subprocess whose cold-start is slow and occasionally
@@ -144,10 +159,20 @@ async function complete({ system, prompt, model, maxTokens = 1024, timeoutMs = n
   // throttling/cold-starts so one blip doesn't abort a multi-hour grind. `attempts`
   // counts every try (retries included) so the budget can charge for them.
   let attempts = 0;
-  const runner = () => { attempts += 1; return withTimeout(laneRunner, effTimeout, `provider(${surface})`); };
+  // Wall-clock of the attempt that SUCCEEDED (each attempt overwrites, so a
+  // retried call reports the latency of the call that actually produced the text,
+  // not the accumulated backoff).
+  let wallMs = null;
+  const runner = () => {
+    attempts += 1;
+    const t0 = Date.now();
+    return withTimeout(laneRunner, effTimeout, `provider(${surface})`)
+      .then((r) => { wallMs = Date.now() - t0; return r; });
+  };
   try {
     const out = await withRetry(runner, { tries: policy.tries, baseDelayMs });
-    return { ...out, surface, provider, attempts };
+    const usage = normalizeUsage({ ...(out.usage || {}), wall_ms: wallMs });
+    return { ...out, usage, wall_ms: wallMs, surface, provider, attempts };
   } catch (e) {
     // Surface the attempt count so a persistently-failing call can be charged for
     // (and, for a timeout, marked failed_timeout by the runner instead of fatal).
@@ -172,13 +197,7 @@ async function completeAnthropicApi({ system, prompt, model, maxTokens, temperat
   if (temperature !== undefined) params.temperature = temperature;
   const resp = await client.messages.create(params);
   const text = (resp.content || []).filter((b) => b.type === 'text').map((b) => b.text).join('');
-  return {
-    text,
-    usage: {
-      input_tokens: (resp.usage && resp.usage.input_tokens) || 0,
-      output_tokens: (resp.usage && resp.usage.output_tokens) || 0,
-    },
-  };
+  return { text, usage: parseAnthropicApiUsage(resp.usage) };
 }
 
 // ── anthropic/cli (claude -p) ──────────────────────────────────────────────────
@@ -189,7 +208,13 @@ function completeClaudeCli({ system, prompt, model, timeoutMs }) {
     const env = { ...process.env };
     delete env.ANTHROPIC_API_KEY;
 
-    const args = ['-p', '--model', resolveModel(model)];
+    // v0.4: `--output-format json` returns ONE JSON object carrying both the
+    // final text (`result`) and the token usage (`usage`) — the default text
+    // output carries no usage at all, which is why usage was previously null on
+    // this lane. The text is read from the parsed object; if the CLI ever emits
+    // something unparseable we fall back to the raw stdout so a run degrades to
+    // the old behaviour (text, no usage) rather than failing.
+    const args = ['-p', '--output-format', 'json', '--model', resolveModel(model)];
     if (system) args.push('--append-system-prompt', system);
 
     const child = spawn('claude', args, { env, stdio: ['pipe', 'pipe', 'pipe'] });
@@ -206,7 +231,10 @@ function completeClaudeCli({ system, prompt, model, timeoutMs }) {
     child.on('close', (code) => {
       clearTimeout(killer);
       if (code !== 0) return reject(new Error(`claude CLI exited ${code}: ${err.slice(0, 400)}`));
-      resolve({ text: out.trim(), usage: null });
+      const parsed = parseClaudeCliJson(out);
+      if (!parsed) return resolve({ text: out.trim(), usage: null });
+      if (parsed.isError) return reject(new Error(`claude CLI reported is_error: ${String(parsed.text || '').slice(0, 300)}`));
+      resolve({ text: String(parsed.text || '').trim(), usage: parsed.usage });
     });
     child.stdin.write(prompt);
     child.stdin.end();
@@ -250,14 +278,7 @@ async function completeOpenaiApi({ system, prompt, model, maxTokens, temperature
   let parsed;
   try { parsed = JSON.parse(raw); } catch (_e) { throw new Error(`openai/api: unparseable response: ${raw.slice(0, 200)}`); }
   const text = ((parsed.choices && parsed.choices[0] && parsed.choices[0].message && parsed.choices[0].message.content) || '');
-  const usage = parsed.usage || {};
-  return {
-    text: String(text).trim(),
-    usage: {
-      input_tokens: usage.prompt_tokens || 0,
-      output_tokens: usage.completion_tokens || 0,
-    },
-  };
+  return { text: String(text).trim(), usage: parseOpenaiApiUsage(parsed.usage) };
 }
 
 // Read the OpenAI provider config (base_url + api-key env) from the registry,
@@ -325,9 +346,15 @@ function completeCodexCli({ system, prompt, model, timeoutMs }) {
     // stdin (per its --help), which is content-agnostic.
     const args = codexFinalArgs({ model, outFile });
 
-    const child = spawn('codex', args, { env: { ...process.env }, stdio: ['pipe', 'ignore', 'pipe'] });
+    // v0.4: stdout is CAPTURED (it was 'ignore'). `--json` streams JSONL events
+    // there, and the terminal `turn.completed` event carries this call's token
+    // usage — the only place codex reports it. The final message still comes from
+    // the -o file (cleaner than scraping the stream); the JSONL is read for usage.
+    const child = spawn('codex', args, { env: { ...process.env }, stdio: ['pipe', 'pipe', 'pipe'] });
     let err = '';
+    let jsonl = '';
     const killer = setTimeout(() => { child.kill('SIGKILL'); }, timeoutMs + 5000);
+    child.stdout.on('data', (d) => { jsonl += d; });
     child.stderr.on('data', (d) => { err += d; });
     child.on('error', (e) => {
       clearTimeout(killer);
@@ -340,7 +367,10 @@ function completeCodexCli({ system, prompt, model, timeoutMs }) {
       try { text = fs.readFileSync(outFile, 'utf8'); } catch (_e) { /* fall through */ }
       try { fs.unlinkSync(outFile); } catch (_e) { /* best effort */ }
       if (code !== 0) return reject(new Error(`codex exec exited ${code}: ${String(err).slice(0, 400)}`));
-      resolve({ text: String(text).trim(), usage: null });
+      const ev = parseCodexJsonl(jsonl);
+      // Prefer the -o file; fall back to the stream's agent_message if it is empty.
+      const finalText = String(text || ev.text || '').trim();
+      resolve({ text: finalText, usage: ev.usage });
     });
     child.stdin.on('error', () => { /* ignore EPIPE if codex exits before reading all input */ });
     child.stdin.write(fullPrompt);

package/lib/receipt.js

@@ -14,7 +14,8 @@ const SCHEMA_FILES = {
   '0.1': 'receipt.v0.1.schema.json',
   '0.2': 'receipt.v0.2.schema.json',
   '0.3': 'receipt.v0.3.schema.json',
-  '0.3.1': 'receipt.schema.json',
+  '0.3.1': 'receipt.v0.3.1.schema.json',
+  '0.4': 'receipt.schema.json',
 };
 
 const _validators = {};
@@ -80,7 +81,7 @@ function aggregate(caseResults) {
 //   run:     { model_id, model_release_date, surface, runner_version, date_utc, judge, registry, transcripts }
 //   cases:   [ { id, mode, outcome, score, mean, stddev, samples, generation_hash, judge_sample_hashes, threshold, reason, judge } ]
 //   editorialReviews: optional [ { url, source, date } ]
-function buildReceipt({ skill, suite, run, cases, verificationLevel = 'TESTED', editorialReviews = null }) {
+function buildReceipt({ skill, suite, run, cases, economics = null, verificationLevel = 'TESTED', editorialReviews = null }) {
   // v0.3.1: cases marked failed_timeout are recorded in results.cases but EXCLUDED
   // from aggregates — a band is never fabricated from a case that did not complete.
   const okCases = cases.filter((c) => c.case_status !== 'failed_timeout');
@@ -138,6 +139,11 @@ function buildReceipt({ skill, suite, run, cases, verificationLevel = 'TESTED',
   //   skill.tokens              — estimated SKILL.md token size (value-per-token axis).
   if (run.surface_overhead_note) receipt.run.surface_overhead_note = run.surface_overhead_note;
   if (skill.tokens != null) receipt.skill.tokens = skill.tokens;
+  // v0.4 economics (additive-optional): the frozen prices this receipt's derived
+  // dollar figures were computed from, and the derived block itself. A receipt
+  // from a surface that reports no usage simply omits both.
+  if (run.pricing_snapshot) receipt.run.pricing_snapshot = run.pricing_snapshot;
+  if (economics) receipt.economics = economics;
   // v0.3.1: mark the receipt incomplete when any case failed (excluded above).
   if (failedCount > 0) {
     receipt.run.status = 'incomplete';

package/lib/run.js

@@ -1,14 +1,16 @@
 // SPDX-License-Identifier: Apache-2.0
 'use strict';
 
-const { complete, resolveModel, surfaceForModel, CODEX_OVERHEAD_NOTE } = require('./provider');
+const { complete, resolveModel, surfaceForModel, isMeteredSurface, CODEX_OVERHEAD_NOTE } = require('./provider');
 const { gradeSamples, judgeSettings } = require('./judge');
 const { buildReceipt } = require('./receipt');
 const { sha256 } = require('./canonical');
-const { registryStatus, providerForModel } = require('./models');
+const { registryStatus, providerForModel, priceForModel } = require('./models');
 const { perCallCostUSD } = require('./cost');
 const { runChecks } = require('./checks');
 const { estimateTokens } = require('./skillCost');
+const { hasUsage, normalizeUsage } = require('./usage');
+const { buildPricingSnapshot, computeEconomics } = require('./value');
 const { RUNNER_VERSION, DEFAULT_JUDGE_SAMPLES } = require('../config');
 
 // Known model release dates (best-effort; null when unknown). Recorded into the
@@ -51,8 +53,8 @@ async function runCase({ skillMd, caseObj, model, withSkill, timeoutMs }) {
     throw e;
   }
   const system = withSkill ? skillMd : undefined;
-  const { text, usage, attempts } = await complete({ system, prompt: caseObj.prompt, model, maxTokens: 1024, timeoutMs });
-  return { text, usage, attempts };
+  const { text, usage, wall_ms, attempts } = await complete({ system, prompt: caseObj.prompt, model, maxTokens: 1024, timeoutMs });
+  return { text, usage, wall_ms, attempts };
 }
 
 // Determine a case outcome from its sampled band and threshold.
@@ -91,6 +93,8 @@ async function judgeCase({ caseObj, response, generationHash, judgeModel, mode,
   // v0.3.1 deterministic post-checks (supplementary; NOT folded into `outcome`).
   const checks = runChecks(response, caseObj.checks);
   if (checks.length) caseResult.checks = checks;
+  // v0.4: grading overhead for this case row, kept OUT of the skill-value math.
+  if (hasUsage(g.usage)) caseResult.judge_usage = g.usage;
   return { caseResult, sampleTexts: g.sample_texts, attempts: g.attempts };
 }
 
@@ -170,6 +174,9 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
       const generationHash = sha256(String(gen.text || ''));
       onProgress({ case: c.id, mode, phase: 'judge', samples });
       const jr = await judgeCase({ caseObj: c, response: gen.text, generationHash, judgeModel, mode, timeoutMs: ct, samples });
+      // v0.4: the GENERATION call's usage is the skill-value measurement (the
+      // judge's own usage rides separately on judge_usage, above).
+      if (hasUsage(gen.usage)) jr.caseResult.usage = normalizeUsage({ ...gen.usage, wall_ms: gen.wall_ms });
       calls += samples;
       // Live budget: count all judge calls (retries included) for this (case, mode).
       if (budget) budget.add((jr.attempts || samples) * perCallCostUSD(judgeModel, 'judge'));
@@ -198,6 +205,24 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
   const transcripts = keepTranscripts ? pairs.map((p) => p.transcript) : null;
 
   const surface = surfaceForModel(modelId);
+  const nowIso = opts.nowIso || new Date().toISOString();
+  // v0.4 economics. The pricing snapshot is frozen HERE, at run time, from the
+  // registry; every derived dollar figure below is computed from the snapshot and
+  // never from the live registry, so this receipt keeps its meaning when prices
+  // later change.
+  const pricingSnapshot = buildPricingSnapshot({
+    models: [modelId, judgeModel],
+    lookup: priceForModel,
+    nowIso,
+  });
+  const economics = computeEconomics({
+    cases: caseResults,
+    modelId,
+    judgeModelId: judgeModel,
+    pricingSnapshot,
+    surface,
+    meteredSurface: isMeteredSurface(surface),
+  });
   const receipt = buildReceipt({
     skill: {
       name: skill.name, version: skill.version, contentHash: skill.contentHash,
@@ -213,12 +238,14 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
       // v0.3.1: on the openai/cli (codex) surface, record the fixed harness preamble.
       surface_overhead_note: surface === 'openai-cli' ? CODEX_OVERHEAD_NOTE : undefined,
       runner_version: RUNNER_VERSION,
-      date_utc: opts.nowIso || new Date().toISOString(),
+      date_utc: nowIso,
       registry: registryStatus(modelId),
       transcripts: keepTranscripts ? 'retained-local' : 'hashes-only',
       judge: judgeSettings(samples, judgeModel),
+      pricing_snapshot: pricingSnapshot,
     },
     cases: caseResults,
+    economics,
     verificationLevel: 'TESTED',
   });
 

package/lib/stub.js

@@ -39,16 +39,37 @@ function stubComplete({ system, prompt }) {
       pass: score >= 0.7,
       reason: `stub judge (DRIFTPROOF_STUB): ${helped ? 'with-skill marker present' : 'baseline'}`,
     };
-    return { text: JSON.stringify(body), usage: null };
+    return { text: JSON.stringify(body), usage: stubUsage({ kind: 'judge', prompt }) };
   }
   // Generation call: canned, marked by mode.
   const mark = system ? MARK_SKILL : MARK_BASE;
   const text = `${mark}\nfeat(stub): canned offline generation for CI (no model was called)`;
-  return { text, usage: null };
+  return { text, usage: stubUsage({ kind: 'gen', system, prompt }) };
+}
+
+// v0.4: deterministic synthetic usage, so a zero-model-call stub run still
+// exercises the whole economics path (usage → pricing snapshot → derived cost/
+// latency fields → the value report's three axes). The numbers are SYNTHETIC and
+// deliberately shaped like the real surfaces: a large fixed harness preamble that
+// is identical in both arms, plus the with-skill arm's extra SKILL.md input and
+// its slightly longer, slower output. No randomness — same input, same usage.
+const STUB_HARNESS_PREAMBLE_TOKENS = 20000;   // the fixed, arm-identical overhead
+function stubUsage({ kind, system, prompt }) {
+  const promptTokens = Math.ceil(String(prompt || '').length / 4);
+  if (kind === 'judge') {
+    return { input_tokens: 1200 + promptTokens, output_tokens: 60, cached_tokens: 800, wall_ms: 900 };
+  }
+  const skillTokens = system ? Math.ceil(String(system).length / 4) : 0;
+  return {
+    input_tokens: STUB_HARNESS_PREAMBLE_TOKENS + promptTokens + skillTokens,
+    output_tokens: system ? 420 : 350,
+    cached_tokens: STUB_HARNESS_PREAMBLE_TOKENS,
+    wall_ms: system ? 4200 : 3600,
+  };
 }
 
 function stubEnabled() {
   return process.env.DRIFTPROOF_STUB === '1';
 }
 
-module.exports = { stubComplete, stubEnabled, MARK_SKILL, MARK_BASE };
+module.exports = { stubComplete, stubEnabled, stubUsage, MARK_SKILL, MARK_BASE, STUB_HARNESS_PREAMBLE_TOKENS };

package/lib/usage.js

@@ -0,0 +1,168 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+// Usage capture — normalizing what each surface reports about a single call.
+//
+// Every lane reports token usage in its own shape, and two of them (the CLI
+// lanes) were previously discarding it entirely: the `claude -p` default text
+// output carries no usage block at all, and `codex exec --json` streams usage on
+// stdout that we were routing to /dev/null. Spec v0.4 captures it.
+//
+// ONE normalized shape, so three substrates are comparable:
+//
+//   { input_tokens, output_tokens, cached_tokens, wall_ms }
+//
+//   input_tokens   TOTAL input presented to the model for this call, INCLUDING
+//                  any portion served from cache. This is the field that most
+//                  needs normalizing: the surfaces disagree about it (below).
+//   cached_tokens  the portion of input_tokens served from cache (null when the
+//                  surface does not surface it — never guessed, never 0-filled).
+//   output_tokens  tokens generated, INCLUDING reasoning/thinking tokens where
+//                  the surface bundles them (both CLI lanes do; the split is
+//                  reported by the surface but deliberately not modelled here —
+//                  see the disclosure in REPORT-STYLE.md).
+//   wall_ms        measured BY US around the call (see lib/provider), not taken
+//                  from the surface. It is the only field whose definition is
+//                  identical across lanes, which is exactly why latency is
+//                  reported as an observed, surface-disclosed number.
+//
+// THE INPUT-TOKEN DISAGREEMENT (verified against real output, 2026-08-18):
+//   - `claude -p --output-format json` reports input_tokens EXCLUDING cache:
+//     total input = input_tokens + cache_creation_input_tokens + cache_read_input_tokens.
+//     (Observed: 10 + 6,974 + 18,134 = 25,118 for a two-word prompt.)
+//   - `codex exec --json` reports input_tokens INCLUDING cache, with
+//     cached_input_tokens as a SUBSET of it. (Observed: 10,807 of which 4,480 cached.)
+//   Normalizing to "total including cache" makes the two comparable; the raw
+//   fixtures both parsers are tested against are checked in under tests/fixtures/.
+//
+// Both CLI surfaces prepend a large fixed harness preamble that we do not
+// control (~25k tokens on claude-cli, ~11k on codex). It is IDENTICAL in the
+// with-skill and baseline arms, so it cancels in the incremental (Δ) figures —
+// which is why Δcost is the honest axis and absolute per-call cost is not.
+
+function n(x) { return Number.isFinite(Number(x)) ? Number(x) : 0; }
+
+// The empty/unknown usage record. Deliberately null (not zeros) so "the surface
+// did not tell us" never reads as "the call cost nothing".
+function emptyUsage() {
+  return { input_tokens: null, output_tokens: null, cached_tokens: null, wall_ms: null };
+}
+
+function hasUsage(u) {
+  return !!(u && (u.input_tokens != null || u.output_tokens != null));
+}
+
+// ── anthropic/cli — `claude -p --output-format json` ──────────────────────────
+// The whole call is one JSON object; `result` carries the final text and `usage`
+// the token counts. input_tokens EXCLUDES cache, so both cache fields are added
+// back to reach the total actually presented to the model.
+function parseClaudeCliJson(stdout) {
+  let j;
+  try { j = JSON.parse(String(stdout || '')); } catch (_e) { return null; }
+  if (!j || typeof j !== 'object') return null;
+  const u = j.usage || {};
+  const cacheRead = n(u.cache_read_input_tokens);
+  const cacheCreate = n(u.cache_creation_input_tokens);
+  const usage = {
+    input_tokens: n(u.input_tokens) + cacheRead + cacheCreate,
+    output_tokens: n(u.output_tokens),
+    // Cache READ is the portion genuinely served from cache. Cache CREATION was
+    // processed fresh this call (and billed at a premium), so it is not cached.
+    cached_tokens: cacheRead,
+    wall_ms: null,
+  };
+  return {
+    text: typeof j.result === 'string' ? j.result : '',
+    usage,
+    isError: j.is_error === true,
+    // The CLI reports its own dollar figure. Recorded here for completeness but
+    // NOT used: costs are computed uniformly from the frozen pricing snapshot so
+    // three substrates are on one basis (see lib/value.js).
+    surfaceReportedCostUsd: Number.isFinite(Number(j.total_cost_usd)) ? Number(j.total_cost_usd) : null,
+  };
+}
+
+// ── openai/cli — `codex exec --json` JSONL event stream ───────────────────────
+// One JSON object per line. Usage rides the terminal `turn.completed` event;
+// input_tokens already INCLUDES cached_input_tokens. Unparseable lines are
+// skipped (the stream also carries progress events we do not model).
+function parseCodexJsonl(stdout) {
+  const lines = String(stdout || '').split('\n').map((l) => l.trim()).filter(Boolean);
+  let usage = null;
+  let text = '';
+  for (const line of lines) {
+    let ev;
+    try { ev = JSON.parse(line); } catch (_e) { continue; }
+    if (!ev || typeof ev !== 'object') continue;
+    if (ev.type === 'turn.completed' && ev.usage) {
+      const u = ev.usage;
+      usage = {
+        input_tokens: n(u.input_tokens),          // already total (cache included)
+        output_tokens: n(u.output_tokens),        // includes reasoning_output_tokens
+        cached_tokens: u.cached_input_tokens == null ? null : n(u.cached_input_tokens),
+        wall_ms: null,
+      };
+    }
+    // The final message is normally read from the -o file; this is a fallback.
+    if (ev.type === 'item.completed' && ev.item && ev.item.type === 'agent_message' && typeof ev.item.text === 'string') {
+      text = ev.item.text;
+    }
+  }
+  return { usage, text };
+}
+
+// ── anthropic/api ─────────────────────────────────────────────────────────────
+// The Messages API reports input_tokens EXCLUDING cache, same as the CLI.
+function parseAnthropicApiUsage(u) {
+  if (!u) return emptyUsage();
+  const cacheRead = n(u.cache_read_input_tokens);
+  const cacheCreate = n(u.cache_creation_input_tokens);
+  return {
+    input_tokens: n(u.input_tokens) + cacheRead + cacheCreate,
+    output_tokens: n(u.output_tokens),
+    cached_tokens: u.cache_read_input_tokens == null ? null : cacheRead,
+    wall_ms: null,
+  };
+}
+
+// ── openai/api (Chat Completions-compatible) ──────────────────────────────────
+// prompt_tokens is the total; the cached portion, when present, is nested under
+// prompt_tokens_details.cached_tokens.
+function parseOpenaiApiUsage(u) {
+  if (!u) return emptyUsage();
+  const details = u.prompt_tokens_details || {};
+  return {
+    input_tokens: n(u.prompt_tokens),
+    output_tokens: n(u.completion_tokens),
+    cached_tokens: details.cached_tokens == null ? null : n(details.cached_tokens),
+    wall_ms: null,
+  };
+}
+
+// Sum a list of usage records into one (used for the N judge samples of a case).
+// Null-safe: unknown fields stay null unless at least one record reported them.
+function sumUsage(list) {
+  const records = (list || []).filter(Boolean);
+  if (!records.length) return emptyUsage();
+  const out = { input_tokens: null, output_tokens: null, cached_tokens: null, wall_ms: null };
+  for (const key of ['input_tokens', 'output_tokens', 'cached_tokens', 'wall_ms']) {
+    const present = records.filter((r) => r[key] != null);
+    if (present.length) out[key] = present.reduce((a, r) => a + n(r[key]), 0);
+  }
+  return out;
+}
+
+// Round a usage record's fields to integers (tokens and ms are whole units).
+function normalizeUsage(u) {
+  if (!u) return emptyUsage();
+  const out = emptyUsage();
+  for (const key of ['input_tokens', 'output_tokens', 'cached_tokens', 'wall_ms']) {
+    if (u[key] != null && Number.isFinite(Number(u[key]))) out[key] = Math.round(Number(u[key]));
+  }
+  return out;
+}
+
+module.exports = {
+  emptyUsage, hasUsage, sumUsage, normalizeUsage,
+  parseClaudeCliJson, parseCodexJsonl, parseAnthropicApiUsage, parseOpenaiApiUsage,
+};