driftproof 0.8.1 → 0.10.0

package/lib/usage.js

@@ -42,6 +42,27 @@
 
 function n(x) { return Number.isFinite(Number(x)) ? Number(x) : 0; }
 
+// ── what answered (spec 026, AC-2 and AC-8) ──────────────────────────────────
+// Every lane also reports, when it can, WHICH model served the call and WHY the
+// reply stopped. Both are read here, never guessed: a surface that says nothing
+// yields null for both, and the receipt records that it said nothing.
+//
+//   reportedModels  the ids the surface named, VERBATIM (the real claude CLI
+//                   keys the call's usage under the undated form and puts a
+//                   dated side entry beside it; an api response names one
+//                   model). Null when the surface names no model. The runner
+//                   compares on canonical ids (canonicalModelId: a trailing
+//                   -YYYYMMDD is not part of the identity), never the reader.
+//   stopReason      the surface's own word for why the reply ended (end_turn,
+//                   max_tokens, stop, length ...). Null when it gives none.
+function canonicalModelId(id) { return String(id || '').replace(/-\d{8}$/, ''); }
+function reportedModelsOf(map) {
+  if (!map || typeof map !== 'object' || Array.isArray(map)) return null;
+  const ids = [...new Set(Object.keys(map).filter((k) => k))].sort();
+  return ids.length ? ids : null;
+}
+function stopReasonOf(v) { return typeof v === 'string' && v ? v : null; }
+
 // The empty/unknown usage record. Deliberately null (not zeros) so "the surface
 // did not tell us" never reads as "the call cost nothing".
 function emptyUsage() {
@@ -75,6 +96,9 @@ function parseClaudeCliJson(stdout) {
     text: typeof j.result === 'string' ? j.result : '',
     usage,
     isError: j.is_error === true,
+    // v0.6: the CLI's own stop reason and the models it says served the call.
+    stopReason: stopReasonOf(j.stop_reason),
+    reportedModels: reportedModelsOf(j.modelUsage),
     // The CLI reports its own dollar figure. Recorded here for completeness but
     // NOT used: costs are computed uniformly from the frozen pricing snapshot so
     // three substrates are on one basis (see lib/value.js).
@@ -85,7 +109,9 @@ function parseClaudeCliJson(stdout) {
 // ── openai/cli — `codex exec --json` JSONL event stream ───────────────────────
 // One JSON object per line. Usage rides the terminal `turn.completed` event;
 // input_tokens already INCLUDES cached_input_tokens. Unparseable lines are
-// skipped (the stream also carries progress events we do not model).
+// skipped (the stream also carries progress events we do not model). The
+// stream names neither the model that served the turn nor a stop reason, so
+// both read null here and the receipt says the surface did not report them.
 function parseCodexJsonl(stdout) {
   const lines = String(stdout || '').split('\n').map((l) => l.trim()).filter(Boolean);
   let usage = null;
@@ -103,16 +129,30 @@ function parseCodexJsonl(stdout) {
         wall_ms: null,
       };
     }
-    // The final message is normally read from the -o file; this is a fallback.
+    // The final message IS the stream: the last `item.completed` /
+    // `agent_message` event carries it. Nothing is read from a file after the
+    // call, on either spawn mode (spec 022 AC-9; the earlier comment here
+    // described an output file the lane no longer reads, F-022-3).
     if (ev.type === 'item.completed' && ev.item && ev.item.type === 'agent_message' && typeof ev.item.text === 'string') {
       text = ev.item.text;
     }
   }
-  return { usage, text };
+  return { usage, text, stopReason: null, reportedModels: null };
 }
 
 // ── anthropic/api ─────────────────────────────────────────────────────────────
 // The Messages API reports input_tokens EXCLUDING cache, same as the CLI.
+// The response object also names the model that served it (`model`) and why
+// it stopped (`stop_reason`); read here from the object, null when absent.
+// Exercised against tests/fixtures/response-anthropic-api.json, which is
+// SYNTHESISED from the API reference, not captured from a call.
+function readAnthropicApiResponse(resp) {
+  const r = resp && typeof resp === 'object' ? resp : {};
+  return {
+    stopReason: stopReasonOf(r.stop_reason),
+    reportedModels: typeof r.model === 'string' && r.model ? [r.model] : null,
+  };
+}
 function parseAnthropicApiUsage(u) {
   if (!u) return emptyUsage();
   const cacheRead = n(u.cache_read_input_tokens);
@@ -127,7 +167,18 @@ function parseAnthropicApiUsage(u) {
 
 // ── openai/api (Chat Completions-compatible) ──────────────────────────────────
 // prompt_tokens is the total; the cached portion, when present, is nested under
-// prompt_tokens_details.cached_tokens.
+// prompt_tokens_details.cached_tokens. The response also names the serving
+// model (`model`) and the first choice's `finish_reason`; read here, null
+// when absent. Exercised against tests/fixtures/response-openai-api.json,
+// SYNTHESISED from the API reference, not captured from a call.
+function readOpenaiApiResponse(resp) {
+  const r = resp && typeof resp === 'object' ? resp : {};
+  const choice = Array.isArray(r.choices) && r.choices[0] && typeof r.choices[0] === 'object' ? r.choices[0] : {};
+  return {
+    stopReason: stopReasonOf(choice.finish_reason),
+    reportedModels: typeof r.model === 'string' && r.model ? [r.model] : null,
+  };
+}
 function parseOpenaiApiUsage(u) {
   if (!u) return emptyUsage();
   const details = u.prompt_tokens_details || {};
@@ -165,4 +216,5 @@ function normalizeUsage(u) {
 module.exports = {
   emptyUsage, hasUsage, sumUsage, normalizeUsage,
   parseClaudeCliJson, parseCodexJsonl, parseAnthropicApiUsage, parseOpenaiApiUsage,
+  readAnthropicApiResponse, readOpenaiApiResponse, canonicalModelId, reportedModelsOf,
 };

package/lib/value.js

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 'use strict';
 
+const { caseFailed } = require('./receipt');
+
 const { EFFECT_FLOOR } = require('../config');
 
 // Economics of a run — what a skill COSTS to run, alongside whether it helps.
@@ -204,7 +206,7 @@ function armEconomics(rows, price) {
 //   surface          the run surface, to label the cost basis honestly
 function computeEconomics({ cases, modelId, judgeModelId, pricingSnapshot, surface, meteredSurface }) {
   const price = (pricingSnapshot && pricingSnapshot.models && pricingSnapshot.models[modelId]) || null;
-  const ok = (cases || []).filter((c) => c.case_status !== 'failed_timeout');
+  const ok = (cases || []).filter((c) => !caseFailed(c));
   const withRows = ok.filter((c) => c.mode === 'with_skill');
   const baseRows = ok.filter((c) => c.mode === 'baseline');
 
@@ -322,7 +324,7 @@ function receiptCostBreakdown(receipt) {
   let judge = 0;
   let missingJudgeRate = null;
   for (const c of ((receipt.results || {}).cases || [])) {
-    if (c.case_status === 'failed_timeout') continue;
+    if (caseFailed(c)) continue;
     const g = costForUsage(c.usage, genRate);
     if (g != null) generation += g;
     if (c.judge_usage) {

package/lib/verdict.js

@@ -43,7 +43,12 @@ function verdictFromReceipt(receipt) {
   // Below-TESTED receipts (imported/DECLARED) and null-delta receipts are never
   // verdicted — we did not run the suite, so we do not certify the outcome.
   const level = (receipt && receipt.verification_level) || 'TESTED';
-  const measured = level === 'TESTED' && typeof cmp.delta === 'number';
+  // Spec 026 AC-4: an INCOMPLETE receipt (its run block's status field reads
+  // incomplete: a case had an arm that could not be measured) is not verdicted
+  // either. spec/RECEIPT.md has said since v0.3.1 that a drift report must not
+  // compute a verdict from one; the badge is the same reader with a shorter path.
+  const incomplete = !!(receipt && receipt.run && receipt.run.status === 'incomplete');
+  const measured = level === 'TESTED' && typeof cmp.delta === 'number' && !incomplete;
   let verdict;
   if (!measured) verdict = 'NOT_MEASURED';
   else if (delta >= EFFECT_FLOOR) verdict = 'PASSED';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "driftproof",
-  "version": "0.8.1",
+  "version": "0.10.0",
   "description": "A dated proof that this skill, this hash, this model, still helps: run a skill's eval suite with and without the skill across model versions, emit hash-verified dated receipts, and diff receipts into drift reports.",
   "license": "Apache-2.0",
   "keywords": [
@@ -45,6 +45,7 @@
     "node": ">=22"
   },
   "devDependencies": {
-    "html-validate": "^11.11.0"
+    "html-validate": "^11.11.0",
+    "playwright": "^1.62.1"
   }
 }

package/spec/RECEIPT.md

@@ -1,5 +1,5 @@
 <!-- SPDX-License-Identifier: Apache-2.0 -->
-# Driftproof receipt — spec v0.5
+# Driftproof receipt — spec v0.6
 
 A **receipt** is a **hash-verified**, dated record of running one agent skill's eval suite
 **with** and **without** the skill on one model version, with the judge **sampled**
@@ -11,16 +11,64 @@ The machine-readable contract is [`receipt.schema.json`](./receipt.schema.json)
 (JSON Schema, draft 2020-12). This document is the human companion. Where they
 disagree, the schema wins.
 
-**Versioning.** The current schema is v0.4
+**Versioning.** The current schema is v0.6
 ([`receipt.schema.json`](./receipt.schema.json)). Prior schemas are kept frozen as
+[`receipt.v0.5.schema.json`](./receipt.v0.5.schema.json),
+[`receipt.v0.4.schema.json`](./receipt.v0.4.schema.json),
 [`receipt.v0.3.1.schema.json`](./receipt.v0.3.1.schema.json),
 [`receipt.v0.3.schema.json`](./receipt.v0.3.schema.json),
 [`receipt.v0.2.schema.json`](./receipt.v0.2.schema.json) and
 [`receipt.v0.1.schema.json`](./receipt.v0.1.schema.json); the validator picks the
-schema by the receipt's own `schema_version`, so v0.1, v0.2, v0.3 and v0.3.1
-receipts still load and validate — including every receipt behind the four
-published reports, which the gate asserts on each run. v0.4 is an **additive**
-bump: everything it adds is optional, and no earlier receipt is invalidated.
+schema by the receipt's own `schema_version`, so v0.1, v0.2, v0.3, v0.3.1, v0.4
+and v0.5 receipts still load and validate — including every receipt behind the
+published reports, which the gate asserts on each run. v0.6 is additive for a
+reader that ignores unknown fields and **not additive for the validator**: a
+v0.5 receipt restamped `"0.6"` is refused, because v0.6 requires the receipt to
+say what answered it (see *What v0.6 adds*). No earlier receipt is invalidated;
+each validates against its own frozen schema.
+
+## What v0.6 adds: the receipt says what answered it
+
+Spec 026 (receipt integrity). Six ways the v0.5 runner wrote a receipt that read
+as more than it was, each closed by a field the receipt now carries and the
+schema now requires:
+
+- **`run.answered_by`** `{ kind, attested, reported_model, reported_models,
+  isolation }`. `kind` is `model` (a model surface answered), `stub` (the
+  canned stub answered and nothing was measured) or `external` (another tool's
+  harness answered; the receipt was imported). `attested` is true only when the
+  surface echoed a model id for every draw and it matched the requested
+  canonical id; `reported_model` and `reported_models` are what it echoed
+  (null when nothing). `isolation` names the spawn path: `eval-user`,
+  `same-user` (`--trusted-skill`) or `none`. A stub run also records
+  `run.surface: "stub"` and `run.judge.surface: "stub"`, and the schema refuses
+  `TESTED` unless `answered_by.kind` is `model`: a stub receipt is
+  `UNVERIFIED` by schema, not only by the runner.
+- **`run.judge.model_id`** and **`run.judge.prompt_template_hash`**: which
+  judge ran, and a sha256 over the grading template with its three slots empty.
+  A different template is a different judge; `diff` names the field and
+  computes no verdict across one. The template hash is null only on an
+  imported receipt.
+- **`stop_reason`** and **`truncated`** on every draw, and
+  **`generation.n_truncated`** per case and arm. A draw cut at the output cap
+  is unmeasured and never judged; a partial answer scored as a whole one was
+  the v0.5 shape.
+- **`case_status: "failed_unmeasured"`**: every draw of the arm was unmeasured
+  for a reason that is not a timeout (an empty generation, a judge output with
+  no score, a truncated draw). Recorded without fabricated samples or hashes
+  exactly as `failed_timeout` is, and excluded pairwise. An absent output is
+  not a measurement; a zero is.
+- **`results.aggregates.band_rule`**: the formula the aggregate band is derived
+  by, stated on the receipt so a reader recomputes it from `results.cases`.
+- **Bands the formula cannot form are null, never `0`.** An arm's `stddev` is
+  null only when its `case_count` is below 2 (the schema refuses a null on a
+  two-case arm); `comparison.delta_uncertainty` is null only beside
+  `delta_uncertainty_unavailable: "single_case"` or `"no_cases"`, and
+  `mean_score` and the comparison are null only under `no_cases`.
+
+A v0.5 receipt carries none of these, so the validator refuses one restamped
+`"0.6"`; v0.5 stays frozen at `receipt.v0.5.schema.json` and every archived
+receipt validates against the version it names.
 
 ## What v0.5 adds: the generation is sampled too
 
@@ -95,7 +143,7 @@ carry a numeric `mean`.
 ## Which schema validates which receipt (the coexistence rule)
 
 - **A producer emits the current version.** `config.js` `RECEIPT_SCHEMA_VERSION`
-  decides it, and at this revision that is **v0.5**.
+  decides it, and at this revision that is **v0.6**.
 - **A reader validates against the receipt's own `schema_version`**, never
   against the newest schema it happens to have. `validateReceipt()` selects the
   schema by that field, which is why a v0.1 receipt from the first report still
@@ -247,7 +295,11 @@ The v0.3.1 schema gained an **additive interop revision** so receipts can be
   outputs were also written to `transcripts/<receipt-id>/` (under
   `--keep-transcripts`), else `"hashes-only"` (the default — only the hashes).
 - **`run.registry`** — `"registered"` when `model_id` resolved in the model
-  registry (`config/models.json`), else `"unregistered"` (the run still executed;
+  registry (`config/models.json`); `"unregistered"` is import-only since v0.6, the runner refuses it (spec 026 AC-10):
+  a run of ours refuses an id the registry does not carry before any call,
+  naming the registry path, so no receipt we write says it; an imported
+  receipt may, its model field being another tool's word (before v0.6 the
+  run was not refused;
   cost was estimated with a conservative default price).
 - These four fields are **required** in a v0.3 receipt. Everything else is
   unchanged from v0.2.
@@ -263,7 +315,7 @@ The v0.3.1 schema gained an **additive interop revision** so receipts can be
 
 ## Fields
 
-### `schema_version` (string, required) — `"0.4"`.
+### `schema_version` (string, required) — `"0.6"`.
 
 ### `skill` (object, required)
 | field | type | notes |
@@ -290,7 +342,7 @@ The v0.3.1 schema gained an **additive interop revision** so receipts can be
 | `surface_overhead_note` | string | **v0.3.1, optional.** On `openai-cli`: the fixed Codex base-instruction preamble (~12–15k input tokens/call) the harness prepends and does not control. |
 | `runner_version` | string | Runner version, for reproducibility. |
 | `date_utc` | ISO 8601 UTC | When the run finished. |
-| `registry` | `"registered"` \| `"unregistered"` | **v0.3.** Whether `model_id` resolved in the model registry (`config/models.json`). |
+| `registry` | `"registered"` \| `"unregistered"` | **v0.3.** Whether `model_id` resolved in the model registry (`config/models.json`). **v0.6:** `"unregistered"` is import-only; the runner refuses an unregistered id before any call (spec 026 AC-10). |
 | `transcripts` | `"retained-local"` \| `"hashes-only"` | **v0.3.** Whether the raw generations + judge outputs were retained on disk (see § Transcripts). |
 | `judge` | object | `{ samples, temperature, sampling, surface }` — see below. |
 
@@ -358,7 +410,12 @@ Every generation and every judge sample is hashed into the receipt
 generations and judge outputs are written to `transcripts/<receipt_hash>/` (one
 JSON per case+mode, plus an `index.json`), and the receipt records
 `transcripts: "retained-local"`. Without the flag the receipt records
-`transcripts: "hashes-only"` and only the hashes are kept. The transcript
+`transcripts: "hashes-only"` and only the hashes are kept. **What is retained
+is the last measured draw of each case and mode only**: the runner keeps one
+transcript per case and mode and overwrites it on every draw, so the per-case
+top-level hashes (which are that draw's) can be re-derived from disk, and the
+draw-level hashes of earlier draws are checkable against nothing the runner
+wrote (spec 026, stated as the bound rather than moved). The transcript
 directory is **gitignored by default** — raw model text is never committed. The
 default for trigger-initiated runs is `retained-local` (disk is cheap, audits are
 not). To verify a retained receipt: re-hash each transcript file and compare