driftproof 0.8.0 → 0.9.0

package/lib/value.js

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 'use strict';
 
+const { caseFailed } = require('./receipt');
+
 const { EFFECT_FLOOR } = require('../config');
 
 // Economics of a run — what a skill COSTS to run, alongside whether it helps.
@@ -204,7 +206,7 @@ function armEconomics(rows, price) {
 //   surface          the run surface, to label the cost basis honestly
 function computeEconomics({ cases, modelId, judgeModelId, pricingSnapshot, surface, meteredSurface }) {
   const price = (pricingSnapshot && pricingSnapshot.models && pricingSnapshot.models[modelId]) || null;
-  const ok = (cases || []).filter((c) => c.case_status !== 'failed_timeout');
+  const ok = (cases || []).filter((c) => !caseFailed(c));
   const withRows = ok.filter((c) => c.mode === 'with_skill');
   const baseRows = ok.filter((c) => c.mode === 'baseline');
 
@@ -322,7 +324,7 @@ function receiptCostBreakdown(receipt) {
   let judge = 0;
   let missingJudgeRate = null;
   for (const c of ((receipt.results || {}).cases || [])) {
-    if (c.case_status === 'failed_timeout') continue;
+    if (caseFailed(c)) continue;
     const g = costForUsage(c.usage, genRate);
     if (g != null) generation += g;
     if (c.judge_usage) {

package/lib/verdict.js

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 'use strict';
 
+const crypto = require('crypto');
 const { EFFECT_FLOOR } = require('../config');
 
 // Single-receipt verdict + shields.io badge.
@@ -42,7 +43,12 @@ function verdictFromReceipt(receipt) {
   // Below-TESTED receipts (imported/DECLARED) and null-delta receipts are never
   // verdicted — we did not run the suite, so we do not certify the outcome.
   const level = (receipt && receipt.verification_level) || 'TESTED';
-  const measured = level === 'TESTED' && typeof cmp.delta === 'number';
+  // Spec 026 AC-4: an INCOMPLETE receipt (its run block's status field reads
+  // incomplete: a case had an arm that could not be measured) is not verdicted
+  // either. spec/RECEIPT.md has said since v0.3.1 that a drift report must not
+  // compute a verdict from one; the badge is the same reader with a shorter path.
+  const incomplete = !!(receipt && receipt.run && receipt.run.status === 'incomplete');
+  const measured = level === 'TESTED' && typeof cmp.delta === 'number' && !incomplete;
   let verdict;
   if (!measured) verdict = 'NOT_MEASURED';
   else if (delta >= EFFECT_FLOOR) verdict = 'PASSED';
@@ -72,14 +78,39 @@ function badgeEndpoint(receipt, { label = 'driftproof' } = {}) {
 }
 
 // Lines suitable for appending to a GitHub Actions $GITHUB_OUTPUT file.
+//
+// Spec 023 (audit A6). Each entry is written in GitHub's multiline form,
+//
+//   name<<ghadelim_<32 hex>
+//   value
+//   ghadelim_<32 hex>
+//
+// with a delimiter drawn fresh from crypto.randomBytes for EVERY entry, so no
+// value can close a block early; and any value carrying \r or \n is refused
+// with a throw before anything is written. The bare `name=value` form this used
+// to emit let a model_id with a newline in it (any string a receipt carries; a
+// .driftproofrc in a skill directory sets it) append a second, attacker-chosen
+// entry, and that entry was then interpolated into the enforce step's shell.
+// Refusing is the honest writer: a line break in a model id is not a value
+// anybody meant to publish. Entries are joined by '\n' with no trailing newline
+// so the caller's console.log stays the one that terminates the text.
+function githubOutputEntry(name, value) {
+  const val = String(value);
+  if (/[\r\n]/.test(val)) {
+    throw new Error(`refusing to write $GITHUB_OUTPUT: the value of "${name}" contains a line break (${JSON.stringify(val)})`);
+  }
+  const delim = `ghadelim_${crypto.randomBytes(16).toString('hex')}`;
+  return `${name}<<${delim}\n${val}\n${delim}`;
+}
+
 function githubOutputLines(receipt) {
   const v = verdictFromReceipt(receipt);
   return [
-    `verdict=${v.verdict}`,
-    `delta=${v.delta}`,
-    `message=${v.message}`,
-    `color=${v.color}`,
-  ].join('\n');
+    ['verdict', v.verdict],
+    ['delta', v.delta],
+    ['message', v.message],
+    ['color', v.color],
+  ].map(([k, val]) => githubOutputEntry(k, val)).join('\n');
 }
 
-module.exports = { verdictFromReceipt, badgeEndpoint, githubOutputLines, shortModel, VERDICTS };
+module.exports = { verdictFromReceipt, badgeEndpoint, githubOutputLines, githubOutputEntry, shortModel, VERDICTS };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "driftproof",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "A dated proof that this skill, this hash, this model, still helps: run a skill's eval suite with and without the skill across model versions, emit hash-verified dated receipts, and diff receipts into drift reports.",
   "license": "Apache-2.0",
   "keywords": [
@@ -45,6 +45,7 @@
     "node": ">=22"
   },
   "devDependencies": {
-    "html-validate": "^11.11.0"
+    "html-validate": "^11.11.0",
+    "playwright": "^1.62.1"
   }
 }

package/spec/RECEIPT.md

@@ -1,5 +1,5 @@
 <!-- SPDX-License-Identifier: Apache-2.0 -->
-# Driftproof receipt — spec v0.5
+# Driftproof receipt — spec v0.6
 
 A **receipt** is a **hash-verified**, dated record of running one agent skill's eval suite
 **with** and **without** the skill on one model version, with the judge **sampled**
@@ -11,16 +11,64 @@ The machine-readable contract is [`receipt.schema.json`](./receipt.schema.json)
 (JSON Schema, draft 2020-12). This document is the human companion. Where they
 disagree, the schema wins.
 
-**Versioning.** The current schema is v0.4
+**Versioning.** The current schema is v0.6
 ([`receipt.schema.json`](./receipt.schema.json)). Prior schemas are kept frozen as
+[`receipt.v0.5.schema.json`](./receipt.v0.5.schema.json),
+[`receipt.v0.4.schema.json`](./receipt.v0.4.schema.json),
 [`receipt.v0.3.1.schema.json`](./receipt.v0.3.1.schema.json),
 [`receipt.v0.3.schema.json`](./receipt.v0.3.schema.json),
 [`receipt.v0.2.schema.json`](./receipt.v0.2.schema.json) and
 [`receipt.v0.1.schema.json`](./receipt.v0.1.schema.json); the validator picks the
-schema by the receipt's own `schema_version`, so v0.1, v0.2, v0.3 and v0.3.1
-receipts still load and validate — including every receipt behind the four
-published reports, which the gate asserts on each run. v0.4 is an **additive**
-bump: everything it adds is optional, and no earlier receipt is invalidated.
+schema by the receipt's own `schema_version`, so v0.1, v0.2, v0.3, v0.3.1, v0.4
+and v0.5 receipts still load and validate — including every receipt behind the
+published reports, which the gate asserts on each run. v0.6 is additive for a
+reader that ignores unknown fields and **not additive for the validator**: a
+v0.5 receipt restamped `"0.6"` is refused, because v0.6 requires the receipt to
+say what answered it (see *What v0.6 adds*). No earlier receipt is invalidated;
+each validates against its own frozen schema.
+
+## What v0.6 adds: the receipt says what answered it
+
+Spec 026 (receipt integrity). Six ways the v0.5 runner wrote a receipt that read
+as more than it was, each closed by a field the receipt now carries and the
+schema now requires:
+
+- **`run.answered_by`** `{ kind, attested, reported_model, reported_models,
+  isolation }`. `kind` is `model` (a model surface answered), `stub` (the
+  canned stub answered and nothing was measured) or `external` (another tool's
+  harness answered; the receipt was imported). `attested` is true only when the
+  surface echoed a model id for every draw and it matched the requested
+  canonical id; `reported_model` and `reported_models` are what it echoed
+  (null when nothing). `isolation` names the spawn path: `eval-user`,
+  `same-user` (`--trusted-skill`) or `none`. A stub run also records
+  `run.surface: "stub"` and `run.judge.surface: "stub"`, and the schema refuses
+  `TESTED` unless `answered_by.kind` is `model`: a stub receipt is
+  `UNVERIFIED` by schema, not only by the runner.
+- **`run.judge.model_id`** and **`run.judge.prompt_template_hash`**: which
+  judge ran, and a sha256 over the grading template with its three slots empty.
+  A different template is a different judge; `diff` names the field and
+  computes no verdict across one. The template hash is null only on an
+  imported receipt.
+- **`stop_reason`** and **`truncated`** on every draw, and
+  **`generation.n_truncated`** per case and arm. A draw cut at the output cap
+  is unmeasured and never judged; a partial answer scored as a whole one was
+  the v0.5 shape.
+- **`case_status: "failed_unmeasured"`**: every draw of the arm was unmeasured
+  for a reason that is not a timeout (an empty generation, a judge output with
+  no score, a truncated draw). Recorded without fabricated samples or hashes
+  exactly as `failed_timeout` is, and excluded pairwise. An absent output is
+  not a measurement; a zero is.
+- **`results.aggregates.band_rule`**: the formula the aggregate band is derived
+  by, stated on the receipt so a reader recomputes it from `results.cases`.
+- **Bands the formula cannot form are null, never `0`.** An arm's `stddev` is
+  null only when its `case_count` is below 2 (the schema refuses a null on a
+  two-case arm); `comparison.delta_uncertainty` is null only beside
+  `delta_uncertainty_unavailable: "single_case"` or `"no_cases"`, and
+  `mean_score` and the comparison are null only under `no_cases`.
+
+A v0.5 receipt carries none of these, so the validator refuses one restamped
+`"0.6"`; v0.5 stays frozen at `receipt.v0.5.schema.json` and every archived
+receipt validates against the version it names.
 
 ## What v0.5 adds: the generation is sampled too
 
@@ -95,7 +143,7 @@ carry a numeric `mean`.
 ## Which schema validates which receipt (the coexistence rule)
 
 - **A producer emits the current version.** `config.js` `RECEIPT_SCHEMA_VERSION`
-  decides it, and at this revision that is **v0.5**.
+  decides it, and at this revision that is **v0.6**.
 - **A reader validates against the receipt's own `schema_version`**, never
   against the newest schema it happens to have. `validateReceipt()` selects the
   schema by that field, which is why a v0.1 receipt from the first report still
@@ -247,7 +295,11 @@ The v0.3.1 schema gained an **additive interop revision** so receipts can be
   outputs were also written to `transcripts/<receipt-id>/` (under
   `--keep-transcripts`), else `"hashes-only"` (the default — only the hashes).
 - **`run.registry`** — `"registered"` when `model_id` resolved in the model
-  registry (`config/models.json`), else `"unregistered"` (the run still executed;
+  registry (`config/models.json`); `"unregistered"` is import-only since v0.6, the runner refuses it (spec 026 AC-10):
+  a run of ours refuses an id the registry does not carry before any call,
+  naming the registry path, so no receipt we write says it; an imported
+  receipt may, its model field being another tool's word (before v0.6 the
+  run was not refused;
   cost was estimated with a conservative default price).
 - These four fields are **required** in a v0.3 receipt. Everything else is
   unchanged from v0.2.
@@ -263,7 +315,7 @@ The v0.3.1 schema gained an **additive interop revision** so receipts can be
 
 ## Fields
 
-### `schema_version` (string, required) — `"0.4"`.
+### `schema_version` (string, required) — `"0.6"`.
 
 ### `skill` (object, required)
 | field | type | notes |
@@ -290,7 +342,7 @@ The v0.3.1 schema gained an **additive interop revision** so receipts can be
 | `surface_overhead_note` | string | **v0.3.1, optional.** On `openai-cli`: the fixed Codex base-instruction preamble (~12–15k input tokens/call) the harness prepends and does not control. |
 | `runner_version` | string | Runner version, for reproducibility. |
 | `date_utc` | ISO 8601 UTC | When the run finished. |
-| `registry` | `"registered"` \| `"unregistered"` | **v0.3.** Whether `model_id` resolved in the model registry (`config/models.json`). |
+| `registry` | `"registered"` \| `"unregistered"` | **v0.3.** Whether `model_id` resolved in the model registry (`config/models.json`). **v0.6:** `"unregistered"` is import-only; the runner refuses an unregistered id before any call (spec 026 AC-10). |
 | `transcripts` | `"retained-local"` \| `"hashes-only"` | **v0.3.** Whether the raw generations + judge outputs were retained on disk (see § Transcripts). |
 | `judge` | object | `{ samples, temperature, sampling, surface }` — see below. |
 
@@ -358,7 +410,12 @@ Every generation and every judge sample is hashed into the receipt
 generations and judge outputs are written to `transcripts/<receipt_hash>/` (one
 JSON per case+mode, plus an `index.json`), and the receipt records
 `transcripts: "retained-local"`. Without the flag the receipt records
-`transcripts: "hashes-only"` and only the hashes are kept. The transcript
+`transcripts: "hashes-only"` and only the hashes are kept. **What is retained
+is the last measured draw of each case and mode only**: the runner keeps one
+transcript per case and mode and overwrites it on every draw, so the per-case
+top-level hashes (which are that draw's) can be re-derived from disk, and the
+draw-level hashes of earlier draws are checkable against nothing the runner
+wrote (spec 026, stated as the bound rather than moved). The transcript
 directory is **gitignored by default** — raw model text is never committed. The
 default for trigger-initiated runs is `retained-local` (disk is cheap, audits are
 not). To verify a retained receipt: re-hash each transcript file and compare