driftproof 0.11.2 → 0.12.0

package/lib/verdict.js

@@ -4,6 +4,18 @@
 const crypto = require('crypto');
 const { EFFECT_FLOOR, POWER_Z } = require('../config');
 const { bandOf } = require('./reuse');
+const { duplicateCaseRows, ambiguityLine, suiteNarrowed } = require('./receipt');
+
+// Spec 050 AC-3. A receipt with two rows for one (id, mode) has no verdict: which row
+// the byId map below kept would decide it. The error carries a code so a caller that
+// renders refusals can tell this one from a defect.
+class AmbiguousReceiptError extends Error {
+  constructor(dups) {
+    super(`ambiguous receipt: ${ambiguityLine(dups)}; a verdict over it would depend on which row was read`);
+    this.code = 'AMBIGUOUS_RECEIPT';
+    this.duplicates = dups;
+  }
+}
 
 // Single-receipt verdict + shields.io badge.
 //
@@ -156,6 +168,11 @@ function notMeasuredRoutes(level, kind, delta, incomplete) {
 // lib/decision.js states. An absent field satisfies "not TESTED" and "not model":
 // a receipt that does not say is not taken to have said the reassuring thing.
 function receiptVerdict(receipt) {
+  // Spec 050 AC-3: refused before anything is read, whatever the level. An existing
+  // receipt written before the loader refused duplicate ids reaches here unchanged,
+  // and a surface that caught nothing must not render a verdict for it.
+  const dups = duplicateCaseRows(receipt);
+  if (dups.length) throw new AmbiguousReceiptError(dups);
   const cmp = (receipt && receipt.comparison) || {};
   // Below-TESTED receipts (imported/DECLARED) and null-delta receipts are never
   // verdicted — we did not run the suite, so we do not certify the outcome.
@@ -169,7 +186,11 @@ function receiptVerdict(receipt) {
   // incomplete: a case had an arm that could not be measured) is not verdicted
   // either. spec/RECEIPT.md has said since v0.3.1 that a drift report must not
   // compute a verdict from one; the badge is the same reader with a shorter path.
-  const incomplete = !!(receipt && receipt.run && receipt.run.status === 'incomplete');
+  //
+  // Spec 062 (register row 4): a receipt that ran fewer cases than its suite holds is incomplete
+  // too, by the same route. Its suite_hash names the whole suite, so a verdict read off it would
+  // be a verdict on cases that were never run.
+  const incomplete = !!(receipt && receipt.run && receipt.run.status === 'incomplete') || suiteNarrowed(receipt);
   // Spec 036 A-036-4: WHICH REFUSAL FIRED. The guard below is one line and says only that
   // some refusal did; a surface that renders NOT_MEASURED in words owes its reader the one
   // that actually fired, and cannot get it from a boolean. `notMeasuredRoutes` names the same
@@ -184,20 +205,7 @@ function receiptVerdict(receipt) {
   // the rung edges. A guard and a route list that disagree read RED there.
   const routes = notMeasuredRoutes(level, kind, cmp.delta, incomplete);
   if (level !== 'TESTED' || kind !== 'model' || typeof cmp.delta !== 'number' || incomplete) return { verdict: 'NOT_MEASURED', cases: [], drawsNeeded: null, notMeasured: routes };
-  const byId = new Map();
-  for (const c of (receipt.results && receipt.results.cases) || []) {
-    if (c.case_status && c.case_status !== 'ok') continue;
-    const e = byId.get(c.id) || {};
-    e[c.mode] = c;
-    byId.set(c.id, e);
-  }
-  const cases = [];
-  for (const [id, e] of byId) {
-    const w = armOf(e.with_skill);
-    const b = armOf(e.baseline);
-    if (!w || !b) continue;
-    cases.push({ id, ...caseRule(b, w) });
-  }
+  const cases = readCases(receipt);
   // Spec 036 A-036-3, THE RUNG R-5 NEVER HAD. R-5 reads its ladder off the cases, and
   // every rung below REGRESSED is a statement about what the cases showed. NO_EFFECT is
   // the bottom rung and it is a MEASUREMENT: "no separation detected at this sample size".
@@ -214,6 +222,31 @@ function receiptVerdict(receipt) {
   return { verdict, cases, drawsNeeded: verdict === 'UNDERPOWERED' ? receiptDrawsNeeded(cases) : null, notMeasured: null };
 }
 
+// R-1 to R-4 over every readable case of a receipt, with none of R-5's refusals applied: a case
+// with a failed arm, or an arm with no readable band, is left out. receiptVerdict reads its ladder
+// off this list; lib/decision.js reads it BEFORE the incomplete rule (spec 062, register row 1),
+// because a case that was measured and separated down is a measured regression however many other
+// cases went unmeasured.
+function readCases(receipt) {
+  const dups = duplicateCaseRows(receipt);
+  if (dups.length) throw new AmbiguousReceiptError(dups);
+  const byId = new Map();
+  for (const c of (receipt && receipt.results && receipt.results.cases) || []) {
+    if (c.case_status && c.case_status !== 'ok') continue;
+    const e = byId.get(c.id) || {};
+    e[c.mode] = c;
+    byId.set(c.id, e);
+  }
+  const cases = [];
+  for (const [id, e] of byId) {
+    const w = armOf(e.with_skill);
+    const b = armOf(e.baseline);
+    if (!w || !b) continue;
+    cases.push({ id, ...caseRule(b, w) });
+  }
+  return cases;
+}
+
 // Derive the verdict object from a receipt. Returns
 //   { verdict, delta, floor, model, message, color, drawsNeeded }
 function verdictFromReceipt(receipt) {
@@ -283,5 +316,6 @@ function githubOutputLines(receipt) {
 
 module.exports = {
   verdictFromReceipt, badgeEndpoint, githubOutputLines, githubOutputEntry, shortModel, VERDICTS,
-  receiptVerdict, caseRule, armOf, drawsLine, receiptDrawsTaken, UNDERPOWERED_LINE, NOT_MEASURED_ROUTES,
+  receiptVerdict, readCases, caseRule, armOf, drawsLine, receiptDrawsTaken, UNDERPOWERED_LINE, NOT_MEASURED_ROUTES,
+  AmbiguousReceiptError,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "driftproof",
-  "version": "0.11.2",
+  "version": "0.12.0",
   "description": "A dated proof that this skill, this hash, this model, still helps: run a skill's eval suite with and without the skill across model versions, emit hash-verified dated receipts, and diff receipts into drift reports.",
   "license": "Apache-2.0",
   "keywords": [

package/spec/RECEIPT.md

@@ -1,5 +1,5 @@
 <!-- SPDX-License-Identifier: Apache-2.0 -->
-# Driftproof receipt, spec v0.7
+# Driftproof receipt, spec v0.9
 
 A **receipt** is a **hash-verified**, dated record of running one agent skill's eval suite
 **with** and **without** the skill on one model version, with the judge **sampled**
@@ -12,8 +12,10 @@ The machine-readable contract is [`receipt.schema.json`](./receipt.schema.json)
 (JSON Schema, draft 2020-12). This document is the human companion. Where they
 disagree, the schema wins.
 
-**Versioning.** The current schema is v0.7
+**Versioning.** The current schema is v0.9
 ([`receipt.schema.json`](./receipt.schema.json)). Prior schemas are kept frozen as
+[`receipt.v0.8.schema.json`](./receipt.v0.8.schema.json),
+[`receipt.v0.7.schema.json`](./receipt.v0.7.schema.json),
 [`receipt.v0.6.schema.json`](./receipt.v0.6.schema.json),
 [`receipt.v0.5.schema.json`](./receipt.v0.5.schema.json),
 [`receipt.v0.4.schema.json`](./receipt.v0.4.schema.json),
@@ -23,12 +25,94 @@ disagree, the schema wins.
 [`receipt.v0.1.schema.json`](./receipt.v0.1.schema.json); the validator picks the
 schema by the receipt's own `schema_version`, so every earlier receipt still loads and
 validates against its own frozen schema, including every receipt behind the published
-reports, which the gate asserts on each run. v0.7 adds no field. It adds a verdict value
-a reader derives from fields every generation-sampled receipt already carries, and a
-v0.6 receipt restamped `"0.7"` is refused by the version constant, which is why v0.6 is
-frozen rather than widened (see *What v0.7 adds*). v0.6 was not additive for the
-validator either: a v0.5 receipt restamped `"0.6"` is refused, because v0.6 requires
-the receipt to say what answered it (see *What v0.6 adds*).
+reports, which the gate asserts on each run. v0.9 is additive for a reader: it adds
+optional fields and widens three values, so a v0.8 receipt restamped `"0.9"` validates;
+v0.8 is frozen because its version constant refuses anything else (see *What v0.9 adds*).
+v0.8 was additive over v0.7 in the same way (see *What v0.8 adds*).
+
+## What v0.9 adds: an imported receipt says what produced it
+
+Spec 049, the importers for the two Anthropic formats: `claude plugin eval`'s
+`aggregate-result.json` (`--from claude-plugin-eval`) and skill-creator's `benchmark.json`,
+which skill-up also writes (`--from skill-creator`). The field mappings are in
+[`docs/interop.md`](../docs/interop.md). Every field below is optional, and none is written
+by a Driftproof run.
+
+- **`run.harness`** `{name, version}`: the agent harness that produced the answers, as the
+  source names it (`claude-code` and `claudeVersion` for a plugin eval; skill-up's engine and
+  its observed CLI version). `version` is null when the source does not record it.
+- **`run.import`** `{tool, format, format_version, source_sha256, imported_at}`: the document
+  the receipt came from, its own version field as written (null when it has none), the sha256
+  of its bytes, and when the import ran. `imported_at` is the only place the import time
+  appears; the run date comes from the source. Two optional lists: `sidecars` (other files
+  read beside the document, by name and sha256, such as skill-up's `result.json`) and
+  `notices` (one sentence per thing the source did not establish, such as an unknown model or a
+  date read from a directory name, and per choice a reader must know, such as errored runs
+  counted).
+- **`skill.unit`**: `"skill"`, or `"plugin"` when a plugin was measured whole, every skill it
+  carries loaded at once (a plugin eval). Absent means a skill.
+- **`results.cases[].activation`**: a list of `{indicator, fired, runs}`, one per unscored
+  skill-invocation indicator the source ran (a `tool_used: Skill` grader with `scored: false`):
+  in how many of the case's measured runs the skill fired. It is never part of a score, band
+  or comparison. On 15 Sep 2026 a native eval scored a skill +0.67 while this indicator read 0
+  of 3: the skill never fired.
+- **`results.cases[].excluded_draws`**: runs the source recorded for the case and arm that are
+  kept out of its samples, each `{draw_index, reason}` (a run with an error, a run whose paid
+  graders were skipped).
+- **`results.cases[].label`**: the source's display name for a case (`eval_name`). Cases are
+  identified by `id`, never by label.
+- **`economics.<arm>.mean_total_tokens`**, and two `economics.basis` values:
+  `source-list-price-estimate` (the source tool's own cost estimate at list price, not a
+  Driftproof figure) and `source-reported` (tokens and time as the source reports them, no
+  cost).
+- **Three widenings**, each for an imported receipt: `run.provider` may be `unknown` (the
+  source records no model); `run.date_utc` may be null, only on an `external` receipt whose
+  source records no date; and an absent model is `run.model_id` `"unknown"`, never a harness's
+  default.
+
+Imported receipts stay DECLARED with surface `external`, every hash null, and `diff` keeps
+refusing them.
+
+## What v0.8 adds: counts that say what they count, and a generation clock
+
+Spec 043, from agentskills discussion #544 (comments 18348575, 18409751, 18409986,
+18530980 and 18531185). Until v0.8 `run.judge.samples` was required, so a receipt had no
+way to say a count was unknown, and the two importers wrote one anyway: agent-skills-eval
+wrote `1` whatever the source said, and skillgrade wrote its largest trial count, which is
+a count of generations, into the judge-sample field.
+
+- **`run.judge.samples` is optional** (#544 comments 18409751 rule 1, 18530980). Absent means unknown, never 1. Every reader in the
+  tree that shows it (the differ, `export --format summary-json`, the receipt summary)
+  says *unknown* or writes `null`.
+- **Two counts, kept apart** (18348488, 18348575, 18530980). `generations_per_arm` (independent candidate generations;
+  0 only where a source establishes zero) and `judge_samples_per_generation` (at least 1),
+  each optional, in a `counts` object. When both `run.judge.samples` and
+  `run.counts.judge_samples_per_generation` are present the loader refuses a receipt
+  where they differ.
+- **The narrowest honest scope** (18530980). `run.counts` when a count is the same for every case of
+  every arm generated in this run; `run.arms.<mode>.counts` when it holds for one arm;
+  `results.cases[].counts` otherwise, and only for the cases where it is known. A reader
+  takes the narrowest scope present (`lib/counts.js` `resolveCount`). Nothing writes a
+  maximum, minimum or mean of differing counts as a count.
+- **Archived arms** (18348488, 18348575, 18409751 rule 2). `run.arms.<mode>` may carry an arm's own `model_id`, `generated_at`
+  and `counts`. An arm with its own `generated_at` was generated in an earlier run: it
+  never inherits `run.counts`, and with no `counts` of its own its counts are unknown.
+- **Clocks** (18409751 rule 2, 18409986, 18530980). `run.generated_at` is stamped by the runner before its first generation
+  call. `run.judged_at` and `run.grader_revision` (the judge template hash and the rubric
+  hashes) appear together, and only on a re-judge. `date_utc` keeps its value; its
+  description now says what the code writes (the row below).
+- **`no_observations`** (18530980, and the operator's answer A-3 in spec 043). A case the source supplied no observation for is listed with
+  `case_status: "no_observations"`, contributes no figure, is excluded from every
+  aggregate and is named in `excluded_cases`. An imported task whose trials list is empty
+  carries `samples: []` and `generations_per_arm: 0`; a task with no trials field carries
+  neither, because its count is unknown.
+- **The importers** (18530857, 18530980; the positive control of 18531185 is read on a native receipt and on uniform skillgrade trials). agent-skills-eval writes no count of either kind: its benchmark
+  format defines neither. skillgrade writes no judge-sample count (its format defines
+  none) and writes its trial counts as generation counts at the narrowest honest scope.
+  Both stay `DECLARED` and `external`, and skillgrade's rewards are kept exactly as
+  supplied.
+- **The runner.** A native receipt carries the judge-sample count it was asked for and
+  each case's generation count as the draws it recorded, placed by the same rule.
 
 ## What v0.7 adds: a receipt that could not resolve the effect floor says so
 
@@ -183,7 +267,7 @@ carry a numeric `mean`.
 ## Which schema validates which receipt (the coexistence rule)
 
 - **A producer emits the current version.** `config.js` `RECEIPT_SCHEMA_VERSION`
-  decides it, and at this revision that is **v0.7**.
+  decides it, and at this revision that is **v0.9**.
 - **A reader validates against the receipt's own `schema_version`**, never
   against the newest schema it happens to have. `validateReceipt()` selects the
   schema by that field, which is why a v0.1 receipt from the first report still
@@ -355,7 +439,7 @@ The v0.3.1 schema gained an **additive interop revision** so receipts can be
 
 ## Fields
 
-### `schema_version` (string, required): `"0.7"`.
+### `schema_version` (string, required): `"0.9"`.
 
 ### `skill` (object, required)
 | field | type | notes |
@@ -363,6 +447,7 @@ The v0.3.1 schema gained an **additive interop revision** so receipts can be
 | `name` | string | From SKILL.md front-matter or its H1. |
 | `version` | string | Skill's declared version. |
 | `content_hash` | sha256 hex | Over `SKILL.md` + every bundled file (the `evals/` dir is **excluded** — hashed separately as `suite_hash`), path + content, path-sorted. |
+| `unit` | `"skill"` \| `"plugin"` | **v0.9**, optional. `"plugin"` when a plugin was measured whole (a `claude plugin eval` import). |
 | `tokens` | integer | **v0.3.1, optional.** Estimated SKILL.md token size (coarse `chars/4` proxy) for the value-per-token axis. |
 
 ### `suite` (object, required)
@@ -377,14 +462,18 @@ The v0.3.1 schema gained an **additive interop revision** so receipts can be
 |---|---|---|
 | `model_id` | string | Canonical model id run on. |
 | `model_release_date` | ISO date or `null` | Release date **if known**, else `null`. |
-| `provider` | `"anthropic"` \| `"openai"` | **v0.3.1.** The two-axis provider (registry `provider`, else inferred from the id). |
+| `provider` | `"anthropic"` \| `"openai"` \| `"unknown"` | **v0.3.1.** The two-axis provider (registry `provider`, else inferred from the id). **v0.9:** `"unknown"` on an imported receipt whose source records no model. |
 | `surface` | `"api"` \| `"claude-cli"` \| `"openai-api"` \| `"openai-cli"` | `api` = Anthropic Messages API; `claude-cli` = spawned `claude -p` (subscription); `openai-api` = Chat-Completions-compatible API (base_url configurable); `openai-cli` = Codex subscription (`codex exec`). |
 | `surface_overhead_note` | string | **v0.3.1, optional.** On `openai-cli`: the fixed Codex base-instruction preamble (~12–15k input tokens/call) the harness prepends and does not control. |
 | `runner_version` | string | Runner version, for reproducibility. |
-| `date_utc` | ISO 8601 UTC | When the run finished. |
+| `date_utc` | ISO 8601 UTC | As `lib/run.js` writes it: when the run's calls finished if the CLI stamps it, or the stamp a caller passes, which for a batch is one stamp shared by every receipt in the batch. For when generation began, read `generated_at` (v0.8). Before v0.8 this row said *when the run finished*, which was true of the CLI and not of a batch. **v0.9:** null only on an `external` receipt whose source records no date. |
 | `registry` | `"registered"` \| `"unregistered"` | **v0.3.** Whether `model_id` resolved in the model registry (`config/models.json`). **v0.6:** `"unregistered"` is import-only; the runner refuses an unregistered id before any call (spec 026 AC-10). |
 | `transcripts` | `"retained-local"` \| `"hashes-only"` | **v0.3.** Whether the raw generations + judge outputs were retained on disk (see § Transcripts). |
-| `judge` | object | `{ samples, temperature, sampling, surface }` — see below. |
+| `judge` | object | `{ samples, temperature, sampling, surface }`, see below. `samples` is optional from v0.8: absent means unknown. |
+| `generated_at` | ISO 8601 UTC | **v0.8**, optional. When generation began, stamped before the first generation call. |
+| `judged_at`, `grader_revision` | ISO 8601 UTC, object | **v0.8**, optional, together or not at all: a re-judge of frozen outputs, and the grading definition it applied. |
+| `counts`, `arms` | object | **v0.8**, optional. See *What v0.8 adds*. |
+| `harness`, `import` | object | **v0.9**, optional, imported receipts. See *What v0.9 adds*. |
 
 **`run.judge`** records how grading was done: `samples` (judge calls per case),
 `temperature` (a number when the surface lets us set it — the `api` surface pins
@@ -411,6 +500,7 @@ receipt says so.
   | `reason` | string | One-line judge rationale (optional). |
   | `checks` | array | **v0.3.1, optional.** Deterministic post-check results, each `{ name, kind, pass }` (`kind ∈ regex/contains/not_contains/min_length`). Run alongside the judge; a **separate column**, **not** folded into `outcome`/the band verdict. |
   | `judge` | object | `{ model_id, rubric_hash }` — who graded and a hash binding the grade to the exact rubric + judge system prompt. |
+  | `activation`, `excluded_draws`, `label` | array, array, string | **v0.9**, optional, imported receipts. Whether the skill fired (never a score), the runs kept out of `samples` with their reasons, and the source's display name. See *What v0.9 adds*. |
 - **`aggregates`** — `{ with_skill, baseline }`, each
   `{ case_count, pass_count, borderline_count?, mean_score, stddev }`. Here
   `stddev` is the **suite dispersion**: the stddev of the per-case means across