driftproof 0.5.0 → 0.7.1

package/lib/sampling.js

@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+// Generation sampling policy — receipt spec v0.5.
+//
+// WHY THIS EXISTS. Until v0.5 a verdict rested on ONE generation draw per arm,
+// and the band a receipt carried was the judge re-scoring that single response.
+// Report #006 measured the other axis directly and found it larger: across-draw
+// sd 0.186 and 0.183, against judge-level noise several times smaller. Every
+// figure this instrument published was error-barred on the smaller of the two
+// noise sources. This module samples the larger one.
+//
+// PURE. No I/O, no provider, no clock. Two functions over a draw list, so the
+// policy is testable from fixtures with no model call and the runner cannot
+// smuggle a different rule past the gate.
+
+const {
+  GENERATION_SAMPLES_MIN, GENERATION_SAMPLES_MAX,
+  GENERATION_SD_THRESHOLD, GENERATION_STABILITY_EPS,
+} = require('../config');
+const { mean, stddev, round, varianceRatio } = require('./stats');
+
+const SAMPLING = {
+  min: GENERATION_SAMPLES_MIN,
+  max: GENERATION_SAMPLES_MAX,
+  sdThreshold: GENERATION_SD_THRESHOLD,
+  stabilityEps: GENERATION_STABILITY_EPS,
+};
+
+// A draw is `measured` or `unmeasured`. An UNMEASURED draw carries no score and
+// is excluded from every statistic — never scored zero (F-009-L). A zero is a
+// measurement claim; a timeout is the absence of one, and filling it with zero
+// invents a regression nobody observed.
+const isMeasured = (d) => d && d.status === 'measured' && typeof d.mean === 'number';
+
+// WHICH null the ratio is (F-014-C). The ratio is generation-sd over mean
+// judge-sd, and at k=1 the per-draw judge spread is 0 by construction — so the
+// number v0.5 exists to produce could not exist at the sampling every pre-v0.5
+// example used, and the receipt recorded a null that read identically to "the
+// judge agreed with itself perfectly". Those are different measurements.
+//
+// EVERY REASON NAMES WHAT WAS OBSERVED AND NONE NAMES A CAUSE (spec 014 AC-7).
+// `judge_samples_unknown` exists for exactly that reason: with no sample list on
+// the draws, the count cannot be established, and picking between the other two
+// would be asserting something this function cannot see.
+function unavailableReason(measured, ratio) {
+  if (ratio !== null) return null;
+  if (!measured.length) return 'no_measured_draws';
+  const counts = measured.map((d) => (Array.isArray(d.samples) ? d.samples.length : null));
+  if (counts.some((c) => c === null)) return 'judge_samples_unknown';
+  if (counts.every((c) => c <= 1)) return 'single_judge_sample';
+  return 'judge_sd_zero';
+}
+
+function acrossDraws(draws) {
+  const all = Array.isArray(draws) ? draws : [];
+  const measured = all.filter(isMeasured);
+  const means = measured.map((d) => d.mean);
+  const judgeSds = measured.map((d) => (typeof d.stddev === 'number' ? d.stddev : 0));
+  const sd = means.length ? stddev(means) : 0;
+  const judgeSdMean = judgeSds.length ? mean(judgeSds) : 0;
+  const ratio = varianceRatio(sd, judgeSdMean);
+  return {
+    n_drawn: all.length,
+    n_measured: measured.length,
+    n_unmeasured: all.length - measured.length,
+    mean: means.length ? round(mean(means)) : null,
+    sd: round(sd),
+    judge_sd_mean: round(judgeSdMean),
+    variance_ratio: ratio,
+    variance_ratio_unavailable: unavailableReason(measured, ratio),
+  };
+}
+
+// Should the runner draw again? Returns the STOPPING REASON as well as the
+// decision: an adaptive rule that does not record why it stopped is unauditable,
+// and #006's own probe stopped at 20 draws because a person decided to.
+function nextAction(draws) {
+  const a = acrossDraws(draws);
+  const drawn = a.n_drawn;
+
+  // Every draw failed. More draws are not obviously wrong, but continuing to
+  // burn calls on a surface that is not answering is, and the receipt should say
+  // that is what happened rather than report an empty mean.
+  if (drawn >= SAMPLING.min && a.n_measured === 0) return { stop: true, reason: 'unmeasured_exhausted' };
+
+  // THE CAP IS CHECKED BEFORE THE MINIMUM. Reversed, a run whose draws mostly
+  // failed reported `below_min` while what actually stopped it was the ceiling —
+  // the receipt named a reason that was not the reason. Approval finding, AC-4.
+  if (drawn >= SAMPLING.max) return { stop: true, reason: 'max_reached' };
+
+  if (a.n_measured < SAMPLING.min) return { stop: false, reason: 'below_min' };
+
+  // Tight enough at the minimum: the extra draws would buy nothing.
+  if (a.sd <= SAMPLING.sdThreshold) return { stop: true, reason: 'min_reached' };
+
+  // (the bounded-escalation ceiling is enforced above, before the minimum)
+
+  // …unless the estimate has stopped moving: two successive prefixes agreeing to
+  // within eps means further draws are not changing what we would report.
+  if (a.n_measured > SAMPLING.min) {
+    const prev = acrossDraws(draws.slice(0, -1));
+    if (prev.n_measured >= SAMPLING.min && Math.abs(a.sd - prev.sd) < SAMPLING.stabilityEps) {
+      return { stop: true, reason: 'stabilised' };
+    }
+  }
+  return { stop: false, reason: 'escalating' };
+}
+
+module.exports = { SAMPLING, acrossDraws, nextAction, isMeasured, unavailableReason };

package/lib/stats.js

@@ -63,4 +63,19 @@ function bandVerdict(meanA, hwA, meanB, hwB) {
   return 'within noise';
 }
 
-module.exports = { mean, stddev, stderr, combineUncertainty, aggregateBands, bandVerdict, round };
+// The variance ratio: how much larger the GENERATION-level spread is than the
+// JUDGE-level spread this instrument has always sampled. The number Report #006
+// wanted and could not produce.
+//
+// A ZERO DENOMINATOR YIELDS null, NEVER A DIVISION RESULT. Infinity (or NaN) is
+// a fabricated finding: it reads as "infinitely noisier" when what actually
+// happened is that the judge agreed with itself perfectly and the ratio is
+// undefined. A null says the ratio could not be formed, which is true.
+function varianceRatio(generationSd, judgeSdMean) {
+  const g = Number(generationSd), j = Number(judgeSdMean);
+  if (!Number.isFinite(g) || !Number.isFinite(j)) return null;
+  if (j === 0) return null;
+  return round(g / j);
+}
+
+module.exports = { mean, stddev, stderr, combineUncertainty, aggregateBands, bandVerdict, round, varianceRatio };

package/lib/value.js

@@ -151,8 +151,33 @@ function meanOf(nums) {
 // One arm = all completed case rows of one mode (with_skill | baseline). Only
 // the GENERATION usage of each row feeds this; judge usage is measurement
 // overhead and is never passed in (see computeEconomics).
+// THE GENERATION USAGE OF A CASE ROW, wherever that version put it (spec 017
+// AC-3, AC-4).
+//
+// v0.4 and earlier record one generation per case and put its usage at case
+// level. v0.5 draws the generation n times and records each draw's usage under
+// `generation.draws[].usage`; the case-level field is then absent. This function
+// read only the case-level one, so `call_count` was 0 and every arm figure null
+// on all three #007 receipts — while the draws carried complete usage. Judge
+// economics never broke, because `judge_usage` stayed at case level, and that
+// asymmetry is what located the defect.
+//
+// A DRAW IS A CALL. An arm that drew three generations made three generation
+// calls, so each measured draw contributes one usage row; the v0.4 path yields
+// exactly one row per case, which is what it always yielded. Unmeasured draws
+// carry no usage and are excluded — a timed-out call has no tokens to count, and
+// counting it as zero would deflate the mean the way scoring it zero would
+// deflate the band (F-009-L, applied to cost).
+function generationUsages(row) {
+  const g = row && row.generation;
+  if (g && Array.isArray(g.draws)) {
+    return g.draws.filter((d) => d && d.status === 'measured' && d.usage).map((d) => d.usage);
+  }
+  return row && row.usage ? [row.usage] : [];
+}
+
 function armEconomics(rows, price) {
-  const usages = rows.map((r) => r.usage).filter(Boolean);
+  const usages = rows.flatMap(generationUsages).filter(Boolean);
   const costs = usages.map((u) => costForUsage(u, price)).filter((c) => c != null);
   const wall = usages.map((u) => (u.wall_ms == null ? null : u.wall_ms)).filter((x) => x != null);
   const q = quartiles(wall);
@@ -491,7 +516,7 @@ function round4(n) { return Math.round(Number(n) * 1e4) / 1e4; }
 function round6(n) { return Math.round(Number(n) * 1e6) / 1e6; }
 
 module.exports = {
-  buildPricingSnapshot, costForUsage, computeEconomics, armEconomics,
+  buildPricingSnapshot, costForUsage, computeEconomics, armEconomics, generationUsages,
   liftIsReportable, costPerLiftPoint, isZeroLooking, dollarsTraceable, runTotalFromReceipts,
   runWallClockFromReceipts,
   receiptCostBreakdown,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "driftproof",
-  "version": "0.5.0",
-  "description": "Continuous verification of agent skills: run a skill's eval suite with and without the skill across model versions, emit signed dated receipts, and diff receipts into drift reports.",
+  "version": "0.7.1",
+  "description": "A dated proof that this skill, this hash, this model, still helps: run a skill's eval suite with and without the skill across model versions, emit hash-verified dated receipts, and diff receipts into drift reports.",
   "license": "Apache-2.0",
   "keywords": [
     "agent-skills",

package/spec/RECEIPT.md

@@ -1,7 +1,7 @@
 <!-- SPDX-License-Identifier: Apache-2.0 -->
-# Driftproof receipt — spec v0.4
+# Driftproof receipt — spec v0.5
 
-A **receipt** is a signed, dated record of running one agent skill's eval suite
+A **receipt** is a **hash-verified**, dated record of running one agent skill's eval suite
 **with** and **without** the skill on one model version, with the judge **sampled**
 so every score carries a confidence band. Receipts are the unit of evidence
 Driftproof produces; diffing two receipts across model releases yields a **drift
@@ -22,6 +22,105 @@ receipts still load and validate — including every receipt behind the four
 published reports, which the gate asserts on each run. v0.4 is an **additive**
 bump: everything it adds is optional, and no earlier receipt is invalidated.
 
+## What v0.5 adds: the generation is sampled too
+
+Until v0.5 a verdict rested on **one generation draw per arm**. The band a
+receipt carried was the *judge* re-scoring that single response — not the spread
+of the model writing a different one. Report #006 measured the second directly
+and found it larger: across-draw **sd 0.186** and **sd 0.183**, against
+judge-level noise several times smaller. Every figure this instrument published
+was error-barred on the smaller of the two noise sources.
+
+v0.5 samples the other axis. Each case and arm carries a `generation` block:
+
+- `draws[]` — **every draw, listed**, each with its own `generation_hash`, its
+  own nested judge `samples`, and its own `mean` and `stddev`. An aggregate a
+  reader cannot re-derive is not evidence; the list is what makes it evidence.
+- `mean`, `sd` — across draws.
+- `judge_sd_mean` — the mean of the per-draw judge stddevs.
+- `variance_ratio` — `sd / judge_sd_mean`: how much larger the generation-level
+  spread is than the judge-level spread. **`null` when the judge sd is zero**,
+  never a division result: an infinity reads as "infinitely noisier" when what
+  happened is that the ratio is undefined.
+- `variance_ratio_unavailable` — **which null it is**, and it is present whenever
+  the ratio is null. One of `single_judge_sample`, `judge_sd_zero`,
+  `no_measured_draws`, `judge_samples_unknown`. Each names what was *observed*
+  and none names a cause.
+- `n_planned`, `n_drawn`, `n_measured`, `n_unmeasured`, `stopping_reason` — the
+  sampling policy **as applied**, so an adaptive run says why it stopped.
+
+**What `--samples 1` costs.** The ratio is generation-sd over mean judge-sd, and
+with **one judge sample per draw the per-draw judge spread is zero by
+construction** — so the number v0.5 exists to produce cannot be formed at `k=1`.
+`k=1` remains legal and the run still emits a receipt; the ratio is `null` and
+`variance_ratio_unavailable` reads **`single_judge_sample`**, which is a
+different statement from `judge_sd_zero` ("the judge agreed with itself perfectly
+across two or more samples"). Before this field the two nulls were the same null.
+Documented sampling is `--samples 5`, the default, and a run whose ratio you
+intend to publish must use at least 2.
+
+## The `generation_sampled` capability flag
+
+A receipt that carries across-draw statistics **must declare
+`generation_sampled: true` at the top level**, and a receipt that declares it
+**must carry at least one case with a `generation` block AND must carry
+`suite.canary`**. All three directions are enforced by the schema, so neither a
+producer nor a third-party emitter can claim the capability without the evidence,
+or carry the evidence without saying so.
+
+**Both blocks, not one.** The exposure this closes named two things v0.5 adds —
+the draw set and the per-suite canary — and binding only the first left a receipt
+able to claim conformance while carrying half of it. A receipt that declares the
+capability has, by construction, run a suite of ours, so it has a canary to
+record.
+
+**Absent means legacy.** A v0.5 receipt that ran no generation sampling — an
+imported `DECLARED` receipt, for instance — omits the field and stays valid, and
+every v0.4-and-earlier receipt is unaffected and unmodified. That is the whole
+reason the requirement is bound to the DECLARATION rather than to
+`verification_level`: binding it to the level would have invalidated hand-built
+fixtures across the archive that never ran a generation, which is a different
+change from the one this flag makes.
+
+**The judge nests inside the draw.** Pooling k×n scores into one list per case is
+exactly what makes generation noise read as judge noise, and it is the defect
+Report #006 exists to name.
+
+**A timed-out draw is `unmeasured`, never zero.** It carries no score and is
+excluded from every mean, sd and ratio. A zero asserts a measurement; a timeout
+is the absence of one, and filling it with zero invents a regression nobody
+observed. The schema enforces this: a draw with `status: "unmeasured"` may not
+carry a numeric `mean`.
+
+## Which schema validates which receipt (the coexistence rule)
+
+- **A producer emits the current version.** `config.js` `RECEIPT_SCHEMA_VERSION`
+  decides it, and at this revision that is **v0.5**.
+- **A reader validates against the receipt's own `schema_version`**, never
+  against the newest schema it happens to have. `validateReceipt()` selects the
+  schema by that field, which is why a v0.1 receipt from the first report still
+  validates today.
+- **v0.4 receipts remain producible and readable.** v0.4 moved from the
+  unversioned filename to `spec/receipt.v0.4.schema.json` when v0.5 took the
+  current pointer; the number still resolves to the schema it meant. Nothing in
+  the archive is retroactively invalidated, and the frozen v0.1, v0.2, v0.3 and
+  v0.3.1 schemas are byte-for-byte unchanged.
+- **A v0.4 reader meeting a v0.5 receipt** finds every field it knows, and the
+  ones it does not know are additive. `mean`, `score` and `stddev` on a case now
+  describe the **draw set** rather than one arbitrary draw, so an older reader
+  reads the aggregate rather than whichever draw happened to be last;
+  `generation_hash`, `samples` and `judge_sample_hashes` continue to describe the
+  last measured draw, which is what they have always described. Older readers
+  that ignore unknown fields therefore stay correct rather than silently wrong.
+
+## Suite canary
+
+`suite.canary` is a GUID derived from the suite's identity and its case ids —
+stable for a suite, distinct across suites, and requiring no registry. It exists
+so a suite that has leaked into training data is **detectable** in a corpus. It
+is a detection aid and not a control: it cannot prevent a leak, and its absence
+proves nothing.
+
 ## What changed in v0.4 (economics — what a skill costs to run)
 
 Reports #001–#004 could say whether a skill still helps. They could not say what
@@ -164,7 +263,7 @@ The v0.3.1 schema gained an **additive interop revision** so receipts can be
 
 ## Fields
 
-### `schema_version` (string, required) — `"0.3.1"`.
+### `schema_version` (string, required) — `"0.4"`.
 
 ### `skill` (object, required)
 | field | type | notes |