driftproof 0.3.0

package/lib/skill.js

@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { sha256Files, sha256Canonical } = require('./canonical');
+const { SUITE_FORMAT } = require('../config');
+
+// Load a skill directory and its eval suite.
+//
+// Expected layout (agentskills.io style):
+//   <skill-dir>/SKILL.md              — the skill instructions (required)
+//   <skill-dir>/evals/evals.json      — the eval suite (required)
+//   <skill-dir>/**                    — any other bundled files contribute to
+//                                       the content hash (references, scripts…)
+//
+// content_hash = sha256 over { SKILL.md, ...bundled files } in canonical (path-
+// sorted) order, so the same skill on any machine yields the same hash and any
+// edit to any bundled file changes it.
+
+const IGNORE_DIRS = new Set(['.git', 'node_modules', 'evals']);
+const IGNORE_FILES = new Set(['.DS_Store']);
+
+function walkFiles(dir, base = dir, acc = []) {
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    if (entry.isDirectory()) {
+      if (IGNORE_DIRS.has(entry.name)) continue;
+      walkFiles(path.join(dir, entry.name), base, acc);
+    } else if (entry.isFile() && !IGNORE_FILES.has(entry.name)) {
+      const abs = path.join(dir, entry.name);
+      acc.push({ path: path.relative(base, abs), bytes: fs.readFileSync(abs) });
+    }
+  }
+  return acc;
+}
+
+function loadSkill(skillDir) {
+  const dir = path.resolve(skillDir);
+  const skillMdPath = path.join(dir, 'SKILL.md');
+  if (!fs.existsSync(skillMdPath)) {
+    throw new Error(`no SKILL.md found in ${dir}`);
+  }
+  const skillMd = fs.readFileSync(skillMdPath, 'utf8');
+
+  // Content hash over SKILL.md + every bundled file (evals/ excluded — the suite
+  // is hashed separately so a suite edit doesn't masquerade as a skill change).
+  const files = walkFiles(dir);
+  const contentHash = sha256Files(files);
+
+  // Parse skill name/version from front-matter or the first H1; fall back to dir.
+  const meta = parseSkillMeta(skillMd);
+  const name = meta.name || path.basename(dir);
+  const version = meta.version || '0.0.0';
+
+  // Load the eval suite.
+  const suitePath = path.join(dir, 'evals', 'evals.json');
+  if (!fs.existsSync(suitePath)) {
+    throw new Error(`no evals/evals.json found in ${dir}`);
+  }
+  const suiteRaw = JSON.parse(fs.readFileSync(suitePath, 'utf8'));
+  const cases = normalizeCases(suiteRaw);
+  const suiteHash = sha256Canonical(cases);
+
+  return {
+    dir,
+    name,
+    version,
+    skillMd,
+    contentHash,
+    suite: {
+      format: SUITE_FORMAT,
+      suiteHash,
+      caseCount: cases.length,
+      cases,
+    },
+  };
+}
+
+// Pull name/version from a YAML-ish front-matter block, else from the H1 line.
+function parseSkillMeta(md) {
+  const meta = {};
+  const fm = md.match(/^---\s*\n([\s\S]*?)\n---/);
+  if (fm) {
+    for (const line of fm[1].split('\n')) {
+      const m = line.match(/^\s*([A-Za-z_]+)\s*:\s*(.+?)\s*$/);
+      if (m) meta[m[1].toLowerCase()] = m[2].replace(/^["']|["']$/g, '');
+    }
+  }
+  if (!meta.name) {
+    const h1 = md.match(/^#\s+(.+)$/m);
+    if (h1) meta.name = h1[1].trim();
+  }
+  return meta;
+}
+
+// Accept a few shapes of the agentskills.io eval suite and normalize each case
+// to { id, prompt, rubric, pass_threshold }. `cases` may live at the top level
+// or under a `cases`/`evals` key.
+function normalizeCases(raw) {
+  const list = Array.isArray(raw) ? raw
+    : Array.isArray(raw.cases) ? raw.cases
+      : Array.isArray(raw.evals) ? raw.evals
+        : null;
+  if (!list) throw new Error('evals.json must be an array or have a `cases`/`evals` array');
+  return list.map((c, i) => {
+    const id = String(c.id || c.name || `case-${i + 1}`);
+    const prompt = c.prompt || c.input || c.task;
+    const rubric = c.rubric || c.criteria || c.expected;
+    if (!prompt) throw new Error(`case "${id}" is missing a prompt/input/task`);
+    if (!rubric) throw new Error(`case "${id}" is missing a rubric/criteria/expected`);
+    const threshold = typeof c.pass_threshold === 'number' ? c.pass_threshold
+      : typeof c.threshold === 'number' ? c.threshold : 0.7;
+    return { id, prompt: String(prompt), rubric: String(rubric), pass_threshold: threshold };
+  });
+}
+
+module.exports = { loadSkill, normalizeCases, parseSkillMeta };

package/lib/stats.js

@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+// Small statistics helpers for sampled judging and confidence bands. Kept
+// dependency-free and deterministic.
+
+function round(n, dp = 6) { const f = Math.pow(10, dp); return Math.round(n * f) / f; }
+
+function mean(xs) {
+  if (!xs.length) return 0;
+  return round(xs.reduce((a, b) => a + b, 0) / xs.length);
+}
+
+// Sample standard deviation (Bessel's n-1). Returns 0 for n < 2 (a single
+// sample has no measurable spread). This is the RAW spread of a set of judge
+// scores — used for the per-case ± band (borderline rule + per-case drift).
+function stddev(xs) {
+  const n = xs.length;
+  if (n < 2) return 0;
+  const m = xs.reduce((a, b) => a + b, 0) / n;
+  const varr = xs.reduce((a, b) => a + (b - m) * (b - m), 0) / (n - 1);
+  return round(Math.sqrt(varr));
+}
+
+// Standard error of a mean: stddev / sqrt(n). This is the uncertainty OF THE
+// MEAN (shrinks as you take more samples) and is the right band for aggregates,
+// which is what run-to-run reproducibility of a headline number actually is.
+function stderr(xs) {
+  const n = xs.length;
+  if (n < 2) return 0;
+  return round(stddev(xs) / Math.sqrt(n));
+}
+
+// Combine independent uncertainties in quadrature: sqrt(a^2 + b^2).
+function combineUncertainty(a, b) {
+  return round(Math.sqrt(a * a + b * b));
+}
+
+// Aggregate a set of per-case measurements, each {mean, stddev, n}, into an
+// overall mean and a band that is the DISPERSION of the per-case means across
+// the suite (sample stddev of the case means) — NOT the standard error of the
+// mean. Rationale: the standard error shrinks with sampling/case-count and makes
+// the headline hypersensitive (a trivial move reads as a real change), which is
+// exactly the cry-wolf failure this project exists to prevent. Suite dispersion
+// is a conventional, honest "mean ± stddev across the suite". The drift HEADLINE
+// verdict is driven by the per-case band-overlap verdicts (see lib/diff.js), not
+// by this aggregate band; this value is a reported summary statistic.
+function aggregateBands(cases) {
+  if (!cases.length) return { mean: 0, stddev: 0 };
+  const means = cases.map((c) => c.mean);
+  return { mean: mean(means), stddev: stddev(means) };
+}
+
+// Do two confidence bands (mean ± half-width) fail to overlap, and in which
+// direction? Returns 'regression' (b below a), 'improvement' (b above a), or
+// 'within noise' (bands touch/overlap). This is the anti-false-positive rule:
+// a change is only claimed when the bands are fully separated.
+//   regression   : meanB + hwB < meanA - hwA
+//   improvement  : meanB - hwB > meanA + hwA
+function bandVerdict(meanA, hwA, meanB, hwB) {
+  if (meanB + hwB < meanA - hwA) return 'regression';
+  if (meanB - hwB > meanA + hwA) return 'improvement';
+  return 'within noise';
+}
+
+module.exports = { mean, stddev, stderr, combineUncertainty, aggregateBands, bandVerdict, round };

package/lib/stub.js

@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+// Offline stub surface (DRIFTPROOF_STUB=1).
+//
+// When set, provider.complete() delegates here and NEVER contacts a model — no
+// API call, no `claude` spawn, zero spend. It exists so the GitHub Action (and
+// CI generally) can be exercised end-to-end for free: the runner still builds a
+// fully-formed, schema-valid, self-hashed receipt with a real with-skill-vs-
+// baseline lift; only the underlying text is canned.
+//
+// Determinism is the whole point, so scores carry no variance (stddev 0). The
+// stub is model- and skill-agnostic: it distinguishes the two things the runner
+// actually varies — whether the SKILL.md was supplied (with_skill vs baseline) —
+// by marking each canned generation, and the canned judge grades on that marker.
+
+// Markers embedded in canned generations. The judge stub reads them back out of
+// the response it is handed, which is how a zero-model-call run still produces a
+// non-trivial (positive) skill lift.
+const MARK_SKILL = 'DRIFTPROOF_STUB_S';   // generation produced WITH the skill
+const MARK_BASE = 'DRIFTPROOF_STUB_B';    // baseline generation (no skill)
+
+// A judge call is recognised structurally (no import of judge.js — that would be
+// circular, since judge.js requires provider.js): the judge prompt always ends by
+// asking for a strict JSON object.
+function isJudgePrompt(prompt) {
+  return /Return ONLY this JSON object/.test(String(prompt || ''));
+}
+
+// Deterministic canned completion. `system` truthy ⇒ a with-skill generation
+// (the runner passes SKILL.md as the system prompt only in with_skill mode).
+function stubComplete({ system, prompt }) {
+  if (isJudgePrompt(prompt)) {
+    // Grade the response we were handed: reward the with-skill marker.
+    const helped = new RegExp(MARK_SKILL).test(String(prompt || ''));
+    const score = helped ? 0.85 : 0.42;
+    const body = {
+      score,
+      pass: score >= 0.7,
+      reason: `stub judge (DRIFTPROOF_STUB): ${helped ? 'with-skill marker present' : 'baseline'}`,
+    };
+    return { text: JSON.stringify(body), usage: null };
+  }
+  // Generation call: canned, marked by mode.
+  const mark = system ? MARK_SKILL : MARK_BASE;
+  const text = `${mark}\nfeat(stub): canned offline generation for CI (no model was called)`;
+  return { text, usage: null };
+}
+
+function stubEnabled() {
+  return process.env.DRIFTPROOF_STUB === '1';
+}
+
+module.exports = { stubComplete, stubEnabled, MARK_SKILL, MARK_BASE };

package/lib/verdict.js

@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+const { EFFECT_FLOOR } = require('../config');
+
+// Single-receipt verdict + shields.io badge.
+//
+// `diff` compares TWO receipts across model releases; a single receipt already
+// carries its own honest verdict: does the skill still help on THIS model? That
+// is exactly `comparison.delta` (with-skill mean minus baseline mean), read
+// through the SAME practical-significance floor the drift rule uses (config.js
+// EFFECT_FLOOR). Band separation is not applied here — a single run has no second
+// band to separate from — but the effect floor is, so a trivial lift below the
+// judge's quantization grid is reported as "no effect", never as "passing".
+//
+//   delta >=  EFFECT_FLOOR → PASSED    (the skill measurably helps)
+//   |delta| <  EFFECT_FLOOR → NO_EFFECT (within the judge's noise floor)
+//   delta <= -EFFECT_FLOOR → REGRESSED (the skill measurably hurts on this model)
+
+const VERDICTS = {
+  PASSED: { color: 'brightgreen', word: 'passing' },
+  NO_EFFECT: { color: 'lightgrey', word: 'no effect' },
+  REGRESSED: { color: 'red', word: 'regressed' },
+};
+
+// Strip a trailing -YYYYMMDD date stamp so the badge reads cleanly
+// (claude-haiku-4-5-20251001 → claude-haiku-4-5).
+function shortModel(modelId) {
+  return String(modelId || 'unknown').replace(/-\d{8}$/, '');
+}
+
+// Derive the verdict object from a receipt. Returns
+//   { verdict, delta, floor, model, message, color }
+function verdictFromReceipt(receipt) {
+  const cmp = (receipt && receipt.comparison) || {};
+  const delta = typeof cmp.delta === 'number' ? cmp.delta : 0;
+  const model = shortModel(receipt && receipt.run && receipt.run.model_id);
+  let verdict;
+  if (delta >= EFFECT_FLOOR) verdict = 'PASSED';
+  else if (delta <= -EFFECT_FLOOR) verdict = 'REGRESSED';
+  else verdict = 'NO_EFFECT';
+  const meta = VERDICTS[verdict];
+  return {
+    verdict,
+    delta,
+    floor: EFFECT_FLOOR,
+    model,
+    message: `${meta.word} on ${model}`,
+    color: meta.color,
+  };
+}
+
+// A shields.io "endpoint" badge object (schemaVersion 1). Committed as JSON and
+// referenced via https://img.shields.io/endpoint?url=<public-url-of-this-json>.
+function badgeEndpoint(receipt, { label = 'driftproof' } = {}) {
+  const v = verdictFromReceipt(receipt);
+  return {
+    schemaVersion: 1,
+    label,
+    message: v.message,
+    color: v.color,
+  };
+}
+
+// Lines suitable for appending to a GitHub Actions $GITHUB_OUTPUT file.
+function githubOutputLines(receipt) {
+  const v = verdictFromReceipt(receipt);
+  return [
+    `verdict=${v.verdict}`,
+    `delta=${v.delta}`,
+    `message=${v.message}`,
+    `color=${v.color}`,
+  ].join('\n');
+}
+
+module.exports = { verdictFromReceipt, badgeEndpoint, githubOutputLines, shortModel, VERDICTS };

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "driftproof",
+  "version": "0.3.0",
+  "description": "Continuous verification of agent skills: run a skill's eval suite with and without the skill across model versions, emit signed dated receipts, and diff receipts into drift reports.",
+  "license": "Apache-2.0",
+  "keywords": [
+    "agent-skills",
+    "evals",
+    "llm",
+    "drift",
+    "verification"
+  ],
+  "homepage": "https://driftproofhq.com",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/driftproofhq/driftproof.git"
+  },
+  "bugs": {
+    "url": "https://github.com/driftproofhq/driftproof/issues"
+  },
+  "bin": {
+    "driftproof": "bin/driftproof"
+  },
+  "files": [
+    "bin/",
+    "lib/",
+    "spec/",
+    "config.js",
+    "config/models.json"
+  ],
+  "scripts": {
+    "gate": "node tests/gate.js",
+    "run": "node bin/driftproof run",
+    "diff": "node bin/driftproof diff"
+  },
+  "dependencies": {
+    "ajv": "^8.17.1"
+  },
+  "optionalDependencies": {
+    "@anthropic-ai/sdk": "^0.39.0"
+  },
+  "engines": {
+    "node": ">=22"
+  }
+}

package/spec/RECEIPT.md

@@ -0,0 +1,246 @@
+<!-- SPDX-License-Identifier: Apache-2.0 -->
+# Driftproof receipt — spec v0.3
+
+A **receipt** is a signed, dated record of running one agent skill's eval suite
+**with** and **without** the skill on one model version, with the judge **sampled**
+so every score carries a confidence band. Receipts are the unit of evidence
+Driftproof produces; diffing two receipts across model releases yields a **drift
+report**.
+
+The machine-readable contract is [`receipt.schema.json`](./receipt.schema.json)
+(JSON Schema, draft 2020-12). This document is the human companion. Where they
+disagree, the schema wins.
+
+**Versioning.** The current schema is v0.3
+([`receipt.schema.json`](./receipt.schema.json)). Prior schemas are kept as
+[`receipt.v0.2.schema.json`](./receipt.v0.2.schema.json) and
+[`receipt.v0.1.schema.json`](./receipt.v0.1.schema.json); the validator picks the
+schema by the receipt's own `schema_version`, so v0.1 and v0.2 receipts still load
+and validate. v0.3 is an **additive** bump — every field it adds is required in a
+fresh run, but older receipts are read unchanged against their own schema.
+
+## Design goals
+
+- **Reproducible.** Every hash is over a *canonical* JSON form (object keys sorted
+  at every level, no insignificant whitespace), so the same inputs produce the
+  same hashes on any machine.
+- **Credible.** An LLM judge is noisy. v0.2 samples the judge N times per case and
+  records the distribution, so a receipt carries a *band*, not a single fragile
+  number — and drift reports only claim a regression when bands don't overlap.
+- **Tamper-evident (lite).** The `receipt_hash` is a self-hash: recompute over the
+  receipt with `receipt_hash` removed and compare. A hand-edit breaks it. This is
+  integrity, **not** authenticity — v0.3 does not sign with a key (see open
+  questions; "signed" is a promissory note until then).
+- **Consumes, doesn't invent.** The eval suite format is `agentskills.io/evals`.
+
+## What changed from v0.2 (transcript auditability + provenance)
+
+- **Per-case `generation_hash`** — sha256 (hex) of the raw model generation that
+  was judged for that (case, mode). Binds the graded case to the exact text.
+- **Per-case `judge_sample_hashes[]`** — sha256 (hex) of each raw judge output,
+  one per judge sample (same length as `samples`). A reader can check every score
+  a verdict rests on against a retained transcript.
+- **`run.transcripts`** — `"retained-local"` when the raw generations + judge
+  outputs were also written to `transcripts/<receipt-id>/` (under
+  `--keep-transcripts`), else `"hashes-only"` (the default — only the hashes).
+- **`run.registry`** — `"registered"` when `model_id` resolved in the model
+  registry (`config/models.json`), else `"unregistered"` (the run still executed;
+  cost was estimated with a conservative default price).
+- These four fields are **required** in a v0.3 receipt. Everything else is
+  unchanged from v0.2.
+
+## What changed from v0.1
+
+- Per-case: added `mean`, `stddev`, `samples[]`; `outcome` gains **`borderline`**.
+- `run`: added a required `judge` block (samples, temperature, sampling, surface).
+- Aggregates: added `stddev` (the run-to-run band) and `borderline_count`.
+- `comparison`: added `delta_uncertainty`.
+- Added optional top-level `editorial_reviews[]`.
+- `score` is retained as an alias of `mean` so v0.1 readers keep working.
+
+## Fields
+
+### `schema_version` (string, required) — `"0.3"`.
+
+### `skill` (object, required)
+| field | type | notes |
+|---|---|---|
+| `name` | string | From SKILL.md front-matter or its H1. |
+| `version` | string | Skill's declared version. |
+| `content_hash` | sha256 hex | Over `SKILL.md` + every bundled file (the `evals/` dir is **excluded** — hashed separately as `suite_hash`), path + content, path-sorted. |
+
+### `suite` (object, required)
+| field | type | notes |
+|---|---|---|
+| `format` | string | Always `"agentskills.io/evals"`. |
+| `suite_hash` | sha256 hex | Over the canonicalized normalized case list (`{id, prompt, rubric, pass_threshold}`). |
+| `case_count` | integer | Cases in the suite (before any `--max-cases` cap). |
+
+### `run` (object, required)
+| field | type | notes |
+|---|---|---|
+| `model_id` | string | Canonical model id run on. |
+| `model_release_date` | ISO date or `null` | Release date **if known**, else `null`. |
+| `surface` | `"api"` \| `"claude-cli"` | `api` = Messages API; `claude-cli` = spawned `claude -p` (subscription). |
+| `runner_version` | string | Runner version, for reproducibility. |
+| `date_utc` | ISO 8601 UTC | When the run finished. |
+| `registry` | `"registered"` \| `"unregistered"` | **v0.3.** Whether `model_id` resolved in the model registry (`config/models.json`). |
+| `transcripts` | `"retained-local"` \| `"hashes-only"` | **v0.3.** Whether the raw generations + judge outputs were retained on disk (see § Transcripts). |
+| `judge` | object | `{ samples, temperature, sampling, surface }` — see below. |
+
+**`run.judge`** records how grading was done: `samples` (judge calls per case),
+`temperature` (a number when the surface lets us set it — the `api` surface pins
+**0** — else `null`), `sampling` (`"api-temperature-0"` or `"surface-controlled"`),
+and `surface`. Determinism where the surface allows: on `api` the judge is pinned
+to temperature 0; on `claude-cli` sampling params are surface-controlled and the
+receipt says so.
+
+### `results` (object, required)
+- **`cases`** — one entry **per (case, mode)**; every case appears twice
+  (`with_skill` and `baseline`):
+  | field | type | notes |
+  |---|---|---|
+  | `id` | string | Eval case id. |
+  | `mode` | `"with_skill"` \| `"baseline"` | Whether SKILL.md was supplied. |
+  | `outcome` | `"pass"` \| `"fail"` \| `"borderline"` \| `"score"` | **`borderline`** = the threshold lies within `mean ± stddev` (the run can't confidently call it). `score` = un-thresholded case. |
+  | `mean` | number 0–1 | Mean of the judge samples. |
+  | `stddev` | number ≥ 0 | Sample stddev of the judge samples — the raw per-case band half-width used by the borderline rule and per-case drift. |
+  | `samples` | number[] | The individual judge scores. |
+  | `generation_hash` | sha256 hex | **v0.3.** sha256 of the raw model generation that was judged for this (case, mode). |
+  | `judge_sample_hashes` | sha256 hex[] | **v0.3.** sha256 of each raw judge output, one per sample (same length as `samples`). |
+  | `score` | number 0–1 | Alias of `mean` (kept for v0.1 readers). |
+  | `threshold` | number or `null` | The case's pass threshold (or `null`). |
+  | `reason` | string | One-line judge rationale (optional). |
+  | `judge` | object | `{ model_id, rubric_hash }` — who graded and a hash binding the grade to the exact rubric + judge system prompt. |
+- **`aggregates`** — `{ with_skill, baseline }`, each
+  `{ case_count, pass_count, borderline_count?, mean_score, stddev }`. Here
+  `stddev` is the **suite dispersion**: the stddev of the per-case means across
+  the suite (a conventional "mean ± stddev across items"). It is a reported
+  summary stat — the drift **headline verdict is driven by the per-case
+  band-overlap verdicts**, not by a separate test on this aggregate band. (Using
+  the standard error of the mean here instead would shrink with sampling and make
+  the headline cry wolf on trivial moves — precisely what this design avoids.)
+
+### `comparison` (object, required)
+| field | type | notes |
+|---|---|---|
+| `with_skill_score` | number 0–1 | Aggregate with-skill mean. |
+| `baseline_score` | number 0–1 | Aggregate baseline mean. |
+| `delta` | number −1–1 | The skill's measured lift. |
+| `delta_uncertainty` | number ≥ 0 | Combined band on the delta (quadrature sum of the two aggregate bands). |
+
+### `verification_level` (string, required)
+Community lattice: **UNVERIFIED** (bare claim) · **DECLARED** (author asserts, no
+run) · **TESTED** (a suite was executed and judged — what Driftproof emits) ·
+**FORMAL** *(reserved / unimplemented; the schema rejects it)*.
+
+### `editorial_reviews` (array, optional)
+Pointers to external one-shot reviews of the skill, each `{ url, source, date }`.
+**Context only — not verification evidence.** A one-shot editorial verdict and a
+dated, model-bound receipt are different things; this field lets a receipt link
+the former without conflating it with the latter.
+
+### `receipt_hash` (sha256 hex, required)
+sha256 over the canonical receipt JSON **with `receipt_hash` removed**.
+
+## Transcripts (v0.3)
+
+Every generation and every judge sample is hashed into the receipt
+(`generation_hash`, `judge_sample_hashes[]`), so a verdict is always checkable
+*in principle*. `--keep-transcripts` makes it checkable *in practice*: the raw
+generations and judge outputs are written to `transcripts/<receipt_hash>/` (one
+JSON per case+mode, plus an `index.json`), and the receipt records
+`transcripts: "retained-local"`. Without the flag the receipt records
+`transcripts: "hashes-only"` and only the hashes are kept. The transcript
+directory is **gitignored by default** — raw model text is never committed. The
+default for trigger-initiated runs is `retained-local` (disk is cheap, audits are
+not). To verify a retained receipt: re-hash each transcript file and compare
+against the receipt's `generation_hash` / `judge_sample_hashes`.
+
+## Drift verdict (how `diff` reads two receipts)
+
+For each case, Driftproof compares the two `with_skill` bands (`mean ± stddev`).
+A verdict requires **two** conditions — band separation **and** a minimum effect:
+
+1. **Band separation** (geometry):
+   - **regression** — `meanB + stddevB < meanA − stddevA` (B's band is entirely below A's)
+   - **improvement** — `meanB − stddevB > meanA + stddevA`
+   - **within noise** — otherwise (the bands touch or overlap)
+2. **Minimum effect floor** — `|meanB − meanA| ≥ EFFECT_FLOOR` (default **0.05**, see
+   `config.js`). A regression/improvement from step 1 whose mean moved by *less*
+   than the floor is downgraded to **`within noise (below effect floor)`**.
+
+**Why the floor.** The LLM judge quantizes scores to a coarse ~0.05–0.1 grid, so a
+confident grade frequently has `stddev 0` — a zero-width "point band." Two point
+bands one quantum apart (e.g. `0.60` vs `0.64`) are technically non-overlapping yet
+represent no meaningful behaviour change. The floor stops that statistically-real-
+but-practically-trivial move from being reported as drift. `0.05` ≈ one judge
+quantization step; the comparison is inclusive (a move of exactly 0.05 counts).
+
+A regression is claimed **only** when both conditions hold. The **headline verdict
+summarizes the per-case verdicts** (e.g. "DRIFT — 2 cases regressed", or "WITHIN
+NOISE" when none moved beyond its band + floor); it does not run a separate, tighter
+test on the aggregate mean. This is the anti-false-positive core: band separation
+**plus a real-sized delta**, not either alone, is what triggers a verdict.
+
+## Example (abridged)
+
+```json
+{
+  "schema_version": "0.3",
+  "skill": { "name": "commit-message-conventions", "version": "0.2.0", "content_hash": "…64 hex…" },
+  "suite": { "format": "agentskills.io/evals", "suite_hash": "…64 hex…", "case_count": 10 },
+  "run": {
+    "model_id": "claude-haiku-4-5-20251001", "model_release_date": "2025-10-01",
+    "surface": "claude-cli", "runner_version": "0.3.0", "date_utc": "2026-07-27T09:15:00.000Z",
+    "registry": "registered", "transcripts": "hashes-only",
+    "judge": { "samples": 5, "temperature": null, "sampling": "surface-controlled", "surface": "claude-cli" }
+  },
+  "results": {
+    "cases": [
+      { "id": "perf-not-refactor", "mode": "with_skill", "outcome": "pass",
+        "score": 0.86, "mean": 0.86, "stddev": 0.05, "samples": [0.9,0.8,0.85,0.9,0.85],
+        "generation_hash": "…64 hex…", "judge_sample_hashes": ["…64 hex…","…","…","…","…"],
+        "threshold": 0.7, "judge": { "model_id": "claude-haiku-4-5-20251001", "rubric_hash": "…64 hex…" } }
+    ],
+    "aggregates": {
+      "with_skill": { "case_count": 10, "pass_count": 7, "borderline_count": 1, "mean_score": 0.81, "stddev": 0.02 },
+      "baseline":   { "case_count": 10, "pass_count": 1, "mean_score": 0.42, "stddev": 0.03 }
+    }
+  },
+  "comparison": { "with_skill_score": 0.81, "baseline_score": 0.42, "delta": 0.39, "delta_uncertainty": 0.036 },
+  "verification_level": "TESTED",
+  "receipt_hash": "…64 hex…"
+}
+```
+
+## Open questions (v0.3)
+
+1. **"Signed" is really "self-hashed".** `receipt_hash` is integrity, not
+   authenticity — anyone can recompute it after an edit. A key signature (Ed25519
+   over the canonical form, or in-toto/sigstore attestation) is the natural next
+   step and the main thing standing between v0.2 and a receipt a third party can
+   *trust*, not just *read*.
+2. **Sampling captures judge variance, not generation variance.** v0.2 samples the
+   judge N times on a **single** generated response per (case, mode). That
+   quantifies judge noise — the dominant, cheapest-to-measure source — but a fresh
+   generation each run would vary too. Bands may therefore be tighter than true
+   run-to-run spread. Sampling generations (and separating the two variance
+   components) is a candidate for v0.3.
+3. **Two different "bands" coexist.** Per-case `stddev` is judge-sample spread
+   (drives borderline + per-case drift); aggregate `stddev` is suite dispersion
+   (a summary stat). Neither is a true run-to-run confidence interval — building
+   one would need repeated full runs (generation + judge), which v0.2 does not do
+   (see #2). The headline sidesteps this by summarizing per-case verdicts rather
+   than testing the aggregate band.
+4. **`model_release_date` provenance is unverified** (small built-in table + a date
+   parsed from the model id). Should cite a source or be omitted.
+5. **`outcome: "score"` overlaps the numeric `score`/`mean`.** Mild redundancy,
+   kept so thresholded and un-thresholded suites both round-trip.
+6. **No environment fingerprint** beyond `runner_version` — fine while the runner
+   is the only producer; matters once third parties emit receipts.
+7. **Baseline construction is fixed** ("same prompt, no SKILL.md"). Skills that
+   assume tools/context a bare baseline never had can show inflated lift; a future
+   spec may need to declare the baseline condition.
+8. **`editorial_reviews` scope.** Deliberately marked context-only; if it starts
+   being treated as evidence, it will need its own trust rules.