driftproof 0.8.1 → 0.9.0

package/README.md

@@ -262,7 +262,7 @@ A receipt is the unit of evidence — one JSON document conforming to
 
 ```jsonc
 {
-  "schema_version": "0.5",
+  "schema_version": "0.6",
   "skill":  { "name": "commit-message-conventions", "version": "0.2.0",
               "content_hash": "…sha256 over SKILL.md + bundled files…" },
   "suite":  { "format": "agentskills.io/evals", "suite_hash": "…", "case_count": 10 },
@@ -271,12 +271,16 @@ A receipt is the unit of evidence — one JSON document conforming to
     "model_release_date": "2025-10-01",
     "provider": "anthropic",
     "surface": "claude-cli",
-    "runner_version": "0.8.1",
+    "runner_version": "0.9.0",
     "date_utc": "2026-07-27T…Z",
     "registry": "registered",
     "transcripts": "hashes-only",
     "judge": { "samples": 5, "temperature": null, "sampling": "surface-controlled",
-               "surface": "claude-cli" }
+               "surface": "claude-cli", "model_id": "claude-haiku-4-5-20251001",
+               "prompt_template_hash": "…sha256 over the grading template…" },
+    "answered_by": { "kind": "model", "attested": true,
+                     "reported_model": "claude-haiku-4-5-20251001",
+                     "reported_models": ["claude-haiku-4-5"], "isolation": "eval-user" }
   },
   "results": {
     "cases": [
@@ -345,7 +349,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: driftproofhq/driftproof@v0.8.1
+      - uses: driftproofhq/driftproof@v0.9.0
         with:
           skill-dir: skills/my-skill
           models: claude-haiku-4-5
@@ -361,7 +365,9 @@ fails the job on `REGRESSED` unless you set `fail-on-regression: 'false'`. For a
 free CI dry-run with **zero model calls**, set `DRIFTPROOF_STUB=1` in the job env —
 the runner returns canned receipts so the wiring can be tested without spend (this
 is exactly how the action's own [self-test](.github/workflows/action-selftest.yml)
-runs). That self-test is also the proof behind the input hardening: on every
+runs). A stub receipt says what it is: `verification_level` `UNVERIFIED`,
+`run.surface` `stub`, `run.answered_by.kind` `stub`, and the verdict on it is
+`NOT_MEASURED` — it proves the wiring, never the skill. That self-test is also the proof behind the input hardening: on every
 run it passes one hostile value (a quote, a semicolon, `$(...)`, a backtick and
 a newline) through each of the action's five inputs on the real runner and
 fails unless every one is refused before anything ran, so a green check there

package/bin/driftproof

@@ -6,13 +6,13 @@ const fs = require('fs');
 const path = require('path');
 const { PROJECT_NAME, RUNNER_VERSION, DEFAULT_JUDGE_SAMPLES, DEV_MAX_USD, DEV_MAX_CALLS } = require('../config');
 const { loadSkill } = require('../lib/skill');
-const { runSkillOnModel, summarizeReceipt, projectCalls } = require('../lib/run');
+const { runSkillOnModel, summarizeReceipt, projectCalls, answeredLine, band, uncertaintyStr } = require('../lib/run');
 const { SAMPLING } = require('../lib/sampling');
 const { validateReceipt, verifyReceiptHash } = require('../lib/receipt');
 const { buildDriftReport, revisionPairProblem } = require('../lib/diff');
 const { surfaceForModel, isSubscriptionSurface, resolveModel, evalUser } = require('../lib/provider');
 const { estimateRunCostUSD, BudgetTracker } = require('../lib/cost');
-const { registryStatus } = require('../lib/models');
+const { registryStatus, assertRegistered, REGISTRY_PATH } = require('../lib/models');
 const { verdictFromReceipt, badgeEndpoint, githubOutputLines } = require('../lib/verdict');
 const { scaffoldInit } = require('../lib/init');
 
@@ -40,6 +40,24 @@ function writeTranscripts(receipt, transcripts) {
 
 const RECEIPTS_DIR = path.join(process.cwd(), 'receipts');
 
+// ── THE DISPLAY VERIFIES BEFORE IT READS (spec 026 AC-20, F7) ────────────────
+// badge, diff and export read a receipt somebody else may have edited. Until
+// this, badge never checked the hash and diff and export printed a warning
+// and proceeded, so a receipt with one digit moved and the hash left as it
+// was rendered `passing` (028's D-5, for the CLI). A receipt whose
+// receipt_hash does not verify is refused here: nothing on stdout, no file,
+// the reason on stderr naming receipt_hash and the file, exit 4. The bound the
+// threat model states does not move: the same edit followed by a re-seal
+// (sealReceipt) verifies and renders, which is what receipt_hash is: integrity
+// since sealing, not authenticity. The check lives here, in the three
+// commands, and not in lib/verdict.js's writer, which the repo gate calls on
+// synthetic unsealed receipts.
+function refuseUnverified(receipt, file, command) {
+  if (verifyReceiptHash(receipt)) return;
+  console.error(`  ✗ REFUSED (${command}): receipt_hash does not verify for ${path.basename(file)} (tampered or hand-edited since it was sealed); nothing rendered.`);
+  process.exit(4);
+}
+
 // Read an optional .driftproofrc (JSON) for per-project run defaults. Looked up
 // in the skill dir first (where `driftproof init` writes it), then the CWD. CLI
 // flags always win over the rc; the rc wins over built-in defaults.
@@ -58,6 +76,81 @@ function loadRc(skillDir) {
 // positional instead of swallowing it as the flag's value.
 const BOOLEAN_FLAGS = new Set(['trusted-skill']);
 
+// ── THE INPUT CONTRACT (spec 026 AC-9, F5) ────────────────────────────────────
+// A COPY of action/lib.sh's RE_* rules, in the shape spec 028's fixture holds
+// them; specs/026-receipt-integrity/fixtures/input-contract.json carries the
+// same copy and AC-9 asserts the three identical, rule for rule, both ways.
+// Consolidating into one shared module is 028's decision 1, the spec after 026.
+//
+// Why a contract at the door: `parseInt('abc', 10)` is NaN, both cost guards
+// are `>` comparisons against it, both are false, and the run completed and
+// wrote a receipt that carried no trace of it (the dollar cap never compared;
+// the call cap silently became the dev default). `--samples abc` was quieter
+// still: `NaN || 5` is 5. Every numeric input, from the command line or the
+// rc, is checked against its rule BEFORE the projection is printed and before
+// any call; a flag given with no value (the parser records `true`) and an
+// empty string are refused the same way, not defaulted.
+const INPUT_CONTRACT = {
+  rules: {
+    models: { re: /^[A-Za-z0-9._-]+(,[A-Za-z0-9._-]+)*$/, message: 'models: expected a comma-separated list of model ids (letters, digits, . _ -)' },
+    max_usd: { re: /^[0-9]+(\.[0-9]+)?$/, message: 'max-usd: expected a positive decimal number such as 20 or 3.53' },
+    max_usd_zero: { re: /^0+(\.0+)?$/, reject_on_match: true, message: 'max-usd: must be greater than zero' },
+    max_calls: { re: /^[1-9][0-9]*$/, message: 'max-calls: expected a positive integer with no leading zero' },
+    skill_dir_control: { re: /[\x00-\x1f\x7f]/, reject_on_match: true, message: 'skill-dir: contains a control character' },
+  },
+  // Which rule each `driftproof run` input is checked against. samples,
+  // max-cases and concurrency are positive integers and reuse max_calls' rule.
+  flags: { 'models': ['models'], 'max-usd': ['max_usd', 'max_usd_zero'], 'max-calls': ['max_calls'], 'samples': ['max_calls'], 'max-cases': ['max_calls'], 'concurrency': ['max_calls'], 'skill-dir': ['skill_dir_control'] },
+};
+
+function shownValue(v) {
+  if (v === true) return '<no value>';
+  return JSON.stringify(String(v));
+}
+// Refuse one input: the rule's own message, the input's name and the value it
+// carried, exit 2. Nothing has been printed or spent when this runs.
+function refuseInput(name, value, message, where) {
+  console.error(`  ✗ REFUSED: ${message}, got ${shownValue(value)} (${where} ${name}); nothing run, no receipt written.`);
+  process.exit(2);
+}
+// Check one input against every rule its flag maps to. `value` is what was
+// given: a string, a number (from the rc), `true` (a flag with no value) or
+// an empty string; only a string that matches every rule passes.
+function checkInput(flag, value, where, shownName = flag) {
+  const rules = INPUT_CONTRACT.flags[flag] || [];
+  const text = typeof value === 'string' ? value : (typeof value === 'number' && Number.isFinite(value)) ? String(value) : null;
+  if (text === null) refuseInput(shownName, value, `${flag}: expected a value`, where);
+  for (const key of rules) {
+    const rule = INPUT_CONTRACT.rules[key];
+    const hit = rule.re.test(text);
+    if (rule.reject_on_match ? hit : !hit) refuseInput(shownName, value, rule.message, where);
+  }
+  return text;
+}
+// Every numeric input the run takes, from the command line first and the rc
+// second, checked before anything is printed. Returns the checked strings.
+// The registry door (spec 026 AC-10, F6): every target and the judge it will
+// run with must be a model the registry knows. lib/models.js assertRegistered
+// is the one rule; lib/run.js asks it too on the path every caller shares, and
+// this door is where bin/driftproof says the registry's path in its own
+// message, before the suite loads.
+function checkRegistered(models, judgeFor) {
+  try {
+    for (const m of models) { assertRegistered(m, 'model'); assertRegistered(judgeFor(m), 'judge model'); }
+  } catch (e) {
+    if (e && e.code === 'UNREGISTERED_MODEL') { console.error(`  ✗ REFUSED (${/judge/.test(e.message) ? 'judge-model / judge_model' : 'models'}): ${e.message}`); process.exit(2); }
+    throw e;
+  }
+}
+function checkNumericInputs(flags, rc) {
+  const out = {};
+  for (const [flag, rcKey] of [['max-calls', 'max_calls'], ['max-usd', 'max_usd'], ['samples', 'samples'], ['max-cases', 'max_cases'], ['concurrency', 'concurrency']]) {
+    if (Object.prototype.hasOwnProperty.call(flags, flag)) out[flag] = checkInput(flag, flags[flag], 'flag --');
+    else if (rc[rcKey] !== undefined && rc[rcKey] !== null) out[flag] = checkInput(flag, rc[rcKey], '.driftproofrc', rcKey);
+  }
+  return out;
+}
+
 function parseArgs(argv) {
   const positional = [];
   const flags = {};
@@ -112,9 +205,10 @@ NOTES
   - --keep-transcripts writes the raw generations + judge outputs to
     transcripts/<receipt-hash>/ (gitignored) and records transcripts:"retained-
     local" in the receipt. Default is "hashes-only" (only the sha256 hashes).
-  - Prices and registry status come from config/models.json; an unregistered
-    model still runs but is marked registry:"unregistered" and costed at the
-    conservative default price.
+  - Prices and registry status come from config/models.json (or the registry
+    DRIFTPROOF_REGISTRY names). An unregistered model id is REFUSED before any call,
+    naming the id and the registry path (so is one not shaped like a model id);
+    registry:"unregistered" is an import-only receipt value, refused for a run.
   - --concurrency runs that many (case,mode) tasks at once (default 1). Higher
     values cut wall-clock on the cli surface (cold-start dominated).
   - ISOLATION (default). On the two cli surfaces every \`claude\` / \`codex\` spawn
@@ -144,23 +238,53 @@ async function cmdRun(positional, flags) {
 
   // Per-project defaults from .driftproofrc (if any); CLI flags override these.
   const rc = loadRc(skillDir);
-  const models = String(flags.models || rc.models || 'haiku').split(',').map((s) => s.trim()).filter(Boolean);
-  const maxCases = flags['max-cases'] ? parseInt(flags['max-cases'], 10) : (rc.max_cases != null ? parseInt(rc.max_cases, 10) : null);
-  const maxCalls = flags['max-calls'] ? parseInt(flags['max-calls'], 10) : (rc.max_calls != null ? parseInt(rc.max_calls, 10) : DEV_MAX_CALLS);
-  const samples = flags.samples ? parseInt(flags.samples, 10) : (rc.samples != null ? parseInt(rc.samples, 10) : DEFAULT_JUDGE_SAMPLES);
+  // The contract, at the door (spec 026 AC-9): every input checked against
+  // its rule before the projection and before any call. A value that fails
+  // its shape is refused naming the input and the value; nothing is defaulted.
+  if (INPUT_CONTRACT.rules.skill_dir_control.re.test(String(skillDir))) refuseInput('skill-dir', skillDir, INPUT_CONTRACT.rules.skill_dir_control.message, 'argument');
+  const checked = checkNumericInputs(flags, rc);
+  const modelsRaw = Object.prototype.hasOwnProperty.call(flags, 'models') ? flags.models : (rc.models !== undefined && rc.models !== null) ? rc.models : 'haiku';
+  const models = String(modelsRaw).split(',').map((s) => s.trim()).filter(Boolean);
   const judgeModel = flags['judge-model'] || rc.judge_model || null;
-  const concurrency = flags.concurrency ? parseInt(flags.concurrency, 10) : (rc.concurrency != null ? parseInt(rc.concurrency, 10) : 1);
-  const maxUsd = flags['max-usd'] ? parseFloat(flags['max-usd']) : (rc.max_usd != null ? parseFloat(rc.max_usd) : DEV_MAX_USD);
+  // Spec 026 AC-10 (F6): every model id, and the judge, must be a model the
+  // registry knows; refused here naming the input, the id and the registry's
+  // absolute path, before the suite loads, before the projection, before any
+  // call. A shape failure is refused the same way (the contract's models rule
+  // below then never sees one). The same check stands in lib/run.js for every
+  // other caller.
+  const judgeFor = (m) => (judgeModel ? resolveModel(judgeModel) : resolveModel(m));
+  checkRegistered(models, judgeFor);
+  const modelsGiven = Object.prototype.hasOwnProperty.call(flags, 'models') ? checkInput('models', flags.models, 'flag --')
+    : (rc.models !== undefined && rc.models !== null) ? checkInput('models', rc.models, '.driftproofrc', 'models') : 'haiku';
+  void modelsGiven;
+  const maxCases = checked['max-cases'] !== undefined ? parseInt(checked['max-cases'], 10) : null;
+  const maxCalls = checked['max-calls'] !== undefined ? parseInt(checked['max-calls'], 10) : DEV_MAX_CALLS;
+  const samples = checked.samples !== undefined ? parseInt(checked.samples, 10) : DEFAULT_JUDGE_SAMPLES;
+  const concurrency = checked.concurrency !== undefined ? parseInt(checked.concurrency, 10) : 1;
+  const maxUsd = checked['max-usd'] !== undefined ? parseFloat(checked['max-usd']) : DEV_MAX_USD;
   const keepTranscripts = !!flags['keep-transcripts'];
-  // spec 022: the same-user spawn exists only behind this flag. Resolving the
-  // eval user here means a bad DRIFTPROOF_EVAL_USER is refused before the banner.
+  // spec 022: the same-user spawn exists only behind this flag.
+  //
+  // THE EVAL USER IS NOT RESOLVED HERE (F-022-4, spec 021). It used to be, on
+  // every run that was not --trusted-skill, including an api-surface-only run
+  // that spawns no CLI at all - so a hostile DRIFTPROOF_EVAL_USER refused a run
+  // that would never have read it. Fail-loud and harmless, and still wrong: a
+  // variable that governs a lane this run does not take should be neither read
+  // nor validated. It is resolved below, once a subscription surface is actually
+  // in the run, which is the same condition the isolation banner already used.
   const trusted = !!flags['trusted-skill'];
-  const isolation = trusted ? 'same-user (--trusted-skill: self-authored skill)' : `eval-user (${evalUser()})`;
   const outDir = flags.out ? path.resolve(flags.out) : RECEIPTS_DIR;
   fs.mkdirSync(outDir, { recursive: true });
 
   const skill = loadSkill(skillDir);
   const nCases = maxCases ? Math.min(maxCases, skill.suite.caseCount) : skill.suite.caseCount;
+  // Spec 026 AC-5 (F2): a suite with no cases measures nothing, and a receipt
+  // over nothing is a receipt about nothing. Refused here, before the
+  // projection and before any call, naming the suite file.
+  if (nCases < 1) {
+    console.error(`  ✗ REFUSED: the suite at ${path.join(path.resolve(skillDir), 'evals', 'evals.json')} has no cases${maxCases ? ` (after --max-cases ${maxCases})` : ''}; nothing to measure, no receipt written.`);
+    process.exit(2);
+  }
   // v0.5 draws the generation up to SAMPLING.max times per arm, so BOTH the
   // printed projection and the dollar guard below must be scaled by it. Left at
   // draws=1 the guard would admit a run costing up to ten times its own
@@ -174,7 +298,13 @@ async function cmdRun(positional, flags) {
   // `api` surface this is real spend, so we refuse if it would exceed --max-usd.
   // On `claude-cli` the metered spend is $0 (subscription); the figure is the
   // hypothetical "if run on the metered API" cost — printed, never blocks.
-  const cost = estimateRunCostUSD({ caseCount: nCases, draws: SAMPLING.max, samples, models: models.map((m) => require('../lib/provider').resolveModel(m)), judgeModel: judgeModel || 'haiku' });
+  // Spec 026 AC-10: the projection is priced with the judge the run will USE
+  // (lib/run.js: --judge-model when given, else the target itself), never a
+  // fixed haiku; each target is priced with its own judge and the figures summed.
+  const targets = models.map((m) => resolveModel(m));
+  const judges = models.map((m) => judgeFor(m));
+  const perModelCost = targets.map((t, i) => estimateRunCostUSD({ caseCount: nCases, draws: SAMPLING.max, samples, models: [targets[i]], judgeModel: judges[i] }));
+  const cost = { totalUSD: Math.round(perModelCost.reduce((a, c) => a + c.totalUSD, 0) * 1e4) / 1e4, perModel: perModelCost.flatMap((c) => c.perModel), judges };
   // Surface is per-model now (a run may mix an Anthropic and an OpenAI target).
   const surfaces = [...new Set(models.map((m) => surfaceForModel(m)))];
   const surface = surfaces.join(', ');
@@ -182,10 +312,17 @@ async function cmdRun(positional, flags) {
 
   const regStatuses = models.map((m) => `${m}:${registryStatus(m)}`);
   console.log(`\n${PROJECT_NAME} run — skill "${skill.name}" v${skill.version}`);
+  // Spec 026 AC-1: a stub run says so before it starts, and its receipt says so
+  // after (surface stub, answered_by.kind stub, UNVERIFIED).
+  if (process.env.DRIFTPROOF_STUB === '1') console.log('  STUB RUN (DRIFTPROOF_STUB=1): nothing will answer; the receipt will be UNVERIFIED and measure nothing');
   console.log(`  content_hash: ${skill.contentHash.slice(0, 16)}…  suite: ${skill.suite.caseCount} cases (${skill.suite.suiteHash.slice(0, 12)}…)`);
   console.log(`  surface: ${surface}   models: ${models.join(', ')}   samples/case: ${samples}   concurrency: ${concurrency}`);
   console.log(`  registry: ${regStatuses.join('  ')}${keepTranscripts ? '   transcripts: retained-local' : ''}`);
-  if (subSurfaces.length) console.log(`  isolation: ${isolation}`);
+  console.log(`  judge: ${[...new Set(cost.judges)].join(', ')}   (${judgeModel ? '--judge-model' : 'the target model itself; set --judge-model to change'})`);
+  if (subSurfaces.length) {
+    const isolation = trusted ? 'same-user (--trusted-skill: self-authored skill)' : `eval-user (${evalUser()})`;
+    console.log(`  isolation: ${isolation}`);
+  }
   console.log(`  projected calls: ${perModelCalls}/model × ${models.length} model(s) = ${totalProjected}   per-model cap: ${maxCalls}`);
   console.log(`  projected cost: ~$${cost.totalUSD.toFixed(2)} (rough upper bound; budget $${maxUsd.toFixed(2)}, hard-stop $${(maxUsd * 1.25).toFixed(2)})`);
   if (subSurfaces.length) console.log(`  actual metered spend on ${subSurfaces.join(', ')}: $0.00 (subscription; the $ figure is the estimated-equivalent API cost, counted against the cap identically)`);
@@ -227,6 +364,12 @@ async function cmdRun(positional, flags) {
         },
       });
     } catch (e) {
+      if (e && e.code === 'SUBSTRATE_MISMATCH') {
+        // Spec 026 AC-2: the surface answered as a different model. The run
+        // stopped before its next call and wrote no receipt.
+        console.error(`\n  ✗ REFUSED (substrate mismatch): ${e.message}`);
+        process.exit(5);
+      }
       if (e && e.code === 'BUDGET_HARDSTOP') {
         console.error(`\n  ✗ ABORT (budget hard-stop): ${e.message}`);
         console.error(`    ${emitted.length} receipt(s) already written to ${path.relative(process.cwd(), outDir)}/.`);
@@ -264,7 +407,14 @@ async function cmdRun(positional, flags) {
     const aw = receipt.results.aggregates.with_skill;
     const ab = receipt.results.aggregates.baseline;
     console.log(`  → ${path.relative(process.cwd(), jsonPath)}  (${calls} calls)`);
-    console.log(`  → skill lift ${cmp.delta >= 0 ? '+' : ''}${cmp.delta.toFixed(3)} ± ${cmp.delta_uncertainty.toFixed(3)}  (with ${aw.mean_score.toFixed(3)} ± ${aw.stddev.toFixed(3)} vs base ${ab.mean_score.toFixed(3)} ± ${ab.stddev.toFixed(3)})\n`);
+    console.log(`  → ${answeredLine(receipt)}; verification_level ${receipt.verification_level}`);
+    // Spec 026 AC-8: how many draws were cut at the output cap and excluded.
+    const nTruncated = receipt.results.cases.reduce((a, c) => a + (((c.generation || {}).n_truncated) || 0), 0);
+    if (nTruncated) console.log(`  → ${nTruncated} draw(s) truncated at the output cap: unmeasured, never judged, excluded from every band`);
+    // Spec 026 AC-6, AC-7: the band rule is named beside the band, and a band
+    // the formula could not form prints as n/a, never as 0.000.
+    if (cmp.delta == null) console.log(`  → skill lift n/a (${cmp.delta_uncertainty_unavailable === 'no_cases' ? 'no case was included on an arm' : 'no comparison'})\n`);
+    else console.log(`  → skill lift ${cmp.delta >= 0 ? '+' : ''}${cmp.delta.toFixed(3)} ± ${uncertaintyStr(cmp)}  (with ${band(aw.mean_score, aw.stddev)} vs base ${band(ab.mean_score, ab.stddev)}; band = sample stddev of the per-case means)\n`);
   }
 
   console.log(`Done. ${emitted.length} receipt(s) emitted to ${path.relative(process.cwd(), outDir)}/`);
@@ -276,10 +426,8 @@ function cmdDiff(positional, flags) {
   const a = JSON.parse(fs.readFileSync(aPath, 'utf8'));
   const b = JSON.parse(fs.readFileSync(bPath, 'utf8'));
 
-  // Warn (do not block) if either receipt fails self-verification.
-  for (const [p, r] of [[aPath, a], [bPath, b]]) {
-    if (!verifyReceiptHash(r)) console.error(`  ⚠ ${path.basename(p)}: receipt_hash does not verify (tampered or hand-edited)`);
-  }
+  // Spec 026 AC-20: a side whose hash does not verify is refused, naming it.
+  for (const [p, r] of [[aPath, a], [bPath, b]]) refuseUnverified(r, p, 'diff');
 
   // --mode revision inverts the axis: the skill text is the variable under test
   // and the substrate is the control. The fields release drift merely warns about
@@ -358,6 +506,7 @@ function cmdBadge(positional, flags) {
   const p = positional[0];
   if (!p) { console.error('usage: driftproof badge <receipt.json> [--out FILE] [--github-output]'); process.exit(2); }
   const receipt = JSON.parse(fs.readFileSync(p, 'utf8'));
+  refuseUnverified(receipt, p, 'badge');
   if (flags['github-output']) { console.log(githubOutputLines(receipt)); return; }
   const badge = badgeEndpoint(receipt);
   const json = JSON.stringify(badge, null, 2);
@@ -408,7 +557,7 @@ function cmdExport(positional, flags) {
   if (to !== 'summary-json') { console.error(`unknown export target "${to}" — supported: summary-json`); process.exit(2); }
   const { toSummaryJson } = require('../lib/export');
   const receipt = JSON.parse(fs.readFileSync(p, 'utf8'));
-  if (!verifyReceiptHash(receipt)) console.error(`  ⚠ ${path.basename(p)}: receipt_hash does not verify (tampered or hand-edited)`);
+  refuseUnverified(receipt, p, 'export');
   const summary = toSummaryJson(receipt, { reportUrl: typeof flags['report-url'] === 'string' ? flags['report-url'] : null });
   const json = JSON.stringify(summary, null, 2);
   if (flags.out) {

package/config.js

@@ -9,7 +9,7 @@ const PROJECT_NAME = 'driftproof';
 // Bumped whenever the runner's behaviour or receipt-generation semantics change
 // in a way that could affect results. Recorded into every receipt as
 // run.runner_version so a receipt is reproducible against a known engine.
-const RUNNER_VERSION = '0.8.1';
+const RUNNER_VERSION = '0.9.0';
 
 // The eval format we CONSUME (we deliberately do not invent our own).
 const SUITE_FORMAT = 'agentskills.io/evals';
@@ -29,7 +29,30 @@ const SUITE_FORMAT = 'agentskills.io/evals';
 //        at run time so derived dollars stay reproducible), and the derived
 //        `economics` block. v0.3.1 is frozen as receipt.v0.3.1.schema.json;
 //        v0.1/v0.2/v0.3/v0.3.1 receipts all still load.
-const RECEIPT_SCHEMA_VERSION = '0.5';
+// v0.5   samples the GENERATION n times per arm (per-case `generation` draw list,
+//        the across-draw band, the variance ratio, the sampling policy applied)
+//        and adds the per-suite canary. v0.4 is frozen as receipt.v0.4.schema.json.
+// v0.6   (spec 026, receipt integrity) says WHAT ANSWERED: run.answered_by, a
+//        `stub` surface, run.judge.model_id + prompt_template_hash, per-draw
+//        stop_reason/truncated and n_truncated, case_status failed_unmeasured,
+//        results.aggregates.band_rule, and bands that are null where the formula
+//        cannot form. NOT additive for the validator: a v0.5 receipt restamped 0.6
+//        is refused. v0.5 is frozen as receipt.v0.5.schema.json.
+const RECEIPT_SCHEMA_VERSION = '0.6';
+
+// Input bounds on the skill loader and the post-checks (spec 026 AC-13, audit
+// A3/A7). lib/skill.js loaded every bundled file with no count, depth, byte,
+// case, prompt or rubric bound, so a 2 MB prompt went to the model at the
+// fixed 250-token projection; lib/checks.js compiled a suite-supplied regex
+// with no bound, and JavaScript cannot time a regex out. Each is set well
+// above every suite this repository holds: they bind pathological input, not
+// the archive. Named in the error a bound raises, and in AUTHORING.md.
+const SKILL_MAX_FILES = 2000;          // bundled files under a skill directory
+const SKILL_MAX_BYTES = 32 * 1024 * 1024; // bundled bytes, all files together
+const SKILL_MAX_DEPTH = 16;            // directory depth below the skill dir
+const SUITE_MAX_CASES = 500;           // cases in one evals.json
+const CASE_MAX_CHARS = 65536;          // characters in one prompt or rubric
+const CHECK_MAX_PATTERN = 256;         // characters in one checks[] regex
 
 // Hard USD budget defaults per entry point (Week 4). --max-usd overrides any of
 // these. The projection is refused before any call if it exceeds the cap, on
@@ -133,6 +156,7 @@ const REPORT_005_JUDGE_MODEL = 'claude-haiku-4-5';
 
 module.exports = {
   PROJECT_NAME, RUNNER_VERSION, SUITE_FORMAT, RECEIPT_SCHEMA_VERSION, DEFAULT_JUDGE_SAMPLES,
+  SKILL_MAX_FILES, SKILL_MAX_BYTES, SKILL_MAX_DEPTH, SUITE_MAX_CASES, CASE_MAX_CHARS, CHECK_MAX_PATTERN,
   EFFECT_FLOOR, DEV_MAX_USD, DEV_MAX_CALLS, REPORT_MAX_USD, TRIGGER_MAX_USD,
   GENERATION_SAMPLES_MIN, GENERATION_SAMPLES_MAX, GENERATION_SD_THRESHOLD, GENERATION_STABILITY_EPS,
   REPORT_002_CLAUDE_MODEL, REPORT_002_GPT_MODEL, REPORT_002_JUDGE_MODEL,

package/lib/checks.js

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 'use strict';
 
+const { CHECK_MAX_PATTERN } = require('../config');
+
 // Deterministic post-checks.
 //
 // A per-case eval suite may declare optional `checks[]`: structural / regex
@@ -19,6 +21,22 @@
 //   contains     — the output includes the literal `value` substring
 //   not_contains — the output does NOT include the literal `value` substring
 //   min_length   — the trimmed output is at least `value` characters long
+//
+// BOUNDED (spec 026 AC-13, audit A7). A suite's regex is third-party input
+// compiled and run against model output, and JavaScript cannot time a regex
+// out. A pattern longer than CHECK_MAX_PATTERN, or one carrying a nested
+// quantifier of the (x+)+ family (a quantified group whose body ends in a
+// quantifier: the shape that is exponential on every backtracking engine),
+// is REFUSED: recorded pass: false with a reason that says so, never
+// evaluated. The test is syntactic and narrow by design; it is not a general
+// ReDoS detector, and the spec says so.
+const NESTED_QUANTIFIER = /\((?:[^()\\]|\\.)*[+*}]\)\s*[+*]|\((?:[^()\\]|\\.)*\|(?:[^()\\]|\\.)*\)\s*[+*]/;
+function refusedPattern(pattern) {
+  const p = String(pattern == null ? '' : pattern);
+  if (p.length > CHECK_MAX_PATTERN) return `refused: pattern of ${p.length} characters exceeds CHECK_MAX_PATTERN (${CHECK_MAX_PATTERN})`;
+  if (NESTED_QUANTIFIER.test(p)) return 'refused: pattern carries a nested quantifier of the (x+)+ family, which is exponential to evaluate';
+  return null;
+}
 
 function runOneCheck(check, output) {
   const text = String(output || '');
@@ -40,11 +58,15 @@ function runOneCheck(check, output) {
 // `checks`). Empty array when the case declares no checks.
 function runChecks(output, checks) {
   if (!Array.isArray(checks) || !checks.length) return [];
-  return checks.map((c) => ({
-    name: String((c && c.name) || (c && c.kind) || 'check'),
-    kind: c && c.kind,
-    pass: !!runOneCheck(c, output),
-  }));
+  return checks.map((c) => {
+    const reason = c && c.kind === 'regex' ? refusedPattern(c.pattern) : null;
+    if (reason) return { name: String((c && c.name) || (c && c.kind) || 'check'), kind: c && c.kind, pass: false, reason };
+    return {
+      name: String((c && c.name) || (c && c.kind) || 'check'),
+      kind: c && c.kind,
+      pass: !!runOneCheck(c, output),
+    };
+  });
 }
 
-module.exports = { runChecks, runOneCheck };
+module.exports = { runChecks, runOneCheck, refusedPattern, NESTED_QUANTIFIER };

package/lib/diff.js

@@ -59,7 +59,10 @@ function withSkillBands(receipt) {
 // which is exactly the v0.1 weakness v0.2 fixes).
 function aggWithBand(receipt) {
   const a = receipt.results.aggregates.with_skill;
-  return { mean: a.mean_score, stddev: a.stddev || 0 };
+  // v0.6 (spec 026 AC-7): a null band is carried as null and rendered as what
+  // it is; a legacy receipt with no aggregate band keeps its 0 (the archive's
+  // rendering is unmoved, AC-16).
+  return { mean: a.mean_score, stddev: a.stddev === null ? null : (a.stddev || 0), cases: a.case_count };
 }
 
 function fmt(n) { return n == null ? 'n/a' : (n >= 0 ? '+' : '') + n.toFixed(3); }
@@ -81,11 +84,41 @@ function pct(n) { return n == null ? 'n/a' : n.toFixed(3); }
 // not from one case's draws).
 
 function bandStr(x) {
+  if (x && x.mean == null) return 'n/a (0 cases)';
+  if (x && x.stddev == null) return `${x.mean.toFixed(3)} ± n/a (${x.cases === 1 ? '1 case' : 'no band'})`;
   const label = x && x.source;
   return `${x.mean.toFixed(3)} ± ${x.stddev.toFixed(3)}${label ? ` (${label})` : ''}`;
 }
 function short(h) { return h ? String(h).slice(0, 12) : 'n/a'; }
 
+// ── THE JUDGE IS PART OF THE INSTRUMENT (spec 026 AC-12, A4) ─────────────────
+// Two receipts graded by different judges are two measurements with different
+// instruments, and a verdict across them says nothing about the skill.
+// lib/reuse.js's triage already says a changed judge means regrade; the
+// differ did not ask. It asks here: run.judge.model_id (on canonical ids, so
+// an alias and its dated form are one judge), run.judge.prompt_template_hash,
+// run.judge.temperature, and every shared case's judge.rubric_hash. A field
+// that differs names itself in the caveats and suppresses every per-case
+// verdict. A pre-v0.6 receipt carries no template hash: the report says the
+// template is unrecorded on that side and still compares the rest.
+const canonicalJudgeId = (id) => String(id == null ? '' : id).replace(/-\d{8}$/, '');
+function judgeDiffers(a, b, ids, aB, bB) {
+  const ja = (a.run && a.run.judge) || {}; const jb = (b.run && b.run.judge) || {};
+  const problems = [];
+  const idA = ja.model_id || (a.results.cases.find((c) => c.judge && c.judge.model_id) || { judge: {} }).judge.model_id;
+  const idB = jb.model_id || (b.results.cases.find((c) => c.judge && c.judge.model_id) || { judge: {} }).judge.model_id;
+  if (idA && idB && canonicalJudgeId(idA) !== canonicalJudgeId(idB)) problems.push(`run.judge.model_id differs (${idA} vs ${idB})`);
+  const unrecorded = [];
+  if (!ja.prompt_template_hash) unrecorded.push('A'); if (!jb.prompt_template_hash) unrecorded.push('B');
+  if (ja.prompt_template_hash && jb.prompt_template_hash && ja.prompt_template_hash !== jb.prompt_template_hash) problems.push(`run.judge.prompt_template_hash differs (${short(ja.prompt_template_hash)} vs ${short(jb.prompt_template_hash)}): the grading template is a different judge`);
+  if (ja.temperature !== undefined && jb.temperature !== undefined && ja.temperature !== jb.temperature) problems.push(`run.judge.temperature differs (${ja.temperature} vs ${jb.temperature})`);
+  const rubricA = Object.fromEntries(a.results.cases.filter((c) => c.mode === 'with_skill' && c.judge).map((c) => [c.id, c.judge.rubric_hash]));
+  const rubricB = Object.fromEntries(b.results.cases.filter((c) => c.mode === 'with_skill' && c.judge).map((c) => [c.id, c.judge.rubric_hash]));
+  const moved = ids.filter((id) => rubricA[id] && rubricB[id] && rubricA[id] !== rubricB[id]);
+  if (moved.length) problems.push(`judge.rubric_hash differs on ${moved.length} shared case(s) (${moved.slice(0, 3).map((x) => `\`${x}\``).join(', ')}${moved.length > 3 ? ', …' : ''})`);
+  return { problems, unrecorded };
+}
+
 // The headline is a SUMMARY of the per-case band-overlap verdicts — the credibility
 // core lives per case (judge-sample bands), and the headline just aggregates it.
 // It deliberately does NOT run a separate band test on the aggregate mean: the
@@ -146,9 +179,15 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B', mode = 'release' }
   // improvement verdicts are never computed from evidence we did not run.
   const levelOf = (r) => r.verification_level || 'TESTED';
   const belowTested = [[labelA, a], [labelB, b]].filter(([, r]) => levelOf(r) !== 'TESTED');
+  // Spec 026 AC-4: an INCOMPLETE receipt on either side computes no per-case
+  // verdict. RECEIPT.md has said so since v0.3.1; the differ did not ask.
+  const incompleteSides = [[labelA, a], [labelB, b]].filter(([, r]) => r.run && r.run.status === 'incomplete');
+  // Spec 026 AC-12: a different judge on either side suppresses the verdict.
+  const judge = judgeDiffers(a, b, ids, aB, bB);
+  const judgeProblem = judge.problems.length ? judge.problems.join('; ') : null;
   // A REFUSAL IS A RESULT, and it suppresses the verdict exactly as an
   // untested input does: no delta is asserted, and the reason travels with it.
-  const measured = belowTested.length === 0 && !refusal;
+  const measured = belowTested.length === 0 && !refusal && incompleteSides.length === 0 && !judgeProblem;
 
   const perCase = ids.map((id) => {
     const before = aB[id] || null;
@@ -250,6 +289,15 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B', mode = 'release' }
   if (belowTested.length) {
     warnings.push(`verdicts NOT computed — ${belowTested.map(([l, r]) => `${l} is ${levelOf(r)}${r.run && r.run.source ? ` (${r.run.source})` : ''}`).join('; ')}. Drift verdicts require TESTED receipts on both sides; declared numbers are shown as context only (see /interop.html).`);
   }
+  if (incompleteSides.length) {
+    warnings.push(`verdicts NOT computed — ${incompleteSides.map(([l, r]) => `${l} is incomplete (run.status incomplete; it excluded ${r.run.failed_case_count || 0} case(s) whose arm could not be measured)`).join('; ')}. A receipt with an unmeasured arm is not evidence for a per-case verdict; its aggregates are shown as context only.`);
+  }
+  if (judgeProblem) {
+    warnings.push(`verdicts NOT computed — the two receipts were graded by different judges: ${judgeProblem}. A verdict across two instruments says nothing about the skill; regrade one side with the other's judge (lib/reuse.js triage: regrade).`);
+  }
+  if (judge.unrecorded.length) {
+    warnings.push(`judge template unrecorded on ${judge.unrecorded.map((x) => (x === 'A' ? labelA : labelB)).join(' and ')} (a pre-v0.6 receipt carries no run.judge.prompt_template_hash); the judge model id and the rubric hashes are compared, the template is not.`);
+  }
   // Cross-provider / cross-surface disclosure (Phase 6). A comparison across
   // providers is a skill-DURABILITY comparison across substrates, not model drift
   // over time; across surfaces, sampling control differs. Both are flagged so a
@@ -281,7 +329,11 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B', mode = 'release' }
     // THE HEADLINE NAMES THE CAUSE THAT ACTUALLY APPLIES. `belowTested.length`
     // was printed unconditionally, so a refused pair got "0 receipt(s) below
     // TESTED" — a statement measurably false of the receipts it was given.
-    if (refusal) {
+    if (incompleteSides.length) {
+      // An incomplete side is a fact about that receipt alone (spec 026 AC-4)
+      // and leads whatever else is wrong with the pair.
+      L.push(`**NOT MEASURED — ${incompleteSides.map(([l]) => l).join(' and ')} incomplete: a case's arm could not be measured, so no per-case verdict is computed.**`);
+    } else if (refusal) {
       // The reason is a complete sentence and already ends by saying no verdict
       // is asserted; prefixing that again produced "REFUSED — no verdict is
       // asserted. the baseline arm did not reproduce…" — a duplicated clause and
@@ -289,6 +341,8 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B', mode = 'release' }
       L.push(`**REFUSED — ${refusal.reason}**`);
     } else if (belowTested.length) {
       L.push(`**NOT MEASURED — ${belowTested.length} receipt(s) below TESTED. Drift verdicts require TESTED receipts on both sides; the declared numbers above are context, not band-verified evidence.**`);
+    } else if (judgeProblem) {
+      L.push(`**NOT MEASURED — different judges: ${judgeProblem}. No verdict crosses two instruments.**`);
     } else {
       L.push('**NOT MEASURED — no verdict is asserted.**');
     }
@@ -327,4 +381,4 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B', mode = 'release' }
   };
 }
 
-module.exports = { buildDriftReport, withSkillBands, revisionPairProblem };
+module.exports = { buildDriftReport, withSkillBands, revisionPairProblem, judgeDiffers };

package/lib/importers.js

@@ -17,7 +17,7 @@
 // compatibility contract) are documented in docs/interop.md.
 
 const { RECEIPT_SCHEMA_VERSION, RUNNER_VERSION, SUITE_FORMAT } = require('../config');
-const { sealReceipt } = require('./receipt');
+const { sealReceipt, BAND_RULE } = require('./receipt');
 const { mean, stddev, round, combineUncertainty, aggregateBands } = require('./stats');
 const { outcomeFor } = require('./run');
 const { inferProvider } = require('./provider');
@@ -64,11 +64,16 @@ function importedReceipt({ tool, skillName, skillVersion, suiteFormat, caseCount
       date_utc: dateUtc,
       registry: registryStatus(modelId),
       transcripts: 'none',
-      judge: judgeBlock,
+      // v0.6: the source tool's grader is the judge that ran; its template is
+      // not ours to hash (null, the one place the schema allows it).
+      judge: { ...judgeBlock, model_id: (cases.find((c) => c.judge && c.judge.model_id) || { judge: { model_id: 'unknown' } }).judge.model_id, prompt_template_hash: null },
+      // v0.6 (spec 026 AC-1): what answered. An external tool's harness did;
+      // nothing here attested a model, and nothing was spawned by us.
+      answered_by: { kind: 'external', attested: false, reported_model: null, reported_models: null, isolation: 'none' },
     },
     results: {
       cases,
-      aggregates: { with_skill: aggregateMode(withSkill), baseline: aggregateMode(baseline) },
+      aggregates: { with_skill: aggregateMode(withSkill), baseline: aggregateMode(baseline), band_rule: BAND_RULE },
     },
     comparison,
     verification_level: 'DECLARED',