driftproof 0.8.1 → 0.10.0

package/lib/checks.js

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 'use strict';
 
+const { CHECK_MAX_PATTERN } = require('../config');
+
 // Deterministic post-checks.
 //
 // A per-case eval suite may declare optional `checks[]`: structural / regex
@@ -19,6 +21,22 @@
 //   contains     — the output includes the literal `value` substring
 //   not_contains — the output does NOT include the literal `value` substring
 //   min_length   — the trimmed output is at least `value` characters long
+//
+// BOUNDED (spec 026 AC-13, audit A7). A suite's regex is third-party input
+// compiled and run against model output, and JavaScript cannot time a regex
+// out. A pattern longer than CHECK_MAX_PATTERN, or one carrying a nested
+// quantifier of the (x+)+ family (a quantified group whose body ends in a
+// quantifier: the shape that is exponential on every backtracking engine),
+// is REFUSED: recorded pass: false with a reason that says so, never
+// evaluated. The test is syntactic and narrow by design; it is not a general
+// ReDoS detector, and the spec says so.
+const NESTED_QUANTIFIER = /\((?:[^()\\]|\\.)*[+*}]\)\s*[+*]|\((?:[^()\\]|\\.)*\|(?:[^()\\]|\\.)*\)\s*[+*]/;
+function refusedPattern(pattern) {
+  const p = String(pattern == null ? '' : pattern);
+  if (p.length > CHECK_MAX_PATTERN) return `refused: pattern of ${p.length} characters exceeds CHECK_MAX_PATTERN (${CHECK_MAX_PATTERN})`;
+  if (NESTED_QUANTIFIER.test(p)) return 'refused: pattern carries a nested quantifier of the (x+)+ family, which is exponential to evaluate';
+  return null;
+}
 
 function runOneCheck(check, output) {
   const text = String(output || '');
@@ -40,11 +58,15 @@ function runOneCheck(check, output) {
 // `checks`). Empty array when the case declares no checks.
 function runChecks(output, checks) {
   if (!Array.isArray(checks) || !checks.length) return [];
-  return checks.map((c) => ({
-    name: String((c && c.name) || (c && c.kind) || 'check'),
-    kind: c && c.kind,
-    pass: !!runOneCheck(c, output),
-  }));
+  return checks.map((c) => {
+    const reason = c && c.kind === 'regex' ? refusedPattern(c.pattern) : null;
+    if (reason) return { name: String((c && c.name) || (c && c.kind) || 'check'), kind: c && c.kind, pass: false, reason };
+    return {
+      name: String((c && c.name) || (c && c.kind) || 'check'),
+      kind: c && c.kind,
+      pass: !!runOneCheck(c, output),
+    };
+  });
 }
 
-module.exports = { runChecks, runOneCheck };
+module.exports = { runChecks, runOneCheck, refusedPattern, NESTED_QUANTIFIER };

package/lib/decision.js

@@ -0,0 +1,425 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+// The decision a RUN reaches, over every receipt it produced (spec 030).
+//
+// `lib/verdict.js` answers a question about ONE receipt: does the skill still
+// help on THIS model? That question is still asked and still answered there,
+// unchanged. This module answers the one the action actually needs and never
+// had a place to ask: a run takes a LIST of models and must reach a single
+// decision over all of them.
+//
+// Until spec 030, `action/run.sh` picked one receipt with `ls … | head -1` and
+// decided the job from it. Receipts are named `<skill>-<model>-<date>.json`, so
+// that pick is ordered by model id: `claude-haiku-4-5` sorts before
+// `claude-sonnet-5`, and a regression on any model that did not sort first was
+// reported as a pass, with a green check and a brightgreen badge. The seam lives
+// here, in lib/, rather than in the shell, because `bin/driftproof badge` needs
+// it too and because a decision state is a property of a receipt.
+
+const fs = require('fs');
+const path = require('path');
+const { EFFECT_FLOOR } = require('../config');
+const { shortModel, githubOutputEntry } = require('./verdict');
+
+// The six decision states (spec 030 AC-4).
+//
+// `helped` is the sixth, and is a RECORDED DEVIATION from the brief's five
+// (spec.md A-030-1): a model on which the skill measurably helped needs a row
+// that says so. `no detected effect` is false about such a model - the effect
+// was detected and it cleared the floor - and widening it to mean "not a
+// regression" would put a measured lift and a measured nothing in one cell.
+const STATES = ['regression', 'refused', 'inconclusive', 'not measured', 'no detected effect', 'helped'];
+
+// WORST FIRST. This is the single definition of the ordering; AC-1's
+// enforcement and AC-3's badge both read it, so they cannot disagree about
+// which of two states governs a run.
+const STATE_ORDER = STATES.slice();
+
+// States that must never render as success on any surface the action writes -
+// the badge, the summary row, or the check title (AC-4).
+const NEVER_SUCCESS = ['inconclusive', 'not measured'];
+
+// States that fail the job. `refused` fails CLOSED (AC-2): a model that
+// produced no receipt is not a model that passed. `regression` fails subject to
+// fail-on-regression. An unmeasured run does not fail on its own (AC-4's rule)
+// - it is not evidence that the skill hurt - but it never renders as success.
+function failsJob(state, { failOnRegression = true } = {}) {
+  // AC-2. `refused` fails CLOSED and is NOT subject to fail-on-regression: that
+  // input says what to do about a measured regression, and a model that
+  // produced no receipt was not measured at all. A run that silently narrowed
+  // to the models that survived would be AC-1's defect with a different cause -
+  // the decision taken over a subset nobody chose.
+  if (state === 'refused') return true;
+  if (state === 'regression') return !!failOnRegression;
+  return false;
+}
+
+// The mapping rule of spec 030 AC-4, in order; the first match wins. A null or
+// missing receipt is `refused`.
+//
+// `not measured` is tested BEFORE `inconclusive` and before any delta is read,
+// because a receipt below TESTED has not measured anything its delta could be
+// about - spec 026 AC-1 is what makes a stub receipt unable to claim TESTED,
+// and reading its delta first would grade a run that never ran.
+// The rule is read off AC-4's table and nothing is supplied that the receipt
+// does not carry. Until this it read
+//
+//   const level = receipt.verification_level || 'TESTED';
+//   ... || (kind && kind !== 'model')
+//
+// - a receipt with no `verification_level` was GRADED ON ITS DELTA as though it
+// had claimed TESTED, and a receipt with no `answered_by` block had the second
+// clause skipped entirely. Both defaults point the same way: they let a receipt
+// that says nothing about how it was answered reach `helped`. AC-4 says each
+// state SHALL be derived "by the rule below and by no other", and the rule says
+// `verification_level != "TESTED"` and `run.answered_by.kind != "model"` - an
+// absent field satisfies both. The receipts the shipped runner writes always
+// carry a level and an answered_by block (lib/run.js answeredByOf always
+// returns a kind), so this changes nothing about a run the action produced; it
+// changes what happens to a receipt from somewhere else, and it changes it in
+// the fail-safe direction.
+function decisionState(receipt) {
+  if (!receipt) return 'refused';
+  const level = receipt.verification_level;
+  const kind = receipt.run && receipt.run.answered_by ? receipt.run.answered_by.kind : undefined;
+  if (level !== 'TESTED' || kind !== 'model') return 'not measured';
+  const cmp = receipt.comparison || {};
+  if ((receipt.run && receipt.run.status === 'incomplete') || typeof cmp.delta !== 'number') return 'inconclusive';
+  if (cmp.delta <= -EFFECT_FLOOR) return 'regression';
+  if (cmp.delta >= EFFECT_FLOOR) return 'helped';
+  return 'no detected effect';
+}
+
+// The worst state in a set, by STATE_ORDER. An empty set has no decision, and
+// says so with null rather than defaulting to something benign.
+function worstState(states) {
+  let worst = null;
+  for (const s of states) {
+    const i = STATE_ORDER.indexOf(s);
+    if (i < 0) continue;
+    if (worst === null || i < STATE_ORDER.indexOf(worst)) worst = s;
+  }
+  return worst;
+}
+
+// Every receipt in a directory, excluding the summaries and the badge writer's
+// own earlier output. Mirrors what action/run.sh's glob selected from, so the
+// set this reads is the set that was there to be read.
+function receiptFiles(dir) {
+  return fs.readdirSync(dir)
+    .filter((f) => f.endsWith('.json') && !f.includes('.summary.') && f !== 'badge.json')
+    .sort()
+    .map((f) => path.join(dir, f));
+}
+
+// Parse the `models` input the same way the runner does: comma-separated, with
+// surrounding whitespace ignored and empty entries dropped.
+function parseModels(csv) {
+  return String(csv || '').split(',').map((s) => s.trim()).filter(Boolean);
+}
+
+// A receipt matches a requested id exactly, or after the release date stamp is
+// stripped from both - `claude-haiku-4-5-20251001` IS `claude-haiku-4-5`, which
+// is the equivalence lib/verdict.js's shortModel already defines for the badge.
+function matchesModel(receiptModelId, requested) {
+  return receiptModelId === requested || shortModel(receiptModelId) === shortModel(requested);
+}
+
+// The same question for a receipt that CANNOT BE READ. `matchesModel` reads
+// `run.model_id`, and for an unreadable receipt that field is inside the bytes
+// that will not parse. The FILENAME is the only thing left that names a model:
+// the runner writes `<skill>-<model>-<date>.json`, so the id appears in the
+// basename as a dash-delimited run.
+//
+// Until this, the unreadable branch matched by POSITION - the first unreadable
+// file in the directory, whichever model it belonged to. With one requested model
+// absent and another's receipt unreadable, the two causes were hung on the wrong
+// models: the absent model was reported as "exists and is unreadable" naming the
+// OTHER model's file, and the model whose file was actually corrupt was reported
+// as having produced nothing. Both still failed closed and both ids still
+// appeared somewhere, which is why AC-2's arms stayed green - they drove one
+// cause at a time and never both at once (F-1 of 2026-09-12).
+function fileNamesModel(file, requested) {
+  const base = path.basename(file);
+  // Both spellings, for the reason `matchesModel` takes both: a receipt for
+  // `claude-haiku-4-5-20251001` answers a request for `claude-haiku-4-5`.
+  for (const id of new Set([requested, shortModel(requested)])) {
+    if (base.includes(`-${id}-`)) return true;
+  }
+  return false;
+}
+
+// The decision over a run: one row per REQUESTED model, in the order requested.
+//
+// Rows come from the requested list rather than from the directory listing, so
+// a model that produced no receipt is a row that says `refused` instead of a
+// row that is simply absent. That is the whole of AC-2: absence is not a pass.
+function decideSet(dir, requestedCsv, { failOnRegression = true } = {}) {
+  const requested = parseModels(requestedCsv);
+  const files = receiptFiles(dir);
+  const loaded = files.map((f) => {
+    let receipt = null, error = null;
+    try { receipt = JSON.parse(fs.readFileSync(f, 'utf8')); } catch (e) { error = e.message; }
+    return { file: f, receipt, error };
+  });
+  const used = new Set();
+  const rows = requested.map((model) => {
+    const hit = loaded.find((l) => !used.has(l.file) && l.receipt
+      && l.receipt.run && matchesModel(l.receipt.run.model_id, model));
+    if (hit) used.add(hit.file);
+    // A receipt that exists and cannot be parsed is UNREADABLE, not absent.
+    // Both fail closed, and the row says which - the register's
+    // absence-vs-unreadable distinction, kept at the point it is decided.
+    const unreadable = !hit && loaded.find((l) => !used.has(l.file) && l.error
+      && fileNamesModel(l.file, model));
+    if (unreadable) used.add(unreadable.file);
+    const receipt = hit ? hit.receipt : null;
+    const state = decisionState(receipt);
+    return {
+      model,
+      state,
+      delta: receipt && receipt.comparison && typeof receipt.comparison.delta === 'number'
+        ? receipt.comparison.delta : null,
+      file: hit ? path.basename(hit.file) : (unreadable ? path.basename(unreadable.file) : null),
+      // The distinction is carried on the ROW, not left in a sentence, because
+      // the enforcement message has to say which of the two happened (AC-2's
+      // mutation class, absence-vs-unreadable).
+      unreadable: unreadable ? unreadable.error : null,
+      // A SHORT CAUSE, not the parser's text. V8's JSON.parse message echoes the
+      // first bytes of the input - "Unexpected token '|', \"|{bad\" is not valid
+      // JSON" - and this string is rendered into a markdown table cell, where one
+      // unescaped `|` adds a column and a newline ends the row. The detail still
+      // reaches the annotation, which is not a table; the cell carries the cause,
+      // and the filename is already beside it in the same cell (F-2 of
+      // 2026-09-12).
+      reason: unreadable ? 'receipt unreadable (not valid JSON)' : (hit ? null : 'no receipt for this model'),
+    };
+  });
+  // Receipts the run wrote for models nobody requested are reported rather than
+  // dropped: they are evidence that the run did something other than what it
+  // was asked for.
+  const unexpected = loaded.filter((l) => !used.has(l.file))
+    // An unreadable file whose name matches no requested model is still reported,
+    // and reported AS unreadable. Calling it a receipt for a model nobody asked
+    // for would state as fact the one thing its bytes could not tell us.
+    .map((l) => (l.error ? `${path.basename(l.file)} (unreadable)` : path.basename(l.file)));
+  const worst = worstState(rows.map((r) => r.state));
+  return {
+    rows,
+    worst,
+    regressed: rows.filter((r) => r.state === 'regression').map((r) => r.model),
+    missing: rows.filter((r) => r.state === 'refused').map((r) => r.model),
+    // `missing` is every model that reached no readable receipt, which is what
+    // the missing_models output has always published. `absent` and `unreadable`
+    // split it by CAUSE: nothing was written for this model, or something was
+    // written and cannot be read. Both fail closed; they are not the same fact,
+    // and the failure has to say which.
+    absent: rows.filter((r) => r.state === 'refused' && !r.unreadable).map((r) => r.model),
+    unreadable: rows.filter((r) => r.state === 'refused' && r.unreadable)
+      .map((r) => ({ model: r.model, file: r.file, error: r.unreadable })),
+    unexpected,
+    receiptCount: files.length,
+    requestedCount: requested.length,
+    fails: rows.some((r) => failsJob(r.state, { failOnRegression })),
+  };
+}
+
+module.exports = {
+  STATES, STATE_ORDER, NEVER_SUCCESS, EFFECT_FLOOR,
+  decisionState, worstState, failsJob, decideSet, receiptFiles, parseModels, matchesModel,
+};
+
+// ── the three surfaces the action writes ────────────────────────────────────
+//
+// AC-4 binds all three: the badge, the summary row and the check title. A state
+// in NEVER_SUCCESS must not render as success on any of them, which is why the
+// renderers live together - three surfaces fed by one table cannot drift apart
+// the way three hand-written strings do.
+
+// How each state renders. `color` is the shields colour; `marker` is the
+// summary row's marker. Only `helped` gets a success marker: `no detected
+// effect` is not a failure but it is not a success either, and the two
+// NEVER_SUCCESS states carry a warning.
+const RENDER = {
+  regression: { word: 'regressed', color: 'red', marker: '\u274c' },
+  refused: { word: 'refused', color: 'red', marker: '\u274c' },
+  inconclusive: { word: 'inconclusive', color: 'yellow', marker: '\u26a0\ufe0f' },
+  'not measured': { word: 'not measured', color: 'lightgrey', marker: '\u26a0\ufe0f' },
+  'no detected effect': { word: 'no effect', color: 'lightgrey', marker: '\u2014' },
+  helped: { word: 'passing', color: 'brightgreen', marker: '\u2705' },
+};
+
+// The one place a state becomes a success claim. AC-4's clause is enforced here
+// rather than restated at each surface.
+function rendersAsSuccess(state) {
+  return state === 'helped';
+}
+
+// The badge over a SET (AC-3): the worst state, naming the model it came from,
+// and how many models share it when more than one does.
+function badgeMessage(d) {
+  const worst = d.worst;
+  if (!worst) return 'no receipts';
+  const r = RENDER[worst];
+  const sharing = d.rows.filter((row) => row.state === worst);
+  const named = sharing[0] ? sharing[0].model : 'unknown';
+  const extra = sharing.length > 1 ? ` (+${sharing.length - 1} more)` : '';
+  return `${r.word} on ${shortModel(named)}${extra}`;
+}
+
+function badgeEndpointForSet(d, { label = 'driftproof' } = {}) {
+  const worst = d.worst;
+  return {
+    schemaVersion: 1,
+    label,
+    message: badgeMessage(d),
+    color: worst ? RENDER[worst].color : 'lightgrey',
+  };
+}
+
+// The step outputs. `verdict` keeps the vocabulary lib/verdict.js publishes, so
+// a workflow reading steps.run.outputs.verdict is not broken by this change;
+// `worst`, `regressed` and `missing` are new and carry what it could not say.
+const VERDICT_WORD = {
+  regression: 'REGRESSED', refused: 'REFUSED', inconclusive: 'INCONCLUSIVE',
+  'not measured': 'NOT_MEASURED', 'no detected effect': 'NO_EFFECT', helped: 'PASSED',
+};
+
+function githubOutputLines(d) {
+  const worst = d.worst;
+  const badge = badgeEndpointForSet(d);
+  const worstRow = d.rows.find((r) => r.state === worst);
+  return [
+    ['verdict', worst ? VERDICT_WORD[worst] : 'NOT_MEASURED'],
+    ['delta', worstRow && worstRow.delta !== null ? worstRow.delta : 0],
+    ['message', badge.message],
+    ['color', badge.color],
+    ['worst_state', worst || 'none'],
+    ['regressed_models', d.regressed.join(',')],
+    ['missing_models', d.missing.join(',')],
+    ['receipt_count', d.receiptCount],
+  ].map(([k, v]) => githubOutputEntry(k, v)).join('\n');
+}
+
+// Anything reaching a table cell from outside this module - a filename, a reason,
+// a model id - is escaped FOR THE CELL it sits in. A markdown row is delimited by
+// `|`, so one unescaped pipe adds a column and the row stops lining up with its
+// header; a newline ends the row outright. The cause AC-4's own finding had was a
+// parser echo, and that echo is gone from the cell - but the filename is still
+// read off a directory, so the row is made to survive the byte rather than made
+// to depend on where the byte came from.
+function cell(s) {
+  return s === null || s === undefined ? '' : String(s).replace(/\r?\n/g, ' ').replace(/\|/g, '\\|');
+}
+
+// The job summary: one row per REQUESTED model, in the order requested (AC-4).
+function summaryMarkdown(d) {
+  const L = [];
+  L.push('### Driftproof');
+  L.push('');
+  L.push('| Model | Decision | Lift | Receipt |');
+  L.push('|---|---|---|---|');
+  for (const row of d.rows) {
+    const r = RENDER[row.state];
+    const delta = row.delta === null ? 'n/a' : (row.delta >= 0 ? '+' : '') + row.delta.toFixed(3);
+    const note = row.reason ? ` <br><sub>${cell(row.reason)}</sub>` : '';
+    L.push(`| \`${cell(row.model)}\` | ${r.marker} ${row.state} | ${delta} | ${cell(row.file) || '\u2014'}${note} |`);
+  }
+  L.push('');
+  L.push(`Decided over ${d.receiptCount} receipt(s) for ${d.requestedCount} requested model(s). `
+    + `Worst: **${d.worst || 'none'}**.`);
+  if (d.unexpected.length) {
+    L.push('');
+    L.push(`Receipts in the directory that no requested model claimed: ${d.unexpected.join(', ')}.`);
+  }
+  return L.join('\n');
+}
+
+// The check title and the human line the enforcement step prints.
+//
+// The message names THE MODELS THAT REGRESSED, never the requested list. The
+// old line interpolated the requested list, so one model's regression was
+// reported against every model asked for - the same defect as the selection,
+// seen from the other side.
+// The parser's message still reaches the ANNOTATION, where the detail is worth
+// having - but a workflow command is ONE LINE: a newline inside it ends the
+// command and drops everything after, and an unbounded echo of a file's bytes
+// does not belong in a check annotation either. One line, bounded, cut marked.
+function oneLine(s, max = 160) {
+  const flat = String(s).replace(/\s+/g, ' ').trim();
+  return flat.length > max ? `${flat.slice(0, max - 1)}\u2026` : flat;
+}
+
+function enforcementLines(d, { failOnRegression = true } = {}) {
+  const L = [];
+  const per = d.rows.map((r) => `${r.model}: ${r.state}`).join(', ');
+  L.push(`Driftproof decision over ${d.receiptCount} receipt(s) for ${d.requestedCount} model(s) - ${per}`);
+  // AC-2: the count is reported before any verdict, and names each model that
+  // reached no readable receipt. Absence is not a pass.
+  //
+  // The two causes are reported apart (absence-vs-unreadable). A model with NO
+  // receipt and a model whose receipt exists and cannot be parsed both fail
+  // closed, but they send the reader to different places - one to the run that
+  // never wrote it, one to the file that is there - so a single message naming
+  // both as "no receipt was produced" would be false about one of them.
+  if (d.absent.length) {
+    L.push(`::error title=Driftproof::No receipt was produced for ${d.absent.join(', ')} - `
+      + `${d.receiptCount} receipt(s) for ${d.requestedCount} requested model(s). `
+      + `Failing closed: a model that was not measured did not pass.`);
+  }
+  if (d.unreadable.length) {
+    L.push(`::error title=Driftproof::The receipt for `
+      + `${d.unreadable.map((u) => `${u.model} exists and is unreadable (${u.file}: ${oneLine(u.error)})`).join('; ')}. `
+      + `Failing closed: a receipt that cannot be read is not a pass, and is not the same as one that was never written.`);
+  }
+  if (d.regressed.length) {
+    const deltas = d.rows.filter((r) => r.state === 'regression')
+      .map((r) => `${r.model} (delta ${r.delta})`).join(', ');
+    if (failOnRegression) {
+      L.push(`::error title=Driftproof::Skill REGRESSED on ${deltas}`);
+    } else {
+      L.push(`::warning title=Driftproof::Skill REGRESSED on ${deltas} (fail-on-regression is false)`);
+    }
+  }
+  // AC-4: an unmeasured run renders as unmeasured, never as passing, and does
+  // not fail the job on its own.
+  const unmeasured = d.rows.filter((r) => NEVER_SUCCESS.includes(r.state));
+  if (unmeasured.length) {
+    // AC-4 binds the TITLE: a `::warning` "whose title names the state and the
+    // models it came from". Until this it named `unmeasured[0].state` and no
+    // model at all - both facts were in the MESSAGE, which is not where the
+    // criterion puts them, and with two never-success states present the title
+    // carried only the first row's. A reader who sees the annotation collapsed
+    // to its title in a check list saw neither the second state nor any model
+    // (F-4 of 2026-09-12).
+    //
+    // Grouped by state, worst first, from STATE_ORDER - the same single ordering
+    // AC-1's enforcement and AC-3's badge read, so the title cannot disagree with
+    // them about which state governs.
+    //
+    // NO COMMA IN THE TITLE, and that is load-bearing rather than a style
+    // choice: GitHub parses a workflow command's properties as a
+    // COMMA-SEPARATED list, so `title=not measured,inconclusive` ends the title
+    // at the comma and leaves the rest to be read as a property GitHub does not
+    // know. States are joined with ' / ' and the models under one state with
+    // ' + ', neither of which GitHub reads.
+    const states = STATE_ORDER.filter((s) => unmeasured.some((r) => r.state === s));
+    const title = states
+      .map((s) => `${s} on ${unmeasured.filter((r) => r.state === s).map((r) => r.model).join(' + ')}`)
+      .join(' / ');
+    L.push(`::warning title=Driftproof: ${title}::`
+      + `${unmeasured.map((r) => `${r.model}: ${r.state}`).join(', ')} - `
+      + `this run did not measure these models, and is not a pass on them.`);
+  }
+  return L;
+}
+
+module.exports.RENDER = RENDER;
+module.exports.rendersAsSuccess = rendersAsSuccess;
+module.exports.badgeMessage = badgeMessage;
+module.exports.badgeEndpointForSet = badgeEndpointForSet;
+module.exports.githubOutputLines = githubOutputLines;
+module.exports.summaryMarkdown = summaryMarkdown;
+module.exports.enforcementLines = enforcementLines;
+module.exports.VERDICT_WORD = VERDICT_WORD;

package/lib/diff.js

@@ -59,7 +59,10 @@ function withSkillBands(receipt) {
 // which is exactly the v0.1 weakness v0.2 fixes).
 function aggWithBand(receipt) {
   const a = receipt.results.aggregates.with_skill;
-  return { mean: a.mean_score, stddev: a.stddev || 0 };
+  // v0.6 (spec 026 AC-7): a null band is carried as null and rendered as what
+  // it is; a legacy receipt with no aggregate band keeps its 0 (the archive's
+  // rendering is unmoved, AC-16).
+  return { mean: a.mean_score, stddev: a.stddev === null ? null : (a.stddev || 0), cases: a.case_count };
 }
 
 function fmt(n) { return n == null ? 'n/a' : (n >= 0 ? '+' : '') + n.toFixed(3); }
@@ -81,11 +84,41 @@ function pct(n) { return n == null ? 'n/a' : n.toFixed(3); }
 // not from one case's draws).
 
 function bandStr(x) {
+  if (x && x.mean == null) return 'n/a (0 cases)';
+  if (x && x.stddev == null) return `${x.mean.toFixed(3)} ± n/a (${x.cases === 1 ? '1 case' : 'no band'})`;
   const label = x && x.source;
   return `${x.mean.toFixed(3)} ± ${x.stddev.toFixed(3)}${label ? ` (${label})` : ''}`;
 }
 function short(h) { return h ? String(h).slice(0, 12) : 'n/a'; }
 
+// ── THE JUDGE IS PART OF THE INSTRUMENT (spec 026 AC-12, A4) ─────────────────
+// Two receipts graded by different judges are two measurements with different
+// instruments, and a verdict across them says nothing about the skill.
+// lib/reuse.js's triage already says a changed judge means regrade; the
+// differ did not ask. It asks here: run.judge.model_id (on canonical ids, so
+// an alias and its dated form are one judge), run.judge.prompt_template_hash,
+// run.judge.temperature, and every shared case's judge.rubric_hash. A field
+// that differs names itself in the caveats and suppresses every per-case
+// verdict. A pre-v0.6 receipt carries no template hash: the report says the
+// template is unrecorded on that side and still compares the rest.
+const canonicalJudgeId = (id) => String(id == null ? '' : id).replace(/-\d{8}$/, '');
+function judgeDiffers(a, b, ids, aB, bB) {
+  const ja = (a.run && a.run.judge) || {}; const jb = (b.run && b.run.judge) || {};
+  const problems = [];
+  const idA = ja.model_id || (a.results.cases.find((c) => c.judge && c.judge.model_id) || { judge: {} }).judge.model_id;
+  const idB = jb.model_id || (b.results.cases.find((c) => c.judge && c.judge.model_id) || { judge: {} }).judge.model_id;
+  if (idA && idB && canonicalJudgeId(idA) !== canonicalJudgeId(idB)) problems.push(`run.judge.model_id differs (${idA} vs ${idB})`);
+  const unrecorded = [];
+  if (!ja.prompt_template_hash) unrecorded.push('A'); if (!jb.prompt_template_hash) unrecorded.push('B');
+  if (ja.prompt_template_hash && jb.prompt_template_hash && ja.prompt_template_hash !== jb.prompt_template_hash) problems.push(`run.judge.prompt_template_hash differs (${short(ja.prompt_template_hash)} vs ${short(jb.prompt_template_hash)}): the grading template is a different judge`);
+  if (ja.temperature !== undefined && jb.temperature !== undefined && ja.temperature !== jb.temperature) problems.push(`run.judge.temperature differs (${ja.temperature} vs ${jb.temperature})`);
+  const rubricA = Object.fromEntries(a.results.cases.filter((c) => c.mode === 'with_skill' && c.judge).map((c) => [c.id, c.judge.rubric_hash]));
+  const rubricB = Object.fromEntries(b.results.cases.filter((c) => c.mode === 'with_skill' && c.judge).map((c) => [c.id, c.judge.rubric_hash]));
+  const moved = ids.filter((id) => rubricA[id] && rubricB[id] && rubricA[id] !== rubricB[id]);
+  if (moved.length) problems.push(`judge.rubric_hash differs on ${moved.length} shared case(s) (${moved.slice(0, 3).map((x) => `\`${x}\``).join(', ')}${moved.length > 3 ? ', …' : ''})`);
+  return { problems, unrecorded };
+}
+
 // The headline is a SUMMARY of the per-case band-overlap verdicts — the credibility
 // core lives per case (judge-sample bands), and the headline just aggregates it.
 // It deliberately does NOT run a separate band test on the aggregate mean: the
@@ -146,9 +179,15 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B', mode = 'release' }
   // improvement verdicts are never computed from evidence we did not run.
   const levelOf = (r) => r.verification_level || 'TESTED';
   const belowTested = [[labelA, a], [labelB, b]].filter(([, r]) => levelOf(r) !== 'TESTED');
+  // Spec 026 AC-4: an INCOMPLETE receipt on either side computes no per-case
+  // verdict. RECEIPT.md has said so since v0.3.1; the differ did not ask.
+  const incompleteSides = [[labelA, a], [labelB, b]].filter(([, r]) => r.run && r.run.status === 'incomplete');
+  // Spec 026 AC-12: a different judge on either side suppresses the verdict.
+  const judge = judgeDiffers(a, b, ids, aB, bB);
+  const judgeProblem = judge.problems.length ? judge.problems.join('; ') : null;
   // A REFUSAL IS A RESULT, and it suppresses the verdict exactly as an
   // untested input does: no delta is asserted, and the reason travels with it.
-  const measured = belowTested.length === 0 && !refusal;
+  const measured = belowTested.length === 0 && !refusal && incompleteSides.length === 0 && !judgeProblem;
 
   const perCase = ids.map((id) => {
     const before = aB[id] || null;
@@ -250,6 +289,15 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B', mode = 'release' }
   if (belowTested.length) {
     warnings.push(`verdicts NOT computed — ${belowTested.map(([l, r]) => `${l} is ${levelOf(r)}${r.run && r.run.source ? ` (${r.run.source})` : ''}`).join('; ')}. Drift verdicts require TESTED receipts on both sides; declared numbers are shown as context only (see /interop.html).`);
   }
+  if (incompleteSides.length) {
+    warnings.push(`verdicts NOT computed — ${incompleteSides.map(([l, r]) => `${l} is incomplete (run.status incomplete; it excluded ${r.run.failed_case_count || 0} case(s) whose arm could not be measured)`).join('; ')}. A receipt with an unmeasured arm is not evidence for a per-case verdict; its aggregates are shown as context only.`);
+  }
+  if (judgeProblem) {
+    warnings.push(`verdicts NOT computed — the two receipts were graded by different judges: ${judgeProblem}. A verdict across two instruments says nothing about the skill; regrade one side with the other's judge (lib/reuse.js triage: regrade).`);
+  }
+  if (judge.unrecorded.length) {
+    warnings.push(`judge template unrecorded on ${judge.unrecorded.map((x) => (x === 'A' ? labelA : labelB)).join(' and ')} (a pre-v0.6 receipt carries no run.judge.prompt_template_hash); the judge model id and the rubric hashes are compared, the template is not.`);
+  }
   // Cross-provider / cross-surface disclosure (Phase 6). A comparison across
   // providers is a skill-DURABILITY comparison across substrates, not model drift
   // over time; across surfaces, sampling control differs. Both are flagged so a
@@ -281,7 +329,11 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B', mode = 'release' }
     // THE HEADLINE NAMES THE CAUSE THAT ACTUALLY APPLIES. `belowTested.length`
     // was printed unconditionally, so a refused pair got "0 receipt(s) below
     // TESTED" — a statement measurably false of the receipts it was given.
-    if (refusal) {
+    if (incompleteSides.length) {
+      // An incomplete side is a fact about that receipt alone (spec 026 AC-4)
+      // and leads whatever else is wrong with the pair.
+      L.push(`**NOT MEASURED — ${incompleteSides.map(([l]) => l).join(' and ')} incomplete: a case's arm could not be measured, so no per-case verdict is computed.**`);
+    } else if (refusal) {
       // The reason is a complete sentence and already ends by saying no verdict
       // is asserted; prefixing that again produced "REFUSED — no verdict is
       // asserted. the baseline arm did not reproduce…" — a duplicated clause and
@@ -289,6 +341,8 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B', mode = 'release' }
       L.push(`**REFUSED — ${refusal.reason}**`);
     } else if (belowTested.length) {
       L.push(`**NOT MEASURED — ${belowTested.length} receipt(s) below TESTED. Drift verdicts require TESTED receipts on both sides; the declared numbers above are context, not band-verified evidence.**`);
+    } else if (judgeProblem) {
+      L.push(`**NOT MEASURED — different judges: ${judgeProblem}. No verdict crosses two instruments.**`);
     } else {
       L.push('**NOT MEASURED — no verdict is asserted.**');
     }
@@ -327,4 +381,4 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B', mode = 'release' }
   };
 }
 
-module.exports = { buildDriftReport, withSkillBands, revisionPairProblem };
+module.exports = { buildDriftReport, withSkillBands, revisionPairProblem, judgeDiffers };

package/lib/importers.js

@@ -17,7 +17,7 @@
 // compatibility contract) are documented in docs/interop.md.
 
 const { RECEIPT_SCHEMA_VERSION, RUNNER_VERSION, SUITE_FORMAT } = require('../config');
-const { sealReceipt } = require('./receipt');
+const { sealReceipt, BAND_RULE } = require('./receipt');
 const { mean, stddev, round, combineUncertainty, aggregateBands } = require('./stats');
 const { outcomeFor } = require('./run');
 const { inferProvider } = require('./provider');
@@ -64,11 +64,16 @@ function importedReceipt({ tool, skillName, skillVersion, suiteFormat, caseCount
       date_utc: dateUtc,
       registry: registryStatus(modelId),
       transcripts: 'none',
-      judge: judgeBlock,
+      // v0.6: the source tool's grader is the judge that ran; its template is
+      // not ours to hash (null, the one place the schema allows it).
+      judge: { ...judgeBlock, model_id: (cases.find((c) => c.judge && c.judge.model_id) || { judge: { model_id: 'unknown' } }).judge.model_id, prompt_template_hash: null },
+      // v0.6 (spec 026 AC-1): what answered. An external tool's harness did;
+      // nothing here attested a model, and nothing was spawned by us.
+      answered_by: { kind: 'external', attested: false, reported_model: null, reported_models: null, isolation: 'none' },
     },
     results: {
       cases,
-      aggregates: { with_skill: aggregateMode(withSkill), baseline: aggregateMode(baseline) },
+      aggregates: { with_skill: aggregateMode(withSkill), baseline: aggregateMode(baseline), band_rule: BAND_RULE },
     },
     comparison,
     verification_level: 'DECLARED',