driftproof 0.5.0 → 0.7.1

package/lib/diff.js

@@ -3,6 +3,8 @@
 
 const { bandVerdict, round } = require('./stats');
 const { EFFECT_FLOOR } = require('../config');
+const { revisionHeadline } = require('./revision');
+const { baselineReproduces, REFUSAL_REASONS, bandOf } = require('./reuse');
 
 // Practical-significance gate applied ON TOP of band separation. bandVerdict()
 // stays a pure geometry test (kept that way so its unit checks are unambiguous);
@@ -31,11 +33,23 @@ function isWithinNoise(v) { return v === 'within noise' || v === WITHIN_NOISE_FL
 
 // Map case-id → { mean, stddev } for a receipt's with_skill cases. Falls back to
 // score/0 for v0.1 receipts that have no per-case band.
+// NO RENDERING SWITCHES. An earlier revision carried two const-true flags so each
+// half of F-015-B's fix could be removed in a probe — and the dead branch of one
+// of them contained, verbatim, the false headline AC-4 exists to forbid. Nothing
+// could fire it, and it was still a defect-restoring branch living in the tree
+// that gets published. The mutation probes patch this source in a disposable copy.
+
 function withSkillBands(receipt) {
   const out = {};
   for (const c of receipt.results.cases) {
     if (c.mode !== 'with_skill') continue;
-    out[c.id] = { mean: c.mean != null ? c.mean : c.score, stddev: c.stddev || 0 };
+  // ONE BAND DEFINITION (spec 016 AC-1). This built its own from the v0.4-shaped
+  // fields, which worked — and that is the point: three copies existed, two
+  // reading the legacy shape and one reading only v0.5, and the one that read
+  // only v0.5 was the one a cross-version control depended on (F-015-C). Routing
+  // every comparison path through `bandOf` means a future shape is added once.
+    const b = bandOf(c);
+    if (b) out[c.id] = { mean: b.mean, stddev: b.sd, source: b.source };
   }
   return out;
 }
@@ -50,7 +64,26 @@ function aggWithBand(receipt) {
 
 function fmt(n) { return n == null ? 'n/a' : (n >= 0 ? '+' : '') + n.toFixed(3); }
 function pct(n) { return n == null ? 'n/a' : n.toFixed(3); }
-function bandStr(x) { return `${x.mean.toFixed(3)} ± ${x.stddev.toFixed(3)}`; }
+// THE BAND SAYS WHICH BAND IT IS (spec 017 AC-7).
+//
+// `bandOf` computes `source: 'legacy' | 'generation'` and carries it onto every
+// band; nothing rendered it, so a reader comparing an archived receipt with a
+// v0.5 one was comparing a JUDGE-SAMPLE spread against an ACROSS-DRAW spread
+// with nothing on the page saying so. They are different statistics over
+// different things, and the comparison is still the only one v0.4 admits — which
+// is exactly why the page has to name them rather than leave them to look alike.
+//
+// THE MARKER IS THE RECEIPT'S OWN WORD — `legacy` or `generation`, exactly as
+// `bandOf` records it — rather than a prettier synonym. A reader who greps the
+// page for what a receipt says should find the same token; a rendering that
+// renames the thing it is disclosing has disclosed a different thing. Omitted
+// when a band carries no source (an aggregate band is computed from case means,
+// not from one case's draws).
+
+function bandStr(x) {
+  const label = x && x.source;
+  return `${x.mean.toFixed(3)} ± ${x.stddev.toFixed(3)}${label ? ` (${label})` : ''}`;
+}
 function short(h) { return h ? String(h).slice(0, 12) : 'n/a'; }
 
 // The headline is a SUMMARY of the per-case band-overlap verdicts — the credibility
@@ -68,7 +101,41 @@ function headlineVerdict(perCase) {
   return 'WITHIN NOISE — no case moved beyond its confidence band; the skill holds up.';
 }
 
-function buildDriftReport(a, b, { labelA = 'A', labelB = 'B' } = {}) {
+// A revision pair is a pair in which the SKILL TEXT is the only thing that
+// moved. `diff` was built for release drift, where the model varies and the text
+// is fixed; this inverts it, so the fields that release drift merely warns about
+// become the preconditions of the comparison. Returns the offending field name,
+// or null when the pair is a valid revision pair.
+function revisionPairProblem(a, b) {
+  if ((a.run.model_id || '') !== (b.run.model_id || '')) return 'run.model_id';
+  if ((a.run.provider || 'anthropic') !== (b.run.provider || 'anthropic')) return 'run.provider';
+  if ((a.run.surface || '') !== (b.run.surface || '')) return 'run.surface';
+  if ((a.suite.suite_hash || '') !== (b.suite.suite_hash || '')) return 'suite.suite_hash';
+  if ((a.skill.content_hash || '') === (b.skill.content_hash || '')) return 'skill.content_hash';
+  return null;
+}
+
+function buildDriftReport(a, b, { labelA = 'A', labelB = 'B', mode = 'release' } = {}) {
+  const revision = mode === 'revision';
+
+  // AC-6 (spec 014) — THE PRECONDITION, IN THE PATH THAT EMITS THE VERDICT.
+  //
+  // A comparison across schema versions assumes the two runs measured the same
+  // thing. The assumption is checkable: the no-skill arm contains no skill text,
+  // so nothing about a skill or a spec revision can move it. If it fails to
+  // reproduce, the comparison has no ground, and Report #006 is what that looks
+  // like when it is checked — 3 cells, 0 measured, 3 refused.
+  //
+  // SCOPED TO CROSS-VERSION PAIRS, which is what AC-6 names. Revision-mode pairs
+  // already carry their own register through revisionPairProblem(); widening
+  // this to every same-version comparison is a live question, recorded in
+  // tasks.md as OPEN-QUESTION-3 rather than decided here.
+  const crossVersion = String(a.schema_version || '') !== String(b.schema_version || '');
+  let refusal = null;
+  if (crossVersion) {
+    const pre = baselineReproduces(a, b);
+    if (!pre.ok) refusal = { key: pre.key, reason: REFUSAL_REASONS[pre.key](pre) };
+  }
   const aB = withSkillBands(a);
   const bB = withSkillBands(b);
   const ids = [...new Set([...Object.keys(aB), ...Object.keys(bB)])];
@@ -79,18 +146,21 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B' } = {}) {
   // improvement verdicts are never computed from evidence we did not run.
   const levelOf = (r) => r.verification_level || 'TESTED';
   const belowTested = [[labelA, a], [labelB, b]].filter(([, r]) => levelOf(r) !== 'TESTED');
-  const measured = belowTested.length === 0;
+  // A REFUSAL IS A RESULT, and it suppresses the verdict exactly as an
+  // untested input does: no delta is asserted, and the reason travels with it.
+  const measured = belowTested.length === 0 && !refusal;
 
   const perCase = ids.map((id) => {
     const before = aB[id] || null;
     const after = bB[id] || null;
     const delta = (before && after) ? round(after.mean - before.mean) : null;
-    const verdict = !measured ? 'not measured'
+    const verdict = refusal ? 'refused'
+      : !measured ? 'not measured'
       : (before && after) ? verdictWithFloor(before, after, delta) : 'n/a';
     return { id, before, after, delta, verdict };
   });
   // Sort worst-first: regressions, then by delta.
-  const order = { regression: 0, 'within noise': 1, [WITHIN_NOISE_FLOOR]: 1, improvement: 2, 'n/a': 3, 'not measured': 3 };
+  const order = { regression: 0, 'within noise': 1, [WITHIN_NOISE_FLOOR]: 1, improvement: 2, 'n/a': 3, 'not measured': 3, refused: 3 };
   perCase.sort((x, y) => (order[x.verdict] - order[y.verdict]) || ((x.delta || 0) - (y.delta || 0)));
 
   const aAgg = aggWithBand(a);
@@ -99,6 +169,32 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B' } = {}) {
   const regressions = perCase.filter((r) => r.verdict === 'regression');
 
   const L = [];
+  if (revision) {
+    // The axis leads. In a release-drift report the model is what moved and it
+    // belongs at the top; here the model is the control and the skill's own text
+    // is the finding, so content_hash is the first row a reader meets.
+    L.push(`# Revision drift report`);
+    L.push('');
+    L.push(`**Skill:** ${a.skill.name} \`${a.skill.version}\``);
+    L.push('');
+    L.push(`**Report type:** revision drift — the skill's own text is the variable under test; the substrate is held fixed.`);
+    L.push('');
+    L.push(`Left column is the **pinned** revision; right column is the **current** upstream revision.`);
+    L.push('');
+    L.push(`| | ${labelA} | ${labelB} |`);
+    L.push(`|---|---|---|`);
+    L.push(`| skill content_hash | \`${short(a.skill.content_hash)}\` | \`${short(b.skill.content_hash)}\` |`);
+    L.push(`| model (held) | \`${a.run.model_id}\` | \`${b.run.model_id}\` |`);
+    L.push(`| provider (held) | ${a.run.provider || 'anthropic'} | ${b.run.provider || 'anthropic'} |`);
+    L.push(`| surface (held) | ${a.run.surface} | ${b.run.surface} |`);
+    L.push(`| suite_hash (held) | \`${short(a.suite.suite_hash)}\` | \`${short(b.suite.suite_hash)}\` |`);
+    L.push(`| run date (UTC) | ${a.run.date_utc} | ${b.run.date_utc} |`);
+    L.push(`| judge samples/case | ${(a.run.judge || {}).samples || 1} | ${(b.run.judge || {}).samples || 1} |`);
+    L.push(`| with_skill (mean ± band) | ${bandStr(aAgg)} | ${bandStr(bAgg)} |`);
+    L.push(`| baseline score | ${pct(a.comparison.baseline_score)} | ${pct(b.comparison.baseline_score)} |`);
+    L.push(`| skill lift (Δ) | ${fmt(a.comparison.delta)} | ${fmt(b.comparison.delta)} |`);
+    L.push('');
+  } else {
   L.push(`# Drift report`);
   L.push('');
   L.push(`**Skill:** ${a.skill.name} \`${a.skill.version}\``);
@@ -115,19 +211,62 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B' } = {}) {
   L.push(`| baseline score | ${pct(a.comparison.baseline_score)} | ${pct(b.comparison.baseline_score)} |`);
   L.push(`| skill lift (Δ) | ${fmt(a.comparison.delta)} | ${fmt(b.comparison.delta)} |`);
   L.push('');
+  }
 
   const warnings = [];
-  if (a.skill.content_hash !== b.skill.content_hash) warnings.push('skill content_hash differs — the skill itself changed between receipts, so drift mixes skill edits with model drift.');
-  if (a.suite.suite_hash !== b.suite.suite_hash) warnings.push('suite_hash differs — the eval suite changed; per-case comparison may be misleading.');
+  // In revision mode the changed skill text is the INDEPENDENT VARIABLE, not
+  // contamination, and the held-constant substrate is what makes the comparison
+  // valid. The release-drift caveat below says the opposite of both, so it is
+  // replaced rather than suppressed: a reader is told what is held, and why a
+  // separated band is attributable to the revision.
+  if (revision) {
+    warnings.push(`model \`${a.run.model_id}\`, provider ${a.run.provider || 'anthropic'}, surface ${a.run.surface} and suite_hash \`${short(a.suite.suite_hash)}\` are held constant across both receipts — the skill text is the only variable under test, so a band-separated move above the ${EFFECT_FLOOR} floor is attributable to the revision.`);
+  }
+  if (!revision && a.skill.content_hash !== b.skill.content_hash) warnings.push('skill content_hash differs — the skill itself changed between receipts, so drift mixes skill edits with model drift.');
+  if (!revision && a.suite.suite_hash !== b.suite.suite_hash) warnings.push('suite_hash differs — the eval suite changed; per-case comparison may be misleading.');
   if (a.skill.name !== b.skill.name) warnings.push(`different skills (${a.skill.name} vs ${b.skill.name}) — comparison is not meaningful.`);
   if ((a.run.judge || {}).samples <= 1 || (b.run.judge || {}).samples <= 1) warnings.push('one or both receipts are single-sample (no bands) — non-overlap can only be trusted when both sides are sampled.');
-  if (!measured) warnings.push(`verdicts NOT computed — ${belowTested.map(([l, r]) => `${l} is ${levelOf(r)}${r.run && r.run.source ? ` (${r.run.source})` : ''}`).join('; ')}. Drift verdicts require TESTED receipts on both sides; declared numbers are shown as context only (see /interop.html).`);
+  // WHAT SUPPRESSED THE VERDICT IS SAID, AND IT IS SAID CORRECTLY (spec 016 AC-3
+  // and AC-4, closing F-015-B).
+  //
+  // TWO DIFFERENT THINGS can suppress a verdict, and this line used to describe
+  // only one of them. A REFUSAL — the baseline-reproduction precondition — set
+  // `measured` false and then the caveat rendered `belowTested`, which on a
+  // refused pair is EMPTY: the page read `verdicts NOT computed — .` under a
+  // headline claiming `0 receipt(s) below TESTED`, on a pair where both receipts
+  // were TESTED. An empty reason and a false statement, in the artifact a report's
+  // verdicts come from. The refusal's own cause-honest reason was computed into
+  // `refusal.reason` and thrown away by the renderer, so nothing a reader could
+  // see said why the comparison had stopped.
+  //
+  // Each half is proved load-bearing by a mutation that patches THIS source in a
+  // disposable copy. An earlier revision guarded them with const-true flags and
+  // this sentence described those; the flags were removed because a
+  // defect-restoring branch resident in the published tree is a hazard, and the
+  // sentence outlived them by one commit.
+  if (refusal) {
+    warnings.push(`verdicts NOT computed — the comparison was REFUSED before any verdict was formed: ${refusal.reason}`);
+  }
+  if (belowTested.length) {
+    warnings.push(`verdicts NOT computed — ${belowTested.map(([l, r]) => `${l} is ${levelOf(r)}${r.run && r.run.source ? ` (${r.run.source})` : ''}`).join('; ')}. Drift verdicts require TESTED receipts on both sides; declared numbers are shown as context only (see /interop.html).`);
+  }
   // Cross-provider / cross-surface disclosure (Phase 6). A comparison across
   // providers is a skill-DURABILITY comparison across substrates, not model drift
   // over time; across surfaces, sampling control differs. Both are flagged so a
   // reader never mistakes one for the other (see docs/neutrality.html).
-  if ((a.run.provider || 'anthropic') !== (b.run.provider || 'anthropic')) warnings.push(`different providers (${a.run.provider || 'anthropic'} vs ${b.run.provider || 'anthropic'}) — this is a cross-substrate durability comparison, not model drift over time; read the delta, not absolute scores (see the neutrality policy).`);
-  if (a.run.surface !== b.run.surface) warnings.push(`different surfaces (${a.run.surface} vs ${b.run.surface}) — sampling control differs between surfaces; compare with care.`);
+  if (!revision && (a.run.provider || 'anthropic') !== (b.run.provider || 'anthropic')) warnings.push(`different providers (${a.run.provider || 'anthropic'} vs ${b.run.provider || 'anthropic'}) — this is a cross-substrate durability comparison, not model drift over time; read the delta, not absolute scores (see the neutrality policy).`);
+  if (!revision && a.run.surface !== b.run.surface) warnings.push(`different surfaces (${a.run.surface} vs ${b.run.surface}) — sampling control differs between surfaces; compare with care.`);
+  // The legend for the band markers, printed whenever either side carries one.
+  // A two-letter marker a reader cannot decode is worse than no marker.
+  // GATED ON WHETHER A LABEL IS ACTUALLY RENDERED, not on whether the data
+  // carries a source. A legend explains markers on the page; if `bandStr` emits
+  // none, the legend is describing something the reader cannot see. Asked of
+  // `bandStr` itself rather than recomputed, so the two cannot disagree.
+  const anyLabelRendered = [...Object.values(aB), ...Object.values(bB)]
+    .some((x) => x && /\([a-z]+\)\s*$/.test(bandStr(x)));
+  if (anyLabelRendered) {
+    warnings.push('band provenance: `(generation)` is an ACROSS-DRAW spread — receipt spec v0.5, n generation draws per arm. `(legacy)` is a JUDGE-SAMPLE spread over a single generation, which is what v0.4 and earlier recorded. They are different statistics. The comparison is the only one the older receipt admits, and it is not like for like.');
+  }
   if (warnings.length) {
     L.push('> **⚠ Caveats**');
     for (const w of warnings) L.push(`> - ${w}`);
@@ -139,10 +278,23 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B' } = {}) {
   L.push(`## Headline`);
   L.push('');
   if (!measured) {
-    L.push(`**NOT MEASURED — ${belowTested.length} receipt(s) below TESTED. Drift verdicts require TESTED receipts on both sides; the declared numbers above are context, not band-verified evidence.**`);
+    // THE HEADLINE NAMES THE CAUSE THAT ACTUALLY APPLIES. `belowTested.length`
+    // was printed unconditionally, so a refused pair got "0 receipt(s) below
+    // TESTED" — a statement measurably false of the receipts it was given.
+    if (refusal) {
+      // The reason is a complete sentence and already ends by saying no verdict
+      // is asserted; prefixing that again produced "REFUSED — no verdict is
+      // asserted. the baseline arm did not reproduce…" — a duplicated clause and
+      // a lower-case sentence start, in the artifact a published report quotes.
+      L.push(`**REFUSED — ${refusal.reason}**`);
+    } else if (belowTested.length) {
+      L.push(`**NOT MEASURED — ${belowTested.length} receipt(s) below TESTED. Drift verdicts require TESTED receipts on both sides; the declared numbers above are context, not band-verified evidence.**`);
+    } else {
+      L.push('**NOT MEASURED — no verdict is asserted.**');
+    }
     L.push('');
   } else {
-    L.push(`**${headlineVerdict(perCase)}**`);
+    L.push(`**${revision ? revisionHeadline(perCase) : headlineVerdict(perCase)}**`);
     L.push('');
     L.push(`with_skill mean moved ${fmt(headlineDelta)} (${bandStr(aAgg)} → ${bandStr(bAgg)}; band = suite dispersion). Per-case band-overlap verdicts: ${regressions.length} regression(s), ${perCase.filter((r) => r.verdict === 'improvement').length} improvement(s), ${nWithin} within noise${nFloor ? ` (${nFloor} of them band-separated but below the ${EFFECT_FLOOR} effect floor)` : ''}.`);
     L.push('');
@@ -167,7 +319,12 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B' } = {}) {
     L.push('');
   }
 
-  return { markdown: L.join('\n'), perCase, headlineDelta, regressions };
+  return {
+    markdown: L.join('\n'), perCase, headlineDelta, regressions,
+    refused: !!refusal,
+    refusal_reason: refusal ? refusal.reason : null,
+    refusal_key: refusal ? refusal.key : null,
+  };
 }
 
-module.exports = { buildDriftReport, withSkillBands };
+module.exports = { buildDriftReport, withSkillBands, revisionPairProblem };

package/lib/hygiene.js

@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// The single definition of what counts as a leak.
+//
+// Two callers scan for the same classes of secret: `tests/gate.js`, which walks
+// the working tree at gate time, and `scripts/merge-check.js`, which walks the
+// candidate tree read out of the commit before permitting a merge. They must
+// agree. Two copies of a deny-list drift, and the copy that quietly stops
+// matching still reads as protection — the same argument the acknowledgment
+// deny-list bite test already makes, applied to the scanner itself.
+//
+// Backlog B-8 records four approval records that reached `dev` carrying a host
+// path. The recorded cause — "the scan walks tracked files" — is false, and this
+// module's existence does not depend on it: `listFiles()` in the repo gate has
+// always enumerated untracked files too, and a planted path in an untracked
+// evidence record does take the repo gate red. The real window is VANTAGE. An
+// approval session measures in a disposable copy made before it writes its
+// record into the shared checkout, so the record is simply not in the tree that
+// was scanned. Scanning the working tree harder cannot close that. Scanning the
+// commit that is about to merge does, and that is what merge-check now uses this
+// for.
+//
+// Patterns are written so a regex literal cannot match its own text: a
+// metacharacter or a character class follows each fixed prefix.
+
+const dec = (b64) => JSON.parse(Buffer.from(b64, 'base64').toString('utf8'));
+
+// Build-box host + private IP, base64 so the plaintext never lands in a file.
+const HOSTIP = dec('WyJpcC0xNzItMzEtNDAtMTU5LmFwLXNvdXRoZWFzdC0xLmNvbXB1dGUuaW50ZXJuYWwiLCIxNzIuMzEuNDAuMTU5Il0=');
+
+const EMAIL_ALLOW = new Set(['example.com', 'example.org', 'driftproofhq.com']);
+
+const PATTERNS = [
+  { name: 'home-path', re: /\/home\/[a-z0-9_-]+\/|\/Users\/[A-Za-z0-9_-]+\// },
+  { name: 'ec2-internal-host', re: /ip-\d+-\d+-\d+-\d+\.[a-z0-9.-]*compute\.(internal|amazonaws\.com)/i },
+  { name: 'private-ip', re: /\b(10\.\d{1,3}\.\d{1,3}\.\d{1,3}|172\.(1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3})\b/ },
+  { name: 'anthropic-key', re: /sk-ant-[A-Za-z0-9_-]{8,}/ },
+  { name: 'github-token', re: /gh[pousr]_[A-Za-z0-9]{20,}/ },
+  { name: 'aws-key', re: /\bAKIA[0-9A-Z]{16}\b/ },
+  { name: 'private-key-block', re: /-----BEGIN [A-Z ]*PRIVATE KEY-----/ },
+  { name: 'env-secret-assignment', re: /\b(ANTHROPIC_API_KEY|AWS_SECRET_ACCESS_KEY|OPENAI_API_KEY)\s*=\s*\S+/ },
+  // Codex subscription auth material (~/.codex/auth.json contents) must NEVER
+  // land in a committed file: the id_token/access_token are JWTs, and an OpenAI
+  // secret key is sk-proj-/sk-svcacct-/sk-admin-. We ban the CONTENTS (tokens),
+  // not the documented path string (`~/.codex/auth.json` is referenced in help
+  // text and docs by design).
+  { name: 'jwt-token', re: /\beyJ[A-Za-z0-9_=-]{10,}\.eyJ[A-Za-z0-9_=-]{10,}\.[A-Za-z0-9_=-]{6,}/ },
+  { name: 'openai-secret-key', re: /\bsk-(proj|svcacct|admin)-[A-Za-z0-9_-]{20,}/ },
+];
+
+// A path is a hit on its own name, with no content read: a committed .env file
+// is a leak whatever it happens to contain.
+function scanPath(rel) {
+  return /(^|\/)\.env(\.|$)/.test(rel) ? [{ file: rel, kind: 'env-file' }] : [];
+}
+
+function scanContent(rel, content) {
+  const hits = [];
+  for (const p of PATTERNS) {
+    const m = content.match(p.re);
+    if (m) hits.push({ file: rel, kind: p.name, sample: m[0].slice(0, 24) });
+  }
+  for (const hv of HOSTIP) if (content.includes(hv)) hits.push({ file: rel, kind: 'build-host-or-ip' });
+  const emails = content.match(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g) || [];
+  for (const e of emails) {
+    const dom = e.split('@')[1].toLowerCase();
+    if (!EMAIL_ALLOW.has(dom) && !dom.endsWith('.example')) hits.push({ file: rel, kind: 'email', sample: e });
+  }
+  return hits;
+}
+
+// scanFiles(files, read, readLink) — `read(rel)` returns the file's text, or
+// throws/returns null for anything unreadable (binary, deleted, a symlink to
+// nowhere), which is skipped exactly as the gate has always skipped it.
+//
+// `readLink(rel)` is optional and returns a symbolic link's TARGET PATH as a
+// string, or null for an entry that is not a link. Where it is supplied, that
+// target is scanned as the entry's content through `scanContent`, so every
+// pattern above applies to it with no second deny-list to drift (spec 011,
+// AC-2). A link's target is a path, and a path is exactly the class of leak the
+// `home-path` pattern exists to catch: P-1 shipped `node_modules ->
+// /home/<user>/... ` into a public tree past a scan that read the link as an
+// unreadable file and skipped it.
+//
+// The argument is optional so a caller supplying nothing behaves exactly as
+// before. That compatibility is deliberate, and it is also the standing risk: a
+// caller added later is blind by default. The repo gate asserts that every call
+// site supplies a reader, with `scripts/merge-check.js` the one exception — its
+// reader is `git show <ref>:<path>`, and git stores a link's blob as its target
+// string, so the target already arrives as content there.
+function scanFiles(files, read, readLink) {
+  const hits = [];
+  for (const rel of files) {
+    hits.push(...scanPath(rel));
+    // The link branch is a single condition on purpose: it is the mutation seam
+    // the spec-011 gate neutralises to prove the scanner goes blind without it.
+    if (typeof readLink === 'function') {
+      let target;
+      try { target = readLink(rel); } catch (_e) { target = null; }
+      if (typeof target === 'string') {
+        hits.push(...scanContent(rel, target));
+        continue;
+      }
+    }
+    let c;
+    try { c = read(rel); } catch (_e) { continue; }
+    if (typeof c !== 'string') continue;
+    hits.push(...scanContent(rel, c));
+  }
+  return hits;
+}
+
+module.exports = { PATTERNS, EMAIL_ALLOW, HOSTIP, scanPath, scanContent, scanFiles };

package/lib/judge.js

@@ -99,7 +99,12 @@ async function gradeOnce({ task, response, rubric, model, timeoutMs, temperature
 //   { samples:[scores], mean, stddev, reason, judge_settings, model_id, rubric_hash }
 // `mean` ± `stddev` is the per-case confidence band (raw spread of the N scores)
 // used by the borderline-outcome rule and per-case drift band-overlap logic.
-async function gradeSamples({ task, response, rubric, model, samples = 5, timeoutMs = 120000 }) {
+// NO DEFAULT TIMEOUT HERE (spec 017 AC-2). This defaulted to 120000, which
+// outranked the per-surface policy exactly as lib/run.js's literal did — so the
+// JUDGE calls timed out on the api policy while running on a CLI surface, which
+// is the second shadowing site and the one #007's prep session had not found.
+// Passing `undefined` through lets lib/provider.js resolve the declared policy.
+async function gradeSamples({ task, response, rubric, model, samples = 5, timeoutMs }) {
   const settings = judgeSettings(samples, model);
   const scores = [];
   const reasons = [];

package/lib/receipt.js

@@ -15,7 +15,12 @@ const SCHEMA_FILES = {
   '0.2': 'receipt.v0.2.schema.json',
   '0.3': 'receipt.v0.3.schema.json',
   '0.3.1': 'receipt.v0.3.1.schema.json',
-  '0.4': 'receipt.schema.json',
+  // v0.4 moved from the unversioned filename to a version-pinned one when v0.5
+  // took the current pointer. Without this the archive would lose v0.4: every
+  // published v0.4 receipt asserts conformance by NUMBER, and the number has to
+  // keep resolving to the schema it meant (AC-10).
+  '0.4': 'receipt.v0.4.schema.json',
+  '0.5': 'receipt.schema.json',
 };
 
 const _validators = {};
@@ -84,15 +89,70 @@ function aggregate(caseResults) {
 function buildReceipt({ skill, suite, run, cases, economics = null, verificationLevel = 'TESTED', editorialReviews = null }) {
   // v0.3.1: cases marked failed_timeout are recorded in results.cases but EXCLUDED
   // from aggregates — a band is never fabricated from a case that did not complete.
-  const okCases = cases.filter((c) => c.case_status !== 'failed_timeout');
-  const failedCount = cases.length - okCases.length;
+  //
+  // PAIRWISE, NOT PER ARM (spec 017 AC-5, AC-6). This filtered a FLAT list and
+  // then split by mode, so a case whose baseline failed kept its with_skill arm:
+  // #007's cell 3 recorded with_skill case_count 7 against baseline 6, and its
+  // headline `delta` of +0.116 was a 7-case mean minus a 6-case mean.
+  //
+  // `comparison.delta` is PAIRED BY CONSTRUCTION — one suite, measured twice —
+  // and a paired statistic computed over unequal sets is not the statistic it
+  // names. So an arm that cannot be measured removes its CASE from both sides.
+  //
+  // Exclusion, not refusal, and the reason is recorded rather than argued: an
+  // aggregate is a summary statistic, not a verdict, so the "refuse rather than
+  // assert" rule that governs verdicts does not reach it; and refusing the whole
+  // aggregate would discard thirteen sound arms because one failed. What the
+  // aggregate owes a reader instead is that it says what it covered, which
+  // `excluded_cases` provides.
+  //
+  // The excluded case STAYS in `results.cases`. It is removed from the mean, not
+  // from the record — deleting the evidence of a failure is a different and worse
+  // defect than averaging over it.
+  const armUnusable = (c) => c.case_status === 'failed_timeout'
+    || (c.mean == null && c.score == null);
+  const excludedIds = new Map();
+  for (const c of cases) {
+    if (!armUnusable(c)) continue;
+    if (excludedIds.has(c.id)) { excludedIds.get(c.id).modes.push(c.mode); continue; }
+    excludedIds.set(c.id, {
+      id: c.id,
+      modes: [c.mode],
+      reason: c.reason
+        || (c.generation && c.generation.stopping_reason === 'unmeasured_exhausted'
+          ? 'every generation draw was unmeasured'
+          : 'the arm has no measured result'),
+    });
+  }
+  const okCases = cases.filter((c) => !excludedIds.has(c.id));
   const withSkill = okCases.filter((c) => c.mode === 'with_skill');
   const baseline = okCases.filter((c) => c.mode === 'baseline');
+  const excludedCases = [...excludedIds.values()];
+  // `run.failed_case_count` COUNTS CASES, and is computed FROM the list it
+  // summarises rather than alongside it, so the two fields cannot disagree.
+  //
+  // It was `cases.length - okCases.length`, which counted ROWS. That was the
+  // same number while exclusion was per arm; once AC-5 made exclusion pairwise
+  // the subtraction removed BOTH arms of every excluded case, so one failed arm
+  // reported 2. Measured on the archive before this fix: #006's writing-plans
+  // receipt recomputed to 4 against the 2 it records, and #007's to 2 against 1
+  // — two published figures doubled by a change that never named this field.
+  //
+  // The field's name says cases and the aggregates exclude by case, so the
+  // count is the length of `results.aggregates.excluded_cases` and nothing
+  // else. A case whose BOTH arms failed is one exclusion and counts once.
+  const failedCount = excludedCases.length;
   const aggWith = aggregate(withSkill);
   const aggBase = aggregate(baseline);
 
   const receipt = {
     schema_version: RECEIPT_SCHEMA_VERSION,
+    // v0.5 CAPABILITY FLAG (F-014-F). DERIVED FROM THE CASES THEMSELVES, never
+    // from a caller's argument: the exposure this closes is a receipt asserting
+    // something it does not carry, and a flag taken on trust from the caller
+    // would be the same defect with an extra step. Absent when no case carries a
+    // draw set, which is what keeps every legacy and imported receipt valid.
+    ...(cases.some((c) => c && c.generation) ? { generation_sampled: true } : {}),
     skill: {
       name: skill.name,
       version: skill.version,
@@ -102,6 +162,12 @@ function buildReceipt({ skill, suite, run, cases, economics = null, verification
       format: suite.format,
       suite_hash: suite.suiteHash,
       case_count: suite.caseCount,
+      // v0.5: the per-suite canary. THIS is the canonical assembly — the runner
+      // built the field and this function dropped it, so the live smoke emitted
+      // `canary: undefined` and the schema, which makes it optional, said
+      // nothing. Caught by reading the receipt a real run produced, not by a
+      // gate; the assertion that would have caught it is added with the fix.
+      ...(suite.canary ? { canary: suite.canary } : {}),
     },
     run: {
       model_id: run.model_id,
@@ -120,7 +186,13 @@ function buildReceipt({ skill, suite, run, cases, economics = null, verification
     },
     results: {
       cases,
-      aggregates: { with_skill: aggWith, baseline: aggBase },
+      aggregates: {
+        with_skill: aggWith,
+        baseline: aggBase,
+        // Present only when something was excluded, so a clean run's receipt is
+        // unchanged and the archive does not acquire an empty field.
+        ...(excludedCases.length ? { excluded_cases: excludedCases } : {}),
+      },
     },
     comparison: {
       with_skill_score: aggWith.mean_score,