driftproof 0.11.2 → 0.12.0

package/lib/counts.js

@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+// Receipt counts (spec 043, receipt spec v0.8). Two counts describe how a receipt's data
+// came to be, and they are kept apart because they measure different things (Report 006:
+// generation spread ran 3 to 7 times judge spread on the same cases):
+//
+//   generations_per_arm           independent candidate generations
+//   judge_samples_per_generation  judge samples taken of each generation
+//
+// A count sits at the narrowest scope that is honest about it: run.counts when it is
+// the same for every case of every arm generated in this run, run.arms.<mode>.counts
+// when it holds for one arm, results.cases[i].counts otherwise. An absent count is
+// unknown, never 1 (agentskills discussion #544, comment 18409751, rule 1).
+
+const KINDS = ['generations_per_arm', 'judge_samples_per_generation'];
+
+// An arm that carries its own generated_at was generated in an earlier run. run.counts
+// describes this run's generations, so an archived arm never inherits it.
+function archived(receipt, mode) {
+  const a = receipt.run && receipt.run.arms && receipt.run.arms[mode];
+  return !!(a && a.generated_at);
+}
+
+const has = (o, k) => !!o && Object.prototype.hasOwnProperty.call(o, k) && Number.isInteger(o[k]);
+
+// Where a reader looks, narrowest first.
+const ORDER = ['case', 'arm', 'run'];
+const SCOPES = {
+  case: (r, c) => c.counts,
+  arm: (r, c) => { const a = r.run && r.run.arms && r.run.arms[c.mode]; return a && a.counts; },
+  run: (r) => r.run && r.run.counts,
+};
+
+// The count of `kind` for results.cases[index], or null when the receipt does not
+// establish it. run.judge.samples is the run-scope judge count (R-5), read only where
+// run.counts is silent and only for an arm generated in this run.
+function resolveCount(receipt, index, kind) {
+  if (!KINDS.includes(kind)) throw new Error(`unknown count kind: ${kind}`);
+  const c = receipt.results.cases[index];
+  for (const scope of ORDER) {
+    const at = SCOPES[scope](receipt, c);
+    if (has(at, kind)) return at[kind];
+    if (scope === 'run' && kind === 'judge_samples_per_generation' && receipt.run && receipt.run.judge && Number.isInteger(receipt.run.judge.samples)) return receipt.run.judge.samples;
+    if (scope === 'arm' && archived(receipt, c.mode)) return null;
+  }
+  return null;
+}
+
+// Which model generated an arm, and when: the arm's override, else the run's.
+function resolveArm(receipt, mode) {
+  const a = (receipt.run && receipt.run.arms && receipt.run.arms[mode]) || {};
+  return {
+    model_id: a.model_id || receipt.run.model_id,
+    generated_at: a.generated_at || receipt.run.generated_at || null,
+    archived: archived(receipt, mode),
+  };
+}
+
+// Write counts at the narrowest honest scope (spec 043 AC-3). `perCase` is one entry per
+// results.cases index: { index, counts: { <kind>: integer } } with a kind left out when
+// the writer cannot establish it. Mutates and returns the receipt; call before sealing.
+function placeCounts(receipt, perCase) {
+  const cases = receipt.results.cases;
+  const fresh = perCase.filter((p) => !archived(receipt, cases[p.index].mode));
+  const put = (obj, kind, v) => { obj.counts = { ...(obj.counts || {}), [kind]: v }; };
+  for (const kind of KINDS) {
+    const entries = fresh.map((p) => ({ p, v: p.counts && Number.isInteger(p.counts[kind]) ? p.counts[kind] : null }));
+    if (!entries.length) continue;
+    const allKnown = entries.length === cases.filter((c) => !archived(receipt, c.mode)).length && entries.every((e) => e.v !== null);
+    const values = new Set(entries.map((e) => e.v));
+    if (allKnown && values.size === 1) { receipt.run.counts = { ...(receipt.run.counts || {}), [kind]: entries[0].v }; continue; }
+    const byMode = new Map();
+    for (const e of entries) { const m = cases[e.p.index].mode; if (!byMode.has(m)) byMode.set(m, []); byMode.get(m).push(e); }
+    for (const [mode, es] of byMode) {
+      const modeKnown = allKnown && es.length === cases.filter((c) => c.mode === mode).length;
+      if (modeKnown && new Set(es.map((e) => e.v)).size === 1) {
+        receipt.run.arms = receipt.run.arms || {};
+        receipt.run.arms[mode] = receipt.run.arms[mode] || {};
+        put(receipt.run.arms[mode], kind, es[0].v);
+      } else {
+        for (const e of es) if (e.v !== null) put(cases[e.p.index], kind, e.v);
+      }
+    }
+  }
+  return receipt;
+}
+
+// The judge-sample count, or null when the receipt does not establish one at run scope.
+function judgeSamplesOrNull(receipt) {
+  const j = receipt && receipt.run && receipt.run.judge;
+  if (j && Number.isInteger(j.samples)) return j.samples;
+  const c = receipt && receipt.run && receipt.run.counts;
+  return has(c, 'judge_samples_per_generation') ? c.judge_samples_per_generation : null;
+}
+
+// v0.8 loader rule (spec 043 AC-2): the two spellings of the run's judge count agree.
+function judgeCountsDisagree(receipt) {
+  const j = receipt && receipt.run && receipt.run.judge;
+  const c = receipt && receipt.run && receipt.run.counts;
+  return !!(j && Number.isInteger(j.samples) && has(c, 'judge_samples_per_generation') && c.judge_samples_per_generation !== j.samples);
+}
+
+module.exports = { KINDS, ORDER, resolveCount, resolveArm, placeCounts, judgeSamplesOrNull, judgeCountsDisagree };

package/lib/decision.js

@@ -17,10 +17,10 @@
 // here, in lib/, rather than in the shell, because `bin/driftproof badge` needs
 // it too and because a decision state is a property of a receipt.
 
-const fs = require('fs');
 const path = require('path');
 const { EFFECT_FLOOR } = require('../config');
-const { shortModel, githubOutputEntry, receiptVerdict, drawsLine, UNDERPOWERED_LINE } = require('./verdict');
+const { shortModel, githubOutputEntry, receiptVerdict, readCases, drawsLine, UNDERPOWERED_LINE } = require('./verdict');
+const { duplicateCaseRows, ambiguityLine, suiteNarrowed, listReceipts, fileSlug } = require('./receipt');
 
 // The six decision states (spec 030 AC-4).
 //
@@ -86,40 +86,57 @@ function failsJob(state, { failOnRegression = true } = {}) {
 // the fail-safe direction.
 function decisionState(receipt) {
   if (!receipt) return 'refused';
+  // Spec 050 AC-3: a receipt with two rows for one (id, mode) is `refused` before
+  // anything else is read. It is not `not measured`, which does not fail a job: the
+  // receipt may well have measured a regression, and which row a reader keeps decides
+  // whether that regression is seen. `refused` fails closed whatever
+  // fail-on-regression says, which is the direction an ambiguity has to fail.
+  if (duplicateCaseRows(receipt).length) return 'refused';
   const level = receipt.verification_level;
   const kind = receipt.run && receipt.run.answered_by ? receipt.run.answered_by.kind : undefined;
   if (level !== 'TESTED' || kind !== 'model') return 'not measured';
+  // Spec 062 (register row 1): THE MEASURED CASES BEFORE THE INCOMPLETE RULE. Until this an
+  // incomplete run read `inconclusive` before any case was read, so one unmeasured case hid a
+  // regression another case had measured: the same receipt read `regression`, exit 1, with every
+  // case measured, and `inconclusive`, exit 0, with one with-skill arm unmeasured (the register's
+  // pass 2 probe 7). A case that separated down was measured whatever happened to the others.
+  if (readCases(receipt).some((c) => c.state === 'separated-down')) return 'regression';
   const cmp = receipt.comparison || {};
-  if ((receipt.run && receipt.run.status === 'incomplete') || typeof cmp.delta !== 'number') return 'inconclusive';
+  // Incomplete: a case failed (run.status), or the receipt ran fewer cases than its suite holds
+  // (spec 062, register row 4).
+  if ((receipt.run && receipt.run.status === 'incomplete') || suiteNarrowed(receipt) || typeof cmp.delta !== 'number') return 'inconclusive';
   // Spec 035: the measured states are the receipt verdict's, read per case by the
   // band rule (lib/verdict.js receiptVerdict), not the aggregate delta against the floor.
-  // receiptVerdict refuses on the same four conditions the two clauses above split
-  // between `not measured` and `inconclusive`, so it cannot answer NOT_MEASURED here.
-  return MEASURED_STATE[receiptVerdict(receipt).verdict];
+  const verdict = receiptVerdict(receipt).verdict;
+  // Spec 062: every verdict has a state. NOT_MEASURED is reachable here - a receipt whose every
+  // case was dropped reads it by spec 036's no_readable_case rung - and until this it had no row
+  // in the table, so the state was undefined and the model left the set (pass 6 V3).
+  if (!Object.hasOwn(MEASURED_STATE, verdict)) throw new Error(`decisionState: the verdict ${JSON.stringify(verdict)} has no decision state`);
+  return MEASURED_STATE[verdict];
 }
 
-const MEASURED_STATE = { REGRESSED: 'regression', PASSED: 'helped', UNDERPOWERED: 'underpowered', NO_EFFECT: 'no detected effect' };
+const MEASURED_STATE = { REGRESSED: 'regression', PASSED: 'helped', UNDERPOWERED: 'underpowered', NO_EFFECT: 'no detected effect', NOT_MEASURED: 'not measured' };
 
 // The worst state in a set, by STATE_ORDER. An empty set has no decision, and
-// says so with null rather than defaulting to something benign.
+// says so with null rather than defaulting to something benign. A state this
+// ordering does not know is refused (spec 062): skipping it dropped its model
+// from the decision, which is a pass by omission.
 function worstState(states) {
   let worst = null;
   for (const s of states) {
     const i = STATE_ORDER.indexOf(s);
-    if (i < 0) continue;
+    if (i < 0) throw new Error(`worstState: unknown decision state ${JSON.stringify(s)}`);
     if (worst === null || i < STATE_ORDER.indexOf(worst)) worst = s;
   }
   return worst;
 }
 
-// Every receipt in a directory, excluding the summaries and the badge writer's
-// own earlier output. Mirrors what action/run.sh's glob selected from, so the
-// set this reads is the set that was there to be read.
+// Every receipt file in a directory, by lib/receipt.js listReceipts (spec 062): the files that
+// parse and carry receipt_hash or results and are no sidecar by name and shape (spec 069), and the
+// files that do not parse. Until spec 062 it was
+// every `.json` but the summaries and badge.json, so a regrade sidecar was read as a receipt.
 function receiptFiles(dir) {
-  return fs.readdirSync(dir)
-    .filter((f) => f.endsWith('.json') && !f.includes('.summary.') && f !== 'badge.json')
-    .sort()
-    .map((f) => path.join(dir, f));
+  return listReceipts(dir).map((e) => e.file);
 }
 
 // Parse the `models` input the same way the runner does: comma-separated, with
@@ -152,9 +169,11 @@ function matchesModel(receiptModelId, requested) {
 function fileNamesModel(file, requested) {
   const base = path.basename(file);
   // Both spellings, for the reason `matchesModel` takes both: a receipt for
-  // `claude-haiku-4-5-20251001` answers a request for `claude-haiku-4-5`.
+  // `claude-haiku-4-5-20251001` answers a request for `claude-haiku-4-5`. Each is
+  // looked for in the form the runner writes it, lib/receipt.js fileSlug (spec 062): an
+  // id with a dot or a capital is not in the file name as itself.
   for (const id of new Set([requested, shortModel(requested)])) {
-    if (base.includes(`-${id}-`)) return true;
+    if (base.includes(`-${fileSlug(id)}-`)) return true;
   }
   return false;
 }
@@ -166,37 +185,49 @@ function fileNamesModel(file, requested) {
 // row that is simply absent. That is the whole of AC-2: absence is not a pass.
 function decideSet(dir, requestedCsv, { failOnRegression = true } = {}) {
   const requested = parseModels(requestedCsv);
-  const files = receiptFiles(dir);
-  const loaded = files.map((f) => {
-    let receipt = null, error = null;
-    try { receipt = JSON.parse(fs.readFileSync(f, 'utf8')); } catch (e) { error = e.message; }
-    return { file: f, receipt, error };
-  });
+  const loaded = listReceipts(dir);
+  const files = loaded.map((l) => l.file);
   const used = new Set();
   const rows = requested.map((model) => {
-    const hit = loaded.find((l) => !used.has(l.file) && l.receipt
+    // Spec 050 AC-6: EVERY readable receipt that matches, not the first. Until this the
+    // row took `loaded.find(...)`, the first match in sorted filename order, and reported
+    // the rest as unexpected. A job with two Driftproof steps shared one directory, so
+    // the second step's model row was filled by whichever skill's receipt sorted first
+    // (an outside correctness audit, finding 2; spec 030's F-1). Two receipts for one
+    // model are a question nobody answered, so the row is `refused`, names every file,
+    // and reads none of them.
+    const matches = loaded.filter((l) => !used.has(l.file) && l.receipt
       && l.receipt.run && matchesModel(l.receipt.run.model_id, model));
-    if (hit) used.add(hit.file);
+    for (const m of matches) used.add(m.file);
+    const many = matches.length > 1;
+    const hit = matches.length === 1 ? matches[0] : null;
     // A receipt that exists and cannot be parsed is UNREADABLE, not absent.
     // Both fail closed, and the row says which - the register's
     // absence-vs-unreadable distinction, kept at the point it is decided.
-    const unreadable = !hit && loaded.find((l) => !used.has(l.file) && l.error
+    const unreadable = !hit && !many && loaded.find((l) => !used.has(l.file) && l.error
       && fileNamesModel(l.file, model));
     if (unreadable) used.add(unreadable.file);
     const receipt = hit ? hit.receipt : null;
+    // Spec 050 AC-3: one receipt, but one with two rows for a case and arm.
+    const dups = receipt ? duplicateCaseRows(receipt) : [];
+    const ambiguous = many
+      ? `${matches.length} receipts for this model (${matches.map((m) => path.basename(m.file)).join(', ')}); none was read`
+      : (dups.length ? `receipt ambiguous: ${ambiguityLine(dups)}` : null);
     const state = decisionState(receipt);
     return {
       model,
       state,
-      delta: receipt && receipt.comparison && typeof receipt.comparison.delta === 'number'
+      // An ambiguous row carries no delta: a delta is a reading, and none was taken.
+      delta: !ambiguous && receipt && receipt.comparison && typeof receipt.comparison.delta === 'number'
         ? receipt.comparison.delta : null,
-      file: hit ? path.basename(hit.file) : (unreadable ? path.basename(unreadable.file) : null),
+      file: many ? null : hit ? path.basename(hit.file) : (unreadable ? path.basename(unreadable.file) : null),
       // Spec 035 AC-3: what an underpowered row would have needed, from the same reading.
       drawsNeeded: state === 'underpowered' ? receiptVerdict(receipt).drawsNeeded : null,
       // The distinction is carried on the ROW, not left in a sentence, because
       // the enforcement message has to say which of the two happened (AC-2's
       // mutation class, absence-vs-unreadable).
       unreadable: unreadable ? unreadable.error : null,
+      ambiguous,
       // A SHORT CAUSE, not the parser's text. V8's JSON.parse message echoes the
       // first bytes of the input - "Unexpected token '|', \"|{bad\" is not valid
       // JSON" - and this string is rendered into a markdown table cell, where one
@@ -204,7 +235,7 @@ function decideSet(dir, requestedCsv, { failOnRegression = true } = {}) {
       // reaches the annotation, which is not a table; the cell carries the cause,
       // and the filename is already beside it in the same cell (F-2 of
       // 2026-09-12).
-      reason: unreadable ? 'receipt unreadable (not valid JSON)' : (hit ? null : 'no receipt for this model'),
+      reason: ambiguous || (unreadable ? 'receipt unreadable (not valid JSON)' : (hit ? null : 'no receipt for this model')),
     };
   });
   // Receipts the run wrote for models nobody requested are reported rather than
@@ -226,9 +257,14 @@ function decideSet(dir, requestedCsv, { failOnRegression = true } = {}) {
     // split it by CAUSE: nothing was written for this model, or something was
     // written and cannot be read. Both fail closed; they are not the same fact,
     // and the failure has to say which.
-    absent: rows.filter((r) => r.state === 'refused' && !r.unreadable).map((r) => r.model),
+    absent: rows.filter((r) => r.state === 'refused' && !r.unreadable && !r.ambiguous).map((r) => r.model),
     unreadable: rows.filter((r) => r.state === 'refused' && r.unreadable)
       .map((r) => ({ model: r.model, file: r.file, error: r.unreadable })),
+    // Spec 050: a third cause of `refused`, apart from the other two for the same reason
+    // they are apart from each other. Something was written, and it can be read, but it
+    // does not say one thing.
+    ambiguous: rows.filter((r) => r.state === 'refused' && r.ambiguous)
+      .map((r) => ({ model: r.model, reason: r.ambiguous })),
     unexpected,
     receiptCount: files.length,
     requestedCount: requested.length,
@@ -390,6 +426,11 @@ function enforcementLines(d, { failOnRegression = true } = {}) {
       + `${d.unreadable.map((u) => `${u.model} exists and is unreadable (${u.file}: ${oneLine(u.error)})`).join('; ')}. `
       + `Failing closed: a receipt that cannot be read is not a pass, and is not the same as one that was never written.`);
   }
+  if (d.ambiguous && d.ambiguous.length) {
+    L.push(`::error title=Driftproof::The receipt set is ambiguous for `
+      + `${d.ambiguous.map((a) => `${a.model}: ${oneLine(a.reason, 240)}`).join('; ')}. `
+      + `Failing closed: a decision that depends on which receipt or row was read is not a pass.`);
+  }
   if (d.regressed.length) {
     const deltas = d.rows.filter((r) => r.state === 'regression')
       .map((r) => `${r.model} (delta ${r.delta})`).join(', ');

package/lib/diff.js

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 'use strict';
 
+const { judgeSamplesOrNull } = require('./counts');
 const { bandVerdict, round, WITHIN_NOISE } = require('./stats');
 const { EFFECT_FLOOR } = require('../config');
 const { revisionHeadline } = require('./revision');
@@ -158,6 +159,12 @@ function revisionPairProblem(a, b) {
   return null;
 }
 
+// The run's judge-sample count as a reader shows it: the number, or 'unknown'.
+function judgeN(r) {
+  const n = judgeSamplesOrNull(r);
+  return n === null ? 'unknown' : n;
+}
+
 function buildDriftReport(a, b, { labelA = 'A', labelB = 'B', mode = 'release' } = {}) {
   const revision = mode === 'revision';
 
@@ -253,7 +260,7 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B', mode = 'release' }
     L.push(`| surface (held) | ${a.run.surface} | ${b.run.surface} |`);
     L.push(`| suite_hash (held) | \`${short(a.suite.suite_hash)}\` | \`${short(b.suite.suite_hash)}\` |`);
     L.push(`| run date (UTC) | ${a.run.date_utc} | ${b.run.date_utc} |`);
-    L.push(`| judge samples/case | ${(a.run.judge || {}).samples || 1} | ${(b.run.judge || {}).samples || 1} |`);
+    L.push(`| judge samples/case | ${judgeN(a)} | ${judgeN(b)} |`);
     L.push(`| with_skill (mean ± band) | ${bandStr(aAgg)} | ${bandStr(bAgg)} |`);
     L.push(`| baseline score | ${pct(a.comparison.baseline_score)} | ${pct(b.comparison.baseline_score)} |`);
     L.push(`| skill lift (Δ) | ${fmt(a.comparison.delta)} | ${fmt(b.comparison.delta)} |`);
@@ -268,7 +275,7 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B', mode = 'release' }
   L.push(`| model | \`${a.run.model_id}\` | \`${b.run.model_id}\` |`);
   L.push(`| run date (UTC) | ${a.run.date_utc} | ${b.run.date_utc} |`);
   L.push(`| surface | ${a.run.surface} | ${b.run.surface} |`);
-  L.push(`| judge samples/case | ${(a.run.judge || {}).samples || 1} | ${(b.run.judge || {}).samples || 1} |`);
+  L.push(`| judge samples/case | ${judgeN(a)} | ${judgeN(b)} |`);
   L.push(`| skill content_hash | \`${short(a.skill.content_hash)}\` | \`${short(b.skill.content_hash)}\` |`);
   L.push(`| suite_hash | \`${short(a.suite.suite_hash)}\` | \`${short(b.suite.suite_hash)}\` |`);
   L.push(`| with_skill (mean ± band) | ${bandStr(aAgg)} | ${bandStr(bAgg)} |`);
@@ -289,7 +296,9 @@ function buildDriftReport(a, b, { labelA = 'A', labelB = 'B', mode = 'release' }
   if (!revision && a.skill.content_hash !== b.skill.content_hash) warnings.push('skill content_hash differs — the skill itself changed between receipts, so drift mixes skill edits with model drift.');
   if (!revision && a.suite.suite_hash !== b.suite.suite_hash) warnings.push('suite_hash differs — the eval suite changed; per-case comparison may be misleading.');
   if (a.skill.name !== b.skill.name) warnings.push(`different skills (${a.skill.name} vs ${b.skill.name}) — comparison is not meaningful.`);
-  if ((a.run.judge || {}).samples <= 1 || (b.run.judge || {}).samples <= 1) warnings.push('one or both receipts are single-sample (no bands) — non-overlap can only be trusted when both sides are sampled.');
+  // v0.8 (spec 043 AC-1): an absent judge count is unknown, said as such, never read as 1.
+  if (judgeN(a) === 'unknown' || judgeN(b) === 'unknown') warnings.push('the judge-sample count is unknown on one or both receipts: the receipt does not establish it, so whether the bands are sampled cannot be read from it.');
+  if (judgeN(a) === 1 || judgeN(b) === 1) warnings.push('one or both receipts are single-sample (no bands): non-overlap can only be trusted when both sides are sampled.');
   // WHAT SUPPRESSED THE VERDICT IS SAID, AND IT IS SAID CORRECTLY (spec 016 AC-3
   // and AC-4, closing F-015-B).
   //

package/lib/export.js

@@ -8,13 +8,16 @@
 // Documented in docs/interop.md; snapshot-tested in the gate.
 
 const { verdictFromReceipt } = require('./verdict');
+const { SUMMARY_SIDECAR } = require('./receipt');
 
-const SUMMARY_FORMAT = 'driftproof/summary';
+const SUMMARY_FORMAT = SUMMARY_SIDECAR.format;
 const SUMMARY_FORMAT_VERSION = '1';
 
 // Build the summary object for one receipt. Deterministic for a given receipt:
 // fixed key order, no export-time timestamps. `reportUrl` is caller-supplied
 // (receipts do not know where their report lives), else null.
+const { judgeSamplesOrNull } = require('./counts');
+
 function toSummaryJson(receipt, { reportUrl = null } = {}) {
   const agg = receipt.results.aggregates;
   const cmp = receipt.comparison || {};
@@ -41,7 +44,8 @@ function toSummaryJson(receipt, { reportUrl = null } = {}) {
     source: receipt.run.source || 'driftproof',
     judge: {
       model_id: (receipt.results.cases.find((c) => c.judge) || { judge: { model_id: 'unknown' } }).judge.model_id,
-      samples: (receipt.run.judge || {}).samples || 1,
+      // v0.8 (spec 043 AC-1): null when the receipt does not establish the count, never 1.
+      samples: judgeSamplesOrNull(receipt),
     },
     receipt_hash: receipt.receipt_hash,
     report_url: reportUrl,