driftproof 0.11.2 → 0.12.0

package/lib/reuse.js

@@ -21,6 +21,9 @@
 //      receipt's own recorded provenance — model, suite and skill content force
 //      a rerun because the generation would differ; judge and rubric force a
 //      regrade because only the scoring would; neither changing permits reuse.
+//      Spec 053 made it one comparison of two provenance records, shared with
+//      `driftproof stale` (lib/stale.js), and added the harness and the judge's
+//      template; see PROVENANCE below.
 //
 // PURE: two receipts in, a decision out. No I/O, no provider. F-009-K's lesson
 // applied — the decision derives from the artifact, never from a filename.
@@ -120,15 +123,147 @@ function compare(older, newer) {
   return { verdict: 'MEASURED', delta: round(b - a), baseline_reproduced: true };
 }
 
+// ── PROVENANCE: ONE COMPARISON FOR TRIAGE AND STALE (spec 053 AC-1, AC-2) ─────
+//
+// What a receipt records about how its numbers were made, and one pure function
+// that compares two such records: two receipts (`triage`), or a receipt and what
+// would run today (`driftproof stale`, lib/stale.js). Before spec 053 `triage`
+// read `run.rubric_hash`, which no receipt carries (rubric hashes are per case),
+// and never looked at the judge's template or the harness, so a rubric or
+// template change was reused and a harness change went unseen.
+//
+// UNKNOWN IS NEVER CURRENT. An axis either side does not record is `unknown`,
+// with the reason, and an arm with an unknown axis is not reused.
+//
+// THE SUITE HASH COVERS PROMPTS AND RUBRICS TOGETHER ({id, prompt, rubric,
+// pass_threshold}), and no receipt records a per-case prompt hash. So a
+// different suite reruns both arms even when only a rubric moved: the record
+// cannot show the prompts did not. A case's rubric hash moving under an equal
+// suite hash (the rubric hash also covers the judge's system prompt) regrades
+// that case (spec 053 R-3).
+
+// The axes, in the order a reader is shown them, with what a difference does
+// and to which arms.
+const AXES = [
+  { axis: 'model', effect: 'rerun', arms: ['with_skill', 'baseline'], why: 'the model differs, so the generation would differ' },
+  { axis: 'harness', effect: 'rerun', arms: ['with_skill', 'baseline'], why: 'the harness differs, so the generation would differ' },
+  { axis: 'skill', effect: 'rerun', arms: ['with_skill'], why: 'the skill content differs, so the with-skill generation would differ; the baseline arm contains no skill text and still stands' },
+  { axis: 'suite', effect: 'rerun', arms: ['with_skill', 'baseline'], why: 'the suite differs, so the prompts may differ' },
+  { axis: 'judge_model', effect: 'regrade', arms: ['with_skill', 'baseline'], why: 'the judge model differs, so the existing generations can be graded again' },
+  { axis: 'judge_template', effect: 'regrade', arms: ['with_skill', 'baseline'], why: 'the judge prompt template differs, so the existing generations can be graded again' },
+  { axis: 'rubric', effect: 'regrade', arms: ['with_skill', 'baseline'], why: 'a case\'s rubric hash differs under the same suite, so that case can be graded again' },
+];
+const UNRECORDED = { model: 'no model id', harness: 'no harness (receipts before v0.9 carry none)', skill: 'no skill content hash', suite: 'no suite hash', judge_model: 'no judge model', judge_template: 'no judge prompt template hash (receipts before v0.6 carry none)', rubric: 'no per-case rubric hash' };
+
+const canonicalId = (id) => (id == null ? null : String(id).replace(/-\d{8}$/, ''));
+
+// THE HARNESS BY SEMVER (spec 053 A-053-1, the operator's instruction of 24 Sep 2026, a departure
+// from the brief's exact match). A major or minor change (2.1 to 2.2) reruns both arms; a patch-only
+// change (2.1.281 to 2.1.282) is an advisory. Claude Code ships a patch release every few days, and
+// an exact match would mark every receipt stale within days. A version that is not semver, or a
+// different harness, must match exactly.
+const semver = (v) => { const m = /^(\d+)\.(\d+)\.(\d+)(.*)$/.exec(String(v)); return m ? [m[1], m[2], m[3], m[4]] : null; };
+function harnessDrift(a, b) {
+  if (a.name !== b.name) return 'moved';
+  if (a.version === b.version) return 'same';
+  const x = semver(a.version); const y = semver(b.version);
+  if (!x || !y) return 'moved';
+  if (x[0] === y[0] && x[1] === y[1]) return x[2] === y[2] && x[3] === y[3] ? 'same' : 'patch';
+  return 'moved';
+}
+
+// The provenance a receipt records. Every field is null when the receipt does
+// not record it; an imported receipt's `unknown` model is null.
+function provenanceOf(r) {
+  const run = (r && r.run) || {};
+  const judge = run.judge || {};
+  const cases = (r && r.results && Array.isArray(r.results.cases)) ? r.results.cases : [];
+  const rubrics = {};
+  for (const c of cases) if (c && c.judge && typeof c.judge.rubric_hash === 'string' && !(c.id in rubrics)) rubrics[c.id] = c.judge.rubric_hash;
+  const firstCaseJudge = (cases.find((c) => c && c.judge && c.judge.model_id) || { judge: {} }).judge.model_id;
+  return {
+    model_id: run.model_id && run.model_id !== 'unknown' ? run.model_id : null,
+    harness: run.harness && typeof run.harness.name === 'string' ? { name: run.harness.name, version: run.harness.version == null ? null : String(run.harness.version) } : null,
+    skill_hash: (r && r.skill && r.skill.content_hash) || null,
+    suite_hash: (r && r.suite && r.suite.suite_hash) || null,
+    case_ids: cases.length ? [...new Set(cases.map((c) => c.id))].sort() : null,
+    judge_model_id: judge.model_id || firstCaseJudge || null,
+    judge_template_hash: judge.prompt_template_hash || null,
+    rubrics: Object.keys(rubrics).length ? rubrics : null,
+  };
+}
+
+const show = (axis, v) => {
+  if (v == null) return null;
+  if (axis === 'harness') return v.version == null ? v.name : `${v.name} ${v.version}`;
+  if (axis === 'rubric' && typeof v === 'object') return `${Object.keys(v).length} case(s)`;
+  return v;
+};
+
+// One entry per axis: { axis, recorded, current, effect, arms, reason[, cases, cases_added, cases_removed] }.
+// `why` optionally carries a reason for a current value that could not be found
+// (lib/stale.js: "no --skill given", "claude --version failed: ...").
+function compareProvenance(rec, cur, why = {}) {
+  const out = [];
+  const entry = (a, recorded, current, same, extra = {}) => {
+    const unknownSide = recorded == null ? `the receipt records ${UNRECORDED[a.axis]}` : current == null ? (why[a.axis] || `the other side records ${UNRECORDED[a.axis]}`) : null;
+    if (unknownSide) return out.push({ axis: a.axis, recorded: show(a.axis, recorded), current: show(a.axis, current), effect: 'unknown', arms: a.arms, reason: unknownSide, ...extra });
+    if (same) return out.push({ axis: a.axis, recorded: show(a.axis, recorded), current: show(a.axis, current), effect: 'current', arms: a.arms, reason: 'unchanged', ...extra });
+    return out.push({ axis: a.axis, recorded: show(a.axis, recorded), current: show(a.axis, current), effect: a.effect, arms: a.arms, reason: a.why, ...extra });
+  };
+  const A = Object.fromEntries(AXES.map((a) => [a.axis, a]));
+  entry(A.model, rec.model_id, cur.model_id, canonicalId(rec.model_id) === canonicalId(cur.model_id));
+  // A harness with no version recorded is unknown, except an API surface, which has no harness to drift.
+  const h = (x) => (x && (x.version != null || x.name === 'api') ? x : null);
+  const drift = h(rec.harness) && h(cur.harness) ? harnessDrift(rec.harness, cur.harness) : null;
+  if (drift === 'patch') out.push({ axis: 'harness', recorded: show('harness', rec.harness), current: show('harness', cur.harness), effect: 'advisory', arms: A.harness.arms, reason: 'a patch release of the same harness: an advisory, not a rerun (spec 053 A-053-1)' });
+  else entry(A.harness, h(rec.harness), h(cur.harness), drift === 'same');
+  entry(A.skill, rec.skill_hash, cur.skill_hash, rec.skill_hash === cur.skill_hash);
+  const added = rec.case_ids && cur.case_ids ? cur.case_ids.filter((id) => !rec.case_ids.includes(id)) : [];
+  const removed = rec.case_ids && cur.case_ids ? rec.case_ids.filter((id) => !cur.case_ids.includes(id)) : [];
+  entry(A.suite, rec.suite_hash, cur.suite_hash, rec.suite_hash === cur.suite_hash, added.length || removed.length ? { cases_added: added, cases_removed: removed } : {});
+  entry(A.judge_model, rec.judge_model_id, cur.judge_model_id, canonicalId(rec.judge_model_id) === canonicalId(cur.judge_model_id));
+  entry(A.judge_template, rec.judge_template_hash, cur.judge_template_hash, rec.judge_template_hash === cur.judge_template_hash);
+  // The rubric, per case, over the cases both sides record.
+  if (rec.rubrics && cur.rubrics) {
+    const moved = Object.keys(rec.rubrics).filter((id) => id in cur.rubrics && rec.rubrics[id] !== cur.rubrics[id]).sort();
+    const suiteMoved = out.find((e) => e.axis === 'suite').effect === 'rerun';
+    if (!moved.length) entry(A.rubric, 'recorded', 'recorded', true);
+    else if (suiteMoved) out.push({ axis: 'rubric', recorded: `${moved.length} case(s)`, current: `${moved.length} case(s)`, effect: 'rerun', arms: A.rubric.arms, cases: moved, reason: 'the rubric moved on these cases, and the suite differs, so the rerun the suite needs grades them again; the receipt records no per-case prompt hash, so a rubric edit cannot be told from a prompt edit' });
+    else out.push({ axis: 'rubric', recorded: `${moved.length} case(s)`, current: `${moved.length} case(s)`, effect: 'regrade', arms: A.rubric.arms, cases: moved, reason: A.rubric.why });
+  } else entry(A.rubric, rec.rubrics, cur.rubrics, false);
+  return out;
+}
+
+// Each arm's decision: rerun, then regrade, then unknown, then reuse.
+const RANK = { rerun: 3, regrade: 2, unknown: 1, current: 0, advisory: 0 };
+function armDecisions(axes) {
+  const arms = {};
+  for (const arm of ['with_skill', 'baseline']) {
+    const on = axes.filter((e) => e.arms.includes(arm));
+    const top = on.reduce((m, e) => (RANK[e.effect] > RANK[m] ? e.effect : m), 'current');
+    const decision = RANK[top] === 0 ? 'reuse' : top;
+    const because = on.filter((e) => e.effect === top && decision !== 'reuse');
+    const cases = decision === 'regrade' && because.every((e) => Array.isArray(e.cases)) ? [...new Set(because.flatMap((e) => e.cases))].sort() : null;
+    arms[arm] = { decision, axes: because.map((e) => e.axis), ...(cases ? { cases } : {}) };
+  }
+  return arms;
+}
+
+// Two receipts' provenance compared (spec 053 AC-2). The decision is the most
+// consequential arm's: rerun, then regrade, then unknown, then reuse. `cases`
+// names the cases a rubric regrade reaches.
 function triage(a, b) {
-  const g = (r, p, d) => p.split('.').reduce((o, k) => (o == null ? o : o[k]), r) ?? d;
-  const changed = (p) => g(a, p) !== g(b, p);
-  if (changed('run.model_id')) return { decision: 'rerun', reason: 'the model id differs, so the generation would differ' };
-  if (changed('suite.suite_hash')) return { decision: 'rerun', reason: 'the suite content differs, so the prompts would differ' };
-  if (changed('skill.content_hash')) return { decision: 'rerun', reason: 'the skill content differs, so the with-skill generation would differ' };
-  if (changed('run.judge.model_id')) return { decision: 'regrade', reason: 'only the judge differs, so the existing generations can be rescored' };
-  if (changed('run.rubric_hash')) return { decision: 'regrade', reason: 'only the rubric differs, so the existing generations can be rescored' };
-  return { decision: 'reuse', reason: 'model, suite, skill, judge and rubric all match, so the earlier result stands' };
+  const axes = compareProvenance(provenanceOf(a), provenanceOf(b));
+  const arms = armDecisions(axes);
+  const decision = ['rerun', 'regrade', 'unknown'].find((d) => arms.with_skill.decision === d || arms.baseline.decision === d) || 'reuse';
+  const first = axes.find((e) => e.effect === decision);
+  const advised = axes.filter((e) => e.effect === 'advisory').map((e) => e.axis);
+  const reason = decision === 'reuse'
+    ? `model, harness, skill, suite, judge, template and rubric all match${advised.length ? ` but for a patch release of the ${advised.join(', ')} (an advisory)` : ''}, so the earlier result stands`
+    : decision === 'unknown' ? `${first.axis} is unknown: ${first.reason}` : first.reason;
+  const cases = decision === 'regrade' ? (arms.with_skill.cases || arms.baseline.cases || null) : null;
+  return { decision, reason, arms, axes, ...(cases ? { cases } : {}) };
 }
 
-module.exports = { compare, baselineReproduces, triage, REFUSAL_REASONS, bandOf };
+module.exports = { compare, baselineReproduces, triage, REFUSAL_REASONS, bandOf, provenanceOf, compareProvenance, armDecisions, AXES };

package/lib/run.js

@@ -1,7 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 'use strict';
 
-const { complete, resolveModel, surfaceForModel, isMeteredSurface, retryPolicyForSurface, CODEX_OVERHEAD_NOTE } = require('./provider');
+const { complete, resolveModel, surfaceForModel, isMeteredSurface, retryPolicyForSurface, CODEX_OVERHEAD_NOTE, buildSpawnPlan } = require('./provider');
+const { spawnSync } = require('child_process');
 const { gradeSamples, judgeSettings, promptTemplateHash, isTruncated } = require('./judge');
 const { buildReceipt, caseFailed, FAILED_STATUSES } = require('./receipt');
 const { sha256 } = require('./canonical');
@@ -32,6 +33,30 @@ const MODEL_RELEASE_DATES = {
   'claude-opus-4-8': '2026-05-28',     // anthropic.com/news/claude-opus-4-8
 };
 
+// The harness that will answer, recorded when a run starts (spec 053 AC-8, R-4): claude-code on
+// claude-cli, codex on openai-cli, each version read from `<bin> --version` through the same spawn
+// plan the run's calls use, so it is the binary that answers; "api" with no version on an API
+// surface. A version that cannot be read is null and the reason is returned for the caller to print;
+// it never stops a run. The receipt has no field for the reason (receipt v0.9's run.harness is
+// {name, version}).
+const HARNESS_OF = { 'claude-cli': ['claude-code', 'claude'], 'openai-cli': ['codex', 'codex'] };
+function harnessFor(surface, trusted) {
+  if (surface === 'api' || surface === 'openai-api') return { harness: { name: 'api', version: null } };
+  const h = HARNESS_OF[surface];
+  if (!h) return { harness: null };
+  const [name, bin] = h;
+  try {
+    const plan = buildSpawnPlan({ bin, args: ['--version'], trusted });
+    const r = spawnSync(plan.file, plan.args, { env: plan.env, encoding: 'utf8', timeout: 30000 });
+    if (r.error) return { harness: { name, version: null }, reason: `${bin} --version could not run: ${r.error.code || r.error.message}` };
+    if (r.status !== 0) return { harness: { name, version: null }, reason: `${bin} --version exited ${r.status}` };
+    const m = /\d+\.\d+\.\d+[0-9A-Za-z.+-]*/.exec(r.stdout || '');
+    return m ? { harness: { name, version: m[0] } } : { harness: { name, version: null }, reason: `${bin} --version printed no version` };
+  } catch (e) {
+    return { harness: { name, version: null }, reason: `${bin} --version could not be planned: ${e.message}` };
+  }
+}
+
 function releaseDateFor(modelId) {
   if (MODEL_RELEASE_DATES[modelId]) return MODEL_RELEASE_DATES[modelId];
   // Derive from a trailing YYYYMMDD in the id if present.
@@ -263,6 +288,12 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
   const reportedAll = new Set();
   const isTimeout = (e) => !!(e && (e.code === 'TIMEOUT' || /tim(e|ed)\s*out/i.test(String((e && e.message) || ''))));
   const genKind = (ws) => (ws ? 'gen_with_skill' : 'gen_baseline');
+  // v0.8 (spec 043 AC-4): when generation began, per receipt, before the first call.
+  // date_utc (nowIso, below) is stamped after the last call, or passed by a batch caller.
+  const generatedAt = new Date().toISOString();
+  // spec 053: the harness, read once as generation begins; a stub run has none.
+  const harnessRead = stubEnabled() ? { harness: null } : harnessFor(surfaceForModel(modelId), trusted);
+  if (harnessRead.reason) console.error(`  ! harness version not recorded: ${harnessRead.reason}; the run carries on with run.harness.version null`);
   const pairs = await mapPool(tasks, concurrency, async ({ c, withSkill }) => {
     const mode = withSkill ? 'with_skill' : 'baseline';
     // A per-case override is an operator's explicit input and still wins, for
@@ -517,8 +548,11 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
       // v0.3.1: on the openai/cli (codex) surface, record the fixed harness
       // preamble. Absent on a stub run: it describes a harness that did not run.
       surface_overhead_note: surface === 'openai-cli' ? CODEX_OVERHEAD_NOTE : undefined,
+      // v0.9, spec 053: the harness that answered; absent on a stub run.
+      harness: surface !== 'stub' && harnessRead.harness ? harnessRead.harness : undefined,
       runner_version: RUNNER_VERSION,
       date_utc: nowIso,
+      generated_at: generatedAt,
       registry: registryStatus(modelId),
       transcripts: keepTranscripts ? 'retained-local' : 'hashes-only',
       judge: judgeBlock,
@@ -572,7 +606,8 @@ function summarizeReceipt(receipt) {
   L.push(`- **run (UTC):** ${receipt.run.date_utc}`);
   L.push(`- **runner:** v${receipt.run.runner_version}`);
   const j = receipt.run.judge || {};
-  L.push(`- **judge:** ${j.samples || 1} samples/case, temperature ${j.temperature == null ? 'n/a' : j.temperature} (${j.sampling || 'single'})`);
+  // v0.8 (spec 043 AC-1): an absent count is unknown, never 1.
+  L.push(`- **judge:** ${Number.isInteger(j.samples) ? j.samples : 'unknown'} samples/case, temperature ${j.temperature == null ? 'n/a' : j.temperature} (${j.sampling || 'single'})`);
   if (receipt.run.registry) L.push(`- **registry:** ${receipt.run.registry}   **transcripts:** ${receipt.run.transcripts || 'hashes-only'}`);
   L.push(`- **skill content_hash:** \`${receipt.skill.content_hash.slice(0, 16)}…\``);
   L.push(`- **suite:** ${receipt.suite.case_count} cases (${receipt.suite.format})`);
@@ -609,7 +644,7 @@ function summarizeReceipt(receipt) {
   // The rule the bands above are derived by, said beside them (spec 026 AC-6).
   L.push(`band rule: each arm's band is the sample stddev of its per-case means${aggs.band_rule ? ` — ${aggs.band_rule}` : ' (unstated on this pre-v0.6 receipt)'}`);
   L.push('');
-  L.push(`## Per-case (mean ± stddev over ${(receipt.run.judge || {}).samples || 1} judge samples)`);
+  L.push(`## Per-case (mean ± stddev over ${Number.isInteger((receipt.run.judge || {}).samples) ? receipt.run.judge.samples : 'an unknown number of'} judge samples)`);
   L.push('');
   L.push(`| case | mode | outcome | mean ± stddev | judge reason |`);
   L.push(`|---|---|---|---|---|`);
@@ -625,4 +660,4 @@ function summarizeReceipt(receipt) {
   return L.join('\n');
 }
 
-module.exports = { runSkillOnModel, summarizeReceipt, releaseDateFor, projectCalls, outcomeFor, resolveCallTimeoutMs, answeredLine, answeredByOf, attest, band, uncertaintyStr };
+module.exports = { runSkillOnModel, judgeCase, summarizeReceipt, releaseDateFor, projectCalls, outcomeFor, resolveCallTimeoutMs, answeredLine, answeredByOf, attest, band, uncertaintyStr };

package/lib/skill.js

@@ -30,23 +30,57 @@ function pastBound(bound, limit, value, what) {
 const IGNORE_DIRS = new Set(['.git', 'node_modules', 'evals']);
 const IGNORE_FILES = new Set(['.DS_Store']);
 
+// Spec 062 (register row 5): A LINK IS REFUSED WHERE THE SKILL IS READ. Until this SKILL.md was
+// read through a link, so a skill directory could make any file the operator can read into the
+// system prompt, while the walk skipped links and content_hash covered none of the bytes used
+// (the SHA-256 of empty input, unchanged when the target changed). A link is neither followed nor
+// skipped: the skill is refused, naming it. The skill directory itself may be reached by a link;
+// what is under it may not.
+function linkRefusal(rel, base) {
+  const e = new Error(`${rel} under ${base} is a symbolic link; a skill is read only from its own directory, so a link is refused rather than followed or skipped`);
+  e.code = 'SKILL_LINK'; e.path = rel;
+  return e;
+}
+// Every component from the skill directory down to `rel` is a plain entry, not a link. An absent
+// component is left to the caller, whose own message names what is missing.
+function assertNoLink(dir, rel) {
+  let cur = dir;
+  for (const part of rel.split('/')) {
+    cur = path.join(cur, part);
+    let st;
+    try { st = fs.lstatSync(cur); } catch (_e) { return; }
+    if (st.isSymbolicLink()) throw linkRefusal(path.relative(dir, cur), dir);
+  }
+}
+
 // Bounded (spec 026 AC-13): the walk stops at SKILL_MAX_DEPTH directories
 // below the skill dir, SKILL_MAX_FILES bundled files, and SKILL_MAX_BYTES of
 // them together, and throws naming the bound the moment one is passed, so a
 // pathological tree is refused before its bytes are read into memory.
-function walkFiles(dir, base = dir, acc = [], state = { bytes: 0 }, depth = 0) {
+//
+// `read` maps a path relative to the skill dir to bytes the caller has already read, so the hash
+// covers those bytes and not a second read of the same path (spec 062: SKILL.md).
+function walkFiles(dir, base = dir, acc = [], state = { bytes: 0 }, depth = 0, read = {}) {
   if (depth > SKILL_MAX_DEPTH) throw pastBound('SKILL_MAX_DEPTH', SKILL_MAX_DEPTH, depth, `directory depth under ${base}`);
   for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    if (entry.isSymbolicLink()) {
+      // A link under an ignored name is never read, so it is left as the name is; `evals` is
+      // checked where the suite is read.
+      if (IGNORE_DIRS.has(entry.name) || IGNORE_FILES.has(entry.name)) continue;
+      throw linkRefusal(path.relative(base, path.join(dir, entry.name)), base);
+    }
     if (entry.isDirectory()) {
       if (IGNORE_DIRS.has(entry.name)) continue;
-      walkFiles(path.join(dir, entry.name), base, acc, state, depth + 1);
+      walkFiles(path.join(dir, entry.name), base, acc, state, depth + 1, read);
     } else if (entry.isFile() && !IGNORE_FILES.has(entry.name)) {
       const abs = path.join(dir, entry.name);
+      const rel = path.relative(base, abs);
       if (acc.length + 1 > SKILL_MAX_FILES) throw pastBound('SKILL_MAX_FILES', SKILL_MAX_FILES, acc.length + 1, `bundled files under ${base}`);
-      const size = fs.statSync(abs).size;
+      const given = Object.hasOwn(read, rel) ? read[rel] : null;
+      const size = given ? given.length : fs.statSync(abs).size;
       state.bytes += size;
       if (state.bytes > SKILL_MAX_BYTES) throw pastBound('SKILL_MAX_BYTES', SKILL_MAX_BYTES, state.bytes, `bundled bytes under ${base}`);
-      acc.push({ path: path.relative(base, abs), bytes: fs.readFileSync(abs) });
+      acc.push({ path: rel, bytes: given || fs.readFileSync(abs) });
     }
   }
   return acc;
@@ -55,14 +89,17 @@ function walkFiles(dir, base = dir, acc = [], state = { bytes: 0 }, depth = 0) {
 function loadSkill(skillDir) {
   const dir = path.resolve(skillDir);
   const skillMdPath = path.join(dir, 'SKILL.md');
+  assertNoLink(dir, 'SKILL.md');
   if (!fs.existsSync(skillMdPath)) {
     throw new Error(`no SKILL.md found in ${dir}`);
   }
-  const skillMd = fs.readFileSync(skillMdPath, 'utf8');
+  const skillMdBytes = fs.readFileSync(skillMdPath);
+  const skillMd = skillMdBytes.toString('utf8');
 
   // Content hash over SKILL.md + every bundled file (evals/ excluded — the suite
   // is hashed separately so a suite edit doesn't masquerade as a skill change).
-  const files = walkFiles(dir);
+  // SKILL.md enters it as the bytes read above, the bytes the run is given.
+  const files = walkFiles(dir, dir, [], { bytes: 0 }, 0, { 'SKILL.md': skillMdBytes });
   const contentHash = sha256Files(files);
 
   // Parse skill name/version from front-matter or the first H1; fall back to dir.
@@ -72,18 +109,12 @@ function loadSkill(skillDir) {
 
   // Load the eval suite.
   const suitePath = path.join(dir, 'evals', 'evals.json');
+  assertNoLink(dir, 'evals/evals.json');
   if (!fs.existsSync(suitePath)) {
     throw new Error(`no evals/evals.json found in ${dir}`);
   }
   const suiteRaw = JSON.parse(fs.readFileSync(suitePath, 'utf8'));
-  const cases = normalizeCases(suiteRaw);
-  // suite_hash is over the CORE case fields only ({id, prompt, rubric,
-  // pass_threshold}); optional annotations (checks, and the claim/grounding the
-  // gate reads straight from the raw suite) are excluded so adding them never
-  // disturbs a receipt's suite_hash. See spec/RECEIPT.md § suite.
-  const suiteHash = sha256Canonical(cases.map((c) => ({
-    id: c.id, prompt: c.prompt, rubric: c.rubric, pass_threshold: c.pass_threshold,
-  })));
+  const { cases, suiteHash } = suiteIdentity(suiteRaw);
 
   return {
     dir,
@@ -100,6 +131,20 @@ function loadSkill(skillDir) {
   };
 }
 
+// A suite's normalised cases and its hash, as a receipt records them. suite_hash
+// is over the CORE case fields only ({id, prompt, rubric, pass_threshold});
+// optional annotations (checks, and the claim/grounding the gate reads straight
+// from the raw suite) are excluded so adding them never disturbs a receipt's
+// suite_hash. See spec/RECEIPT.md § suite. Exported for `driftproof stale
+// --suite` (spec 053), so a suite file is hashed by the same code as a run.
+function suiteIdentity(suiteRaw) {
+  const cases = normalizeCases(suiteRaw);
+  const suiteHash = sha256Canonical(cases.map((c) => ({
+    id: c.id, prompt: c.prompt, rubric: c.rubric, pass_threshold: c.pass_threshold,
+  })));
+  return { cases, suiteHash };
+}
+
 // Pull name/version from a YAML-ish front-matter block, else from the H1 line.
 function parseSkillMeta(md) {
   const meta = {};
@@ -128,8 +173,18 @@ function normalizeCases(raw) {
   if (!list) throw new Error('evals.json must be an array or have a `cases`/`evals` array');
   // Bounded (spec 026 AC-13): the case count, and each prompt and rubric.
   if (list.length > SUITE_MAX_CASES) throw pastBound('SUITE_MAX_CASES', SUITE_MAX_CASES, list.length, 'cases in the suite');
+  // Spec 050 AC-1: normalized ids are unique. The receipt and every reader of it key a
+  // case by this string, so two cases that share it are measured, paid for, and then
+  // one is silently dropped from the verdict. Refused here, before any call. The
+  // comparison is on the id as assigned, including a `case-<n>` given to an unnamed
+  // case, which an explicit id can collide with.
+  const firstAt = new Map();
   return list.map((c, i) => {
     const id = String(c.id || c.name || `case-${i + 1}`);
+    if (firstAt.has(id)) {
+      throw Object.assign(new Error(`case ids must be unique: cases ${firstAt.get(id)} and ${i + 1} both have the id ${JSON.stringify(id)}; rename one and run again`), { code: 'DUPLICATE_CASE_ID' });
+    }
+    firstAt.set(id, i + 1);
     const prompt = c.prompt || c.input || c.task;
     const rubric = c.rubric || c.criteria || c.expected;
     if (!prompt) throw new Error(`case "${id}" is missing a prompt/input/task`);
@@ -146,4 +201,4 @@ function normalizeCases(raw) {
   });
 }
 
-module.exports = { loadSkill, normalizeCases, parseSkillMeta, pastBound };
+module.exports = { loadSkill, normalizeCases, parseSkillMeta, pastBound, suiteIdentity };

package/lib/stale.js

@@ -0,0 +1,194 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+// `driftproof stale` (spec 053): for each receipt, does its conclusion still stand under what would
+// run today, or does it need a rerun of one or both arms, or only a regrade?
+//
+// What a receipt recorded is read by lib/reuse.js provenanceOf; what would run today is built here,
+// from the same places `driftproof run` reads (flags, then .driftproofrc; a value nobody set is
+// unknown, never a default); the two are compared by lib/reuse.js compareProvenance, the function
+// `triage()` uses, so the command and the triage cannot disagree about an axis.
+//
+// No model call and no network. The one thing spawned is `claude --version` or `codex --version`,
+// and only when neither --harness-version nor --no-harness-check is given.
+//
+// THE OUTPUT IS A PUBLIC CONTRACT (spec/stale.v1.schema.json, `driftproof.stale/1`): fields are
+// only ever added. It carries no clock, so the same inputs print the same bytes.
+
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { provenanceOf, compareProvenance, armDecisions } = require('./reuse');
+const { loadSkill, suiteIdentity } = require('./skill');
+const { rubricHash, promptTemplateHash } = require('./judge');
+const { resolveModel } = require('./provider');
+const { loadRegistry } = require('./models');
+const { validateReceipt, verifyReceiptHash } = require('./receipt');
+const { RUNNER_VERSION, PROJECT_NAME } = require('../config');
+
+const SCHEMA_ID = 'driftproof.stale/1';
+const HARNESS_BIN = { 'claude-code': 'claude', codex: 'codex' };
+const SURFACE_HARNESS = { 'claude-cli': 'claude-code', 'openai-cli': 'codex', api: 'api', 'openai-api': 'api' };
+
+// Today's version of a harness, read from its binary on PATH (spec 053 R-5).
+function harnessVersionNow(name) {
+  const bin = HARNESS_BIN[name];
+  if (!bin) return { version: null, why: `no way to read today's ${name} version` };
+  const r = spawnSync(bin, ['--version'], { encoding: 'utf8', timeout: 10000 });
+  if (r.error) return { version: null, why: `${bin} --version could not run: ${r.error.code || r.error.message}` };
+  if (r.status !== 0) return { version: null, why: `${bin} --version exited ${r.status}` };
+  const m = /\d+\.\d+\.\d+[0-9A-Za-z.+-]*/.exec(r.stdout || '');
+  return m ? { version: m[0] } : { version: null, why: `${bin} --version printed no version` };
+}
+
+// What would run today, for one receipt. `why` carries the reason for each value that could not be found.
+function currentProvenance(receipt, opts, cache) {
+  const why = {};
+  const rec = provenanceOf(receipt);
+  const rc = opts.rc || {};
+  // Model: --model, else the rc's models; the receipt's model is current when the rc names it.
+  let model = null;
+  if (opts.model) model = resolveModel(opts.model);
+  else if (rc.models !== undefined && rc.models !== null) {
+    const list = String(rc.models).split(',').map((x) => x.trim()).filter(Boolean).map((x) => resolveModel(x));
+    model = rec.model_id && list.includes(resolveModel(rec.model_id)) ? resolveModel(rec.model_id) : list.join(',') || null;
+  }
+  if (!model) why.model = 'no --model given and no models in .driftproofrc';
+  // Judge: --judge, else the rc's judge_model.
+  const judgeRaw = opts.judge || rc.judge_model || null;
+  const judge = judgeRaw ? resolveModel(judgeRaw) : null;
+  if (!judge) why.judge_model = 'no --judge given and no judge_model in .driftproofrc';
+  // Skill and suite.
+  let skillHash = null; let suite = null;
+  if (opts.skill) {
+    const s = cache.skill || (cache.skill = loadSkill(opts.skill));
+    skillHash = s.contentHash;
+    if (!opts.suite) suite = { suiteHash: s.suite.suiteHash, cases: s.suite.cases };
+  } else why.skill = 'no --skill given';
+  if (opts.suite) suite = cache.suite || (cache.suite = suiteIdentity(JSON.parse(fs.readFileSync(opts.suite, 'utf8'))));
+  if (!suite) { why.suite = 'no --suite or --skill given'; why.rubric = why.suite; }
+  // Harness: the receipt's harness name, else its surface's.
+  const name = (rec.harness && rec.harness.name) || SURFACE_HARNESS[(receipt.run || {}).surface] || null;
+  let harness = null;
+  if (!name) why.harness = 'the receipt names no harness and no surface with one';
+  else if (name === 'api') harness = { name: 'api', version: null };
+  else if (opts.noHarnessCheck) why.harness = 'not checked (--no-harness-check)';
+  else if (opts.harnessVersion) harness = { name, version: String(opts.harnessVersion) };
+  else {
+    const v = cache[`h:${name}`] || (cache[`h:${name}`] = harnessVersionNow(name));
+    harness = { name, version: v.version };
+    if (v.version == null) why.harness = v.why;
+  }
+  const rubrics = suite ? Object.fromEntries(suite.cases.map((c) => [c.id, rubricHash(c.rubric)])) : null;
+  return {
+    prov: {
+      model_id: model, harness, skill_hash: skillHash, suite_hash: suite ? suite.suiteHash : null,
+      case_ids: suite ? [...new Set(suite.cases.map((c) => c.id))].sort() : null,
+      judge_model_id: judge, judge_template_hash: promptTemplateHash(), rubrics,
+    },
+    why,
+  };
+}
+
+// The newer-model advisory (spec 053 R-9): same family, registered, released after both the run's
+// date and the recorded model's release date. No advisory when either date is missing.
+const dated = (m) => !!m && typeof m.released === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(m.released);
+function advisories(receipt) {
+  const reg = loadRegistry();
+  const id = (receipt.run || {}).model_id;
+  const row = id ? reg.byId[resolveModel(id)] || reg.byId[id] : null;
+  const runDay = typeof (receipt.run || {}).date_utc === 'string' ? receipt.run.date_utc.slice(0, 10) : null;
+  if (!dated(row) || !runDay) return [];
+  return reg.models
+    .filter((m) => m.id !== row.id && m.family === row.family && !m.auto_added && dated(m) && String(m.released) > runDay && String(m.released) > String(row.released))
+    .sort((a, b) => (a.released === b.released ? (a.id < b.id ? -1 : 1) : (a.released < b.released ? 1 : -1)))
+    .map((m) => ({ model_id: m.id, released: m.released }));
+}
+
+// The command to run next (spec 053 R-7).
+function nextCommand(file, receipt, arms, opts, current) {
+  const skillArg = opts.skill || '<skill dir>';
+  const d = [arms.with_skill.decision, arms.baseline.decision];
+  if (d.includes('rerun')) return `${PROJECT_NAME} run ${skillArg} --model ${current.model_id && !current.model_id.includes(',') ? current.model_id : receipt.run.model_id}`;
+  if (d.includes('regrade')) return `${PROJECT_NAME} regrade ${file} --skill ${skillArg} --answers <answers.json> --judge-model ${current.judge_model_id || receipt.run.judge.model_id}`;
+  return null;
+}
+
+function assess(file, receipt, opts, cache) {
+  const { prov, why } = currentProvenance(receipt, opts, cache);
+  const axes = compareProvenance(provenanceOf(receipt), prov, why);
+  const arms = armDecisions(axes);
+  const d = [arms.with_skill.decision, arms.baseline.decision];
+  const status = d.includes('rerun') || d.includes('regrade') ? 'stale' : d.includes('unknown') ? 'unknown' : 'current';
+  const run = receipt.run || {};
+  return {
+    path: file,
+    receipt_hash: receipt.receipt_hash || null,
+    skill: (receipt.skill && receipt.skill.name) || null,
+    model_id: run.model_id || null,
+    date_utc: run.date_utc || null,
+    verification_level: receipt.verification_level || null,
+    status,
+    arms,
+    axes,
+    advisories: advisories(receipt),
+    runner_version: { recorded: run.runner_version || null, current: RUNNER_VERSION },
+    next: nextCommand(file, receipt, arms, opts, prov),
+  };
+}
+
+// The whole report: every receipt read, validated and assessed, and the exit code (spec 053 R-8).
+function staleReport(files, opts = {}) {
+  const cache = {};
+  const receipts = files.map((file) => {
+    let r;
+    try { r = JSON.parse(fs.readFileSync(file, 'utf8')); } catch (e) { return { path: file, error: e.code === 'ENOENT' ? 'the file does not exist' : `the file is not a readable JSON document: ${e.message}` }; }
+    const v = validateReceipt(r);
+    if (!v.valid) return { path: file, error: `the receipt does not validate: ${(v.errors || []).slice(0, 2).map((x) => (typeof x === 'string' ? x : `${x.instancePath || ''} ${x.message || ''}`.trim())).join('; ')}` };
+    // As `driftproof validate` reads a receipt: the schema, and the hash over its content.
+    if (!verifyReceiptHash(r)) return { path: file, error: 'the receipt_hash does not match the receipt\'s content' };
+    try { return assess(file, r, opts, cache); } catch (e) { return { path: file, error: `the receipt could not be assessed: ${e.message}` }; }
+  });
+  const ok = receipts.filter((x) => !x.error);
+  // Advisories: a newer model, and an axis that moved by a patch release only (A-053-1).
+  const summary = { receipts: receipts.length, current: ok.filter((x) => x.status === 'current').length, stale: ok.filter((x) => x.status === 'stale').length, unknown: ok.filter((x) => x.status === 'unknown').length, errors: receipts.length - ok.length, advisories: ok.reduce((n, x) => n + x.advisories.length + x.axes.filter((e) => e.effect === 'advisory').length, 0) };
+  let exit = summary.errors ? 2 : summary.stale ? 1 : summary.unknown || summary.advisories ? 3 : 0;
+  if (opts.strict && exit === 3) exit = 1;
+  return { schema: SCHEMA_ID, runner_version: RUNNER_VERSION, strict: !!opts.strict, receipts, summary, exit_code: exit };
+}
+
+// ── the human output ─────────────────────────────────────────────────────────
+const LABEL = { model: 'model', harness: 'harness', skill: 'skill', suite: 'suite', judge_model: 'judge model', judge_template: 'judge template', rubric: 'rubric' };
+const short = (v) => (typeof v === 'string' && /^[0-9a-f]{64}$/.test(v) ? v.slice(0, 8) : v == null ? 'unrecorded' : String(v));
+function effectText(e) {
+  if (e.effect === 'unknown') return `unknown: ${e.reason}`;
+  if (e.effect === 'advisory') return 'a patch release only (advisory)';
+  if (e.effect === 'regrade') return e.cases ? `regrade case(s) ${e.cases.join(', ')}` : 'regrade both arms';
+  if (e.effect === 'rerun') {
+    if (e.arms.length === 1) return 'rerun the with-skill arm; baseline still valid';
+    const extra = [e.cases_added && e.cases_added.length ? `added ${e.cases_added.join(', ')}` : '', e.cases_removed && e.cases_removed.length ? `removed ${e.cases_removed.join(', ')}` : '', e.cases ? `rubric moved on ${e.cases.join(', ')}` : ''].filter(Boolean).join('; ');
+    return `rerun both arms${extra ? ` (${extra})` : ''}`;
+  }
+  return 'unchanged';
+}
+function renderText(doc) {
+  const L = [];
+  for (const x of doc.receipts) {
+    if (x.error) { L.push(`${x.path}   ERROR`, `  ${x.error}`, ''); continue; }
+    L.push(`${x.skill || '(no skill)'}   ${x.model_id || '(no model)'}   ${x.date_utc ? String(x.date_utc).slice(0, 10) : '(no date)'}   ${x.status.toUpperCase()}`);
+    for (const e of x.axes) {
+      if (e.effect === 'current' && e.axis !== 'judge_model') continue;
+      const now = e.current == null ? 'not known' : short(e.current);
+      const move = e.effect === 'current' ? `${now}, unchanged` : `${short(e.recorded)} to ${now}`;
+      // Two spaces at least between the move and its effect, however long the move (approval F-1).
+      L.push(`  ${LABEL[e.axis].padEnd(14)}${(move + '  ').padEnd(38)}${e.effect === 'current' ? '' : effectText(e)}`.trimEnd());
+    }
+    for (const a of x.advisories) L.push(`  ${'newer model'.padEnd(14)}${a.model_id}, released ${a.released} (advisory)`);
+    L.push(`  ${'next'.padEnd(14)}${x.next || (x.status === 'current' ? 'nothing to run' : 'nothing can be decided until the unknown axes are known')}`, '');
+  }
+  const s = doc.summary;
+  L.push(`${s.receipts} receipt(s): ${s.current} current, ${s.stale} stale, ${s.unknown} unknown, ${s.errors} error(s); exit ${doc.exit_code}${doc.strict ? ' (--strict)' : ''}`);
+  return L.join('\n') + '\n';
+}
+
+module.exports = { staleReport, renderText, currentProvenance, advisories, SCHEMA_ID };