driftproof 0.11.2 → 0.12.0

package/lib/receipt.js

@@ -5,6 +5,7 @@ const fs = require('fs');
 const path = require('path');
 const { canonicalize, sha256 } = require('./canonical');
 const { RECEIPT_SCHEMA_VERSION } = require('../config');
+const { placeCounts, judgeCountsDisagree } = require('./counts');
 const { aggregateBands, combineUncertainty, round } = require('./stats');
 
 // Schema file per receipt version. The current schema is receipt.schema.json;
@@ -29,7 +30,15 @@ const SCHEMA_FILES = {
   // verdict token a reader derives (UNDERPOWERED), and a v0.6 receipt restamped 0.7
   // is refused by the const, so the number keeps resolving to the schema it meant.
   '0.6': 'receipt.v0.6.schema.json',
-  '0.7': 'receipt.schema.json',
+  // v0.7 moved the same way when v0.8 took the current pointer (spec 043): v0.8 makes
+  // run.judge.samples optional and adds counts and clocks, and its const refuses a v0.7
+  // receipt that has not been restamped, so the number keeps resolving to the schema it meant.
+  '0.7': 'receipt.v0.7.schema.json',
+  // v0.8 moved the same way when v0.9 took the current pointer (spec 049): v0.9 adds the
+  // import record, the harness, the unit and activation, and its const refuses a v0.8 receipt
+  // that has not been restamped, so the number keeps resolving to the schema it meant.
+  '0.8': 'receipt.v0.8.schema.json',
+  '0.9': 'receipt.schema.json',
 };
 
 // THE BAND RULE, stated on every receipt (results.aggregates.band_rule, spec
@@ -48,20 +57,28 @@ const BAND_RULE = 'per arm: the sample standard deviation (n-1) of the per-case
 const FAILED_STATUSES = ['failed_timeout', 'failed_unmeasured'];
 function caseFailed(c) { return !!(c && c.case_status && c.case_status !== 'ok'); }
 
-const _validators = {};
+// Spec 062 (register row 2): the table is read by OWN key and the cache is a Map. Until this
+// the lookup was `SCHEMA_FILES[version]`, so a receipt whose schema_version named an
+// Object.prototype property ("constructor", "toString") found something truthy, fell back to
+// the current schema's cached validator and read VALID; and a version nobody knew was checked
+// against the current schema as though it had claimed it. A version that is not a row here has
+// no schema, and a receipt with no schema is not valid.
+const _validators = new Map();
 // Lazily compile the JSON Schema validator (ajv) for a given version. Kept lazy
 // so the library can be required without ajv present (pure hashing utilities).
+// Returns null for a version the table does not hold.
 function getValidator(version) {
-  const v = SCHEMA_FILES[version] ? version : RECEIPT_SCHEMA_VERSION;
-  if (_validators[v]) return _validators[v];
+  if (typeof version !== 'string' || !Object.hasOwn(SCHEMA_FILES, version)) return null;
+  if (_validators.has(version)) return _validators.get(version);
   let Ajv;
   // The schema is JSON Schema draft 2020-12, so use ajv's 2020 build.
   try { Ajv = require('ajv/dist/2020'); }
   catch (_e) { throw new Error('receipt validation requires the `ajv` package (npm install)'); }
-  const schema = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'spec', SCHEMA_FILES[v]), 'utf8'));
+  const schema = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'spec', SCHEMA_FILES[version]), 'utf8'));
   const ajv = new Ajv({ allErrors: true, strict: false });
-  _validators[v] = ajv.compile(schema);
-  return _validators[v];
+  const validate = ajv.compile(schema);
+  _validators.set(version, validate);
+  return validate;
 }
 
 // Compute the receipt_hash: sha256 over the canonical receipt JSON with the
@@ -82,13 +99,145 @@ function verifyReceiptHash(receipt) {
   return receipt.receipt_hash === computeReceiptHash(receipt);
 }
 
+// Spec 050: every (id, mode) pair that occurs more than once in results.cases, with
+// the index of each row that carries it. A receipt is one row per case and arm; two
+// rows for one pair are two measurements that every reader keyed by (id, mode) would
+// collapse into one, keeping whichever came last. So a regression measured in the
+// first row could vanish from the verdict while the receipt still validated and its
+// hash still verified (an outside correctness audit, finding 1). This is the one
+// definition of an ambiguous receipt: validation, the verdict, the decision and the
+// CLI all call it, so none of them can disagree about what counts.
+function duplicateCaseRows(receipt) {
+  const cases = receipt && receipt.results && Array.isArray(receipt.results.cases) ? receipt.results.cases : [];
+  const seen = new Map();
+  cases.forEach((c, i) => {
+    const key = JSON.stringify([c && c.id, c && c.mode]);
+    if (!seen.has(key)) seen.set(key, { id: c && c.id, mode: c && c.mode, rows: [] });
+    seen.get(key).rows.push(i);
+  });
+  return [...seen.values()].filter((d) => d.rows.length > 1);
+}
+
+// The sentence every refusal of an ambiguous receipt uses, so the CLI, the verdict
+// and the decision say the same thing about the same receipt.
+function ambiguityLine(dups) {
+  return dups.map((d) => `case ${JSON.stringify(d.id)} has ${d.rows.length} ${d.mode} rows (results.cases ${d.rows.join(', ')})`).join('; ');
+}
+
 // Validate against the schema matching the receipt's own schema_version (so both
 // v0.1 and v0.2 receipts validate). Returns { valid, errors, version }.
+//
+// Spec 062: a schema_version that is absent, or is not a version the table holds, is invalid,
+// with no fallback to the current schema. Absent is not read as current: every receipt the
+// runner and the importers write carries one.
 function validateReceipt(receipt) {
-  const version = (receipt && receipt.schema_version) || RECEIPT_SCHEMA_VERSION;
+  const version = receipt && typeof receipt === 'object' ? receipt.schema_version : undefined;
   const validate = getValidator(version);
+  if (!validate) {
+    return { valid: false, errors: [{ instancePath: '/schema_version', message: `unknown schema_version ${JSON.stringify(version === undefined ? null : version)}: not a version this validator holds (${Object.keys(SCHEMA_FILES).join(', ')}); no other schema was tried` }], version };
+  }
   const valid = validate(receipt);
-  return { valid, errors: valid ? [] : (validate.errors || []), version };
+  const errors = valid ? [] : (validate.errors || []);
+  // v0.8 (spec 043 AC-2): run.judge.samples and run.counts.judge_samples_per_generation
+  // are two spellings of one count; a receipt that gives both must give one value. v0.9
+  // keeps both fields, so it keeps the rule.
+  if (version === '0.8' || version === '0.9') {
+    if (judgeCountsDisagree(receipt)) errors.push({ instancePath: '/run/counts/judge_samples_per_generation', message: 'differs from run.judge.samples' });
+  }
+  // Spec 050 AC-2: one row per (id, mode), in every schema version. No schema can say
+  // it (uniqueness over a pair of fields is outside JSON Schema), so it is said here.
+  for (const d of duplicateCaseRows(receipt)) {
+    for (const i of d.rows.slice(1)) errors.push({ instancePath: `/results/cases/${i}`, message: `duplicate case id and mode: ${JSON.stringify(d.id)} ${d.mode} is also at /results/cases/${d.rows[0]}` });
+  }
+  return { valid: errors.length === 0, errors, version };
+}
+
+// Spec 062 (register row 4): whether a receipt ran fewer cases than its suite holds. The receipt
+// records the ids it ran (results.cases) and the suite's count (suite.case_count), not the
+// suite's ids, so a receipt narrows its suite when either arm names fewer distinct case ids than
+// the suite counts (R-5). A failed case keeps its rows, so it is not a cut. A
+// narrowed receipt is not a reading of the suite its suite_hash names, and both readers treat it
+// as incomplete (lib/verdict.js receiptVerdict, lib/decision.js decisionState).
+function suiteNarrowed(receipt) {
+  const n = receipt && receipt.suite ? receipt.suite.case_count : undefined;
+  if (!Number.isInteger(n)) return false;
+  const ids = { with_skill: new Set(), baseline: new Set() };
+  for (const c of (receipt.results && receipt.results.cases) || []) if (c && ids[c.mode]) ids[c.mode].add(c.id);
+  return ids.with_skill.size < n || ids.baseline.size < n;
+}
+
+// ── receipt files (spec 062, register row 3) ────────────────────────────────
+// The writer's naming rule and the readers' listing rule, in one module. Until this the name was
+// assembled in bin/driftproof and read back by lib/decision.js with a different rule: a second run
+// on the same day overwrote the first, a regrade sidecar in the directory was read as a receipt
+// that did not verify, and a model id with a dot was looked for in a name that held its slug.
+
+// The file-name form of a skill name or a model id. Named for what it is rather than `slug`: the
+// probe-copies rule reads an exported name, and spec 054's probe has a heading-anchor `slug` of its
+// own that is not this function (spec 062 A-062-1).
+function fileSlug(s) { return String(s).toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, ''); }
+
+// <skill>-<model>[-<tag>]-<date>[-<hash12>]. `hash` is the first 12 characters of the receipt's
+// own receipt_hash for a run, so two runs on one day name two files; an import passes its source
+// document's hash, which keeps the name spec 049 gave it.
+function receiptBaseName({ skill, model, date, hash = null, tag = null }) {
+  return [fileSlug(skill), fileSlug(model), ...(tag ? [tag] : []), date, ...(hash ? [String(hash).slice(0, 12)] : [])].join('-');
+}
+
+// The regrade provenance sidecar: its name is the receipt's base name with this suffix, and its
+// `format` names this version. `regrade`'s writers read both from here (lib/regrade.js the format,
+// bin/driftproof the name), so the listing rule below is the writer's own (spec 069).
+const REGRADE_SIDECAR = { suffix: '.regrade.json', format: 'driftproof-regrade/2' };
+const REGRADE_FAMILY = REGRADE_SIDECAR.format.split('/')[0];
+
+// The summary export kept beside its receipt: `export --out <dir>` names it the receipt's base name
+// with this suffix, and lib/export.js stamps this format on it (spec 107). It carries the receipt's
+// receipt_hash as a reference, so the listing rule below skips it by name and shape.
+const SUMMARY_SIDECAR = { suffix: '.summary.json', format: 'driftproof/summary' };
+
+// The files a writer puts beside a receipt that carry its receipt_hash and are not receipts. A file
+// is one of these only when its name AND its shape match (spec 069, the operator's ruling Q1): a
+// name alone or a shape alone is not enough, so a receipt renamed, or a sidecar renamed, is not
+// passed over. None of them carries `results`.
+//   - regrade provenance: `format` is a version of the regrade format. The published Report 011
+//     sidecars carry the version before the writer's current one, and read as sidecars.
+//   - surface record (spec 042's run): `receipt` names the receipt it sits beside, which is its own
+//     name with the suffix replaced by `.json`.
+//   - summary export (spec 107): `format` is the summary format and `format_version` a string of
+//     digits.
+const RECEIPT_SIDECARS = [
+  { kind: 'regrade provenance', suffix: REGRADE_SIDECAR.suffix,
+    shape: (doc) => typeof doc.format === 'string' && new RegExp(`^${REGRADE_FAMILY}/[0-9]+$`).test(doc.format) },
+  { kind: 'surface record', suffix: '.surface.json',
+    shape: (doc, name) => doc.receipt === `${name.slice(0, -'.surface.json'.length)}.json` },
+  { kind: 'summary export', suffix: SUMMARY_SIDECAR.suffix,
+    shape: (doc) => doc.format === SUMMARY_SIDECAR.format && typeof doc.format_version === 'string' && /^[0-9]+$/.test(doc.format_version) },
+];
+function sidecarKind(name, doc) {
+  if (Object.hasOwn(doc, 'results')) return null;
+  const s = RECEIPT_SIDECARS.find((x) => name.endsWith(x.suffix) && name.length > x.suffix.length && x.shape(doc, name));
+  return s ? s.kind : null;
+}
+
+// Every receipt in a directory, sorted by file name: { file, receipt, error }. A `.json` file that
+// parses to an object carrying `receipt_hash` or `results` is returned, unless it is a sidecar by
+// name and shape (above), so the caller's hash check refuses by name a receipt with either field
+// removed, rather than reading it as absent (spec 062 A-062-3 for `receipt_hash`, spec 069 for
+// `results`). An object carrying neither is not a receipt and is not returned: a badge, a stale
+// document. A `.json` file that does not parse IS returned, with receipt null and the
+// parser's message: it may be a receipt, and spec 030 AC-2 fails its model closed as unreadable.
+function listReceipts(dir) {
+  const out = [];
+  for (const name of fs.readdirSync(dir).filter((f) => f.endsWith('.json')).sort()) {
+    const file = path.join(dir, name);
+    let receipt;
+    try { receipt = JSON.parse(fs.readFileSync(file, 'utf8')); } catch (e) { out.push({ file, receipt: null, error: e.message }); continue; }
+    if (!receipt || typeof receipt !== 'object' || Array.isArray(receipt)) continue;
+    if (!Object.hasOwn(receipt, 'results') && !Object.hasOwn(receipt, 'receipt_hash')) continue;
+    if (sidecarKind(name, receipt)) continue;
+    out.push({ file, receipt, error: null });
+  }
+  return out;
 }
 
 function mean(nums) {
@@ -224,11 +373,19 @@ function buildReceipt({ skill, suite, run, cases, economics = null, verification
       surface: run.surface,
       runner_version: run.runner_version,
       date_utc: run.date_utc,
+      // v0.8 (spec 043 AC-4): when generation began, stamped by the runner before its
+      // first generation call; date_utc keeps its meaning.
+      ...(run.generated_at ? { generated_at: run.generated_at } : {}),
+      // v0.8 (spec 043 AC-4; spec 044): a re-judge of frozen outputs. Its archived arms
+      // are carried before placeCounts reads them, and the two clocks travel together.
+      ...(run.arms ? { arms: run.arms } : {}),
+      ...(run.judged_at ? { judged_at: run.judged_at, grader_revision: run.grader_revision } : {}),
       // v0.3: registry provenance + transcript-retention mode. Defaults keep the
       // honest, cheapest interpretation when a caller omits them.
       registry: run.registry || 'unregistered',
       transcripts: run.transcripts || 'hashes-only',
-      judge: run.judge || { samples: 1, temperature: null, sampling: 'single', surface: run.surface },
+      // v0.8: no default judge-sample count; a caller that says nothing leaves it unknown.
+      judge: run.judge || { temperature: null, sampling: 'single', surface: run.surface },
       // v0.6 (spec 026 AC-1, AC-2): what answered. Carried from the caller as
       // given and never defaulted: a receipt that does not say what answered it
       // is refused by the schema, which is the point.
@@ -255,6 +412,8 @@ function buildReceipt({ skill, suite, run, cases, economics = null, verification
   //   skill.tokens              — estimated SKILL.md token size (value-per-token axis).
   if (run.surface_overhead_note) receipt.run.surface_overhead_note = run.surface_overhead_note;
   if (skill.tokens != null) receipt.skill.tokens = skill.tokens;
+  // v0.9, spec 053: the harness the runner read as the run began ({name, version}; absent on a stub run).
+  if (run.harness && typeof run.harness.name === 'string') receipt.run.harness = { name: run.harness.name, version: run.harness.version == null ? null : String(run.harness.version) };
   // v0.4 economics (additive-optional): the frozen prices this receipt's derived
   // dollar figures were computed from, and the derived block itself. A receipt
   // from a surface that reports no usage simply omits both.
@@ -266,9 +425,22 @@ function buildReceipt({ skill, suite, run, cases, economics = null, verification
     receipt.run.failed_case_count = failedCount;
   }
   if (editorialReviews && editorialReviews.length) receipt.editorial_reviews = editorialReviews;
+  // v0.8 counts (spec 043 AC-3, AC-7): what the runner establishes, per case. The judge
+  // count is the samples the run was asked for; the generation count is the draws the
+  // case recorded. A case without a draw set (failed, or a legacy caller) establishes
+  // no generation count, so none is written for it.
+  const judgeN = receipt.run.judge && Number.isInteger(receipt.run.judge.samples) ? receipt.run.judge.samples : null;
+  placeCounts(receipt, cases.map((c, index) => ({
+    index,
+    counts: {
+      ...(c.generation && Array.isArray(c.generation.draws) ? { generations_per_arm: c.generation.draws.length } : {}),
+      ...(judgeN !== null && !caseFailed(c) ? { judge_samples_per_generation: judgeN } : {}),
+    },
+  })));
   return sealReceipt(receipt);
 }
 
 module.exports = {
   buildReceipt, sealReceipt, computeReceiptHash, verifyReceiptHash, validateReceipt, BAND_RULE, comparisonOf, caseFailed, FAILED_STATUSES,
+  duplicateCaseRows, ambiguityLine, suiteNarrowed, fileSlug, receiptBaseName, listReceipts, REGRADE_SIDECAR, SUMMARY_SIDECAR,
 };

package/lib/regrade.js

@@ -0,0 +1,266 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+// Regrade: the executor behind lib/reuse.js's `regrade` decision (spec 044).
+//
+// When only the judge (or the grading template) differs between two receipts,
+// triage says the existing generations can be rescored. This does it: it takes a
+// sealed receipt, the skill it was run on and the answers its draws graded, and
+// judges every draw again with the judge it is given. The generation side of the
+// new receipt is the original's, draw for draw; the judge side is new.
+//
+// THE SAME CODE A RUN JUDGES WITH. Each draw goes through lib/run.js judgeCase,
+// its replies through lib/run.js attest, each case through lib/sampling.js
+// acrossDraws, and the receipt through lib/receipt.js buildReceipt. Nothing here
+// grades, aggregates or seals by a second route.
+//
+// THE DRAW SET IS THE ORIGINAL'S. The sampling rule's escalation decides how many
+// generations to draw; with the generations fixed there is nothing for it to
+// decide, so `n_planned` and `stopping_reason` are carried and no draw is added.
+//
+// A DRAW WITH NO ANSWER TO GRADE IS CARRIED, NOT GRADED: one with no
+// generation_hash (the generation timed out) or one cut at the output cap (spec
+// 026 AC-8). A draw whose original judge gave no score has an answer and is graded.
+
+const { sha256 } = require('./canonical');
+const { judgeCase, attest, outcomeFor, resolveCallTimeoutMs } = require('./run');
+const { judgeSettings, promptTemplateHash, rubricHash } = require('./judge');
+const { buildReceipt, verifyReceiptHash, FAILED_STATUSES, REGRADE_SIDECAR } = require('./receipt');
+const { acrossDraws } = require('./sampling');
+const { resolveModel, surfaceForModel, isMeteredSurface } = require('./provider');
+const { priceForModel, assertRegistered } = require('./models');
+const { perCallCostUSD } = require('./cost');
+const { buildPricingSnapshot, computeEconomics } = require('./value');
+const { canonicalModelId } = require('./usage');
+const { RUNNER_VERSION } = require('../config');
+
+const isTimeout = (e) => !!(e && (e.code === 'TIMEOUT' || /tim(e|ed)\s*out/i.test(String((e && e.message) || ''))));
+
+// Whether a draw carries an answer the judge can grade.
+function gradable(d) { return !!(d && d.generation_hash && d.truncated !== true); }
+
+// Every draw to grade, with its case and mode, in receipt order.
+function drawsToGrade(receipt) {
+  const out = [];
+  for (const c of receipt.results.cases) for (const d of ((c.generation || {}).draws || [])) if (gradable(d)) out.push({ c, d });
+  return out;
+}
+
+// The answer check (AC-4): every draw to grade has an answer, and each answer is
+// the text its generation_hash was taken over. An answer that fails either is a
+// regrade of something no model wrote.
+function answerProblems(receipt, answers) {
+  const bad = [];
+  for (const { c, d } of drawsToGrade(receipt)) {
+    const where = `${c.id}/${c.mode} draw ${d.draw_index}`;
+    const text = answers[d.generation_hash];
+    if (typeof text !== 'string') bad.push(`${where}: no answer for generation_hash ${d.generation_hash.slice(0, 12)}`);
+    else if (sha256(text) !== d.generation_hash) bad.push(`${where}: the answer's sha256 is not its generation_hash ${d.generation_hash.slice(0, 12)}`);
+  }
+  return bad;
+}
+
+// Everything that must hold before the first call, and what the regrade will
+// cost. No call is made. -> { problems, draws, calls, usd }
+function planRegrade({ receipt, skill, answers, judgeModel, samples }) {
+  const problems = [];
+  if (!verifyReceiptHash(receipt)) problems.push('the receipt_hash does not verify: the receipt was edited after it was sealed');
+  if (skill.contentHash !== receipt.skill.content_hash) problems.push(`the skill's content_hash ${String(skill.contentHash).slice(0, 12)} is not the receipt's ${String(receipt.skill.content_hash).slice(0, 12)}`);
+  if (skill.suite.suiteHash !== receipt.suite.suite_hash) problems.push(`the suite's hash ${String(skill.suite.suiteHash).slice(0, 12)} is not the receipt's ${String(receipt.suite.suite_hash).slice(0, 12)}`);
+  // One row per (case, mode) (spec 044 A-044-4): a receipt with two rows for one case and mode
+  // is ambiguous, which spec 050's validation refuses; refused here before any call, not after the
+  // spend.
+  const rows = new Set();
+  for (const c of receipt.results.cases) {
+    const key = `${c.id}\u0000${c.mode}`;
+    if (rows.has(key)) problems.push(`case ${c.id}/${c.mode} appears in more than one row; the receipt is ambiguous`);
+    rows.add(key);
+  }
+  for (const c of receipt.results.cases) {
+    if (!skill.suite.cases.some((k) => k.id === c.id)) problems.push(`case ${c.id} is not in the suite`);
+    if (!c.generation || !Array.isArray(c.generation.draws)) problems.push(`case ${c.id}/${c.mode} carries no draw set; a pre-v0.5 receipt cannot be regraded draw by draw`);
+  }
+  if (!problems.length) problems.push(...answerProblems(receipt, answers));
+  const draws = problems.length ? 0 : drawsToGrade(receipt).length;
+  const calls = draws * samples;
+  const usd = Math.round(calls * perCallCostUSD(resolveModel(judgeModel), 'judge') * 1e4) / 1e4;
+  return { problems, draws, calls, usd };
+}
+
+// The generation side of a draw, carried as it was.
+function generationSide(d) {
+  const g = { stop_reason: d.stop_reason === undefined ? null : d.stop_reason, truncated: d.truncated === true, reported_model: d.reported_model === undefined ? null : d.reported_model };
+  return g;
+}
+
+// Regrade one sealed receipt. opts: { trusted, budget, timeoutMs, onProgress, nowIso }.
+// -> { receipt, provenance, calls }
+async function regradeReceipt({ receipt, skill, answers, judgeModel, samples, opts = {} }) {
+  const judge = resolveModel(judgeModel);
+  const modelId = receipt.run.model_id;
+  assertRegistered(judge, 'judge model');
+  const trusted = !!opts.trusted;
+  const budget = opts.budget || null;
+  const onProgress = opts.onProgress || (() => {});
+  const timeoutMs = resolveCallTimeoutMs(surfaceForModel(judge), opts);
+  const judgedAt = new Date().toISOString();
+
+  let calls = 0;
+  const replies = [];
+  const cases = [];
+  for (const orig of receipt.results.cases) {
+    const caseObj = skill.suite.cases.find((k) => k.id === orig.id);
+    const mode = orig.mode;
+    const draws = [];
+    let last = null;
+    for (const d of orig.generation.draws) {
+      if (!gradable(d)) { draws.push(JSON.parse(JSON.stringify(d))); continue; }
+      const gen = generationSide(d);
+      const usage = d.usage ? { usage: d.usage } : {};
+      onProgress({ case: orig.id, mode, phase: 'judge', draw: d.draw_index, samples });
+      let jr;
+      try {
+        jr = await judgeCase({ caseObj, response: answers[d.generation_hash], generationHash: d.generation_hash, judgeModel: judge, mode, timeoutMs, samples, trusted });
+      } catch (e) {
+        if (!isTimeout(e)) throw e;
+        if (budget) budget.add((e.judgeAttempts || 1) * perCallCostUSD(judge, 'judge'));
+        draws.push({ draw_index: d.draw_index, generation_hash: d.generation_hash, status: 'unmeasured', reason: String((e && e.message) || 'timeout').slice(0, 200), samples: [], mean: null, stddev: null, ...gen, ...usage });
+        continue;
+      }
+      calls += samples;
+      for (const reply of (jr.replies || [])) {
+        if (!reply) continue;
+        replies.push(reply);
+        attest(reply, judgeModel, 'judge');
+      }
+      if (budget) budget.add((jr.attempts || samples) * perCallCostUSD(judge, 'judge'));
+      if (jr.unmeasured) {
+        const u = { draw_index: d.draw_index, generation_hash: d.generation_hash, status: 'unmeasured', reason: String(jr.reason || '').slice(0, 200), samples: [], mean: null, stddev: null, ...gen };
+        if ((jr.sampleHashes || []).length) u.judge_sample_hashes = jr.sampleHashes;
+        Object.assign(u, usage);
+        if (jr.judge_usage) u.judge_usage = jr.judge_usage;
+        draws.push(u);
+        continue;
+      }
+      const m = {
+        draw_index: d.draw_index, generation_hash: d.generation_hash, status: 'measured',
+        samples: jr.caseResult.samples, judge_sample_hashes: jr.caseResult.judge_sample_hashes,
+        mean: jr.caseResult.mean, stddev: jr.caseResult.stddev, ...gen, ...usage,
+      };
+      if (jr.caseResult.judge_usage) m.judge_usage = jr.caseResult.judge_usage;
+      draws.push(m);
+      last = jr;
+    }
+    const agg = acrossDraws(draws);
+    const generation = {
+      n_planned: orig.generation.n_planned,
+      n_drawn: agg.n_drawn,
+      n_measured: agg.n_measured,
+      n_unmeasured: agg.n_unmeasured,
+      stopping_reason: orig.generation.stopping_reason,
+      mean: agg.mean,
+      sd: agg.sd,
+      judge_sd_mean: agg.judge_sd_mean,
+      variance_ratio: agg.variance_ratio,
+      variance_ratio_unavailable: agg.variance_ratio_unavailable,
+      n_truncated: draws.filter((x) => x.truncated === true).length,
+      draws,
+    };
+    if (!last) {
+      const nonTimeout = draws.some((x) => x.status === 'unmeasured' && !/tim(e|ed)\s*out/i.test(String(x.reason || '')));
+      const reason = [...draws].reverse().map((x) => x.reason).find(Boolean) || 'timeout';
+      cases.push({ id: orig.id, mode, case_status: nonTimeout ? FAILED_STATUSES[1] : FAILED_STATUSES[0], reason, generation });
+      continue;
+    }
+    const caseResult = { ...last.caseResult, generation };
+    caseResult.mean = agg.mean;
+    caseResult.score = agg.mean;
+    caseResult.stddev = agg.sd;
+    caseResult.outcome = outcomeFor(agg.mean, agg.sd, caseObj.pass_threshold);
+    onProgress({ case: orig.id, mode, phase: 'done', outcome: caseResult.outcome, score: caseResult.mean, stddev: caseResult.stddev });
+    cases.push(caseResult);
+  }
+
+  // What answered the judge. A stub judge graded nothing: its receipt is UNVERIFIED
+  // with judge surface stub, whatever answered the generations (spec 026 AC-1).
+  const judgeStub = replies.length > 0 && replies.every((r) => r.answeredBy === 'stub');
+  const judgeModelAnswered = replies.length > 0 && replies.every((r) => r.answeredBy === 'model');
+  const judgeBlock = { ...judgeSettings(samples, judge), ...(judgeStub ? { surface: 'stub' } : {}), model_id: judge, prompt_template_hash: promptTemplateHash() };
+  const nowIso = opts.nowIso || new Date().toISOString();
+  const surface = receipt.run.surface;
+  const pricingSnapshot = buildPricingSnapshot({ models: [modelId, judge], lookup: priceForModel, nowIso });
+  const economics = computeEconomics({ cases, modelId, judgeModelId: judge, pricingSnapshot, surface, meteredSurface: isMeteredSurface(surface) });
+
+  // v0.8 (spec 043 AC-4). Every arm was generated in an earlier run, so each is archived:
+  // its own model and generation time, carried from the original (the original's own arm
+  // override where it has one, else its run's), and its own counts, because an archived arm
+  // never inherits run.counts. lib/counts.js placeCounts places counts for arms generated in
+  // this run and none is, so they are placed here by its rule: at arm scope when every case
+  // of the arm has the same count, else on each case that has one. Nothing is generated in
+  // a regrade, so the receipt carries no run.generated_at and no run.counts.
+  const arms = {};
+  for (const mode of ['with_skill', 'baseline']) {
+    const modeCases = cases.filter((c) => c.mode === mode);
+    if (!modeCases.length) continue;
+    const a = (receipt.run.arms && receipt.run.arms[mode]) || {};
+    arms[mode] = { model_id: a.model_id || modelId, generated_at: a.generated_at || receipt.run.generated_at || receipt.run.date_utc };
+    const perCase = modeCases.map((c) => ({
+      c,
+      counts: {
+        ...(c.generation && Array.isArray(c.generation.draws) ? { generations_per_arm: c.generation.draws.length } : {}),
+        ...(!c.case_status ? { judge_samples_per_generation: samples } : {}),
+      },
+    }));
+    for (const kind of ['generations_per_arm', 'judge_samples_per_generation']) {
+      const values = perCase.map((x) => (Number.isInteger(x.counts[kind]) ? x.counts[kind] : null));
+      if (values.every((v) => v !== null) && new Set(values).size === 1) arms[mode].counts = { ...(arms[mode].counts || {}), [kind]: values[0] };
+      else for (const x of perCase) if (Number.isInteger(x.counts[kind])) x.c.counts = { ...(x.c.counts || {}), [kind]: x.counts[kind] };
+    }
+  }
+  const graderRevision = {
+    prompt_template_hash: judgeBlock.prompt_template_hash,
+    rubric_hashes: cases.map((c) => { const k = skill.suite.cases.find((x) => x.id === c.id); return k ? rubricHash(k.rubric) : null; }),
+  };
+  const out = buildReceipt({
+    skill: { name: receipt.skill.name, version: receipt.skill.version, contentHash: receipt.skill.content_hash, tokens: receipt.skill.tokens },
+    suite: { format: receipt.suite.format, suiteHash: receipt.suite.suite_hash, caseCount: receipt.suite.case_count, canary: receipt.suite.canary },
+    run: {
+      model_id: modelId,
+      model_release_date: receipt.run.model_release_date,
+      provider: receipt.run.provider,
+      surface,
+      surface_overhead_note: receipt.run.surface_overhead_note,
+      runner_version: RUNNER_VERSION,
+      date_utc: nowIso,
+      registry: receipt.run.registry,
+      transcripts: 'hashes-only',
+      judge: judgeBlock,
+      pricing_snapshot: pricingSnapshot,
+      answered_by: receipt.run.answered_by,
+      arms,
+      judged_at: judgedAt,
+      grader_revision: graderRevision,
+    },
+    cases,
+    economics,
+    verificationLevel: receipt.verification_level === 'TESTED' && judgeModelAnswered ? 'TESTED' : 'UNVERIFIED',
+  });
+
+  // Where the regrade came from: what v0.8 has no field for. The judge time, the grader
+  // revision and the archived arms are in the receipt (spec 043 AC-4) and are not repeated
+  // here; this sidecar is bound to the receipt by its hash.
+  const provenance = {
+    format: REGRADE_SIDECAR.format,
+    receipt_hash: out.receipt_hash,
+    regraded_from: { receipt_hash: receipt.receipt_hash, runner_version: receipt.run.runner_version, date_utc: receipt.run.date_utc, judge: receipt.run.judge },
+    generated_at_basis: receipt.run.generated_at || (receipt.run.arms && Object.values(receipt.run.arms).some((x) => x && x.generated_at))
+      ? "the original receipt's own generated_at"
+      : "the original receipt's run.date_utc: a receipt of that schema records no separate generation time",
+    draws: { graded: drawsToGrade(receipt).length, carried: receipt.results.cases.reduce((a, c) => a + c.generation.draws.length, 0) - drawsToGrade(receipt).length },
+    judge_reported_models: [...new Set(replies.flatMap((r) => (r.reportedModels || []).map((id) => canonicalModelId(id))))].sort(),
+    calls,
+  };
+  return { receipt: out, provenance, calls };
+}
+
+module.exports = { regradeReceipt, planRegrade, answerProblems, drawsToGrade, gradable };