driftproof 0.6.0 → 0.7.1

package/lib/receipt.js

@@ -15,7 +15,12 @@ const SCHEMA_FILES = {
   '0.2': 'receipt.v0.2.schema.json',
   '0.3': 'receipt.v0.3.schema.json',
   '0.3.1': 'receipt.v0.3.1.schema.json',
-  '0.4': 'receipt.schema.json',
+  // v0.4 moved from the unversioned filename to a version-pinned one when v0.5
+  // took the current pointer. Without this the archive would lose v0.4: every
+  // published v0.4 receipt asserts conformance by NUMBER, and the number has to
+  // keep resolving to the schema it meant (AC-10).
+  '0.4': 'receipt.v0.4.schema.json',
+  '0.5': 'receipt.schema.json',
 };
 
 const _validators = {};
@@ -84,15 +89,70 @@ function aggregate(caseResults) {
 function buildReceipt({ skill, suite, run, cases, economics = null, verificationLevel = 'TESTED', editorialReviews = null }) {
   // v0.3.1: cases marked failed_timeout are recorded in results.cases but EXCLUDED
   // from aggregates — a band is never fabricated from a case that did not complete.
-  const okCases = cases.filter((c) => c.case_status !== 'failed_timeout');
-  const failedCount = cases.length - okCases.length;
+  //
+  // PAIRWISE, NOT PER ARM (spec 017 AC-5, AC-6). This filtered a FLAT list and
+  // then split by mode, so a case whose baseline failed kept its with_skill arm:
+  // #007's cell 3 recorded with_skill case_count 7 against baseline 6, and its
+  // headline `delta` of +0.116 was a 7-case mean minus a 6-case mean.
+  //
+  // `comparison.delta` is PAIRED BY CONSTRUCTION — one suite, measured twice —
+  // and a paired statistic computed over unequal sets is not the statistic it
+  // names. So an arm that cannot be measured removes its CASE from both sides.
+  //
+  // Exclusion, not refusal, and the reason is recorded rather than argued: an
+  // aggregate is a summary statistic, not a verdict, so the "refuse rather than
+  // assert" rule that governs verdicts does not reach it; and refusing the whole
+  // aggregate would discard thirteen sound arms because one failed. What the
+  // aggregate owes a reader instead is that it says what it covered, which
+  // `excluded_cases` provides.
+  //
+  // The excluded case STAYS in `results.cases`. It is removed from the mean, not
+  // from the record — deleting the evidence of a failure is a different and worse
+  // defect than averaging over it.
+  const armUnusable = (c) => c.case_status === 'failed_timeout'
+    || (c.mean == null && c.score == null);
+  const excludedIds = new Map();
+  for (const c of cases) {
+    if (!armUnusable(c)) continue;
+    if (excludedIds.has(c.id)) { excludedIds.get(c.id).modes.push(c.mode); continue; }
+    excludedIds.set(c.id, {
+      id: c.id,
+      modes: [c.mode],
+      reason: c.reason
+        || (c.generation && c.generation.stopping_reason === 'unmeasured_exhausted'
+          ? 'every generation draw was unmeasured'
+          : 'the arm has no measured result'),
+    });
+  }
+  const okCases = cases.filter((c) => !excludedIds.has(c.id));
   const withSkill = okCases.filter((c) => c.mode === 'with_skill');
   const baseline = okCases.filter((c) => c.mode === 'baseline');
+  const excludedCases = [...excludedIds.values()];
+  // `run.failed_case_count` COUNTS CASES, and is computed FROM the list it
+  // summarises rather than alongside it, so the two fields cannot disagree.
+  //
+  // It was `cases.length - okCases.length`, which counted ROWS. That was the
+  // same number while exclusion was per arm; once AC-5 made exclusion pairwise
+  // the subtraction removed BOTH arms of every excluded case, so one failed arm
+  // reported 2. Measured on the archive before this fix: #006's writing-plans
+  // receipt recomputed to 4 against the 2 it records, and #007's to 2 against 1
+  // — two published figures doubled by a change that never named this field.
+  //
+  // The field's name says cases and the aggregates exclude by case, so the
+  // count is the length of `results.aggregates.excluded_cases` and nothing
+  // else. A case whose BOTH arms failed is one exclusion and counts once.
+  const failedCount = excludedCases.length;
   const aggWith = aggregate(withSkill);
   const aggBase = aggregate(baseline);
 
   const receipt = {
     schema_version: RECEIPT_SCHEMA_VERSION,
+    // v0.5 CAPABILITY FLAG (F-014-F). DERIVED FROM THE CASES THEMSELVES, never
+    // from a caller's argument: the exposure this closes is a receipt asserting
+    // something it does not carry, and a flag taken on trust from the caller
+    // would be the same defect with an extra step. Absent when no case carries a
+    // draw set, which is what keeps every legacy and imported receipt valid.
+    ...(cases.some((c) => c && c.generation) ? { generation_sampled: true } : {}),
     skill: {
       name: skill.name,
       version: skill.version,
@@ -102,6 +162,12 @@ function buildReceipt({ skill, suite, run, cases, economics = null, verification
       format: suite.format,
       suite_hash: suite.suiteHash,
       case_count: suite.caseCount,
+      // v0.5: the per-suite canary. THIS is the canonical assembly — the runner
+      // built the field and this function dropped it, so the live smoke emitted
+      // `canary: undefined` and the schema, which makes it optional, said
+      // nothing. Caught by reading the receipt a real run produced, not by a
+      // gate; the assertion that would have caught it is added with the fix.
+      ...(suite.canary ? { canary: suite.canary } : {}),
     },
     run: {
       model_id: run.model_id,
@@ -120,7 +186,13 @@ function buildReceipt({ skill, suite, run, cases, economics = null, verification
     },
     results: {
       cases,
-      aggregates: { with_skill: aggWith, baseline: aggBase },
+      aggregates: {
+        with_skill: aggWith,
+        baseline: aggBase,
+        // Present only when something was excluded, so a clean run's receipt is
+        // unchanged and the archive does not acquire an empty field.
+        ...(excludedCases.length ? { excluded_cases: excludedCases } : {}),
+      },
     },
     comparison: {
       with_skill_score: aggWith.mean_score,

package/lib/reuse.js

@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+// Rerun / regrade / reuse, and the baseline-reproduction precondition.
+// Receipt spec v0.5.
+//
+// TWO THINGS LIVE HERE, both decided from what two receipts RECORD rather than
+// from a flag someone passed:
+//
+//   1. THE PRECONDITION (AC-6). Report #006's whole result. The no-skill arm
+//      contains no skill text, so a skill revision cannot move it. If it fails
+//      to reproduce the arm an earlier receipt measured on the same model, the
+//      same surface and the same suite, then whatever else changed, the two
+//      receipts are not measuring the same thing — and a verdict computed across
+//      them would be a comparison whose premise was never checked. That is the
+//      F-009-X class. The comparison is REFUSED and no verdict is asserted.
+//
+//   2. THE TRIAGE (AC-8). Terminal-Bench 4.0 distinguishes rerunning a task,
+//      regrading an existing transcript, and reusing a prior result. Adopted,
+//      translated: our unit is a (case, arm) draw set, and the trigger is the
+//      receipt's own recorded provenance — model, suite and skill content force
+//      a rerun because the generation would differ; judge and rubric force a
+//      regrade because only the scoring would; neither changing permits reuse.
+//
+// PURE: two receipts in, a decision out. No I/O, no provider. F-009-K's lesson
+// applied — the decision derives from the artifact, never from a filename.
+
+const { round } = require('./stats');
+
+// EVERY REASON STATES WHAT WAS OBSERVED AND STOPS THERE (F-009-N). The control
+// proves non-reproduction; it cannot say why. A reason that named a cause — "the
+// skill regressed", "the model got worse" — would be a finding this instrument
+// did not measure, which is the one thing it must never publish.
+const REFUSAL_REASONS = {
+  baseline_did_not_reproduce: ({ observed, expected } = {}) =>
+    `the baseline arm did not reproduce: this run observed ${observed}, the earlier receipt recorded ${expected}, and the bands do not overlap. No verdict is asserted; the control shows non-reproduction and cannot establish a cause.`,
+  baseline_missing: () =>
+    'no baseline arm is present in one of the two receipts, so reproduction cannot be checked and no verdict is asserted.',
+  baseline_unmeasured: () =>
+    'the baseline arm has no measured draws, so reproduction cannot be checked and no verdict is asserted.',
+  incomparable_surface: () =>
+    'the two receipts record different surfaces, so the earlier measurement cannot stand as a control here and no verdict is asserted.',
+};
+
+const casesOf = (r) => ((r && r.results && r.results.cases) || []);
+const armOf = (r, mode) => casesOf(r).filter((c) => c.mode === mode);
+
+// THE ONE BAND DEFINITION, and it resolves the shape the receipt actually
+// recorded (spec 016 AC-1, closing F-015-C).
+//
+// WHAT WENT WRONG. This read `c.generation.mean` and nothing else. That block
+// exists on v0.5 receipts and on no earlier one — v0.4 records the same
+// measurement as `c.mean` and `c.stddev` — so every v0.4 case resolved to `null`
+// and `baselineReproduces` returned `baseline_unmeasured` for EVERY v0.4→v0.5
+// pair, always, whatever the baselines had done. The precondition written
+// specifically for cross-version comparison could not pass against the archive
+// it exists to compare against, and Report #007's entire comparison step is
+// v0.4-against-v0.5.
+//
+// It survived 487 repo assertions and three approval rounds because every
+// assertion that ever exercised it handed it a fixture built to the CURRENT
+// schema. That is the seventh narrowing class, `fixture-vs-real-artifact`.
+//
+// THE BAND CARRIES WHICH SHAPE IT CAME FROM, and the two are not the same
+// statistic: v0.5's `sd` is spread ACROSS DRAWS, v0.4's `stddev` is spread across
+// JUDGE SAMPLES of one draw. Comparing them is the only comparison v0.4 admits —
+// it is the band that version measured — but a reader is owed the fact that the
+// older side is a judge-level band, so `source` travels with it and the differ
+// says so rather than presenting them as like for like.
+//
+// NO SWITCH GUARDS THE LEGACY PATH. An earlier revision carried a const-true
+// `LEGACY_BAND_ENABLED` so a probe could disable the fallback without editing the
+// logic; that leaves the pre-fix behaviour resident in the shipped tree, which an
+// approval called what it is — a hazard a comment does not remove. The mutation
+// probe patches this source in a disposable copy instead, and fails loudly if its
+// anchor moves.
+function bandOf(c) {
+  if (!c) return null;
+  const g = c.generation;
+  if (g && typeof g.mean === 'number') {
+    const sd = typeof g.sd === 'number' ? g.sd : 0;
+    return { mean: g.mean, sd, lo: g.mean - sd, hi: g.mean + sd, n: g.n_measured, source: 'generation' };
+  }
+  // v0.4 and earlier: the same measurement, under the names that version used.
+  // `score` is v0.1's spelling of the mean and is accepted for the same reason.
+  const mean = typeof c.mean === 'number' ? c.mean : (typeof c.score === 'number' ? c.score : null);
+  if (mean === null) return null;
+  const sd = typeof c.stddev === 'number' ? c.stddev : 0;
+  return { mean, sd, lo: mean - sd, hi: mean + sd, n: Array.isArray(c.samples) ? c.samples.length : null, source: 'legacy' };
+}
+
+function baselineReproduces(older, newer) {
+  const a = armOf(older, 'baseline');
+  const b = armOf(newer, 'baseline');
+  if (!a.length || !b.length) return { ok: false, key: 'baseline_missing' };
+  const bad = [];
+  for (const nb of b) {
+    const ob = a.find((x) => x.id === nb.id);
+    if (!ob) continue;
+    const bandA = bandOf(ob); const bandB = bandOf(nb);
+    if (!bandA || !bandB) return { ok: false, key: 'baseline_unmeasured' };
+    const overlap = bandA.lo <= bandB.hi && bandB.lo <= bandA.hi;
+    if (!overlap) bad.push({ id: nb.id, observed: round(bandB.mean), expected: round(bandA.mean) });
+  }
+  if (bad.length) return { ok: false, key: 'baseline_did_not_reproduce', ...bad[0], cases: bad };
+  return { ok: true };
+}
+
+// The verdict path. REFUSED is a RESULT, not an error: it carries no delta, and
+// it is what #006 published three times.
+function compare(older, newer) {
+  const pre = baselineReproduces(older, newer);
+  if (!pre.ok) {
+    const reason = REFUSAL_REASONS[pre.key]({ observed: pre.observed, expected: pre.expected });
+    return { verdict: 'REFUSED', reason, reason_key: pre.key, delta: null, cases: pre.cases || [] };
+  }
+  const ws = (r) => { const arm = armOf(r, 'with_skill'); const b = arm.map(bandOf).filter(Boolean); return b.length ? b.reduce((s, x) => s + x.mean, 0) / b.length : null; };
+  const a = ws(older); const b = ws(newer);
+  if (a === null || b === null) return { verdict: 'REFUSED', reason: REFUSAL_REASONS.baseline_unmeasured(), reason_key: 'baseline_unmeasured', delta: null };
+  return { verdict: 'MEASURED', delta: round(b - a), baseline_reproduced: true };
+}
+
+function triage(a, b) {
+  const g = (r, p, d) => p.split('.').reduce((o, k) => (o == null ? o : o[k]), r) ?? d;
+  const changed = (p) => g(a, p) !== g(b, p);
+  if (changed('run.model_id')) return { decision: 'rerun', reason: 'the model id differs, so the generation would differ' };
+  if (changed('suite.suite_hash')) return { decision: 'rerun', reason: 'the suite content differs, so the prompts would differ' };
+  if (changed('skill.content_hash')) return { decision: 'rerun', reason: 'the skill content differs, so the with-skill generation would differ' };
+  if (changed('run.judge.model_id')) return { decision: 'regrade', reason: 'only the judge differs, so the existing generations can be rescored' };
+  if (changed('run.rubric_hash')) return { decision: 'regrade', reason: 'only the rubric differs, so the existing generations can be rescored' };
+  return { decision: 'reuse', reason: 'model, suite, skill, judge and rubric all match, so the earlier result stands' };
+}
+
+module.exports = { compare, baselineReproduces, triage, REFUSAL_REASONS, bandOf };

package/lib/revision.js

@@ -2,6 +2,7 @@
 'use strict';
 
 const { bandVerdict, round } = require('./stats');
+const { bandOf } = require('./reuse');
 const { EFFECT_FLOOR } = require('../config');
 
 // Revision drift — the fifth report type (spec 009, Report #006).
@@ -104,7 +105,13 @@ function baselineBands(receipt) {
   const out = {};
   for (const c of receipt.results.cases) {
     if (c.mode !== 'baseline') continue;
-    out[c.id] = { mean: c.mean != null ? c.mean : c.score, stddev: c.stddev || 0 };
+  // ONE BAND DEFINITION (spec 016 AC-1). This built its own from the v0.4-shaped
+  // fields, which worked — and that is the point: three copies existed, two
+  // reading the legacy shape and one reading only v0.5, and the one that read
+  // only v0.5 was the one a cross-version control depended on (F-015-C). Routing
+  // every comparison path through `bandOf` means a future shape is added once.
+    const b = bandOf(c);
+    if (b) out[c.id] = { mean: b.mean, stddev: b.sd, source: b.source };
   }
   return out;
 }

package/lib/run.js

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 'use strict';
 
-const { complete, resolveModel, surfaceForModel, isMeteredSurface, CODEX_OVERHEAD_NOTE } = require('./provider');
+const { complete, resolveModel, surfaceForModel, isMeteredSurface, retryPolicyForSurface, CODEX_OVERHEAD_NOTE } = require('./provider');
 const { gradeSamples, judgeSettings } = require('./judge');
 const { buildReceipt } = require('./receipt');
 const { sha256 } = require('./canonical');
@@ -11,7 +11,9 @@ const { runChecks } = require('./checks');
 const { estimateTokens } = require('./skillCost');
 const { hasUsage, normalizeUsage } = require('./usage');
 const { buildPricingSnapshot, computeEconomics } = require('./value');
-const { RUNNER_VERSION, DEFAULT_JUDGE_SAMPLES } = require('../config');
+const { RUNNER_VERSION, DEFAULT_JUDGE_SAMPLES, DEV_MAX_CALLS } = require('../config');
+const { SAMPLING, acrossDraws, nextAction } = require('./sampling');
+const { suiteCanary } = require('./canary');
 
 // Known model release dates (best-effort; null when unknown). Recorded into the
 // receipt so drift reports can order runs by model age. Dateless model ids
@@ -36,9 +38,14 @@ function releaseDateFor(modelId) {
 }
 
 // Calls for one model run: each case does 2 generations (with_skill + baseline)
-// and 2×samples judge calls (each generation judged `samples` times).
-function projectCalls(caseCount, samples) {
-  return caseCount * (2 + 2 * samples);
+// and 2×samples judge calls (each generation judged `samples` times) — times the
+// number of GENERATION DRAWS per arm (v0.5).
+//
+// `draws` DEFAULTS TO 1 so every existing caller projects exactly what it
+// projected before; the runner's own cost guard passes the sampling MAXIMUM,
+// which is the fail-safe direction for a guard that decides whether to spend.
+function projectCalls(caseCount, samples, draws = 1) {
+  return caseCount * draws * (2 + 2 * samples);
 }
 
 // Ask the target model to perform one eval case. `withSkill` decides whether the
@@ -126,14 +133,45 @@ async function mapPool(items, concurrency, fn) {
 //   keepTranscripts — when true, the run records transcripts:"retained-local"
 //                     and returns the raw generations + judge outputs so the
 //                     caller can write them to transcripts/<receipt-id>/.
+// THE ONE PLACE A CALL TIMEOUT IS DECIDED (spec 017 AC-1, AC-2).
+//
+// PURE: a surface name and the run's options in, milliseconds out. No I/O, no
+// clock, no provider — so a probe can ask what a run WOULD use without making a
+// call, which is what makes this criterion assertable at all.
+//
+// An operator-supplied `opts.timeoutMs` still wins; what is gone is the silent
+// literal that used to stand in for a policy. `Number.isFinite` rather than a
+// truthiness test, so an explicit 0 is a value and not a fall-through.
+// Declared as a NAMED FUNCTION EXPRESSION bound to a const, not as a bare
+// declaration. The binding the module exports and the name inside the function
+// are then separable, so a mutation probe can rename the inner function to prove
+// the export is load-bearing without the module failing to load on an undefined
+// identifier. The inner name is kept for stack traces.
+const resolveCallTimeoutMs = function resolveCallTimeoutMs(surface, opts = {}) {
+  if (opts && Number.isFinite(opts.timeoutMs)) return opts.timeoutMs;
+  return retryPolicyForSurface(surface).timeoutMs;
+};
+
 async function runSkillOnModel({ skill, model, opts = {} }) {
   const modelId = resolveModel(model);
   const judgeModel = opts.judgeModel ? resolveModel(opts.judgeModel) : modelId;
-  const timeoutMs = opts.timeoutMs || 120000;
+  // THE SURFACE'S OWN DECLARED TIMEOUT, resolved per arm (spec 017 AC-1).
+  //
+  // This read `opts.timeoutMs || 120000`, and `lib/provider.js` documents that an
+  // explicit caller value wins — so the literal outranked the policy and the
+  // 300 s claude-cli timeout, written for cold-start-dominated CLI subprocesses,
+  // never executed. Report #007 lost 25 of 160 draws to
+  // `provider(claude-cli) timed out after 120000ms`, and one arm entirely.
+  //
+  // RESOLVED SEPARATELY FOR GENERATION AND JUDGE, because they can be different
+  // models on different surfaces: #007 generated on claude-fable-5 and judged on
+  // claude-haiku-4-5. One shared timeout would apply one surface's policy to both.
+  const genTimeoutMs = resolveCallTimeoutMs(surfaceForModel(modelId), opts);
+  const judgeTimeoutMs = resolveCallTimeoutMs(surfaceForModel(judgeModel), opts);
   // Optional per-case timeout overrides { caseId: ms }; a slow case can get a
   // longer budget without lengthening every other case's per-call timeout.
   const caseTimeoutMs = opts.caseTimeoutMs || {};
-  const maxCalls = opts.maxCalls || 200;
+  const maxCalls = opts.maxCalls || DEV_MAX_CALLS;
   const samples = opts.samples || DEFAULT_JUDGE_SAMPLES;
   const concurrency = Math.max(1, opts.concurrency || 1);
   const onProgress = opts.onProgress || (() => {});
@@ -145,7 +183,10 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
 
   // Cost guard: project the whole run up front and refuse before spending a
   // single call if it would blow the cap.
-  const projected = projectCalls(cases.length, samples);
+  // The WORST CASE, deliberately: escalation is adaptive and a guard that
+  // projects the floor would wave through a run that then draws ten times.
+  // Refusing a run that would have fit is recoverable; overspending is not.
+  const projected = projectCalls(cases.length, samples, SAMPLING.max);
   if (projected > maxCalls) {
     const e = new Error(`cost guard: projected ${projected} calls exceeds cap ${maxCalls} (${cases.length} cases × (2 + 2×${samples} samples)). Raise --max-calls or lower --max-cases/--samples.`);
     e.code = 'CALL_CAP';
@@ -163,43 +204,111 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
   const genKind = (ws) => (ws ? 'gen_with_skill' : 'gen_baseline');
   const pairs = await mapPool(tasks, concurrency, async ({ c, withSkill }) => {
     const mode = withSkill ? 'with_skill' : 'baseline';
-    const ct = caseTimeoutMs[c.id] || timeoutMs;
-    try {
-      onProgress({ case: c.id, mode, phase: 'generate' });
-      const gen = await runCase({ skillMd: skill.skillMd, caseObj: c, model: modelId, withSkill, timeoutMs: ct });
-      calls += 1;
-      // Live budget: count the generation call INCLUDING retries, then hard-stop
-      // if over 1.25× cap.
-      if (budget) budget.add((gen.attempts || 1) * perCallCostUSD(modelId, genKind(withSkill)));
-      const generationHash = sha256(String(gen.text || ''));
-      onProgress({ case: c.id, mode, phase: 'judge', samples });
-      const jr = await judgeCase({ caseObj: c, response: gen.text, generationHash, judgeModel, mode, timeoutMs: ct, samples });
-      // v0.4: the GENERATION call's usage is the skill-value measurement (the
-      // judge's own usage rides separately on judge_usage, above).
-      if (hasUsage(gen.usage)) jr.caseResult.usage = normalizeUsage({ ...gen.usage, wall_ms: gen.wall_ms });
-      calls += samples;
-      // Live budget: count all judge calls (retries included) for this (case, mode).
-      if (budget) budget.add((jr.attempts || samples) * perCallCostUSD(judgeModel, 'judge'));
-      onProgress({ case: c.id, mode, phase: 'done', outcome: jr.caseResult.outcome, score: jr.caseResult.mean, stddev: jr.caseResult.stddev });
-      const transcript = keepTranscripts
-        ? { id: c.id, mode, generation: String(gen.text || ''), judge_outputs: jr.sampleTexts }
-        : null;
-      return { caseResult: jr.caseResult, transcript };
-    } catch (e) {
-      if (e && e.code === 'BUDGET_HARDSTOP') throw e;   // budget hard-stop stays fatal
-      if (!isTimeout(e)) throw e;                        // non-timeout errors stay fatal
-      // Persistent timeout → NON-FATAL: charge the consumed attempts, record the
-      // case as failed_timeout (no fabricated samples), and continue the run.
-      if (budget) {
-        try {
-          if (e.phase === 'judge') budget.add((e.judgeAttempts || 1) * perCallCostUSD(judgeModel, 'judge'));
-          else budget.add((e.attempts || 1) * perCallCostUSD(modelId, genKind(withSkill)));
-        } catch (be) { if (be && be.code === 'BUDGET_HARDSTOP') throw be; }
+    // A per-case override is an operator's explicit input and still wins, for
+    // both arms. Otherwise each arm uses its own surface's declared policy.
+    const ctGen = caseTimeoutMs[c.id] || genTimeoutMs;
+    const ctJudge = caseTimeoutMs[c.id] || judgeTimeoutMs;
+    // v0.5 — DRAW THE GENERATION n TIMES, and keep each draw's judge samples
+    // INSIDE that draw. Pooling k×n scores into one list is precisely what made
+    // generation noise read as judge noise: it is the defect Report #006 exists
+    // to name, and the nesting is the whole measurement.
+    const draws = [];
+    let last = null;             // last MEASURED draw — carries the v0.4-shaped fields
+    let lastTranscript = null;
+    let action = { stop: false, reason: 'below_min' };
+    let fatal = null;
+
+    while (!action.stop && draws.length < SAMPLING.max) {
+      const drawIndex = draws.length;
+      try {
+        onProgress({ case: c.id, mode, phase: 'generate', draw: drawIndex });
+        const gen = await runCase({ skillMd: skill.skillMd, caseObj: c, model: modelId, withSkill, timeoutMs: ctGen });
+        calls += 1;
+        if (budget) budget.add((gen.attempts || 1) * perCallCostUSD(modelId, genKind(withSkill)));
+        const generationHash = sha256(String(gen.text || ''));
+        onProgress({ case: c.id, mode, phase: 'judge', samples, draw: drawIndex });
+        const jr = await judgeCase({ caseObj: c, response: gen.text, generationHash, judgeModel, mode, timeoutMs: ctJudge, samples });
+        calls += samples;
+        if (budget) budget.add((jr.attempts || samples) * perCallCostUSD(judgeModel, 'judge'));
+        const draw = {
+          draw_index: drawIndex,
+          generation_hash: generationHash,
+          status: 'measured',
+          samples: jr.caseResult.samples,
+          judge_sample_hashes: jr.caseResult.judge_sample_hashes,
+          mean: jr.caseResult.mean,
+          stddev: jr.caseResult.stddev,
+        };
+        if (hasUsage(gen.usage)) draw.usage = normalizeUsage({ ...gen.usage, wall_ms: gen.wall_ms });
+        if (jr.caseResult.judge_usage) draw.judge_usage = jr.caseResult.judge_usage;
+        draws.push(draw);
+        last = jr;
+        if (keepTranscripts) lastTranscript = { id: c.id, mode, generation: String(gen.text || ''), judge_outputs: jr.sampleTexts };
+      } catch (e) {
+        if (e && e.code === 'BUDGET_HARDSTOP') throw e;   // budget hard-stop stays fatal
+        if (!isTimeout(e)) { fatal = e; break; }          // non-timeout errors stay fatal
+        if (budget) {
+          try {
+            if (e.phase === 'judge') budget.add((e.judgeAttempts || 1) * perCallCostUSD(judgeModel, 'judge'));
+            else budget.add((e.attempts || 1) * perCallCostUSD(modelId, genKind(withSkill)));
+          } catch (be) { if (be && be.code === 'BUDGET_HARDSTOP') throw be; }
+        }
+        // F-009-L: the draw is UNMEASURED. No score, no fabricated samples, and
+        // it is excluded from every statistic rather than counted as a zero — a
+        // zero asserts a measurement, and a timeout is the absence of one.
+        draws.push({
+          draw_index: drawIndex,
+          generation_hash: null,
+          status: 'unmeasured',
+          reason: String((e && e.message) || 'timeout').slice(0, 200),
+          samples: [],
+          mean: null,
+          stddev: null,
+        });
+        onProgress({ case: c.id, mode, phase: 'failed', draw: drawIndex, reason: String((e && e.message) || 'timeout') });
       }
+      action = nextAction(draws);
+    }
+    if (fatal) throw fatal;
+
+    const agg = acrossDraws(draws);
+    const generation = {
+      n_planned: SAMPLING.min,
+      n_drawn: agg.n_drawn,
+      n_measured: agg.n_measured,
+      n_unmeasured: agg.n_unmeasured,
+      stopping_reason: action.reason,
+      mean: agg.mean,
+      sd: agg.sd,
+      judge_sd_mean: agg.judge_sd_mean,
+      variance_ratio: agg.variance_ratio,
+      // WHICH null, when it is null (F-014-C). Copied through explicitly rather
+      // than spread from `agg`: this assembly names its keys one by one, and the
+      // canary was dropped by exactly such an assembly silently gaining a field
+      // upstream that nothing here carried down (F-014-D).
+      variance_ratio_unavailable: agg.variance_ratio_unavailable,
+      draws,
+    };
+
+    // Every draw failed: the case is recorded failed_timeout as before, but it
+    // now carries the draw list showing WHAT failed and how often.
+    if (!last) {
       failedCases += 1;
-      onProgress({ case: c.id, mode, phase: 'failed', reason: String((e && e.message) || 'timeout') });
-      return { caseResult: { id: c.id, mode, case_status: 'failed_timeout', reason: String((e && e.message) || 'timeout').slice(0, 200) }, transcript: null };
+      return { caseResult: { id: c.id, mode, case_status: 'failed_timeout', reason: (draws[draws.length - 1] || {}).reason || 'timeout', generation }, transcript: null };
     }
+
+    // The v0.4-shaped fields now describe the DRAW SET, not one arbitrary draw,
+    // so a v0.4 reader pointed at a v0.5 receipt reads the aggregate rather than
+    // whichever draw happened to be last. `generation_hash`, `samples` and
+    // `judge_sample_hashes` continue to describe the last measured draw, which
+    // is the one they have always described; RECEIPT.md states this.
+    const caseResult = { ...last.caseResult, generation };
+    caseResult.mean = agg.mean;
+    caseResult.score = agg.mean;
+    caseResult.stddev = agg.sd;
+    caseResult.outcome = outcomeFor(agg.mean, agg.sd, c.pass_threshold);
+    onProgress({ case: c.id, mode, phase: 'done', outcome: caseResult.outcome, score: caseResult.mean, stddev: caseResult.stddev });
+    return { caseResult, transcript: lastTranscript };
   });
   const caseResults = pairs.map((p) => p.caseResult);
   const transcripts = keepTranscripts ? pairs.map((p) => p.transcript) : null;
@@ -229,7 +338,18 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
       // v0.3.1 value-per-token axis: estimated SKILL.md token size.
       tokens: estimateTokens(skill.skillMd),
     },
-    suite: { format: skill.suite.format, suiteHash: skill.suite.suiteHash, caseCount: skill.suite.caseCount },
+    // v0.5: the suite canary. Derived from the suite identity and its case ids,
+    // so it is stable without a registry and distinct across suites — a leaked
+    // suite is detectable in a corpus. A detection aid, not a control.
+    suite: {
+      format: skill.suite.format,
+      suiteHash: skill.suite.suiteHash,
+      caseCount: skill.suite.caseCount,
+      // The FULL suite, never the post---max-cases list: a canary derived from a
+      // truncated run is not stable for the suite, and a canary that moves
+      // cannot say which suite leaked. Approval finding, spec 014.
+      canary: suiteCanary({ id: skill.name || skill.suite.suiteHash, cases: (skill.suite.cases || cases).map((c) => ({ id: c.id })) }),
+    },
     run: {
       model_id: modelId,
       model_release_date: releaseDateFor(modelId),
@@ -277,7 +397,7 @@ function summarizeReceipt(receipt) {
   const aggs = receipt.results.aggregates;
   const sign = cmp.delta >= 0 ? '+' : '';
   if (receipt.run.status === 'incomplete') {
-    L.push(`> ⏱ **INCOMPLETE** — ${receipt.run.failed_case_count} case(s) failed (timed out after retries) and are EXCLUDED from the aggregates below; this receipt must not be used to compute a drift/durability verdict.`);
+    L.push(`> ⏱ **INCOMPLETE** — ${receipt.run.failed_case_count} case(s) had an arm that could not be measured and are EXCLUDED from the aggregates below, BOTH arms together; this receipt must not be used to compute a drift/durability verdict.`);
     L.push('');
   }
   L.push(`with_skill **${band(aggs.with_skill.mean_score, aggs.with_skill.stddev)}** vs baseline **${band(aggs.baseline.mean_score, aggs.baseline.stddev)}**`);
@@ -300,4 +420,4 @@ function summarizeReceipt(receipt) {
   return L.join('\n');
 }
 
-module.exports = { runSkillOnModel, summarizeReceipt, releaseDateFor, projectCalls, outcomeFor };
+module.exports = { runSkillOnModel, summarizeReceipt, releaseDateFor, projectCalls, outcomeFor, resolveCallTimeoutMs };