driftproof 0.5.0 → 0.7.1

package/lib/reuse.js

@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+// Rerun / regrade / reuse, and the baseline-reproduction precondition.
+// Receipt spec v0.5.
+//
+// TWO THINGS LIVE HERE, both decided from what two receipts RECORD rather than
+// from a flag someone passed:
+//
+//   1. THE PRECONDITION (AC-6). Report #006's whole result. The no-skill arm
+//      contains no skill text, so a skill revision cannot move it. If it fails
+//      to reproduce the arm an earlier receipt measured on the same model, the
+//      same surface and the same suite, then whatever else changed, the two
+//      receipts are not measuring the same thing — and a verdict computed across
+//      them would be a comparison whose premise was never checked. That is the
+//      F-009-X class. The comparison is REFUSED and no verdict is asserted.
+//
+//   2. THE TRIAGE (AC-8). Terminal-Bench 4.0 distinguishes rerunning a task,
+//      regrading an existing transcript, and reusing a prior result. Adopted,
+//      translated: our unit is a (case, arm) draw set, and the trigger is the
+//      receipt's own recorded provenance — model, suite and skill content force
+//      a rerun because the generation would differ; judge and rubric force a
+//      regrade because only the scoring would; neither changing permits reuse.
+//
+// PURE: two receipts in, a decision out. No I/O, no provider. F-009-K's lesson
+// applied — the decision derives from the artifact, never from a filename.
+
+const { round } = require('./stats');
+
+// EVERY REASON STATES WHAT WAS OBSERVED AND STOPS THERE (F-009-N). The control
+// proves non-reproduction; it cannot say why. A reason that named a cause — "the
+// skill regressed", "the model got worse" — would be a finding this instrument
+// did not measure, which is the one thing it must never publish.
+const REFUSAL_REASONS = {
+  baseline_did_not_reproduce: ({ observed, expected } = {}) =>
+    `the baseline arm did not reproduce: this run observed ${observed}, the earlier receipt recorded ${expected}, and the bands do not overlap. No verdict is asserted; the control shows non-reproduction and cannot establish a cause.`,
+  baseline_missing: () =>
+    'no baseline arm is present in one of the two receipts, so reproduction cannot be checked and no verdict is asserted.',
+  baseline_unmeasured: () =>
+    'the baseline arm has no measured draws, so reproduction cannot be checked and no verdict is asserted.',
+  incomparable_surface: () =>
+    'the two receipts record different surfaces, so the earlier measurement cannot stand as a control here and no verdict is asserted.',
+};
+
+const casesOf = (r) => ((r && r.results && r.results.cases) || []);
+const armOf = (r, mode) => casesOf(r).filter((c) => c.mode === mode);
+
+// THE ONE BAND DEFINITION, and it resolves the shape the receipt actually
+// recorded (spec 016 AC-1, closing F-015-C).
+//
+// WHAT WENT WRONG. This read `c.generation.mean` and nothing else. That block
+// exists on v0.5 receipts and on no earlier one — v0.4 records the same
+// measurement as `c.mean` and `c.stddev` — so every v0.4 case resolved to `null`
+// and `baselineReproduces` returned `baseline_unmeasured` for EVERY v0.4→v0.5
+// pair, always, whatever the baselines had done. The precondition written
+// specifically for cross-version comparison could not pass against the archive
+// it exists to compare against, and Report #007's entire comparison step is
+// v0.4-against-v0.5.
+//
+// It survived 487 repo assertions and three approval rounds because every
+// assertion that ever exercised it handed it a fixture built to the CURRENT
+// schema. That is the seventh narrowing class, `fixture-vs-real-artifact`.
+//
+// THE BAND CARRIES WHICH SHAPE IT CAME FROM, and the two are not the same
+// statistic: v0.5's `sd` is spread ACROSS DRAWS, v0.4's `stddev` is spread across
+// JUDGE SAMPLES of one draw. Comparing them is the only comparison v0.4 admits —
+// it is the band that version measured — but a reader is owed the fact that the
+// older side is a judge-level band, so `source` travels with it and the differ
+// says so rather than presenting them as like for like.
+//
+// NO SWITCH GUARDS THE LEGACY PATH. An earlier revision carried a const-true
+// `LEGACY_BAND_ENABLED` so a probe could disable the fallback without editing the
+// logic; that leaves the pre-fix behaviour resident in the shipped tree, which an
+// approval called what it is — a hazard a comment does not remove. The mutation
+// probe patches this source in a disposable copy instead, and fails loudly if its
+// anchor moves.
+function bandOf(c) {
+  if (!c) return null;
+  const g = c.generation;
+  if (g && typeof g.mean === 'number') {
+    const sd = typeof g.sd === 'number' ? g.sd : 0;
+    return { mean: g.mean, sd, lo: g.mean - sd, hi: g.mean + sd, n: g.n_measured, source: 'generation' };
+  }
+  // v0.4 and earlier: the same measurement, under the names that version used.
+  // `score` is v0.1's spelling of the mean and is accepted for the same reason.
+  const mean = typeof c.mean === 'number' ? c.mean : (typeof c.score === 'number' ? c.score : null);
+  if (mean === null) return null;
+  const sd = typeof c.stddev === 'number' ? c.stddev : 0;
+  return { mean, sd, lo: mean - sd, hi: mean + sd, n: Array.isArray(c.samples) ? c.samples.length : null, source: 'legacy' };
+}
+
+function baselineReproduces(older, newer) {
+  const a = armOf(older, 'baseline');
+  const b = armOf(newer, 'baseline');
+  if (!a.length || !b.length) return { ok: false, key: 'baseline_missing' };
+  const bad = [];
+  for (const nb of b) {
+    const ob = a.find((x) => x.id === nb.id);
+    if (!ob) continue;
+    const bandA = bandOf(ob); const bandB = bandOf(nb);
+    if (!bandA || !bandB) return { ok: false, key: 'baseline_unmeasured' };
+    const overlap = bandA.lo <= bandB.hi && bandB.lo <= bandA.hi;
+    if (!overlap) bad.push({ id: nb.id, observed: round(bandB.mean), expected: round(bandA.mean) });
+  }
+  if (bad.length) return { ok: false, key: 'baseline_did_not_reproduce', ...bad[0], cases: bad };
+  return { ok: true };
+}
+
+// The verdict path. REFUSED is a RESULT, not an error: it carries no delta, and
+// it is what #006 published three times.
+function compare(older, newer) {
+  const pre = baselineReproduces(older, newer);
+  if (!pre.ok) {
+    const reason = REFUSAL_REASONS[pre.key]({ observed: pre.observed, expected: pre.expected });
+    return { verdict: 'REFUSED', reason, reason_key: pre.key, delta: null, cases: pre.cases || [] };
+  }
+  const ws = (r) => { const arm = armOf(r, 'with_skill'); const b = arm.map(bandOf).filter(Boolean); return b.length ? b.reduce((s, x) => s + x.mean, 0) / b.length : null; };
+  const a = ws(older); const b = ws(newer);
+  if (a === null || b === null) return { verdict: 'REFUSED', reason: REFUSAL_REASONS.baseline_unmeasured(), reason_key: 'baseline_unmeasured', delta: null };
+  return { verdict: 'MEASURED', delta: round(b - a), baseline_reproduced: true };
+}
+
+function triage(a, b) {
+  const g = (r, p, d) => p.split('.').reduce((o, k) => (o == null ? o : o[k]), r) ?? d;
+  const changed = (p) => g(a, p) !== g(b, p);
+  if (changed('run.model_id')) return { decision: 'rerun', reason: 'the model id differs, so the generation would differ' };
+  if (changed('suite.suite_hash')) return { decision: 'rerun', reason: 'the suite content differs, so the prompts would differ' };
+  if (changed('skill.content_hash')) return { decision: 'rerun', reason: 'the skill content differs, so the with-skill generation would differ' };
+  if (changed('run.judge.model_id')) return { decision: 'regrade', reason: 'only the judge differs, so the existing generations can be rescored' };
+  if (changed('run.rubric_hash')) return { decision: 'regrade', reason: 'only the rubric differs, so the existing generations can be rescored' };
+  return { decision: 'reuse', reason: 'model, suite, skill, judge and rubric all match, so the earlier result stands' };
+}
+
+module.exports = { compare, baselineReproduces, triage, REFUSAL_REASONS, bandOf };

package/lib/revision.js

@@ -0,0 +1,169 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+const { bandVerdict, round } = require('./stats');
+const { bandOf } = require('./reuse');
+const { EFFECT_FLOOR } = require('../config');
+
+// Revision drift — the fifth report type (spec 009, Report #006).
+//
+// Every other report type holds the skill text fixed and moves something
+// underneath it: the model release (#001, #003), the vendor surface (#002), the
+// capability tier (#004), the axes and the price (#005). This one inverts the
+// design. The substrate is held still — same model, same provider, same surface,
+// same suite, same fixed judge, same sampling — and the SKILL'S OWN TEXT moves,
+// from the revision a report pinned to the revision upstream ships today.
+//
+// This module holds the report type's LANGUAGE as code rather than as
+// hand-written page copy. That is deliberate: the fairness rule below is the one
+// a measurement project is most tempted to apply in one direction only, and a
+// rule that lives in prose cannot be gated before the run that would tempt it.
+
+// ── the cell headline ────────────────────────────────────────────────────────
+// A summary of the per-case band-overlap verdicts, worded about the REVISION.
+// The release-drift headline says "the skill is measurably weaker", which is a
+// sentence about a skill under a moving model. Here the model is the control.
+function revisionHeadline(perCase) {
+  const reg = perCase.filter((r) => r.verdict === 'regression').length;
+  const imp = perCase.filter((r) => r.verdict === 'improvement').length;
+  const s = (n) => (n === 1 ? '' : 's');
+  if (reg && imp) {
+    return `MIXED — the revision improved ${imp} case${s(imp)} and regressed ${reg} on non-overlapping bands.`;
+  }
+  if (reg) {
+    return `REVISION REGRESSED — ${reg} case${s(reg)} scored lower under the current upstream text (bands do not overlap).`;
+  }
+  if (imp) {
+    return `REVISION IMPROVED — ${imp} case${s(imp)} scored higher under the current upstream text (bands do not overlap); none regressed.`;
+  }
+  return 'WITHIN NOISE — the revision moved no case beyond its confidence band; the pinned text and the current text measure the same.';
+}
+
+// Classification word for a cell, from its headline. Kept separate so a caller
+// can branch on the class without parsing prose.
+function revisionClass(perCase) {
+  const h = revisionHeadline(perCase);
+  return h.split(' —')[0];
+}
+
+// ── the fairness sentence ────────────────────────────────────────────────────
+// Spec 009 § Fairness, clauses 1 and 4. Where a revision IMPROVED a skill, the
+// published #005 figure understates the pack a reader can install today; where it
+// REGRESSED one, #005 overstates it. Both sentences are generated by the same
+// function, from the same template, so the disclosure cannot quietly become a
+// one-directional courtesy — the symmetry is a property of the code, and the
+// gate asserts it.
+//
+// A cell within noise gets NO sentence. #005's figure stands unamended, because
+// nothing was measured that would amend it, and manufacturing a hedge for a null
+// result is how a report launders noise into a finding.
+function fairnessSentence({ slug, classification, report005Delta, measuredDelta }) {
+  const cls = String(classification || '');
+  if (cls !== 'REVISION IMPROVED' && cls !== 'REVISION REGRESSED') return null;
+  const improved = cls === 'REVISION IMPROVED';
+  const direction = improved ? 'understates' : 'overstates';
+  const d = (n) => (n == null ? 'n/a' : (n >= 0 ? '+' : '') + Number(n).toFixed(3));
+  return `Report #005 measured ${slug} at ${d(report005Delta)} on the text it had pinned. `
+    + `Report #006 measures the current upstream revision at ${d(measuredDelta)} on the same substrate and the same suite. `
+    + `#005's published figure therefore ${direction} the pack upstream ships today for this skill, `
+    + `and is amended by this report rather than corrected in place.`;
+}
+
+// ── per-cell scoping disclosures ─────────────────────────────────────────────
+// Spec 009 AC-10. One cell in Report #006 measures something other than what its
+// upstream author changed it to do, and the reader looking at that row is the
+// reader who needs to be told.
+const SCOPING_NOTES = {
+  'git-workflow-and-versioning':
+    'This revision changes the frontmatter `description:` line. In a skill runtime a description is a '
+    + 'routing trigger: it decides whether the skill loads, and never reaches the model as guidance. '
+    + 'Driftproof makes no routing decision — it always injects the skill, and passes the whole file, '
+    + 'frontmatter included, as the system prompt. This cell therefore measures the revision as added '
+    + 'context and cannot measure it as a trigger.',
+};
+function scopingNote(slug) {
+  return SCOPING_NOTES[slug] || null;
+}
+
+// ── the baseline-reproduction control ────────────────────────────────────────
+// Spec 009 AC-6, and the thing that makes the free pinned arm honest.
+//
+// Report #006 reuses #005's receipts as the pinned-text arm. That is valid only
+// if the substrate has not moved, and `run.model_release_date` is null on every
+// #005 receipt, so id equality is the only version evidence a receipt carries. A
+// provider that re-points a concrete id at a new snapshot is invisible to it.
+//
+// It does not have to be. Every fresh run emits a BASELINE arm: the same cases,
+// the same substrate, and no skill text at all. The revision cannot touch it by
+// construction, so comparing the fresh baseline against the reused receipt's
+// baseline re-measures exactly the thing id equality could not prove — at no
+// extra cost, because that arm is already paid for.
+//
+// A cell whose baselines do not reproduce is NOT MEASURED. The reuse is a tested
+// prediction, not an assumption the report asks the reader to grant.
+function baselineBands(receipt) {
+  const out = {};
+  for (const c of receipt.results.cases) {
+    if (c.mode !== 'baseline') continue;
+  // ONE BAND DEFINITION (spec 016 AC-1). This built its own from the v0.4-shaped
+  // fields, which worked — and that is the point: three copies existed, two
+  // reading the legacy shape and one reading only v0.5, and the one that read
+  // only v0.5 was the one a cross-version control depended on (F-015-C). Routing
+  // every comparison path through `bandOf` means a future shape is added once.
+    const b = bandOf(c);
+    if (b) out[c.id] = { mean: b.mean, stddev: b.sd, source: b.source };
+  }
+  return out;
+}
+
+function baselineControl(reused, fresh) {
+  const A = baselineBands(reused);
+  const B = baselineBands(fresh);
+  const ids = [...new Set([...Object.keys(A), ...Object.keys(B)])];
+
+  const perCase = ids.map((id) => {
+    const before = A[id] || null;
+    const after = B[id] || null;
+    if (!before || !after) return { id, before, after, delta: null, moved: false, missing: true };
+    const delta = round(after.mean - before.mean);
+    // The same rule the study uses everywhere else: band separation AND the
+    // effect floor. A baseline that wobbles inside its band has not moved.
+    const raw = bandVerdict(before.mean, before.stddev, after.mean, after.stddev);
+    const separated = raw === 'regression' || raw === 'improvement';
+    return { id, before, after, delta, moved: separated && Math.abs(delta) >= EFFECT_FLOOR, missing: false };
+  });
+
+  const movedCases = perCase.filter((r) => r.moved);
+  const missing = perCase.filter((r) => r.missing);
+  const reproduced = movedCases.length === 0 && missing.length === 0;
+  const aggDelta = round(
+    (fresh.comparison && fresh.comparison.baseline_score != null ? fresh.comparison.baseline_score : 0)
+    - (reused.comparison && reused.comparison.baseline_score != null ? reused.comparison.baseline_score : 0),
+  );
+
+  return {
+    reproduced,
+    blocked: !reproduced,
+    verdict: reproduced ? 'MEASURED' : 'NOT MEASURED',
+    moved_cases: movedCases.map((r) => r.id),
+    missing_cases: missing.map((r) => r.id),
+    aggregate_baseline_delta: aggDelta,
+    floor: EFFECT_FLOOR,
+    // THE CONTROL PROVES NON-REPRODUCTION. IT CANNOT SAY WHY. These strings used
+    // to read 'the substrate moved' and 'the substrate held still' — a cause,
+    // asserted by a comparison that measures two baseline arms and nothing else.
+    // A 120-call stability probe then found generation-level sampling noise large
+    // enough to account for every gap this control saw, with no substrate movement
+    // required, and the report page retracted the claim while three committed
+    // control records still carried it (approval finding F-009-N). Reason strings
+    // only: no score, sample, hash or verdict changed with this edit.
+    reason: reproduced
+      ? 'the fresh baseline reproduces the reused receipt\'s baseline within the band and the floor, so the pinned-arm reuse stands for this cell'
+      : `the fresh baseline does not reproduce the reused receipt's baseline (${movedCases.length} case(s) moved beyond the band and the ${EFFECT_FLOOR} floor${missing.length ? `, ${missing.length} case(s) absent on one side` : ''}) — the reused pinned arm is not comparable to the fresh arm, so revision drift cannot be separated from whatever else changed in this cell; the control establishes non-reproduction and does not identify a cause`,
+    perCase,
+  };
+}
+
+module.exports = {
+  revisionHeadline, revisionClass, fairnessSentence, scopingNote, baselineControl,
+};

package/lib/run.js

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 'use strict';
 
-const { complete, resolveModel, surfaceForModel, isMeteredSurface, CODEX_OVERHEAD_NOTE } = require('./provider');
+const { complete, resolveModel, surfaceForModel, isMeteredSurface, retryPolicyForSurface, CODEX_OVERHEAD_NOTE } = require('./provider');
 const { gradeSamples, judgeSettings } = require('./judge');
 const { buildReceipt } = require('./receipt');
 const { sha256 } = require('./canonical');
@@ -11,7 +11,9 @@ const { runChecks } = require('./checks');
 const { estimateTokens } = require('./skillCost');
 const { hasUsage, normalizeUsage } = require('./usage');
 const { buildPricingSnapshot, computeEconomics } = require('./value');
-const { RUNNER_VERSION, DEFAULT_JUDGE_SAMPLES } = require('../config');
+const { RUNNER_VERSION, DEFAULT_JUDGE_SAMPLES, DEV_MAX_CALLS } = require('../config');
+const { SAMPLING, acrossDraws, nextAction } = require('./sampling');
+const { suiteCanary } = require('./canary');
 
 // Known model release dates (best-effort; null when unknown). Recorded into the
 // receipt so drift reports can order runs by model age. Dateless model ids
@@ -36,9 +38,14 @@ function releaseDateFor(modelId) {
 }
 
 // Calls for one model run: each case does 2 generations (with_skill + baseline)
-// and 2×samples judge calls (each generation judged `samples` times).
-function projectCalls(caseCount, samples) {
-  return caseCount * (2 + 2 * samples);
+// and 2×samples judge calls (each generation judged `samples` times) — times the
+// number of GENERATION DRAWS per arm (v0.5).
+//
+// `draws` DEFAULTS TO 1 so every existing caller projects exactly what it
+// projected before; the runner's own cost guard passes the sampling MAXIMUM,
+// which is the fail-safe direction for a guard that decides whether to spend.
+function projectCalls(caseCount, samples, draws = 1) {
+  return caseCount * draws * (2 + 2 * samples);
 }
 
 // Ask the target model to perform one eval case. `withSkill` decides whether the
@@ -126,14 +133,45 @@ async function mapPool(items, concurrency, fn) {
 //   keepTranscripts — when true, the run records transcripts:"retained-local"
 //                     and returns the raw generations + judge outputs so the
 //                     caller can write them to transcripts/<receipt-id>/.
+// THE ONE PLACE A CALL TIMEOUT IS DECIDED (spec 017 AC-1, AC-2).
+//
+// PURE: a surface name and the run's options in, milliseconds out. No I/O, no
+// clock, no provider — so a probe can ask what a run WOULD use without making a
+// call, which is what makes this criterion assertable at all.
+//
+// An operator-supplied `opts.timeoutMs` still wins; what is gone is the silent
+// literal that used to stand in for a policy. `Number.isFinite` rather than a
+// truthiness test, so an explicit 0 is a value and not a fall-through.
+// Declared as a NAMED FUNCTION EXPRESSION bound to a const, not as a bare
+// declaration. The binding the module exports and the name inside the function
+// are then separable, so a mutation probe can rename the inner function to prove
+// the export is load-bearing without the module failing to load on an undefined
+// identifier. The inner name is kept for stack traces.
+const resolveCallTimeoutMs = function resolveCallTimeoutMs(surface, opts = {}) {
+  if (opts && Number.isFinite(opts.timeoutMs)) return opts.timeoutMs;
+  return retryPolicyForSurface(surface).timeoutMs;
+};
+
 async function runSkillOnModel({ skill, model, opts = {} }) {
   const modelId = resolveModel(model);
   const judgeModel = opts.judgeModel ? resolveModel(opts.judgeModel) : modelId;
-  const timeoutMs = opts.timeoutMs || 120000;
+  // THE SURFACE'S OWN DECLARED TIMEOUT, resolved per arm (spec 017 AC-1).
+  //
+  // This read `opts.timeoutMs || 120000`, and `lib/provider.js` documents that an
+  // explicit caller value wins — so the literal outranked the policy and the
+  // 300 s claude-cli timeout, written for cold-start-dominated CLI subprocesses,
+  // never executed. Report #007 lost 25 of 160 draws to
+  // `provider(claude-cli) timed out after 120000ms`, and one arm entirely.
+  //
+  // RESOLVED SEPARATELY FOR GENERATION AND JUDGE, because they can be different
+  // models on different surfaces: #007 generated on claude-fable-5 and judged on
+  // claude-haiku-4-5. One shared timeout would apply one surface's policy to both.
+  const genTimeoutMs = resolveCallTimeoutMs(surfaceForModel(modelId), opts);
+  const judgeTimeoutMs = resolveCallTimeoutMs(surfaceForModel(judgeModel), opts);
   // Optional per-case timeout overrides { caseId: ms }; a slow case can get a
   // longer budget without lengthening every other case's per-call timeout.
   const caseTimeoutMs = opts.caseTimeoutMs || {};
-  const maxCalls = opts.maxCalls || 200;
+  const maxCalls = opts.maxCalls || DEV_MAX_CALLS;
   const samples = opts.samples || DEFAULT_JUDGE_SAMPLES;
   const concurrency = Math.max(1, opts.concurrency || 1);
   const onProgress = opts.onProgress || (() => {});
@@ -145,7 +183,10 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
 
   // Cost guard: project the whole run up front and refuse before spending a
   // single call if it would blow the cap.
-  const projected = projectCalls(cases.length, samples);
+  // The WORST CASE, deliberately: escalation is adaptive and a guard that
+  // projects the floor would wave through a run that then draws ten times.
+  // Refusing a run that would have fit is recoverable; overspending is not.
+  const projected = projectCalls(cases.length, samples, SAMPLING.max);
   if (projected > maxCalls) {
     const e = new Error(`cost guard: projected ${projected} calls exceeds cap ${maxCalls} (${cases.length} cases × (2 + 2×${samples} samples)). Raise --max-calls or lower --max-cases/--samples.`);
     e.code = 'CALL_CAP';
@@ -163,43 +204,111 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
   const genKind = (ws) => (ws ? 'gen_with_skill' : 'gen_baseline');
   const pairs = await mapPool(tasks, concurrency, async ({ c, withSkill }) => {
     const mode = withSkill ? 'with_skill' : 'baseline';
-    const ct = caseTimeoutMs[c.id] || timeoutMs;
-    try {
-      onProgress({ case: c.id, mode, phase: 'generate' });
-      const gen = await runCase({ skillMd: skill.skillMd, caseObj: c, model: modelId, withSkill, timeoutMs: ct });
-      calls += 1;
-      // Live budget: count the generation call INCLUDING retries, then hard-stop
-      // if over 1.25× cap.
-      if (budget) budget.add((gen.attempts || 1) * perCallCostUSD(modelId, genKind(withSkill)));
-      const generationHash = sha256(String(gen.text || ''));
-      onProgress({ case: c.id, mode, phase: 'judge', samples });
-      const jr = await judgeCase({ caseObj: c, response: gen.text, generationHash, judgeModel, mode, timeoutMs: ct, samples });
-      // v0.4: the GENERATION call's usage is the skill-value measurement (the
-      // judge's own usage rides separately on judge_usage, above).
-      if (hasUsage(gen.usage)) jr.caseResult.usage = normalizeUsage({ ...gen.usage, wall_ms: gen.wall_ms });
-      calls += samples;
-      // Live budget: count all judge calls (retries included) for this (case, mode).
-      if (budget) budget.add((jr.attempts || samples) * perCallCostUSD(judgeModel, 'judge'));
-      onProgress({ case: c.id, mode, phase: 'done', outcome: jr.caseResult.outcome, score: jr.caseResult.mean, stddev: jr.caseResult.stddev });
-      const transcript = keepTranscripts
-        ? { id: c.id, mode, generation: String(gen.text || ''), judge_outputs: jr.sampleTexts }
-        : null;
-      return { caseResult: jr.caseResult, transcript };
-    } catch (e) {
-      if (e && e.code === 'BUDGET_HARDSTOP') throw e;   // budget hard-stop stays fatal
-      if (!isTimeout(e)) throw e;                        // non-timeout errors stay fatal
-      // Persistent timeout → NON-FATAL: charge the consumed attempts, record the
-      // case as failed_timeout (no fabricated samples), and continue the run.
-      if (budget) {
-        try {
-          if (e.phase === 'judge') budget.add((e.judgeAttempts || 1) * perCallCostUSD(judgeModel, 'judge'));
-          else budget.add((e.attempts || 1) * perCallCostUSD(modelId, genKind(withSkill)));
-        } catch (be) { if (be && be.code === 'BUDGET_HARDSTOP') throw be; }
+    // A per-case override is an operator's explicit input and still wins, for
+    // both arms. Otherwise each arm uses its own surface's declared policy.
+    const ctGen = caseTimeoutMs[c.id] || genTimeoutMs;
+    const ctJudge = caseTimeoutMs[c.id] || judgeTimeoutMs;
+    // v0.5 — DRAW THE GENERATION n TIMES, and keep each draw's judge samples
+    // INSIDE that draw. Pooling k×n scores into one list is precisely what made
+    // generation noise read as judge noise: it is the defect Report #006 exists
+    // to name, and the nesting is the whole measurement.
+    const draws = [];
+    let last = null;             // last MEASURED draw — carries the v0.4-shaped fields
+    let lastTranscript = null;
+    let action = { stop: false, reason: 'below_min' };
+    let fatal = null;
+
+    while (!action.stop && draws.length < SAMPLING.max) {
+      const drawIndex = draws.length;
+      try {
+        onProgress({ case: c.id, mode, phase: 'generate', draw: drawIndex });
+        const gen = await runCase({ skillMd: skill.skillMd, caseObj: c, model: modelId, withSkill, timeoutMs: ctGen });
+        calls += 1;
+        if (budget) budget.add((gen.attempts || 1) * perCallCostUSD(modelId, genKind(withSkill)));
+        const generationHash = sha256(String(gen.text || ''));
+        onProgress({ case: c.id, mode, phase: 'judge', samples, draw: drawIndex });
+        const jr = await judgeCase({ caseObj: c, response: gen.text, generationHash, judgeModel, mode, timeoutMs: ctJudge, samples });
+        calls += samples;
+        if (budget) budget.add((jr.attempts || samples) * perCallCostUSD(judgeModel, 'judge'));
+        const draw = {
+          draw_index: drawIndex,
+          generation_hash: generationHash,
+          status: 'measured',
+          samples: jr.caseResult.samples,
+          judge_sample_hashes: jr.caseResult.judge_sample_hashes,
+          mean: jr.caseResult.mean,
+          stddev: jr.caseResult.stddev,
+        };
+        if (hasUsage(gen.usage)) draw.usage = normalizeUsage({ ...gen.usage, wall_ms: gen.wall_ms });
+        if (jr.caseResult.judge_usage) draw.judge_usage = jr.caseResult.judge_usage;
+        draws.push(draw);
+        last = jr;
+        if (keepTranscripts) lastTranscript = { id: c.id, mode, generation: String(gen.text || ''), judge_outputs: jr.sampleTexts };
+      } catch (e) {
+        if (e && e.code === 'BUDGET_HARDSTOP') throw e;   // budget hard-stop stays fatal
+        if (!isTimeout(e)) { fatal = e; break; }          // non-timeout errors stay fatal
+        if (budget) {
+          try {
+            if (e.phase === 'judge') budget.add((e.judgeAttempts || 1) * perCallCostUSD(judgeModel, 'judge'));
+            else budget.add((e.attempts || 1) * perCallCostUSD(modelId, genKind(withSkill)));
+          } catch (be) { if (be && be.code === 'BUDGET_HARDSTOP') throw be; }
+        }
+        // F-009-L: the draw is UNMEASURED. No score, no fabricated samples, and
+        // it is excluded from every statistic rather than counted as a zero — a
+        // zero asserts a measurement, and a timeout is the absence of one.
+        draws.push({
+          draw_index: drawIndex,
+          generation_hash: null,
+          status: 'unmeasured',
+          reason: String((e && e.message) || 'timeout').slice(0, 200),
+          samples: [],
+          mean: null,
+          stddev: null,
+        });
+        onProgress({ case: c.id, mode, phase: 'failed', draw: drawIndex, reason: String((e && e.message) || 'timeout') });
       }
+      action = nextAction(draws);
+    }
+    if (fatal) throw fatal;
+
+    const agg = acrossDraws(draws);
+    const generation = {
+      n_planned: SAMPLING.min,
+      n_drawn: agg.n_drawn,
+      n_measured: agg.n_measured,
+      n_unmeasured: agg.n_unmeasured,
+      stopping_reason: action.reason,
+      mean: agg.mean,
+      sd: agg.sd,
+      judge_sd_mean: agg.judge_sd_mean,
+      variance_ratio: agg.variance_ratio,
+      // WHICH null, when it is null (F-014-C). Copied through explicitly rather
+      // than spread from `agg`: this assembly names its keys one by one, and the
+      // canary was dropped by exactly such an assembly silently gaining a field
+      // upstream that nothing here carried down (F-014-D).
+      variance_ratio_unavailable: agg.variance_ratio_unavailable,
+      draws,
+    };
+
+    // Every draw failed: the case is recorded failed_timeout as before, but it
+    // now carries the draw list showing WHAT failed and how often.
+    if (!last) {
       failedCases += 1;
-      onProgress({ case: c.id, mode, phase: 'failed', reason: String((e && e.message) || 'timeout') });
-      return { caseResult: { id: c.id, mode, case_status: 'failed_timeout', reason: String((e && e.message) || 'timeout').slice(0, 200) }, transcript: null };
+      return { caseResult: { id: c.id, mode, case_status: 'failed_timeout', reason: (draws[draws.length - 1] || {}).reason || 'timeout', generation }, transcript: null };
     }
+
+    // The v0.4-shaped fields now describe the DRAW SET, not one arbitrary draw,
+    // so a v0.4 reader pointed at a v0.5 receipt reads the aggregate rather than
+    // whichever draw happened to be last. `generation_hash`, `samples` and
+    // `judge_sample_hashes` continue to describe the last measured draw, which
+    // is the one they have always described; RECEIPT.md states this.
+    const caseResult = { ...last.caseResult, generation };
+    caseResult.mean = agg.mean;
+    caseResult.score = agg.mean;
+    caseResult.stddev = agg.sd;
+    caseResult.outcome = outcomeFor(agg.mean, agg.sd, c.pass_threshold);
+    onProgress({ case: c.id, mode, phase: 'done', outcome: caseResult.outcome, score: caseResult.mean, stddev: caseResult.stddev });
+    return { caseResult, transcript: lastTranscript };
   });
   const caseResults = pairs.map((p) => p.caseResult);
   const transcripts = keepTranscripts ? pairs.map((p) => p.transcript) : null;
@@ -229,7 +338,18 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
       // v0.3.1 value-per-token axis: estimated SKILL.md token size.
       tokens: estimateTokens(skill.skillMd),
     },
-    suite: { format: skill.suite.format, suiteHash: skill.suite.suiteHash, caseCount: skill.suite.caseCount },
+    // v0.5: the suite canary. Derived from the suite identity and its case ids,
+    // so it is stable without a registry and distinct across suites — a leaked
+    // suite is detectable in a corpus. A detection aid, not a control.
+    suite: {
+      format: skill.suite.format,
+      suiteHash: skill.suite.suiteHash,
+      caseCount: skill.suite.caseCount,
+      // The FULL suite, never the post---max-cases list: a canary derived from a
+      // truncated run is not stable for the suite, and a canary that moves
+      // cannot say which suite leaked. Approval finding, spec 014.
+      canary: suiteCanary({ id: skill.name || skill.suite.suiteHash, cases: (skill.suite.cases || cases).map((c) => ({ id: c.id })) }),
+    },
     run: {
       model_id: modelId,
       model_release_date: releaseDateFor(modelId),
@@ -277,7 +397,7 @@ function summarizeReceipt(receipt) {
   const aggs = receipt.results.aggregates;
   const sign = cmp.delta >= 0 ? '+' : '';
   if (receipt.run.status === 'incomplete') {
-    L.push(`> ⏱ **INCOMPLETE** — ${receipt.run.failed_case_count} case(s) failed (timed out after retries) and are EXCLUDED from the aggregates below; this receipt must not be used to compute a drift/durability verdict.`);
+    L.push(`> ⏱ **INCOMPLETE** — ${receipt.run.failed_case_count} case(s) had an arm that could not be measured and are EXCLUDED from the aggregates below, BOTH arms together; this receipt must not be used to compute a drift/durability verdict.`);
     L.push('');
   }
   L.push(`with_skill **${band(aggs.with_skill.mean_score, aggs.with_skill.stddev)}** vs baseline **${band(aggs.baseline.mean_score, aggs.baseline.stddev)}**`);
@@ -300,4 +420,4 @@ function summarizeReceipt(receipt) {
   return L.join('\n');
 }
 
-module.exports = { runSkillOnModel, summarizeReceipt, releaseDateFor, projectCalls, outcomeFor };
+module.exports = { runSkillOnModel, summarizeReceipt, releaseDateFor, projectCalls, outcomeFor, resolveCallTimeoutMs };