driftproof 0.8.1 → 0.9.0

package/lib/run.js

@@ -2,16 +2,18 @@
 'use strict';
 
 const { complete, resolveModel, surfaceForModel, isMeteredSurface, retryPolicyForSurface, CODEX_OVERHEAD_NOTE } = require('./provider');
-const { gradeSamples, judgeSettings } = require('./judge');
-const { buildReceipt } = require('./receipt');
+const { gradeSamples, judgeSettings, promptTemplateHash, isTruncated } = require('./judge');
+const { buildReceipt, caseFailed, FAILED_STATUSES } = require('./receipt');
 const { sha256 } = require('./canonical');
-const { registryStatus, providerForModel, priceForModel } = require('./models');
+const { registryStatus, providerForModel, priceForModel, assertRegistered } = require('./models');
 const { perCallCostUSD } = require('./cost');
 const { runChecks } = require('./checks');
 const { estimateTokens } = require('./skillCost');
 const { hasUsage, normalizeUsage } = require('./usage');
 const { buildPricingSnapshot, computeEconomics } = require('./value');
 const { RUNNER_VERSION, DEFAULT_JUDGE_SAMPLES, DEV_MAX_CALLS } = require('../config');
+const { stubEnabled } = require('./stub');
+const { canonicalModelId } = require('./usage');
 const { SAMPLING, acrossDraws, nextAction } = require('./sampling');
 const { suiteCanary } = require('./canary');
 
@@ -60,8 +62,44 @@ async function runCase({ skillMd, caseObj, model, withSkill, timeoutMs, trusted
     throw e;
   }
   const system = withSkill ? skillMd : undefined;
-  const { text, usage, wall_ms, attempts } = await complete({ system, prompt: caseObj.prompt, model, maxTokens: 1024, timeoutMs, trusted });
-  return { text, usage, wall_ms, attempts };
+  const out = await complete({ system, prompt: caseObj.prompt, model, maxTokens: 1024, timeoutMs, trusted });
+  // The whole reply travels: what answered, what it said served the call, why
+  // it stopped, and which spawn path was taken (spec 026 AC-1, AC-2, AC-8).
+  return { text: out.text, usage: out.usage, wall_ms: out.wall_ms, attempts: out.attempts, answeredBy: out.answeredBy, surface: out.surface, reportedModels: out.reportedModels, stopReason: out.stopReason, isolation: out.isolation };
+}
+
+// ── what answered (spec 026, AC-2) ───────────────────────────────────────────
+// The surface's echo is compared with the requested id on CANONICAL ids (the
+// real claude CLI keys its usage by the undated form). A surface that names a
+// DIFFERENT model stops the run before the next call; no receipt is written,
+// because a receipt naming a model that did not answer is the defect this
+// exists to close. A surface that names nothing is recorded as attested: false.
+function attest(reply, requestedId, phase) {
+  const reported = Array.isArray(reply && reply.reportedModels) ? reply.reportedModels : null;
+  if (!reported || !reported.length) return { attested: false, reported };
+  const want = canonicalModelId(resolveModel(requestedId));
+  const other = reported.find((id) => canonicalModelId(id) !== want);
+  if (other) {
+    const e = new Error(`substrate mismatch: the ${phase} surface answered as "${other}" where "${requestedId}" (canonical "${want}") was requested; the run stops before the next call and no receipt is written`);
+    e.code = 'SUBSTRATE_MISMATCH';
+    e.requested = requestedId; e.reported = other;
+    throw e;
+  }
+  return { attested: true, reported };
+}
+
+// The run's answered_by block, derived from every reply the run received and
+// never from the surface the runner would have chosen (spec 026 AC-1: a stub
+// run used to record the real surface name because the name was computed
+// from the model id, not from what answered). With no reply at all (every
+// call failed before answering) the kind is what the process would have
+// answered with, which is the one thing still known.
+function answeredByOf(replies, { attestedGen = false, reportedAll = new Set() } = {}) {
+  const kinds = new Set(replies.map((r) => r && r.answeredBy).filter(Boolean));
+  const kind = replies.length ? (kinds.size === 1 && kinds.has('stub') ? 'stub' : 'model') : (stubEnabled() ? 'stub' : 'model');
+  const iso = replies.map((r) => r && r.isolation).find(Boolean) || 'none';
+  const reportedModels = reportedAll.size ? [...reportedAll].sort() : null;
+  return { kind, attested: kind === 'model' && attestedGen, reported_models: reportedModels, isolation: iso };
 }
 
 // Determine a case outcome from its sampled band and threshold.
@@ -80,6 +118,12 @@ function outcomeFor(mean, stddev, threshold) {
 // under --keep-transcripts; never part of the receipt).
 async function judgeCase({ caseObj, response, generationHash, judgeModel, mode, timeoutMs, samples, trusted = false }) {
   const g = await gradeSamples({ task: caseObj.prompt, response, rubric: caseObj.rubric, model: judgeModel, samples, timeoutMs, trusted });
+  // A judge that produced no score produced no measurement (spec 026 AC-3):
+  // the draw is unmeasured with the judge's reason, and no sample is kept.
+  // The judge calls that WERE made travel with the unmeasured result: their
+  // output hashes (transcript auditability: the judge said something, and a
+  // reader can check what) and their usage, so the receipt records every call.
+  if (g.unmeasured) return { unmeasured: true, reason: g.reason, sampleTexts: g.sample_texts, sampleHashes: g.sample_hashes || [], attempts: g.attempts, replies: g.replies || [], judge_usage: hasUsage(g.usage) ? g.usage : null };
   const outcome = outcomeFor(g.mean, g.stddev, caseObj.pass_threshold);
   const caseResult = {
     id: caseObj.id,
@@ -102,7 +146,7 @@ async function judgeCase({ caseObj, response, generationHash, judgeModel, mode,
   if (checks.length) caseResult.checks = checks;
   // v0.4: grading overhead for this case row, kept OUT of the skill-value math.
   if (hasUsage(g.usage)) caseResult.judge_usage = g.usage;
-  return { caseResult, sampleTexts: g.sample_texts, attempts: g.attempts };
+  return { caseResult, sampleTexts: g.sample_texts, attempts: g.attempts, replies: g.replies || [] };
 }
 
 // Run up to `concurrency` async tasks at a time, preserving input order in the
@@ -155,6 +199,13 @@ const resolveCallTimeoutMs = function resolveCallTimeoutMs(surface, opts = {}) {
 async function runSkillOnModel({ skill, model, opts = {} }) {
   const modelId = resolveModel(model);
   const judgeModel = opts.judgeModel ? resolveModel(opts.judgeModel) : modelId;
+  // Spec 026 AC-10 (F6): the target and the judge must be models the registry
+  // knows, checked HERE, on the path the CLI, the report scripts and the
+  // trigger all share, before the cost guard and before any call. bin/driftproof
+  // makes the same check at its door so the refusal names the registry path in
+  // its own message; this one is the door every caller passes.
+  assertRegistered(modelId, 'model');
+  assertRegistered(judgeModel, 'judge model');
   // THE SURFACE'S OWN DECLARED TIMEOUT, resolved per arm (spec 017 AC-1).
   //
   // This read `opts.timeoutMs || 120000`, and `lib/provider.js` documents that an
@@ -204,6 +255,12 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
 
   let calls = 0;
   let failedCases = 0;
+  // Spec 026 AC-1, AC-2: every reply the run received, for the answered_by
+  // block; whether every generation reply attested the requested model (read
+  // from the surface's echo, AC-2); every canonical id any reply named.
+  const replies = [];
+  let attestedGen = true;
+  const reportedAll = new Set();
   const isTimeout = (e) => !!(e && (e.code === 'TIMEOUT' || /tim(e|ed)\s*out/i.test(String((e && e.message) || ''))));
   const genKind = (ws) => (ws ? 'gen_with_skill' : 'gen_baseline');
   const pairs = await mapPool(tasks, concurrency, async ({ c, withSkill }) => {
@@ -221,6 +278,27 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
     let lastTranscript = null;
     let action = { stop: false, reason: 'below_min' };
     let fatal = null;
+    // Spec 026 AC-3: an unmeasured draw that was NOT a timeout (an empty
+    // generation, a judge with no score) makes the case failed_unmeasured
+    // rather than failed_timeout when no draw measured.
+    let nonTimeout = false;
+
+    // An UNMEASURED draw carries no score, no fabricated samples, and the
+    // reason it carries none; it is excluded from every statistic rather than
+    // counted as a zero. v0.6 adds what the surface said about the reply. The
+    // generation's own hash is kept when there was text.
+    const unmeasuredDraw = (drawIndex, reason, gen) => ({
+      draw_index: drawIndex,
+      generation_hash: gen && String(gen.text || '') ? sha256(String(gen.text || '')) : null,
+      status: 'unmeasured',
+      reason: String(reason || '').slice(0, 200),
+      samples: [],
+      mean: null,
+      stddev: null,
+      stop_reason: gen ? (gen.stopReason || null) : null,
+      truncated: !!(gen && gen.truncated === true),
+      reported_model: gen && gen.attested ? modelId : null,
+    });
 
     while (!action.stop && draws.length < SAMPLING.max) {
       const drawIndex = draws.length;
@@ -229,11 +307,64 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
         const gen = await runCase({ skillMd: skill.skillMd, caseObj: c, model: modelId, withSkill, timeoutMs: ctGen, trusted });
         calls += 1;
         if (budget) budget.add((gen.attempts || 1) * perCallCostUSD(modelId, genKind(withSkill)));
+        replies.push(gen);
+        // What answered, on canonical ids; a different model stops the run.
+        const a = attest(gen, modelId, 'generation');
+        gen.attested = a.attested;
+        if (!a.attested) attestedGen = false;
+        for (const id of (a.reported || [])) reportedAll.add(canonicalModelId(id));
+        gen.truncated = isTruncated(gen.stopReason);
+        // Spec 026 AC-8 (F4): a generation cut at the output cap is a partial
+        // answer, and a partial answer graded as a whole one is a score about
+        // something the model did not write. Recorded truncated, unmeasured,
+        // never judged.
+        if (gen.truncated === true) {
+          nonTimeout = true;
+          const d = unmeasuredDraw(drawIndex, `generation truncated at the output cap (stop_reason ${gen.stopReason})`, gen);
+          if (hasUsage(gen.usage)) d.usage = normalizeUsage({ ...gen.usage, wall_ms: gen.wall_ms });
+          draws.push(d);
+          onProgress({ case: c.id, mode, phase: 'failed', draw: drawIndex, reason: d.reason });
+          action = nextAction(draws);
+          continue;
+        }
+        // Spec 026 AC-3 (F2): an empty generation is the absence of an output,
+        // not an output that scored zero. It is not sent to the judge.
+        if (!String(gen.text || '').trim()) {
+          nonTimeout = true;
+          const d = unmeasuredDraw(drawIndex, 'empty generation (the surface returned no text)', gen);
+          if (hasUsage(gen.usage)) d.usage = normalizeUsage({ ...gen.usage, wall_ms: gen.wall_ms });
+          draws.push(d);
+          onProgress({ case: c.id, mode, phase: 'failed', draw: drawIndex, reason: d.reason });
+          action = nextAction(draws);
+          continue;
+        }
         const generationHash = sha256(String(gen.text || ''));
         onProgress({ case: c.id, mode, phase: 'judge', samples, draw: drawIndex });
         const jr = await judgeCase({ caseObj: c, response: gen.text, generationHash, judgeModel, mode, timeoutMs: ctJudge, samples, trusted });
         calls += samples;
         if (budget) budget.add((jr.attempts || samples) * perCallCostUSD(judgeModel, 'judge'));
+        for (const r of (jr.replies || [])) {
+          if (!r) continue;
+          replies.push(r);
+          const ja = attest(r, judgeModel, 'judge');
+          for (const id of (ja.reported || [])) reportedAll.add(canonicalModelId(id));
+        }
+        if (jr.unmeasured) {
+          // The judge returned no score for this draw (empty, unparseable,
+          // non-numeric, out of range): unmeasured, naming the judge's reason;
+          // the generation's own hash is kept.
+          nonTimeout = true;
+          const d = unmeasuredDraw(drawIndex, jr.reason, gen);
+          // The judge samples taken before the draw was called unmeasured are
+          // recorded by hash (no score entered any statistic; the calls happened).
+          if ((jr.sampleHashes || []).length) d.judge_sample_hashes = jr.sampleHashes;
+          if (hasUsage(gen.usage)) d.usage = normalizeUsage({ ...gen.usage, wall_ms: gen.wall_ms });
+          if (jr.judge_usage) d.judge_usage = jr.judge_usage;
+          draws.push(d);
+          onProgress({ case: c.id, mode, phase: 'failed', draw: drawIndex, reason: d.reason });
+          action = nextAction(draws);
+          continue;
+        }
         const draw = {
           draw_index: drawIndex,
           generation_hash: generationHash,
@@ -242,6 +373,12 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
           judge_sample_hashes: jr.caseResult.judge_sample_hashes,
           mean: jr.caseResult.mean,
           stddev: jr.caseResult.stddev,
+          // v0.6 (spec 026 AC-2, AC-8): why the generation stopped, that it
+          // was not cut (a cut draw never reaches here), and what the surface
+          // said served it (null when it said nothing).
+          stop_reason: gen.stopReason || null,
+          truncated: false,
+          reported_model: gen.attested ? modelId : null,
         };
         if (hasUsage(gen.usage)) draw.usage = normalizeUsage({ ...gen.usage, wall_ms: gen.wall_ms });
         if (jr.caseResult.judge_usage) draw.judge_usage = jr.caseResult.judge_usage;
@@ -250,7 +387,7 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
         if (keepTranscripts) lastTranscript = { id: c.id, mode, generation: String(gen.text || ''), judge_outputs: jr.sampleTexts };
       } catch (e) {
         if (e && e.code === 'BUDGET_HARDSTOP') throw e;   // budget hard-stop stays fatal
-        if (!isTimeout(e)) { fatal = e; break; }          // non-timeout errors stay fatal
+        if (!isTimeout(e)) { fatal = e; break; }          // non-timeout errors stay fatal (a SUBSTRATE_MISMATCH among them)
         if (budget) {
           try {
             if (e.phase === 'judge') budget.add((e.judgeAttempts || 1) * perCallCostUSD(judgeModel, 'judge'));
@@ -268,6 +405,9 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
           samples: [],
           mean: null,
           stddev: null,
+          stop_reason: null,
+          truncated: false,
+          reported_model: null,
         });
         onProgress({ case: c.id, mode, phase: 'failed', draw: drawIndex, reason: String((e && e.message) || 'timeout') });
       }
@@ -291,14 +431,22 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
       // canary was dropped by exactly such an assembly silently gaining a field
       // upstream that nothing here carried down (F-014-D).
       variance_ratio_unavailable: agg.variance_ratio_unavailable,
+      // v0.6 (spec 026 AC-8): draws cut at the output cap, counted here so a
+      // reader sees it without walking the draw list.
+      n_truncated: draws.filter((d) => d.truncated === true).length,
       draws,
     };
 
-    // Every draw failed: the case is recorded failed_timeout as before, but it
-    // now carries the draw list showing WHAT failed and how often.
+    // Every draw failed: the case is recorded failed, carrying the draw list
+    // showing WHAT failed and how often. failed_timeout when every failure was
+    // a timeout; failed_unmeasured when any draw was unmeasured for another
+    // reason (spec 026 AC-3). The two share one predicate, caseFailed, in
+    // lib/receipt.js, and no reader excludes by either literal.
     if (!last) {
       failedCases += 1;
-      return { caseResult: { id: c.id, mode, case_status: 'failed_timeout', reason: (draws[draws.length - 1] || {}).reason || 'timeout', generation }, transcript: null };
+      const status = nonTimeout ? FAILED_STATUSES[1] : FAILED_STATUSES[0];
+      const lastReason = [...draws].reverse().map((d) => d.reason).find(Boolean) || 'timeout';
+      return { caseResult: { id: c.id, mode, case_status: status, reason: lastReason, generation }, transcript: null };
     }
 
     // The v0.4-shaped fields now describe the DRAW SET, not one arbitrary draw,
@@ -317,7 +465,14 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
   const caseResults = pairs.map((p) => p.caseResult);
   const transcripts = keepTranscripts ? pairs.map((p) => p.transcript) : null;
 
-  const surface = surfaceForModel(modelId);
+  // THE SURFACE IS WHAT ANSWERED, not what the runner would have chosen for
+  // the model id (spec 026 AC-1, F1). A stub run records surface stub, judge
+  // surface stub, answered_by.kind stub, and is UNVERIFIED: it measured nothing.
+  const answered = answeredByOf(replies, { attestedGen: attestedGen && replies.some((r) => r && r.answeredBy === 'model'), reportedAll });
+  const answeredBy = { kind: answered.kind, attested: answered.attested, reported_model: answered.attested ? modelId : null, reported_models: answered.reported_models, isolation: answered.isolation };
+  const surface = answered.kind === 'stub' ? 'stub' : surfaceForModel(modelId);
+  // v0.6 (spec 026 AC-11): which judge ran, and the template it graded with.
+  const judgeBlock = { ...judgeSettings(samples, judgeModel), ...(answered.kind === 'stub' ? { surface: 'stub' } : {}), model_id: judgeModel, prompt_template_hash: promptTemplateHash() };
   const nowIso = opts.nowIso || new Date().toISOString();
   // v0.4 economics. The pricing snapshot is frozen HERE, at run time, from the
   // registry; every derived dollar figure below is computed from the snapshot and
@@ -359,24 +514,52 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
       model_release_date: releaseDateFor(modelId),
       provider: providerForModel(modelId),
       surface,
-      // v0.3.1: on the openai/cli (codex) surface, record the fixed harness preamble.
+      // v0.3.1: on the openai/cli (codex) surface, record the fixed harness
+      // preamble. Absent on a stub run: it describes a harness that did not run.
       surface_overhead_note: surface === 'openai-cli' ? CODEX_OVERHEAD_NOTE : undefined,
       runner_version: RUNNER_VERSION,
       date_utc: nowIso,
       registry: registryStatus(modelId),
       transcripts: keepTranscripts ? 'retained-local' : 'hashes-only',
-      judge: judgeSettings(samples, judgeModel),
+      judge: judgeBlock,
       pricing_snapshot: pricingSnapshot,
+      answered_by: answeredBy,
     },
     cases: caseResults,
     economics,
-    verificationLevel: 'TESTED',
+    // The level is DERIVED from what answered: only a model-answered run may
+    // read TESTED. The schema refuses TESTED on a stub receipt as the second,
+    // independent control (spec 026 AC-1).
+    verificationLevel: answered.kind === 'model' ? 'TESTED' : 'UNVERIFIED',
   });
 
   return { receipt, calls, transcripts, failedCases };
 }
 
-function band(mean, sd) { return `${mean.toFixed(3)} ± ${sd.toFixed(3)}`; }
+// A band the formula could not form is printed as what it is, never as 0.000
+// (spec 026 AC-7): one included case has a mean and no dispersion; none has
+// neither.
+function band(mean, sd) {
+  if (mean == null) return 'n/a (0 cases)';
+  if (sd == null) return `${mean.toFixed(3)} ± n/a (1 case)`;
+  return `${mean.toFixed(3)} ± ${sd.toFixed(3)}`;
+}
+// The comparison band, or the reason there is none.
+function uncertaintyStr(cmp) {
+  if (cmp.delta_uncertainty != null) return cmp.delta_uncertainty.toFixed(3);
+  return `n/a (${cmp.delta_uncertainty_unavailable === 'single_case' ? '1 case' : cmp.delta_uncertainty_unavailable === 'no_cases' ? '0 cases' : 'no band'})`;
+}
+// The answered_by block, said in one line for the summary and the CLI.
+function answeredLine(receipt) {
+  const ab = (receipt.run && receipt.run.answered_by) || null;
+  if (!ab) return 'answered by: unrecorded (pre-v0.6 receipt)';
+  if (ab.kind === 'stub') return 'answered by: stub (DRIFTPROOF_STUB) — nothing answered; this run measured nothing (UNVERIFIED)';
+  if (ab.kind === 'external') return 'answered by: an external tool (imported)';
+  const iso = ab.isolation === 'eval-user' ? 'the isolated eval-user hop' : ab.isolation === 'same-user' ? 'the same-user spawn (--trusted-skill)' : 'no spawn (api surface)';
+  return ab.attested
+    ? `answered by: model ${ab.reported_model} (attested by the surface; ${iso})`
+    : `answered by: model, but the surface did not report which model answered (attested: false; ${iso})`;
+}
 
 // Render a short human-readable markdown summary of a receipt.
 function summarizeReceipt(receipt) {
@@ -385,6 +568,7 @@ function summarizeReceipt(receipt) {
   L.push('');
   L.push(`- **model:** \`${receipt.run.model_id}\`${receipt.run.model_release_date ? ` (released ${receipt.run.model_release_date})` : ''}`);
   L.push(`- **surface:** ${receipt.run.surface}`);
+  L.push(`- **${answeredLine(receipt)}**`);
   L.push(`- **run (UTC):** ${receipt.run.date_utc}`);
   L.push(`- **runner:** v${receipt.run.runner_version}`);
   const j = receipt.run.judge || {};
@@ -399,22 +583,39 @@ function summarizeReceipt(receipt) {
   L.push('');
   const cmp = receipt.comparison;
   const aggs = receipt.results.aggregates;
-  const sign = cmp.delta >= 0 ? '+' : '';
+  if ((receipt.run.answered_by || {}).kind === 'stub') {
+    L.push('> **STUB RUN** — DRIFTPROOF_STUB=1: nothing answered, the text was canned, and this run measured nothing. The receipt is UNVERIFIED and verdicts nothing.');
+    L.push('');
+  }
   if (receipt.run.status === 'incomplete') {
     L.push(`> ⏱ **INCOMPLETE** — ${receipt.run.failed_case_count} case(s) had an arm that could not be measured and are EXCLUDED from the aggregates below, BOTH arms together; this receipt must not be used to compute a drift/durability verdict.`);
     L.push('');
   }
+  // Spec 026 AC-8: draws cut at the output cap, said once for the run.
+  const nTruncated = receipt.results.cases.reduce((a, c) => a + (((c.generation || {}).n_truncated) || 0), 0);
+  if (nTruncated) {
+    L.push(`> ✂ **${nTruncated} draw(s) truncated** at the output cap: each is unmeasured and excluded from every band below.`);
+    L.push('');
+  }
   L.push(`with_skill **${band(aggs.with_skill.mean_score, aggs.with_skill.stddev)}** vs baseline **${band(aggs.baseline.mean_score, aggs.baseline.stddev)}**`);
   L.push('');
-  L.push(`skill lift **${sign}${cmp.delta.toFixed(3)}** (combined uncertainty ± ${cmp.delta_uncertainty.toFixed(3)})`);
+  if (cmp.delta == null) {
+    L.push(`skill lift **n/a** (${cmp.delta_uncertainty_unavailable === 'no_cases' ? 'no case was included on an arm' : 'no comparison'})`);
+  } else {
+    const sign = cmp.delta >= 0 ? '+' : '';
+    L.push(`skill lift **${sign}${cmp.delta.toFixed(3)}** (combined uncertainty ± ${uncertaintyStr(cmp)})`);
+  }
+  L.push('');
+  // The rule the bands above are derived by, said beside them (spec 026 AC-6).
+  L.push(`band rule: each arm's band is the sample stddev of its per-case means${aggs.band_rule ? ` — ${aggs.band_rule}` : ' (unstated on this pre-v0.6 receipt)'}`);
   L.push('');
   L.push(`## Per-case (mean ± stddev over ${(receipt.run.judge || {}).samples || 1} judge samples)`);
   L.push('');
   L.push(`| case | mode | outcome | mean ± stddev | judge reason |`);
   L.push(`|---|---|---|---|---|`);
   for (const c of receipt.results.cases) {
-    if (c.case_status === 'failed_timeout') {
-      L.push(`| \`${c.id}\` | ${c.mode} | ⏱ failed_timeout | — (not measured) | ${c.reason || 'timed out'} |`);
+    if (caseFailed(c)) {
+      L.push(`| \`${c.id}\` | ${c.mode} | ⏱ ${c.case_status} | — (not measured) | ${c.reason || 'not measured'} |`);
       continue;
     }
     const flag = c.outcome === 'borderline' ? ' ⚠' : '';
@@ -424,4 +625,4 @@ function summarizeReceipt(receipt) {
   return L.join('\n');
 }
 
-module.exports = { runSkillOnModel, summarizeReceipt, releaseDateFor, projectCalls, outcomeFor, resolveCallTimeoutMs };
+module.exports = { runSkillOnModel, summarizeReceipt, releaseDateFor, projectCalls, outcomeFor, resolveCallTimeoutMs, answeredLine, answeredByOf, attest, band, uncertaintyStr };

package/lib/skill.js

@@ -4,7 +4,16 @@
 const fs = require('fs');
 const path = require('path');
 const { sha256Files, sha256Canonical } = require('./canonical');
-const { SUITE_FORMAT } = require('../config');
+const { SUITE_FORMAT, SKILL_MAX_FILES, SKILL_MAX_BYTES, SKILL_MAX_DEPTH, SUITE_MAX_CASES, CASE_MAX_CHARS } = require('../config');
+
+// A bound was exceeded: the error names the bound and the value (spec 026
+// AC-13), so the refusal says what to change rather than that something is
+// too big.
+function pastBound(bound, limit, value, what) {
+  const e = new Error(`${what}: ${value} exceeds ${bound} (${limit}); refusing to load`);
+  e.code = 'INPUT_BOUND'; e.bound = bound; e.limit = limit; e.value = value;
+  return e;
+}
 
 // Load a skill directory and its eval suite.
 //
@@ -21,13 +30,22 @@ const { SUITE_FORMAT } = require('../config');
 const IGNORE_DIRS = new Set(['.git', 'node_modules', 'evals']);
 const IGNORE_FILES = new Set(['.DS_Store']);
 
-function walkFiles(dir, base = dir, acc = []) {
+// Bounded (spec 026 AC-13): the walk stops at SKILL_MAX_DEPTH directories
+// below the skill dir, SKILL_MAX_FILES bundled files, and SKILL_MAX_BYTES of
+// them together, and throws naming the bound the moment one is passed, so a
+// pathological tree is refused before its bytes are read into memory.
+function walkFiles(dir, base = dir, acc = [], state = { bytes: 0 }, depth = 0) {
+  if (depth > SKILL_MAX_DEPTH) throw pastBound('SKILL_MAX_DEPTH', SKILL_MAX_DEPTH, depth, `directory depth under ${base}`);
   for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
     if (entry.isDirectory()) {
       if (IGNORE_DIRS.has(entry.name)) continue;
-      walkFiles(path.join(dir, entry.name), base, acc);
+      walkFiles(path.join(dir, entry.name), base, acc, state, depth + 1);
     } else if (entry.isFile() && !IGNORE_FILES.has(entry.name)) {
       const abs = path.join(dir, entry.name);
+      if (acc.length + 1 > SKILL_MAX_FILES) throw pastBound('SKILL_MAX_FILES', SKILL_MAX_FILES, acc.length + 1, `bundled files under ${base}`);
+      const size = fs.statSync(abs).size;
+      state.bytes += size;
+      if (state.bytes > SKILL_MAX_BYTES) throw pastBound('SKILL_MAX_BYTES', SKILL_MAX_BYTES, state.bytes, `bundled bytes under ${base}`);
       acc.push({ path: path.relative(base, abs), bytes: fs.readFileSync(abs) });
     }
   }
@@ -108,12 +126,16 @@ function normalizeCases(raw) {
       : Array.isArray(raw.evals) ? raw.evals
         : null;
   if (!list) throw new Error('evals.json must be an array or have a `cases`/`evals` array');
+  // Bounded (spec 026 AC-13): the case count, and each prompt and rubric.
+  if (list.length > SUITE_MAX_CASES) throw pastBound('SUITE_MAX_CASES', SUITE_MAX_CASES, list.length, 'cases in the suite');
   return list.map((c, i) => {
     const id = String(c.id || c.name || `case-${i + 1}`);
     const prompt = c.prompt || c.input || c.task;
     const rubric = c.rubric || c.criteria || c.expected;
     if (!prompt) throw new Error(`case "${id}" is missing a prompt/input/task`);
     if (!rubric) throw new Error(`case "${id}" is missing a rubric/criteria/expected`);
+    if (String(prompt).length > CASE_MAX_CHARS) throw pastBound('CASE_MAX_CHARS', CASE_MAX_CHARS, String(prompt).length, `case "${id}" prompt characters`);
+    if (String(rubric).length > CASE_MAX_CHARS) throw pastBound('CASE_MAX_CHARS', CASE_MAX_CHARS, String(rubric).length, `case "${id}" rubric characters`);
     const threshold = typeof c.pass_threshold === 'number' ? c.pass_threshold
       : typeof c.threshold === 'number' ? c.threshold : 0.7;
     const norm = { id, prompt: String(prompt), rubric: String(rubric), pass_threshold: threshold };
@@ -124,4 +146,4 @@ function normalizeCases(raw) {
   });
 }
 
-module.exports = { loadSkill, normalizeCases, parseSkillMeta };
+module.exports = { loadSkill, normalizeCases, parseSkillMeta, pastBound };

package/lib/stats.js

@@ -31,8 +31,11 @@ function stderr(xs) {
   return round(stddev(xs) / Math.sqrt(n));
 }
 
-// Combine independent uncertainties in quadrature: sqrt(a^2 + b^2).
+// Combine independent uncertainties in quadrature: sqrt(a^2 + b^2). Null when
+// either band is null: a combination of a band that could not form cannot form
+// either (spec 026 AC-7), and the receipt says why beside it.
 function combineUncertainty(a, b) {
+  if (a == null || b == null) return null;
   return round(Math.sqrt(a * a + b * b));
 }
 
@@ -45,10 +48,16 @@ function combineUncertainty(a, b) {
 // is a conventional, honest "mean ± stddev across the suite". The drift HEADLINE
 // verdict is driven by the per-case band-overlap verdicts (see lib/diff.js), not
 // by this aggregate band; this value is a reported summary statistic.
+//
+// A BAND THE FORMULA CANNOT FORM IS NULL, NEVER 0 (spec 026 AC-7, F3). The
+// sample standard deviation of one value is undefined, and the mean of no
+// values is not a number; printing 0.000 for either asserted a precision that
+// was never measured (the one-case run's `± 0.000`). The rule the receipt
+// states (results.aggregates.band_rule) is this function.
 function aggregateBands(cases) {
-  if (!cases.length) return { mean: 0, stddev: 0 };
+  if (!cases.length) return { mean: null, stddev: null };
   const means = cases.map((c) => c.mean);
-  return { mean: mean(means), stddev: stddev(means) };
+  return { mean: mean(means), stddev: means.length < 2 ? null : stddev(means) };
 }
 
 // Do two confidence bands (mean ± half-width) fail to overlap, and in which

package/lib/usage.js

@@ -42,6 +42,27 @@
 
 function n(x) { return Number.isFinite(Number(x)) ? Number(x) : 0; }
 
+// ── what answered (spec 026, AC-2 and AC-8) ──────────────────────────────────
+// Every lane also reports, when it can, WHICH model served the call and WHY the
+// reply stopped. Both are read here, never guessed: a surface that says nothing
+// yields null for both, and the receipt records that it said nothing.
+//
+//   reportedModels  the ids the surface named, VERBATIM (the real claude CLI
+//                   keys the call's usage under the undated form and puts a
+//                   dated side entry beside it; an api response names one
+//                   model). Null when the surface names no model. The runner
+//                   compares on canonical ids (canonicalModelId: a trailing
+//                   -YYYYMMDD is not part of the identity), never the reader.
+//   stopReason      the surface's own word for why the reply ended (end_turn,
+//                   max_tokens, stop, length ...). Null when it gives none.
+function canonicalModelId(id) { return String(id || '').replace(/-\d{8}$/, ''); }
+function reportedModelsOf(map) {
+  if (!map || typeof map !== 'object' || Array.isArray(map)) return null;
+  const ids = [...new Set(Object.keys(map).filter((k) => k))].sort();
+  return ids.length ? ids : null;
+}
+function stopReasonOf(v) { return typeof v === 'string' && v ? v : null; }
+
 // The empty/unknown usage record. Deliberately null (not zeros) so "the surface
 // did not tell us" never reads as "the call cost nothing".
 function emptyUsage() {
@@ -75,6 +96,9 @@ function parseClaudeCliJson(stdout) {
     text: typeof j.result === 'string' ? j.result : '',
     usage,
     isError: j.is_error === true,
+    // v0.6: the CLI's own stop reason and the models it says served the call.
+    stopReason: stopReasonOf(j.stop_reason),
+    reportedModels: reportedModelsOf(j.modelUsage),
     // The CLI reports its own dollar figure. Recorded here for completeness but
     // NOT used: costs are computed uniformly from the frozen pricing snapshot so
     // three substrates are on one basis (see lib/value.js).
@@ -85,7 +109,9 @@ function parseClaudeCliJson(stdout) {
 // ── openai/cli — `codex exec --json` JSONL event stream ───────────────────────
 // One JSON object per line. Usage rides the terminal `turn.completed` event;
 // input_tokens already INCLUDES cached_input_tokens. Unparseable lines are
-// skipped (the stream also carries progress events we do not model).
+// skipped (the stream also carries progress events we do not model). The
+// stream names neither the model that served the turn nor a stop reason, so
+// both read null here and the receipt says the surface did not report them.
 function parseCodexJsonl(stdout) {
   const lines = String(stdout || '').split('\n').map((l) => l.trim()).filter(Boolean);
   let usage = null;
@@ -103,16 +129,30 @@ function parseCodexJsonl(stdout) {
         wall_ms: null,
       };
     }
-    // The final message is normally read from the -o file; this is a fallback.
+    // The final message IS the stream: the last `item.completed` /
+    // `agent_message` event carries it. Nothing is read from a file after the
+    // call, on either spawn mode (spec 022 AC-9; the earlier comment here
+    // described an output file the lane no longer reads, F-022-3).
     if (ev.type === 'item.completed' && ev.item && ev.item.type === 'agent_message' && typeof ev.item.text === 'string') {
       text = ev.item.text;
     }
   }
-  return { usage, text };
+  return { usage, text, stopReason: null, reportedModels: null };
 }
 
 // ── anthropic/api ─────────────────────────────────────────────────────────────
 // The Messages API reports input_tokens EXCLUDING cache, same as the CLI.
+// The response object also names the model that served it (`model`) and why
+// it stopped (`stop_reason`); read here from the object, null when absent.
+// Exercised against tests/fixtures/response-anthropic-api.json, which is
+// SYNTHESISED from the API reference, not captured from a call.
+function readAnthropicApiResponse(resp) {
+  const r = resp && typeof resp === 'object' ? resp : {};
+  return {
+    stopReason: stopReasonOf(r.stop_reason),
+    reportedModels: typeof r.model === 'string' && r.model ? [r.model] : null,
+  };
+}
 function parseAnthropicApiUsage(u) {
   if (!u) return emptyUsage();
   const cacheRead = n(u.cache_read_input_tokens);
@@ -127,7 +167,18 @@ function parseAnthropicApiUsage(u) {
 
 // ── openai/api (Chat Completions-compatible) ──────────────────────────────────
 // prompt_tokens is the total; the cached portion, when present, is nested under
-// prompt_tokens_details.cached_tokens.
+// prompt_tokens_details.cached_tokens. The response also names the serving
+// model (`model`) and the first choice's `finish_reason`; read here, null
+// when absent. Exercised against tests/fixtures/response-openai-api.json,
+// SYNTHESISED from the API reference, not captured from a call.
+function readOpenaiApiResponse(resp) {
+  const r = resp && typeof resp === 'object' ? resp : {};
+  const choice = Array.isArray(r.choices) && r.choices[0] && typeof r.choices[0] === 'object' ? r.choices[0] : {};
+  return {
+    stopReason: stopReasonOf(choice.finish_reason),
+    reportedModels: typeof r.model === 'string' && r.model ? [r.model] : null,
+  };
+}
 function parseOpenaiApiUsage(u) {
   if (!u) return emptyUsage();
   const details = u.prompt_tokens_details || {};
@@ -165,4 +216,5 @@ function normalizeUsage(u) {
 module.exports = {
   emptyUsage, hasUsage, sumUsage, normalizeUsage,
   parseClaudeCliJson, parseCodexJsonl, parseAnthropicApiUsage, parseOpenaiApiUsage,
+  readAnthropicApiResponse, readOpenaiApiResponse, canonicalModelId, reportedModelsOf,
 };

package/lib/value.js

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 'use strict';
 
+const { caseFailed } = require('./receipt');
+
 const { EFFECT_FLOOR } = require('../config');
 
 // Economics of a run — what a skill COSTS to run, alongside whether it helps.
@@ -204,7 +206,7 @@ function armEconomics(rows, price) {
 //   surface          the run surface, to label the cost basis honestly
 function computeEconomics({ cases, modelId, judgeModelId, pricingSnapshot, surface, meteredSurface }) {
   const price = (pricingSnapshot && pricingSnapshot.models && pricingSnapshot.models[modelId]) || null;
-  const ok = (cases || []).filter((c) => c.case_status !== 'failed_timeout');
+  const ok = (cases || []).filter((c) => !caseFailed(c));
   const withRows = ok.filter((c) => c.mode === 'with_skill');
   const baseRows = ok.filter((c) => c.mode === 'baseline');
 
@@ -322,7 +324,7 @@ function receiptCostBreakdown(receipt) {
   let judge = 0;
   let missingJudgeRate = null;
   for (const c of ((receipt.results || {}).cases || [])) {
-    if (c.case_status === 'failed_timeout') continue;
+    if (caseFailed(c)) continue;
     const g = costForUsage(c.usage, genRate);
     if (g != null) generation += g;
     if (c.judge_usage) {

package/lib/verdict.js

@@ -43,7 +43,12 @@ function verdictFromReceipt(receipt) {
   // Below-TESTED receipts (imported/DECLARED) and null-delta receipts are never
   // verdicted — we did not run the suite, so we do not certify the outcome.
   const level = (receipt && receipt.verification_level) || 'TESTED';
-  const measured = level === 'TESTED' && typeof cmp.delta === 'number';
+  // Spec 026 AC-4: an INCOMPLETE receipt (its run block's status field reads
+  // incomplete: a case had an arm that could not be measured) is not verdicted
+  // either. spec/RECEIPT.md has said since v0.3.1 that a drift report must not
+  // compute a verdict from one; the badge is the same reader with a shorter path.
+  const incomplete = !!(receipt && receipt.run && receipt.run.status === 'incomplete');
+  const measured = level === 'TESTED' && typeof cmp.delta === 'number' && !incomplete;
   let verdict;
   if (!measured) verdict = 'NOT_MEASURED';
   else if (delta >= EFFECT_FLOOR) verdict = 'PASSED';