driftproof 0.3.0 → 0.4.0

package/lib/provider.js

@@ -1,20 +1,33 @@
 // SPDX-License-Identifier: Apache-2.0
 'use strict';
 
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
 const { spawn } = require('child_process');
 const { withRetry, withTimeout } = require('./json');
 const { stubComplete, stubEnabled } = require('./stub');
 
-// Provider abstraction: one `complete()` call, two surfaces.
+// Provider abstraction: one `complete()` call over a TWO-AXIS lane model —
+// provider (anthropic | openai) × surface (api | cli). Four concrete lanes:
 //
-//   CLAUDE_PROVIDER=api   → Anthropic Messages API (needs ANTHROPIC_API_KEY).
-//   CLAUDE_PROVIDER=cli   → spawn `claude -p` with ANTHROPIC_API_KEY STRIPPED
-//                           from the child env, so dev runs draw on the local
-//                           subscription session credit rather than metered API
-//                           billing. This is the DEFAULT for dev.
+//   anthropic/api   → Anthropic Messages API (ANTHROPIC_API_KEY)   label "api"
+//   anthropic/cli   → spawn `claude -p` on the subscription        label "claude-cli"
+//   openai/api      → Chat Completions-compatible API (OPENAI_API_KEY, base_url
+//                     configurable — generic so a future provider is just a
+//                     registry entry with a base_url)               label "openai-api"
+//   openai/cli      → spawn `codex exec` on the ChatGPT subscription label "openai-cli"
+//
+// The provider of a call is inferred from the MODEL id (so a run can generate on
+// an OpenAI model while the fixed Haiku judge still runs on Anthropic). The
+// surface for each provider is chosen from the environment:
+//
+//   anthropic:  CLAUDE_PROVIDER = api | cli      (default cli — subscription)
+//   openai:     OPENAI_SURFACE  = api | cli      (default: api iff OPENAI_API_KEY
+//                                                 is present, else cli/codex)
 //
 // The surface actually used is returned so the runner can stamp it into the
-// receipt (run.surface = "api" | "claude-cli").
+// receipt (run.surface), together with run.provider.
 
 // Short model aliases → canonical ids. Kept tiny and explicit; unknown values
 // are passed through verbatim so a full model id always works.
@@ -24,47 +37,131 @@ const MODEL_ALIASES = {
   'sonnet-5': 'claude-sonnet-5',
   'sonnet-4-6': 'claude-sonnet-4-6',  // previous Sonnet point release
   opus: 'claude-opus-4-8',
+  // OpenAI convenience aliases (full ids always work too).
+  gpt: 'gpt-5.6-sol',
+  'gpt-flagship': 'gpt-5.6-sol',
 };
 
 function resolveModel(m) {
   return MODEL_ALIASES[m] || m;
 }
 
+// Infer the provider from a (resolved) model id. Pure and self-contained (no
+// registry require) so provider.js has no cycle with lib/models.js; the registry
+// `provider` field is authoritative where a model is registered (see
+// lib/models.providerForModel), and registered OpenAI ids match this prefix too.
+function inferProvider(modelId) {
+  const id = String(resolveModel(modelId) || '').toLowerCase();
+  if (/^(gpt-|o[1-9]|chatgpt|codex|text-|davinci|omni)/.test(id)) return 'openai';
+  return 'anthropic';
+}
+
+// ── surface selection per provider ───────────────────────────────────────────
+function anthropicSurface() {
+  return (process.env.CLAUDE_PROVIDER || 'cli').toLowerCase() === 'api' ? 'api' : 'claude-cli';
+}
+
+function openaiSurface() {
+  const pref = (process.env.OPENAI_SURFACE || '').toLowerCase();
+  if (pref === 'api') return 'openai-api';
+  if (pref === 'cli' || pref === 'codex') return 'openai-cli';
+  // Auto: prefer the metered API when a key is present (published-run default),
+  // else fall back to the Codex subscription surface.
+  return process.env.OPENAI_API_KEY ? 'openai-api' : 'openai-cli';
+}
+
+function surfaceForModel(modelId) {
+  return inferProvider(modelId) === 'openai' ? openaiSurface() : anthropicSurface();
+}
+
+function providerForSurface(surface) {
+  return (surface === 'openai-api' || surface === 'openai-cli') ? 'openai' : 'anthropic';
+}
+
+// Backward-compatible default surface label (the Anthropic axis). Kept so callers
+// that stamp a surface without a model (legacy) keep their prior meaning.
+function surfaceLabel() { return anthropicSurface(); }
+
 function providerName() {
   return (process.env.CLAUDE_PROVIDER || 'cli').toLowerCase();
 }
 
-// The receipt-facing label for the surface in use.
-function surfaceLabel() {
-  return providerName() === 'api' ? 'api' : 'claude-cli';
-}
-
-// Send a single-turn prompt and return { text, usage, surface }.
-// usage is { input_tokens, output_tokens } when the surface reports it (api),
-// else null (cli does not expose token counts to us).
-// `temperature` is honoured ONLY on the api surface (the Messages API exposes
-// it). On the cli surface sampling params are surface-controlled — we cannot set
-// them, so temperature is ignored there and the receipt records that fact.
-async function complete({ system, prompt, model, maxTokens = 1024, timeoutMs = 120000, temperature = undefined }) {
-  const surface = surfaceLabel();
-  // Offline stub surface: return a canned completion with zero model calls. The
-  // receipt still records the real surface label so a stub run is not mistaken
-  // for a genuine one at read time — only run.judge/generation TEXT is canned.
-  if (stubEnabled()) return { ...stubComplete({ system, prompt }), surface };
-  const runner = () => (surface === 'api'
-    ? completeApi({ system, prompt, model, maxTokens, temperature })
-    : completeCli({ system, prompt, model, timeoutMs }));
+// A surface whose spend is metered (real dollars) vs a subscription surface
+// (metered spend $0; the $ figure is the estimated-equivalent API cost).
+function isMeteredSurface(surface) { return surface === 'api' || surface === 'openai-api'; }
+function isSubscriptionSurface(surface) { return !isMeteredSurface(surface); }
+
+// Per-surface retry/timeout policy. A cli/subscription surface spawns a first-party
+// CLI subprocess (cold-start-dominated, occasionally hangs), so it gets a longer
+// per-call timeout (300s) and a longer exponential backoff (baseDelay 30s → 30/60/
+// 120s across the 4 retries); api surfaces keep the tighter 120s / 3s defaults.
+function retryPolicyForSurface(surface) {
+  const cli = isSubscriptionSurface(surface);
+  return { timeoutMs: cli ? 300000 : 120000, baseDelayMs: cli ? 30000 : 3000, tries: 4 };
+}
+
+// The fixed harness preamble Codex prepends to every `codex exec` call. Recorded
+// into openai/cli receipts as run.surface_overhead_note so a reader knows the
+// per-call input-token count is dominated by a constant we do not control.
+const CODEX_OVERHEAD_NOTE =
+  'openai/cli surface (codex exec): every call carries a fixed Codex base-instruction '
+  + 'preamble of ~12,000–15,000 input tokens that the harness prepends and we do not '
+  + 'control; the model id is set by us via -m (it is not echoed in the JSONL stream). '
+  + 'Approval prompting is off by default on `codex exec` (no -a flag is passed).';
+
+// Send a single-turn prompt and return { text, usage, surface, provider }.
+// usage is { input_tokens, output_tokens } when the surface reports it (api
+// surfaces), else null (cli surfaces do not expose token counts to us).
+// `temperature` is honoured ONLY on api surfaces; cli surfaces control sampling
+// themselves, so temperature is ignored there and the receipt records that fact.
+async function complete({ system, prompt, model, maxTokens = 1024, timeoutMs = null, temperature = undefined }) {
+  const surface = surfaceForModel(model);
+  const provider = providerForSurface(surface);
+  // Offline stub surface: canned completion, zero model calls. The receipt still
+  // records the real surface/provider so a stub run is not mistaken for a genuine
+  // one at read time — only the generation/judge TEXT is canned.
+  if (stubEnabled()) return { ...stubComplete({ system, prompt }), surface, provider, attempts: 1 };
+
+  // Per-surface tolerance (see retryPolicyForSurface): a cli/subscription surface
+  // spawns a first-party CLI subprocess whose cold-start is slow and occasionally
+  // hangs, so it gets a longer per-call timeout (300s) and a longer exponential
+  // backoff between the 4 retries (30s / 60s / 120s); api surfaces keep the tighter
+  // defaults. An explicit caller timeoutMs still wins.
+  const policy = retryPolicyForSurface(surface);
+  const effTimeout = timeoutMs != null ? timeoutMs : policy.timeoutMs;
+  const baseDelayMs = policy.baseDelayMs;
+
+  const laneRunner = () => {
+    switch (surface) {
+      case 'api': return completeAnthropicApi({ system, prompt, model, maxTokens, temperature });
+      case 'claude-cli': return completeClaudeCli({ system, prompt, model, timeoutMs: effTimeout });
+      case 'openai-api': return completeOpenaiApi({ system, prompt, model, maxTokens, temperature, timeoutMs: effTimeout });
+      case 'openai-cli': return completeCodexCli({ system, prompt, model, timeoutMs: effTimeout });
+      default: throw new Error(`unknown surface: ${surface}`);
+    }
+  };
   // A published run makes ~1000s of calls; be patient with transient
-  // throttling/cold-starts so one blip doesn't abort a multi-hour grind.
-  const out = await withRetry(() => withTimeout(runner, timeoutMs, `provider(${surface})`), { tries: 4, baseDelayMs: 3000 });
-  return { ...out, surface };
+  // throttling/cold-starts so one blip doesn't abort a multi-hour grind. `attempts`
+  // counts every try (retries included) so the budget can charge for them.
+  let attempts = 0;
+  const runner = () => { attempts += 1; return withTimeout(laneRunner, effTimeout, `provider(${surface})`); };
+  try {
+    const out = await withRetry(runner, { tries: policy.tries, baseDelayMs });
+    return { ...out, surface, provider, attempts };
+  } catch (e) {
+    // Surface the attempt count so a persistently-failing call can be charged for
+    // (and, for a timeout, marked failed_timeout by the runner instead of fatal).
+    if (e && typeof e === 'object') e.attempts = attempts;
+    throw e;
+  }
 }
 
-async function completeApi({ system, prompt, model, maxTokens, temperature }) {
+// ── anthropic/api ─────────────────────────────────────────────────────────────
+async function completeAnthropicApi({ system, prompt, model, maxTokens, temperature }) {
   let Anthropic;
   try { Anthropic = require('@anthropic-ai/sdk'); }
-  catch (_e) { throw new Error('CLAUDE_PROVIDER=api requires the @anthropic-ai/sdk package (npm i @anthropic-ai/sdk)'); }
-  if (!process.env.ANTHROPIC_API_KEY) throw new Error('CLAUDE_PROVIDER=api requires ANTHROPIC_API_KEY');
+  catch (_e) { throw new Error('the anthropic/api surface requires the @anthropic-ai/sdk package (npm i @anthropic-ai/sdk)'); }
+  if (!process.env.ANTHROPIC_API_KEY) throw new Error('the anthropic/api surface requires ANTHROPIC_API_KEY');
   const client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
   const params = {
     model: resolveModel(model),
@@ -84,7 +181,8 @@ async function completeApi({ system, prompt, model, maxTokens, temperature }) {
   };
 }
 
-function completeCli({ system, prompt, model, timeoutMs }) {
+// ── anthropic/cli (claude -p) ──────────────────────────────────────────────────
+function completeClaudeCli({ system, prompt, model, timeoutMs }) {
   return new Promise((resolve, reject) => {
     // Strip ANTHROPIC_API_KEY so the CLI uses the subscription session, not the
     // metered API key. Everything else in the env is preserved.
@@ -102,7 +200,7 @@ function completeCli({ system, prompt, model, timeoutMs }) {
     child.stderr.on('data', (d) => { err += d; });
     child.on('error', (e) => {
       clearTimeout(killer);
-      if (e.code === 'ENOENT') reject(new Error("CLAUDE_PROVIDER=cli requires the `claude` CLI on PATH. Install it or set CLAUDE_PROVIDER=api."));
+      if (e.code === 'ENOENT') reject(new Error("the anthropic/cli surface requires the `claude` CLI on PATH. Install it or set CLAUDE_PROVIDER=api."));
       else reject(e);
     });
     child.on('close', (code) => {
@@ -115,4 +213,149 @@ function completeCli({ system, prompt, model, timeoutMs }) {
   });
 }
 
-module.exports = { complete, resolveModel, surfaceLabel, providerName, MODEL_ALIASES };
+// ── openai/api (Chat Completions-compatible) ───────────────────────────────────
+// Generic OpenAI-compatible client using global fetch (Node ≥ 18). The base_url
+// is configurable — env OPENAI_BASE_URL wins, else the registry's provider
+// base_url (lib/models.providerConfig), else api.openai.com/v1. Building it
+// base_url-first is deliberate: a future provider is a registry entry with a
+// base_url + api-key env, not a new code path.
+async function completeOpenaiApi({ system, prompt, model, maxTokens, temperature, timeoutMs }) {
+  const cfg = openaiProviderConfig();
+  const apiKey = process.env[cfg.apiKeyEnv] || process.env.OPENAI_API_KEY;
+  if (!apiKey) throw new Error(`the openai/api surface requires ${cfg.apiKeyEnv} (or run the codex CLI surface). Set OPENAI_SURFACE=cli to use the subscription instead.`);
+  const baseUrl = (process.env.OPENAI_BASE_URL || cfg.baseUrl || 'https://api.openai.com/v1').replace(/\/+$/, '');
+  const messages = [];
+  if (system) messages.push({ role: 'system', content: system });
+  messages.push({ role: 'user', content: prompt });
+  const body = { model: resolveModel(model), messages };
+  // Newer OpenAI models use `max_completion_tokens`; classic Chat Completions use
+  // `max_tokens`. Send the modern field; harmless on classic-compatible servers
+  // that ignore unknown fields, and correct for current OpenAI models.
+  if (maxTokens) body.max_completion_tokens = maxTokens;
+  if (temperature !== undefined) body.temperature = temperature;
+
+  const controller = new AbortController();
+  const t = setTimeout(() => controller.abort(), timeoutMs);
+  let resp;
+  try {
+    resp = await fetch(`${baseUrl}/chat/completions`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json', authorization: `Bearer ${apiKey}` },
+      body: JSON.stringify(body),
+      signal: controller.signal,
+    });
+  } finally { clearTimeout(t); }
+  const raw = await resp.text();
+  if (!resp.ok) throw new Error(`openai/api ${resp.status}: ${raw.slice(0, 300)}`);
+  let parsed;
+  try { parsed = JSON.parse(raw); } catch (_e) { throw new Error(`openai/api: unparseable response: ${raw.slice(0, 200)}`); }
+  const text = ((parsed.choices && parsed.choices[0] && parsed.choices[0].message && parsed.choices[0].message.content) || '');
+  const usage = parsed.usage || {};
+  return {
+    text: String(text).trim(),
+    usage: {
+      input_tokens: usage.prompt_tokens || 0,
+      output_tokens: usage.completion_tokens || 0,
+    },
+  };
+}
+
+// Read the OpenAI provider config (base_url + api-key env) from the registry,
+// lazily (require inside the fn) so provider.js keeps no load-time cycle with
+// lib/models.js. Falls back to the public defaults if the registry omits it.
+function openaiProviderConfig() {
+  try {
+    const cfg = require('./models').providerConfig('openai');
+    if (cfg) return { baseUrl: cfg.base_url, apiKeyEnv: cfg.api_key_env || 'OPENAI_API_KEY' };
+  } catch (_e) { /* fall through to defaults */ }
+  return { baseUrl: 'https://api.openai.com/v1', apiKeyEnv: 'OPENAI_API_KEY' };
+}
+
+// ── openai/cli (codex exec — ChatGPT subscription) ─────────────────────────────
+// Invocation template (from the verified recon, reference_codex_cli):
+//   codex exec --json -s read-only --skip-git-repo-check --ephemeral -o <tmp> [-m <model>] "PROMPT"
+// Honoured facts:
+//   - `-a/--ask-for-approval` is INVALID on `codex exec` — it is NEVER passed;
+//     exec already defaults to approval_policy "never".
+//   - the model id is NOT echoed in the JSONL stream; we set -m so the receipt
+//     records what WE requested (run.model_id).
+//   - auth lives at ~/.codex/auth.json; we do a PRESENCE check only and NEVER
+//     read or print its contents.
+//   - `codex exec` has no system-prompt flag, so a system prompt (the SKILL.md in
+//     with_skill mode) is folded into the prompt text.
+//   - stderr emits a harmless "Reading additional input from stdin…" notice.
+const CODEX_EXEC_ARGS = ['exec', '--json', '-s', 'read-only', '--skip-git-repo-check', '--ephemeral'];
+
+function codexAuthPresent(homeDir) {
+  const home = homeDir || process.env.CODEX_HOME_DIR || os.homedir();
+  // CODEX_HOME overrides the auth location if the user set it (codex convention).
+  const base = process.env.CODEX_HOME || path.join(home, '.codex');
+  return fs.existsSync(path.join(base, 'auth.json'));
+}
+
+// Build the exact argv for a codex exec call. Exposed for the gate to assert the
+// template (and that `-a` never appears). `outFile` receives the final message.
+function buildCodexArgs({ model, outFile }) {
+  const args = [...CODEX_EXEC_ARGS, '-o', outFile];
+  if (model) args.push('-m', resolveModel(model));
+  return args;
+}
+
+// The FULL argv for a codex call: the template plus a single `-` positional that
+// means "read the prompt from stdin". The prompt is NEVER a positional argv —
+// real SKILL.md files start with `---` (YAML frontmatter) and codex rejects a
+// positional beginning with `--`. Exposed so the gate can lock this in.
+function codexFinalArgs({ model, outFile }) {
+  return [...buildCodexArgs({ model, outFile }), '-'];
+}
+
+function completeCodexCli({ system, prompt, model, timeoutMs }) {
+  return new Promise((resolve, reject) => {
+    if (!codexAuthPresent()) {
+      return reject(new Error('the openai/cli surface requires codex auth (~/.codex/auth.json). Run `codex login` (or `codex login --device-auth` on a headless box).'));
+    }
+    // codex exec has no system-prompt flag → fold the system prompt into the
+    // prompt text (this is how the SKILL.md reaches the model in with_skill mode).
+    const fullPrompt = system ? `${system}\n\n---\n\n${prompt}` : prompt;
+    const outFile = path.join(os.tmpdir(), `driftproof-codex-${process.pid}-${Date.now()}-${Math.floor(codexCounter())}.txt`);
+    // The prompt is delivered on STDIN, not as a positional argv: real SKILL.md
+    // files begin with `---` (YAML frontmatter), and a positional argument that
+    // starts with `--` is rejected by codex's arg parser ("unexpected argument
+    // '---'"). `-` as the positional tells `codex exec` to read instructions from
+    // stdin (per its --help), which is content-agnostic.
+    const args = codexFinalArgs({ model, outFile });
+
+    const child = spawn('codex', args, { env: { ...process.env }, stdio: ['pipe', 'ignore', 'pipe'] });
+    let err = '';
+    const killer = setTimeout(() => { child.kill('SIGKILL'); }, timeoutMs + 5000);
+    child.stderr.on('data', (d) => { err += d; });
+    child.on('error', (e) => {
+      clearTimeout(killer);
+      if (e.code === 'ENOENT') reject(new Error('the openai/cli surface requires the `codex` CLI on PATH (npm i -g @openai/codex).'));
+      else reject(e);
+    });
+    child.on('close', (code) => {
+      clearTimeout(killer);
+      let text = '';
+      try { text = fs.readFileSync(outFile, 'utf8'); } catch (_e) { /* fall through */ }
+      try { fs.unlinkSync(outFile); } catch (_e) { /* best effort */ }
+      if (code !== 0) return reject(new Error(`codex exec exited ${code}: ${String(err).slice(0, 400)}`));
+      resolve({ text: String(text).trim(), usage: null });
+    });
+    child.stdin.on('error', () => { /* ignore EPIPE if codex exits before reading all input */ });
+    child.stdin.write(fullPrompt);
+    child.stdin.end();
+  });
+}
+
+// A monotonically-increasing counter for temp-file uniqueness that does not use
+// Math.random (kept deterministic-friendly for any harness that forbids it).
+let _codexCounter = 0;
+function codexCounter() { _codexCounter += 1; return _codexCounter; }
+
+module.exports = {
+  complete, resolveModel, surfaceLabel, providerName, MODEL_ALIASES,
+  inferProvider, surfaceForModel, providerForSurface,
+  isMeteredSurface, isSubscriptionSurface, retryPolicyForSurface,
+  CODEX_OVERHEAD_NOTE, CODEX_EXEC_ARGS, buildCodexArgs, codexFinalArgs, codexAuthPresent,
+};

package/lib/receipt.js

@@ -13,7 +13,8 @@ const { aggregateBands, combineUncertainty, round } = require('./stats');
 const SCHEMA_FILES = {
   '0.1': 'receipt.v0.1.schema.json',
   '0.2': 'receipt.v0.2.schema.json',
-  '0.3': 'receipt.schema.json',
+  '0.3': 'receipt.v0.3.schema.json',
+  '0.3.1': 'receipt.schema.json',
 };
 
 const _validators = {};
@@ -80,8 +81,12 @@ function aggregate(caseResults) {
 //   cases:   [ { id, mode, outcome, score, mean, stddev, samples, generation_hash, judge_sample_hashes, threshold, reason, judge } ]
 //   editorialReviews: optional [ { url, source, date } ]
 function buildReceipt({ skill, suite, run, cases, verificationLevel = 'TESTED', editorialReviews = null }) {
-  const withSkill = cases.filter((c) => c.mode === 'with_skill');
-  const baseline = cases.filter((c) => c.mode === 'baseline');
+  // v0.3.1: cases marked failed_timeout are recorded in results.cases but EXCLUDED
+  // from aggregates — a band is never fabricated from a case that did not complete.
+  const okCases = cases.filter((c) => c.case_status !== 'failed_timeout');
+  const failedCount = cases.length - okCases.length;
+  const withSkill = okCases.filter((c) => c.mode === 'with_skill');
+  const baseline = okCases.filter((c) => c.mode === 'baseline');
   const aggWith = aggregate(withSkill);
   const aggBase = aggregate(baseline);
 
@@ -100,6 +105,9 @@ function buildReceipt({ skill, suite, run, cases, verificationLevel = 'TESTED',
     run: {
       model_id: run.model_id,
       model_release_date: run.model_release_date == null ? null : run.model_release_date,
+      // v0.3.1: two-axis provider (registry `provider`, else inferred). Defaults
+      // to anthropic for any legacy caller that omits it.
+      provider: run.provider || 'anthropic',
       surface: run.surface,
       runner_version: run.runner_version,
       date_utc: run.date_utc,
@@ -124,6 +132,17 @@ function buildReceipt({ skill, suite, run, cases, verificationLevel = 'TESTED',
     verification_level: verificationLevel,
     receipt_hash: '',
   };
+  // v0.3.1 additive-optional fields (canonicalization sorts keys, so placement
+  // here does not affect the hash):
+  //   run.surface_overhead_note — the fixed harness preamble on the openai/cli surface.
+  //   skill.tokens              — estimated SKILL.md token size (value-per-token axis).
+  if (run.surface_overhead_note) receipt.run.surface_overhead_note = run.surface_overhead_note;
+  if (skill.tokens != null) receipt.skill.tokens = skill.tokens;
+  // v0.3.1: mark the receipt incomplete when any case failed (excluded above).
+  if (failedCount > 0) {
+    receipt.run.status = 'incomplete';
+    receipt.run.failed_case_count = failedCount;
+  }
   if (editorialReviews && editorialReviews.length) receipt.editorial_reviews = editorialReviews;
   return sealReceipt(receipt);
 }

package/lib/run.js

@@ -1,12 +1,14 @@
 // SPDX-License-Identifier: Apache-2.0
 'use strict';
 
-const { complete, resolveModel, surfaceLabel } = require('./provider');
+const { complete, resolveModel, surfaceForModel, CODEX_OVERHEAD_NOTE } = require('./provider');
 const { gradeSamples, judgeSettings } = require('./judge');
 const { buildReceipt } = require('./receipt');
 const { sha256 } = require('./canonical');
-const { registryStatus } = require('./models');
+const { registryStatus, providerForModel } = require('./models');
 const { perCallCostUSD } = require('./cost');
+const { runChecks } = require('./checks');
+const { estimateTokens } = require('./skillCost');
 const { RUNNER_VERSION, DEFAULT_JUDGE_SAMPLES } = require('../config');
 
 // Known model release dates (best-effort; null when unknown). Recorded into the
@@ -20,6 +22,8 @@ const MODEL_RELEASE_DATES = {
   'claude-haiku-4-5': '2025-10-01',
   'claude-sonnet-5': '2026-06-30',     // anthropic.com/news/claude-sonnet-5
   'claude-sonnet-4-6': '2026-02-17',   // anthropic.com/news/claude-sonnet-4-6
+  'claude-opus-5': '2026-07-24',       // anthropic.com/news/claude-opus-5
+  'claude-opus-4-8': '2026-05-28',     // anthropic.com/news/claude-opus-4-8
 };
 
 function releaseDateFor(modelId) {
@@ -39,9 +43,16 @@ function projectCalls(caseCount, samples) {
 // SKILL.md is prepended as a system prompt (the whole point: measure the skill's
 // marginal effect vs a bare baseline).
 async function runCase({ skillMd, caseObj, model, withSkill, timeoutMs }) {
+  // Test seam (gate only): force a persistent timeout for a named case id so the
+  // failed_timeout path is exercised deterministically without any live call.
+  if (process.env.DRIFTPROOF_TEST_TIMEOUT_CASEID && process.env.DRIFTPROOF_TEST_TIMEOUT_CASEID === caseObj.id) {
+    const e = new Error('provider timed out (test seam) after retries');
+    e.code = 'TIMEOUT'; e.attempts = 4;
+    throw e;
+  }
   const system = withSkill ? skillMd : undefined;
-  const { text, usage } = await complete({ system, prompt: caseObj.prompt, model, maxTokens: 1024, timeoutMs });
-  return { text, usage };
+  const { text, usage, attempts } = await complete({ system, prompt: caseObj.prompt, model, maxTokens: 1024, timeoutMs });
+  return { text, usage, attempts };
 }
 
 // Determine a case outcome from its sampled band and threshold.
@@ -77,7 +88,10 @@ async function judgeCase({ caseObj, response, generationHash, judgeModel, mode,
     reason: g.reason,
     judge: { model_id: g.model_id, rubric_hash: g.rubric_hash },
   };
-  return { caseResult, sampleTexts: g.sample_texts };
+  // v0.3.1 deterministic post-checks (supplementary; NOT folded into `outcome`).
+  const checks = runChecks(response, caseObj.checks);
+  if (checks.length) caseResult.checks = checks;
+  return { caseResult, sampleTexts: g.sample_texts, attempts: g.attempts };
 }
 
 // Run up to `concurrency` async tasks at a time, preserving input order in the
@@ -112,6 +126,9 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
   const modelId = resolveModel(model);
   const judgeModel = opts.judgeModel ? resolveModel(opts.judgeModel) : modelId;
   const timeoutMs = opts.timeoutMs || 120000;
+  // Optional per-case timeout overrides { caseId: ms }; a slow case can get a
+  // longer budget without lengthening every other case's per-call timeout.
+  const caseTimeoutMs = opts.caseTimeoutMs || {};
   const maxCalls = opts.maxCalls || 200;
   const samples = opts.samples || DEFAULT_JUDGE_SAMPLES;
   const concurrency = Math.max(1, opts.concurrency || 1);
@@ -137,46 +154,75 @@ async function runSkillOnModel({ skill, model, opts = {} }) {
   for (const c of cases) for (const withSkill of [true, false]) tasks.push({ c, withSkill });
 
   let calls = 0;
+  let failedCases = 0;
+  const isTimeout = (e) => !!(e && (e.code === 'TIMEOUT' || /tim(e|ed)\s*out/i.test(String((e && e.message) || ''))));
+  const genKind = (ws) => (ws ? 'gen_with_skill' : 'gen_baseline');
   const pairs = await mapPool(tasks, concurrency, async ({ c, withSkill }) => {
     const mode = withSkill ? 'with_skill' : 'baseline';
-    onProgress({ case: c.id, mode, phase: 'generate' });
-    const { text } = await runCase({ skillMd: skill.skillMd, caseObj: c, model: modelId, withSkill, timeoutMs });
-    calls += 1;
-    // Live budget: count the generation call, then hard-stop if over 1.25× cap.
-    if (budget) budget.add(perCallCostUSD(modelId, withSkill ? 'gen_with_skill' : 'gen_baseline'));
-    const generationHash = sha256(String(text || ''));
-    onProgress({ case: c.id, mode, phase: 'judge', samples });
-    const { caseResult, sampleTexts } = await judgeCase({ caseObj: c, response: text, generationHash, judgeModel, mode, timeoutMs, samples });
-    calls += samples;
-    // Live budget: count all `samples` judge calls for this (case, mode).
-    if (budget) budget.add(samples * perCallCostUSD(judgeModel, 'judge'));
-    onProgress({ case: c.id, mode, phase: 'done', outcome: caseResult.outcome, score: caseResult.mean, stddev: caseResult.stddev });
-    const transcript = keepTranscripts
-      ? { id: c.id, mode, generation: String(text || ''), judge_outputs: sampleTexts }
-      : null;
-    return { caseResult, transcript };
+    const ct = caseTimeoutMs[c.id] || timeoutMs;
+    try {
+      onProgress({ case: c.id, mode, phase: 'generate' });
+      const gen = await runCase({ skillMd: skill.skillMd, caseObj: c, model: modelId, withSkill, timeoutMs: ct });
+      calls += 1;
+      // Live budget: count the generation call INCLUDING retries, then hard-stop
+      // if over 1.25× cap.
+      if (budget) budget.add((gen.attempts || 1) * perCallCostUSD(modelId, genKind(withSkill)));
+      const generationHash = sha256(String(gen.text || ''));
+      onProgress({ case: c.id, mode, phase: 'judge', samples });
+      const jr = await judgeCase({ caseObj: c, response: gen.text, generationHash, judgeModel, mode, timeoutMs: ct, samples });
+      calls += samples;
+      // Live budget: count all judge calls (retries included) for this (case, mode).
+      if (budget) budget.add((jr.attempts || samples) * perCallCostUSD(judgeModel, 'judge'));
+      onProgress({ case: c.id, mode, phase: 'done', outcome: jr.caseResult.outcome, score: jr.caseResult.mean, stddev: jr.caseResult.stddev });
+      const transcript = keepTranscripts
+        ? { id: c.id, mode, generation: String(gen.text || ''), judge_outputs: jr.sampleTexts }
+        : null;
+      return { caseResult: jr.caseResult, transcript };
+    } catch (e) {
+      if (e && e.code === 'BUDGET_HARDSTOP') throw e;   // budget hard-stop stays fatal
+      if (!isTimeout(e)) throw e;                        // non-timeout errors stay fatal
+      // Persistent timeout → NON-FATAL: charge the consumed attempts, record the
+      // case as failed_timeout (no fabricated samples), and continue the run.
+      if (budget) {
+        try {
+          if (e.phase === 'judge') budget.add((e.judgeAttempts || 1) * perCallCostUSD(judgeModel, 'judge'));
+          else budget.add((e.attempts || 1) * perCallCostUSD(modelId, genKind(withSkill)));
+        } catch (be) { if (be && be.code === 'BUDGET_HARDSTOP') throw be; }
+      }
+      failedCases += 1;
+      onProgress({ case: c.id, mode, phase: 'failed', reason: String((e && e.message) || 'timeout') });
+      return { caseResult: { id: c.id, mode, case_status: 'failed_timeout', reason: String((e && e.message) || 'timeout').slice(0, 200) }, transcript: null };
+    }
   });
   const caseResults = pairs.map((p) => p.caseResult);
   const transcripts = keepTranscripts ? pairs.map((p) => p.transcript) : null;
 
+  const surface = surfaceForModel(modelId);
   const receipt = buildReceipt({
-    skill: { name: skill.name, version: skill.version, contentHash: skill.contentHash },
+    skill: {
+      name: skill.name, version: skill.version, contentHash: skill.contentHash,
+      // v0.3.1 value-per-token axis: estimated SKILL.md token size.
+      tokens: estimateTokens(skill.skillMd),
+    },
     suite: { format: skill.suite.format, suiteHash: skill.suite.suiteHash, caseCount: skill.suite.caseCount },
     run: {
       model_id: modelId,
       model_release_date: releaseDateFor(modelId),
-      surface: surfaceLabel(),
+      provider: providerForModel(modelId),
+      surface,
+      // v0.3.1: on the openai/cli (codex) surface, record the fixed harness preamble.
+      surface_overhead_note: surface === 'openai-cli' ? CODEX_OVERHEAD_NOTE : undefined,
       runner_version: RUNNER_VERSION,
       date_utc: opts.nowIso || new Date().toISOString(),
       registry: registryStatus(modelId),
       transcripts: keepTranscripts ? 'retained-local' : 'hashes-only',
-      judge: judgeSettings(samples),
+      judge: judgeSettings(samples, judgeModel),
     },
     cases: caseResults,
     verificationLevel: 'TESTED',
   });
 
-  return { receipt, calls, transcripts };
+  return { receipt, calls, transcripts, failedCases };
 }
 
 function band(mean, sd) { return `${mean.toFixed(3)} ± ${sd.toFixed(3)}`; }
@@ -191,7 +237,7 @@ function summarizeReceipt(receipt) {
   L.push(`- **run (UTC):** ${receipt.run.date_utc}`);
   L.push(`- **runner:** v${receipt.run.runner_version}`);
   const j = receipt.run.judge || {};
-  L.push(`- **judge:** ${j.samples || 1} samples/case, temperature ${j.temperature == null ? 'n/a (surface-controlled)' : j.temperature} (${j.sampling || 'single'})`);
+  L.push(`- **judge:** ${j.samples || 1} samples/case, temperature ${j.temperature == null ? 'n/a' : j.temperature} (${j.sampling || 'single'})`);
   if (receipt.run.registry) L.push(`- **registry:** ${receipt.run.registry}   **transcripts:** ${receipt.run.transcripts || 'hashes-only'}`);
   L.push(`- **skill content_hash:** \`${receipt.skill.content_hash.slice(0, 16)}…\``);
   L.push(`- **suite:** ${receipt.suite.case_count} cases (${receipt.suite.format})`);
@@ -203,6 +249,10 @@ function summarizeReceipt(receipt) {
   const cmp = receipt.comparison;
   const aggs = receipt.results.aggregates;
   const sign = cmp.delta >= 0 ? '+' : '';
+  if (receipt.run.status === 'incomplete') {
+    L.push(`> ⏱ **INCOMPLETE** — ${receipt.run.failed_case_count} case(s) failed (timed out after retries) and are EXCLUDED from the aggregates below; this receipt must not be used to compute a drift/durability verdict.`);
+    L.push('');
+  }
   L.push(`with_skill **${band(aggs.with_skill.mean_score, aggs.with_skill.stddev)}** vs baseline **${band(aggs.baseline.mean_score, aggs.baseline.stddev)}**`);
   L.push('');
   L.push(`skill lift **${sign}${cmp.delta.toFixed(3)}** (combined uncertainty ± ${cmp.delta_uncertainty.toFixed(3)})`);
@@ -212,6 +262,10 @@ function summarizeReceipt(receipt) {
   L.push(`| case | mode | outcome | mean ± stddev | judge reason |`);
   L.push(`|---|---|---|---|---|`);
   for (const c of receipt.results.cases) {
+    if (c.case_status === 'failed_timeout') {
+      L.push(`| \`${c.id}\` | ${c.mode} | ⏱ failed_timeout | — (not measured) | ${c.reason || 'timed out'} |`);
+      continue;
+    }
     const flag = c.outcome === 'borderline' ? ' ⚠' : '';
     L.push(`| \`${c.id}\` | ${c.mode} | ${c.outcome}${flag} | ${band(c.mean, c.stddev || 0)} | ${c.reason || ''} |`);
   }