driftproof 0.3.0

package/lib/diff.js

@@ -0,0 +1,152 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+const { bandVerdict, round } = require('./stats');
+const { EFFECT_FLOOR } = require('../config');
+
+// Practical-significance gate applied ON TOP of band separation. bandVerdict()
+// stays a pure geometry test (kept that way so its unit checks are unambiguous);
+// this wrapper additionally requires |delta| >= EFFECT_FLOOR before a verdict is
+// claimed. A separated-but-trivial move (below the judge's quantization floor)
+// becomes "within noise (below effect floor)".
+const WITHIN_NOISE_FLOOR = 'within noise (below effect floor)';
+function verdictWithFloor(before, after, delta) {
+  const raw = bandVerdict(before.mean, before.stddev, after.mean, after.stddev);
+  if ((raw === 'regression' || raw === 'improvement') && Math.abs(delta) < EFFECT_FLOOR) {
+    return WITHIN_NOISE_FLOOR;
+  }
+  return raw;
+}
+function isWithinNoise(v) { return v === 'within noise' || v === WITHIN_NOISE_FLOOR; }
+
+// Build a drift report (markdown) between two receipts.
+//
+// CREDIBILITY CORE — the anti-false-positive rule: a per-case regression (or
+// improvement) is claimed ONLY when (1) the two confidence bands do NOT overlap
+// AND (2) the mean moved by at least EFFECT_FLOOR (practical-significance floor,
+// see config.js). When the bands overlap OR the move is below the floor, the
+// change is reported as "within noise" and is NOT counted as a regression. A tool
+// that cries wolf is worse than useless, so band separation PLUS a real-sized
+// delta — not either alone — is what triggers a verdict.
+
+// Map case-id → { mean, stddev } for a receipt's with_skill cases. Falls back to
+// score/0 for v0.1 receipts that have no per-case band.
+function withSkillBands(receipt) {
+  const out = {};
+  for (const c of receipt.results.cases) {
+    if (c.mode !== 'with_skill') continue;
+    out[c.id] = { mean: c.mean != null ? c.mean : c.score, stddev: c.stddev || 0 };
+  }
+  return out;
+}
+
+// Aggregate with_skill band for a receipt (mean_score ± stddev). v0.1 receipts
+// have no aggregate stddev → 0 (bands collapse to points; every move looks real,
+// which is exactly the v0.1 weakness v0.2 fixes).
+function aggWithBand(receipt) {
+  const a = receipt.results.aggregates.with_skill;
+  return { mean: a.mean_score, stddev: a.stddev || 0 };
+}
+
+function fmt(n) { return n == null ? 'n/a' : (n >= 0 ? '+' : '') + n.toFixed(3); }
+function pct(n) { return n == null ? 'n/a' : n.toFixed(3); }
+function bandStr(x) { return `${x.mean.toFixed(3)} ± ${x.stddev.toFixed(3)}`; }
+function short(h) { return h ? String(h).slice(0, 12) : 'n/a'; }
+
+// The headline is a SUMMARY of the per-case band-overlap verdicts — the credibility
+// core lives per case (judge-sample bands), and the headline just aggregates it.
+// It deliberately does NOT run a separate band test on the aggregate mean: the
+// aggregate band is suite dispersion, and a separate test there would either cry
+// wolf (if too tight) or mask real per-case drift (if too wide).
+function headlineVerdict(perCase) {
+  const reg = perCase.filter((r) => r.verdict === 'regression').length;
+  const imp = perCase.filter((r) => r.verdict === 'improvement').length;
+  const s = (n) => (n === 1 ? '' : 's');
+  if (reg && imp) return `MIXED — ${reg} case regression${s(reg)} and ${imp} improvement${s(imp)} on non-overlapping bands.`;
+  if (reg) return `DRIFT — ${reg} case${s(reg)} regressed (bands do not overlap); the skill is measurably weaker on ${reg} case${s(reg)}.`;
+  if (imp) return `IMPROVED — ${imp} case${s(imp)} improved (bands do not overlap); none regressed.`;
+  return 'WITHIN NOISE — no case moved beyond its confidence band; the skill holds up.';
+}
+
+function buildDriftReport(a, b, { labelA = 'A', labelB = 'B' } = {}) {
+  const aB = withSkillBands(a);
+  const bB = withSkillBands(b);
+  const ids = [...new Set([...Object.keys(aB), ...Object.keys(bB)])];
+
+  const perCase = ids.map((id) => {
+    const before = aB[id] || null;
+    const after = bB[id] || null;
+    const delta = (before && after) ? round(after.mean - before.mean) : null;
+    const verdict = (before && after) ? verdictWithFloor(before, after, delta) : 'n/a';
+    return { id, before, after, delta, verdict };
+  });
+  // Sort worst-first: regressions, then by delta.
+  const order = { regression: 0, 'within noise': 1, [WITHIN_NOISE_FLOOR]: 1, improvement: 2, 'n/a': 3 };
+  perCase.sort((x, y) => (order[x.verdict] - order[y.verdict]) || ((x.delta || 0) - (y.delta || 0)));
+
+  const aAgg = aggWithBand(a);
+  const bAgg = aggWithBand(b);
+  const headlineDelta = round(bAgg.mean - aAgg.mean);
+  const regressions = perCase.filter((r) => r.verdict === 'regression');
+
+  const L = [];
+  L.push(`# Drift report`);
+  L.push('');
+  L.push(`**Skill:** ${a.skill.name} \`${a.skill.version}\``);
+  L.push('');
+  L.push(`| | ${labelA} | ${labelB} |`);
+  L.push(`|---|---|---|`);
+  L.push(`| model | \`${a.run.model_id}\` | \`${b.run.model_id}\` |`);
+  L.push(`| run date (UTC) | ${a.run.date_utc} | ${b.run.date_utc} |`);
+  L.push(`| surface | ${a.run.surface} | ${b.run.surface} |`);
+  L.push(`| judge samples/case | ${(a.run.judge || {}).samples || 1} | ${(b.run.judge || {}).samples || 1} |`);
+  L.push(`| skill content_hash | \`${short(a.skill.content_hash)}\` | \`${short(b.skill.content_hash)}\` |`);
+  L.push(`| suite_hash | \`${short(a.suite.suite_hash)}\` | \`${short(b.suite.suite_hash)}\` |`);
+  L.push(`| with_skill (mean ± band) | ${bandStr(aAgg)} | ${bandStr(bAgg)} |`);
+  L.push(`| baseline score | ${pct(a.comparison.baseline_score)} | ${pct(b.comparison.baseline_score)} |`);
+  L.push(`| skill lift (Δ) | ${fmt(a.comparison.delta)} | ${fmt(b.comparison.delta)} |`);
+  L.push('');
+
+  const warnings = [];
+  if (a.skill.content_hash !== b.skill.content_hash) warnings.push('skill content_hash differs — the skill itself changed between receipts, so drift mixes skill edits with model drift.');
+  if (a.suite.suite_hash !== b.suite.suite_hash) warnings.push('suite_hash differs — the eval suite changed; per-case comparison may be misleading.');
+  if (a.skill.name !== b.skill.name) warnings.push(`different skills (${a.skill.name} vs ${b.skill.name}) — comparison is not meaningful.`);
+  if ((a.run.judge || {}).samples <= 1 || (b.run.judge || {}).samples <= 1) warnings.push('one or both receipts are single-sample (no bands) — non-overlap can only be trusted when both sides are sampled.');
+  if (warnings.length) {
+    L.push('> **⚠ Caveats**');
+    for (const w of warnings) L.push(`> - ${w}`);
+    L.push('');
+  }
+
+  const nWithin = perCase.filter((r) => isWithinNoise(r.verdict)).length;
+  const nFloor = perCase.filter((r) => r.verdict === WITHIN_NOISE_FLOOR).length;
+  L.push(`## Headline`);
+  L.push('');
+  L.push(`**${headlineVerdict(perCase)}**`);
+  L.push('');
+  L.push(`with_skill mean moved ${fmt(headlineDelta)} (${bandStr(aAgg)} → ${bandStr(bAgg)}; band = suite dispersion). Per-case band-overlap verdicts: ${regressions.length} regression(s), ${perCase.filter((r) => r.verdict === 'improvement').length} improvement(s), ${nWithin} within noise${nFloor ? ` (${nFloor} of them band-separated but below the ${EFFECT_FLOOR} effect floor)` : ''}.`);
+  L.push('');
+
+  L.push(`## Per-case with_skill (band overlap → verdict)`);
+  L.push('');
+  L.push(`| case | ${labelA} (mean ± sd) | ${labelB} (mean ± sd) | Δ | verdict |`);
+  L.push(`|---|---|---|---|---|`);
+  for (const r of perCase) {
+    const flag = r.verdict === 'regression' ? '🔻 regression' : r.verdict === 'improvement' ? '🔼 improvement' : r.verdict === WITHIN_NOISE_FLOOR ? 'within noise (below floor)' : r.verdict === 'within noise' ? 'within noise' : 'n/a';
+    L.push(`| \`${r.id}\` | ${r.before ? bandStr(r.before) : 'n/a'} | ${r.after ? bandStr(r.after) : 'n/a'} | ${fmt(r.delta)} | ${flag} |`);
+  }
+  L.push('');
+
+  if (regressions.length) {
+    L.push(`## Regressions (${regressions.length}) — bands do not overlap`);
+    L.push('');
+    for (const r of regressions) {
+      L.push(`- \`${r.id}\`: ${bandStr(r.before)} → ${bandStr(r.after)} (${fmt(r.delta)})`);
+    }
+    L.push('');
+  }
+
+  return { markdown: L.join('\n'), perCase, headlineDelta, regressions };
+}
+
+module.exports = { buildDriftReport, withSkillBands };

package/lib/init.js

@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+// `driftproof init <dir>` scaffolding.
+//
+// Produces a complete, runnable skill skeleton the way agentskills.io expects it:
+//   <dir>/SKILL.md            — the skill instructions (a stub, if none exists)
+//   <dir>/evals/evals.json    — an eval suite with 3 example cases, each rubric
+//                               anchored at 0.80 (the scoring convention the
+//                               example suite and Report #001 use)
+//   <dir>/.driftproofrc       — per-project run defaults (budget, models, samples)
+//
+// NEVER overwrites an existing file — every write is guarded, and an existing
+// path is reported as "skipped". Safe to re-run.
+
+// JSON carries no comments, so guidance rides in `_`-prefixed keys. The runner's
+// suite loader (lib/skill.js normalizeCases) reads only id/prompt/rubric/
+// pass_threshold and ignores everything else, so these keys are inert at run time
+// and exist purely to guide the author editing the file.
+function evalsTemplate(skillName) {
+  return {
+    skill: skillName,
+    version: '0.1.0',
+    format: 'agentskills.io/evals',
+    _guidance: [
+      'Each case must be grounded in a claim your SKILL.md actually makes.',
+      'Aim for graded difficulty: a good skill should land with_skill ~0.7-0.9, not 1.0 (saturation hides drift).',
+      "Anchor every rubric at 0.80 = 'fully correct'; reserve 0.81-1.00 for exemplary work only, and cap clear errors low.",
+      'The baseline is the SAME prompt with no SKILL.md — so a case only measures the skill if the skill is what makes it pass.',
+      'See AUTHORING.md (https://driftproofhq.com/authoring.html) for the full fair-suite guide.',
+    ],
+    cases: [
+      {
+        id: 'example-core-claim',
+        _comment: 'Replace with a case that exercises the MAIN thing your skill teaches. The prompt should be answerable badly without the skill and well with it.',
+        prompt: 'Describe the task here — a realistic request a user would make that your skill is meant to help with.',
+        rubric: "State exactly what a good response must do, drawn from your SKILL.md. SCORING ANCHOR (apply strictly): a fully correct, idiomatic response scores 0.80; award 0.81-0.90 only if it is ALSO exemplary; 0.91-1.00 only if flawless and exceptional (rare). Subtract ~0.2 for each concrete error the skill is supposed to prevent.",
+        pass_threshold: 0.7,
+      },
+      {
+        id: 'example-common-mistake',
+        _comment: 'Target a specific mistake your skill is supposed to prevent. The baseline (no skill) should fall into the trap; the skill should avoid it.',
+        prompt: 'Describe a task where the obvious answer is wrong in a way your skill corrects.',
+        rubric: "Identify the specific correct behaviour your skill mandates here. SCORING ANCHOR (apply strictly): if the response makes the mistake the skill exists to prevent, cap at 0.3; a correct response scores 0.80; 0.81-1.00 only if exemplary. Subtract ~0.2 per additional error.",
+        pass_threshold: 0.7,
+      },
+      {
+        id: 'example-edge-case',
+        _comment: 'A harder edge case with real headroom, so the suite is not saturated. Keep it fair: no undocumented expectations, no gotchas the SKILL.md never mentions.',
+        prompt: 'Describe an edge case your skill handles that a naive answer would get subtly wrong.',
+        rubric: "Name the subtle requirement here, grounded in a claim the SKILL.md makes. SCORING ANCHOR (apply strictly): a correct handling scores 0.80; 0.81-1.00 only if additionally flawless and exceptional; cap a response that misses the edge case at 0.5.",
+        pass_threshold: 0.7,
+      },
+    ],
+  };
+}
+
+function skillTemplate(skillName) {
+  return `---
+name: ${skillName}
+version: 0.1.0
+description: One line describing what this skill teaches an agent to do.
+---
+
+# ${skillName}
+
+Replace this stub with your skill's instructions. A skill is the guidance you
+would give an agent so it follows your conventions — the more concrete and
+checkable the claims, the better a suite can measure whether the model still
+honours them.
+
+## Rules
+
+1. State the first rule your skill enforces.
+2. State the second.
+3. Keep each rule specific enough that an eval case can check it.
+
+## Notes
+
+Every case in \`evals/evals.json\` should be grounded in a rule above. If a case
+tests something this file never says, the suite is unfair — see AUTHORING.md.
+`;
+}
+
+function rcTemplate() {
+  return {
+    _comment: 'Per-project driftproof defaults. CLI flags override these. See `driftproof help`.',
+    models: 'claude-haiku-4-5',
+    samples: 5,
+    max_usd: 2,
+  };
+}
+
+// Write `content` to `file` only if it does not already exist. Records the path
+// in `created` or `skipped`.
+function writeIfAbsent(file, content, created, skipped) {
+  if (fs.existsSync(file)) { skipped.push(file); return; }
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, content);
+  created.push(file);
+}
+
+// Scaffold into `targetDir`. Returns { dir, created:[abs...], skipped:[abs...] }.
+function scaffoldInit(targetDir) {
+  const dir = path.resolve(targetDir);
+  const skillName = path.basename(dir);
+  const created = [];
+  const skipped = [];
+
+  writeIfAbsent(path.join(dir, 'SKILL.md'), skillTemplate(skillName), created, skipped);
+  writeIfAbsent(
+    path.join(dir, 'evals', 'evals.json'),
+    JSON.stringify(evalsTemplate(skillName), null, 2) + '\n',
+    created, skipped,
+  );
+  writeIfAbsent(
+    path.join(dir, '.driftproofrc'),
+    JSON.stringify(rcTemplate(), null, 2) + '\n',
+    created, skipped,
+  );
+
+  return { dir, created, skipped };
+}
+
+module.exports = { scaffoldInit, evalsTemplate, skillTemplate, rcTemplate };

package/lib/json.js

@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+// JSON salvage + retry/timeout helpers.
+//
+// PORTED (then genericized) from a private codebase's aiHelpers.js:
+//   - safeParseJSON  (trailing-comma + unescaped-newline repair)
+//   - extractJsonArray  (streaming brace-matcher that salvages a truncated
+//                        array of objects — LLMs routinely overrun max_tokens)
+// The private origin carried no business logic in these functions; they are pure
+// string/JSON utilities. Scanned clean against the confidentiality deny-list.
+
+function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+// Parse JSON with basic repair for common LLM output quirks. Throws if the text
+// is unsalvageable so callers can fall back rather than silently accept garbage.
+function safeParseJSON(str) {
+  try { return JSON.parse(str); } catch (_e) { /* fall through to repairs */ }
+  // Strip a ```json ... ``` fence if present.
+  const fence = String(str).match(/```(?:json)?\s*([\s\S]*?)```/i);
+  let s = fence ? fence[1] : String(str);
+  // Repair trailing commas before ] or }.
+  let fixed = s.replace(/,\s*([\]}])/g, '$1');
+  try { return JSON.parse(fixed); } catch (_e) { /* keep trying */ }
+  // Repair unescaped newlines inside string values.
+  fixed = fixed.replace(/(?<=": ")([\s\S]*?)(?="[,}])/g, (m) => m.replace(/\n/g, '\\n'));
+  try { return JSON.parse(fixed); } catch (_e) { /* give up */ }
+  throw new Error('Invalid JSON (unsalvageable)');
+}
+
+// Extract as many complete objects as possible from a JSON array in `text`,
+// even when the array is truncated mid-stream. Returns { items, truncated }.
+// A well-formed array parses on the happy path; otherwise a brace-matcher walks
+// the string and keeps every object that closed cleanly.
+function extractJsonArray(text) {
+  if (!text || typeof text !== 'string') return { items: [], truncated: false };
+  let s = text.trim();
+  const fence = s.match(/```(?:json)?\s*([\s\S]*?)```/i);
+  if (fence) s = fence[1].trim();
+  try {
+    const whole = JSON.parse(s);
+    if (Array.isArray(whole)) return { items: whole, truncated: false };
+  } catch (_e) { /* salvage below */ }
+  const start = s.indexOf('[');
+  if (start === -1) return { items: [], truncated: false };
+  const items = [];
+  let depth = 0, inStr = false, esc = false, objStart = -1, closed = false;
+  for (let i = start + 1; i < s.length; i++) {
+    const c = s[i];
+    if (inStr) {
+      if (esc) esc = false;
+      else if (c === '\\') esc = true;
+      else if (c === '"') inStr = false;
+      continue;
+    }
+    if (c === '"') { inStr = true; continue; }
+    if (c === '{') { if (depth === 0) objStart = i; depth++; }
+    else if (c === '}') {
+      depth--;
+      if (depth === 0 && objStart !== -1) {
+        try { items.push(JSON.parse(s.slice(objStart, i + 1))); } catch (_e) { /* skip broken object */ }
+        objStart = -1;
+      }
+    } else if (c === ']' && depth === 0) { closed = true; break; }
+  }
+  return { items, truncated: !closed };
+}
+
+// Extract a single JSON object (the first balanced {...}) from free text, with
+// salvage. Returns the parsed object or throws.
+function extractJsonObject(text) {
+  if (!text || typeof text !== 'string') throw new Error('no text to parse');
+  let s = text.trim();
+  const fence = s.match(/```(?:json)?\s*([\s\S]*?)```/i);
+  if (fence) s = fence[1].trim();
+  const start = s.indexOf('{');
+  if (start === -1) throw new Error('no JSON object found');
+  let depth = 0, inStr = false, esc = false;
+  for (let i = start; i < s.length; i++) {
+    const c = s[i];
+    if (inStr) {
+      if (esc) esc = false;
+      else if (c === '\\') esc = true;
+      else if (c === '"') inStr = false;
+      continue;
+    }
+    if (c === '"') inStr = true;
+    else if (c === '{') depth++;
+    else if (c === '}') {
+      depth--;
+      if (depth === 0) return safeParseJSON(s.slice(start, i + 1));
+    }
+  }
+  // Unterminated object — hand the tail to the repairer.
+  return safeParseJSON(s.slice(start));
+}
+
+// Run an async thunk with a wall-clock timeout. Rejects with a TIMEOUT error if
+// it does not settle in time.
+function withTimeout(promiseFactory, ms, label = 'operation') {
+  return new Promise((resolve, reject) => {
+    let done = false;
+    const t = setTimeout(() => {
+      if (done) return;
+      done = true;
+      const e = new Error(`${label} timed out after ${ms}ms`);
+      e.code = 'TIMEOUT';
+      reject(e);
+    }, ms);
+    Promise.resolve()
+      .then(promiseFactory)
+      .then((v) => { if (!done) { done = true; clearTimeout(t); resolve(v); } })
+      .catch((err) => { if (!done) { done = true; clearTimeout(t); reject(err); } });
+  });
+}
+
+// Retry an async thunk with exponential backoff. Retries only when shouldRetry
+// returns true for the caught error (default: 429 / rate-limit / timeout).
+async function withRetry(fn, { tries = 3, baseDelayMs = 1000, shouldRetry } = {}) {
+  const retryable = shouldRetry || ((err) => {
+    const msg = String((err && err.message) || err || '');
+    return (err && err.status === 429) || /429|rate.?limit|timeout|ETIMEDOUT|ECONNRESET/i.test(msg);
+  });
+  let lastErr;
+  for (let attempt = 0; attempt < tries; attempt++) {
+    try {
+      return await fn(attempt);
+    } catch (err) {
+      lastErr = err;
+      if (attempt === tries - 1 || !retryable(err)) throw err;
+      await sleep(baseDelayMs * Math.pow(2, attempt));
+    }
+  }
+  throw lastErr;
+}
+
+module.exports = {
+  sleep, safeParseJSON, extractJsonArray, extractJsonObject, withTimeout, withRetry,
+};

package/lib/judge.js

@@ -0,0 +1,126 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+const { complete, surfaceLabel } = require('./provider');
+const { extractJsonObject } = require('./json');
+const { sha256 } = require('./canonical');
+const { mean, stddev } = require('./stats');
+
+// Rubric-based LLM judge.
+//
+// REWRITTEN from scratch. Only the generic SHAPE of a private "gap report"
+// grader was reused: (system role) + (material + rubric) + "return ONLY JSON" +
+// salvage-parse + clamp/validate the score. All domain content (sales/committee/
+// deal) was dropped — none of it applies here. Scanned clean against the deny-list.
+//
+// The judge reads a model's OUTPUT for one eval case and grades it against that
+// case's rubric, returning a normalized score in [0,1] plus a short reason.
+
+const JUDGE_SYSTEM =
+  'You are a strict, fair grader. You are given a TASK, a model RESPONSE to that task, '
+  + 'and a RUBRIC describing what a good response must do. Grade only against the rubric. '
+  + 'Be objective and specific: reward exactly what the rubric asks for and nothing else. '
+  + 'Return your grade as JSON only.';
+
+// Build the grading prompt. Kept deterministic so the same (task, response,
+// rubric) always yields the same prompt and thus a stable rubric_hash.
+function buildJudgePrompt({ task, response, rubric }) {
+  return [
+    'TASK GIVEN TO THE MODEL:',
+    '"""',
+    String(task || '').trim(),
+    '"""',
+    '',
+    'MODEL RESPONSE TO GRADE:',
+    '"""',
+    String(response || '').trim(),
+    '"""',
+    '',
+    'RUBRIC (grade strictly against this):',
+    '"""',
+    String(rubric || '').trim(),
+    '"""',
+    '',
+    'Return ONLY this JSON object, no prose before or after:',
+    '{',
+    '  "score": <number 0.0 to 1.0, fraction of the rubric satisfied>,',
+    '  "pass": <true if the response substantially meets the rubric, else false>,',
+    '  "reason": "<one sentence, <=30 words, citing the specific rubric points met or missed>"',
+    '}',
+  ].join('\n');
+}
+
+// The rubric_hash recorded in the receipt binds a grade to the EXACT grading
+// instruction used, so a later reader can tell whether two receipts were graded
+// the same way. It hashes the judge system prompt + the case rubric text.
+function rubricHash(rubric) {
+  return sha256(JUDGE_SYSTEM + '\n---\n' + String(rubric || '').trim());
+}
+
+function clamp01(n) {
+  const x = Number(n);
+  if (!Number.isFinite(x)) return 0;
+  return Math.max(0, Math.min(1, x));
+}
+
+// Judge settings for the CURRENT surface. Determinism where the surface allows:
+// on the api surface we pin temperature 0 for judge calls (and record it); on
+// the cli surface sampling params are surface-controlled and cannot be set, so
+// temperature is null and `sampling` says so. Recorded into every receipt.
+function judgeSettings(samples) {
+  const surface = surfaceLabel();
+  if (surface === 'api') {
+    return { samples, temperature: 0, sampling: 'api-temperature-0', surface };
+  }
+  return { samples, temperature: null, sampling: 'surface-controlled', surface };
+}
+
+// Grade one response once. Returns { score, reason, raw }.
+// `raw` is the judge's verbatim output text (hashed into the receipt for
+// transcript auditability, and optionally retained under --keep-transcripts).
+async function gradeOnce({ task, response, rubric, model, timeoutMs, temperature }) {
+  const prompt = buildJudgePrompt({ task, response, rubric });
+  const { text } = await complete({ system: JUDGE_SYSTEM, prompt, model, maxTokens: 400, timeoutMs, temperature });
+  let parsed;
+  try {
+    parsed = extractJsonObject(text);
+  } catch (_e) {
+    // Unsalvageable judge output → conservative 0 (a judge that can't be parsed
+    // must never silently "pass"), tagged so the caller can see it happened.
+    return { score: 0, reason: 'judge output unparseable', unparsed: true, raw: String(text || '') };
+  }
+  return { score: clamp01(parsed.score), reason: String(parsed.reason || '').slice(0, 300), raw: String(text || '') };
+}
+
+// Grade a response N times and return the sampled distribution:
+//   { samples:[scores], mean, stddev, reason, judge_settings, model_id, rubric_hash }
+// `mean` ± `stddev` is the per-case confidence band (raw spread of the N scores)
+// used by the borderline-outcome rule and per-case drift band-overlap logic.
+async function gradeSamples({ task, response, rubric, model, samples = 5, timeoutMs = 120000 }) {
+  const settings = judgeSettings(samples);
+  const scores = [];
+  const reasons = [];
+  const rawTexts = [];
+  for (let i = 0; i < samples; i++) {
+    const r = await gradeOnce({ task, response, rubric, model, timeoutMs, temperature: settings.temperature === null ? undefined : settings.temperature });
+    scores.push(r.score);
+    reasons.push(r.reason);
+    rawTexts.push(r.raw || '');
+  }
+  return {
+    samples: scores,
+    mean: mean(scores),
+    stddev: stddev(scores),
+    reason: reasons[0] || '',
+    // v0.3 transcript auditability: verbatim judge outputs + their sha256 hashes,
+    // one per sample. `sample_texts` is transient (retained only under
+    // --keep-transcripts); `sample_hashes` goes into the receipt.
+    sample_texts: rawTexts,
+    sample_hashes: rawTexts.map((t) => sha256(t)),
+    judge_settings: settings,
+    model_id: model,
+    rubric_hash: rubricHash(rubric),
+  };
+}
+
+module.exports = { gradeOnce, gradeSamples, judgeSettings, rubricHash, buildJudgePrompt, JUDGE_SYSTEM };

package/lib/models.js

@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { resolveModel } = require('./provider');
+
+// The model registry (config/models.json) is the single source of truth for
+// which models Driftproof knows about, their per-MTok prices, tier, and whether
+// they are eligible to serve as the judge. The cost guard reads prices here; the
+// runner marks each receipt registered/unregistered against it; the release
+// trigger appends auto-discovered ids to it.
+
+// The registry ships inside the package (config/models.json), so it resolves
+// correctly from a global/npx install — path is relative to this module, not the
+// caller's CWD. A user can point at their own registry with DRIFTPROOF_REGISTRY.
+const REGISTRY_PATH = process.env.DRIFTPROOF_REGISTRY
+  ? path.resolve(process.env.DRIFTPROOF_REGISTRY)
+  : path.join(__dirname, '..', 'config', 'models.json');
+
+// Conservative default price for an UNREGISTERED model, per MTok. A budget guard
+// must never UNDER-estimate, so an unknown id is costed as if it were the most
+// expensive model we know about (Fable-tier). This is deliberately pessimistic:
+// an unregistered id should over-project and, if anything, abort early rather
+// than quietly overspend. Documented in config/models.json.default_price_note.
+const DEFAULT_PRICE = { input: 10.0, output: 50.0 };
+
+let _cache = null;
+function loadRegistry(force) {
+  if (_cache && !force) return _cache;
+  const raw = JSON.parse(fs.readFileSync(REGISTRY_PATH, 'utf8'));
+  const models = Array.isArray(raw.models) ? raw.models : [];
+  const byId = Object.create(null);
+  for (const m of models) byId[m.id] = m;
+  _cache = { ...raw, models, byId };
+  return _cache;
+}
+
+// Infer the model family from an id. Used to find a same-family predecessor when
+// the release trigger discovers a new model.
+function familyOf(id) {
+  const m = String(id).match(/claude-(fable|mythos|opus|sonnet|haiku)/i);
+  return m ? m[1].toLowerCase() : 'unknown';
+}
+
+// Resolve a (possibly aliased) model id against the registry.
+//   returns { id: <canonical>, entry: <registry row|null>, registered: bool }
+function resolveRegistry(modelId) {
+  const reg = loadRegistry();
+  const canonical = resolveModel(modelId);
+  const entry = reg.byId[canonical] || reg.byId[modelId] || null;
+  return { id: canonical, entry, registered: !!entry };
+}
+
+// The value stamped into receipt.run.registry.
+function registryStatus(modelId) {
+  return resolveRegistry(modelId).registered ? 'registered' : 'unregistered';
+}
+
+// Price per MTok for a model: the registry price when registered, else the
+// conservative default. Returns { input, output, registered }.
+function priceForModel(modelId) {
+  const { entry } = resolveRegistry(modelId);
+  if (entry) return { input: entry.input_price, output: entry.output_price, registered: true };
+  return { input: DEFAULT_PRICE.input, output: DEFAULT_PRICE.output, registered: false };
+}
+
+// Whether a model is allowed to be the judge (fixed-judge policy — only the
+// cheap Haiku judge is judge_eligible; see docs/judge-policy.html).
+function isJudgeEligible(modelId) {
+  const { entry } = resolveRegistry(modelId);
+  return !!(entry && entry.judge_eligible);
+}
+
+// The servable same-family predecessor of a model: the registered model in the
+// same family with the most recent `released` date strictly before this one
+// (falling back to any other same-family model). Null when the family has no
+// other member — the trigger then falls back to a cross-family "capability gap"
+// pairing. `modelId` need not itself be in the registry.
+function familyPredecessor(modelId) {
+  const reg = loadRegistry();
+  const canonical = resolveModel(modelId);
+  const fam = familyOf(canonical);
+  const self = reg.byId[canonical] || null;
+  const selfDate = self && self.released ? self.released : null;
+  let pool = reg.models.filter((m) => familyOf(m.id) === fam && m.id !== canonical);
+  if (!pool.length) return null;
+  // Prefer members released strictly before the new model when we know its date.
+  if (selfDate) {
+    const before = pool.filter((m) => m.released && m.released < selfDate);
+    if (before.length) pool = before;
+  }
+  const dated = pool.filter((m) => m.released);
+  const chooseFrom = dated.length ? dated : pool;
+  chooseFrom.sort((a, b) => String(b.released || '').localeCompare(String(a.released || '')));
+  return chooseFrom[0].id;
+}
+
+// Append a newly-discovered model id to the registry file (release trigger).
+// `firstSeen` is the first-seen date recorded as `released`. Idempotent: a no-op
+// if the id already exists. Returns the added entry (or null if already present).
+function addDiscoveredModel(id, { firstSeen = null, registryPath = REGISTRY_PATH } = {}) {
+  const raw = JSON.parse(fs.readFileSync(registryPath, 'utf8'));
+  if (raw.models.some((m) => m.id === id)) return null;
+  const family = familyOf(id);
+  // Best-effort tier from family; unknown families default to frontier pricing.
+  const tier = family === 'haiku' ? 'cheap' : family === 'sonnet' ? 'standard' : 'frontier';
+  const price = family === 'haiku' ? { input: 1.0, output: 5.0 }
+    : family === 'sonnet' ? { input: 3.0, output: 15.0 }
+      : { input: 5.0, output: 25.0 };
+  const entry = {
+    id, family, provider: 'anthropic', released: firstSeen,
+    input_price: price.input, output_price: price.output, tier,
+    judge_eligible: false, auto_added: true,
+  };
+  raw.models.push(entry);
+  fs.writeFileSync(registryPath, JSON.stringify(raw, null, 2) + '\n');
+  _cache = null; // invalidate cache so subsequent lookups see the new entry
+  return entry;
+}
+
+module.exports = {
+  loadRegistry, resolveRegistry, registryStatus, priceForModel, isJudgeEligible,
+  familyOf, familyPredecessor, addDiscoveredModel, DEFAULT_PRICE, REGISTRY_PATH,
+};