driftproof 0.3.0 → 0.5.0

package/lib/importers.js

@@ -0,0 +1,207 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+// Receipt interop — importers (Phase 7). Convert results from neighboring eval
+// tools into valid Driftproof receipts with HONEST epistemics:
+//
+//   - verification_level DECLARED (never TESTED — we did not run, hash, or
+//     judge the generations),
+//   - run.surface "external" + run.source "imported/<tool>",
+//   - generation/judge-sample hashes OMITTED, content/suite/rubric hashes null —
+//     hashes are never fabricated,
+//   - no baseline mode in the source tool → empty baseline aggregate + null
+//     comparison, never a fabricated 0-baseline.
+//
+// The per-tool field mappings (and the assumed-shape disclosure — neither tool
+// publishes a frozen results schema, so the checked-in fixtures ARE the
+// compatibility contract) are documented in docs/interop.md.
+
+const { RECEIPT_SCHEMA_VERSION, RUNNER_VERSION, SUITE_FORMAT } = require('../config');
+const { sealReceipt } = require('./receipt');
+const { mean, stddev, round, combineUncertainty, aggregateBands } = require('./stats');
+const { outcomeFor } = require('./run');
+const { inferProvider } = require('./provider');
+const { registryStatus } = require('./models');
+
+const IMPORT_TOOLS = ['agent-skills-eval', 'skillgrade'];
+
+// Aggregate a list of imported cases the same way lib/receipt.aggregate does
+// (suite dispersion band). An empty mode aggregates to the conventional empty
+// aggregate (case_count 0) — its absence is signalled there, and in the null
+// comparison, not by fabricated numbers.
+function aggregateMode(cases) {
+  const band = aggregateBands(cases.map((c) => ({ mean: c.mean, stddev: c.stddev || 0, n: c.samples.length })));
+  return {
+    case_count: cases.length,
+    pass_count: cases.filter((c) => c.outcome === 'pass').length,
+    borderline_count: cases.filter((c) => c.outcome === 'borderline').length,
+    mean_score: band.mean,
+    stddev: band.stddev,
+  };
+}
+
+// Shared receipt shell for both importers. `judgeBlock` describes the SOURCE
+// tool's grading (samples = what it actually did), never our sampled judge.
+function importedReceipt({ tool, skillName, skillVersion, suiteFormat, caseCount, modelId, dateUtc, judgeBlock, cases, comparison }) {
+  const withSkill = cases.filter((c) => c.mode === 'with_skill');
+  const baseline = cases.filter((c) => c.mode === 'baseline');
+  const receipt = {
+    schema_version: RECEIPT_SCHEMA_VERSION,
+    skill: {
+      name: skillName,
+      version: skillVersion || 'unknown',
+      // Never seen the skill bytes — null, never fabricated.
+      content_hash: null,
+    },
+    suite: { format: suiteFormat, suite_hash: null, case_count: caseCount },
+    run: {
+      model_id: modelId,
+      model_release_date: null,
+      provider: inferProvider(modelId),
+      surface: 'external',
+      source: `imported/${tool}`,
+      runner_version: RUNNER_VERSION,
+      date_utc: dateUtc,
+      registry: registryStatus(modelId),
+      transcripts: 'none',
+      judge: judgeBlock,
+    },
+    results: {
+      cases,
+      aggregates: { with_skill: aggregateMode(withSkill), baseline: aggregateMode(baseline) },
+    },
+    comparison,
+    verification_level: 'DECLARED',
+    receipt_hash: '',
+  };
+  return sealReceipt(receipt);
+}
+
+// ── agent-skills-eval (darkrishabh/agent-skills-eval) ────────────────────────
+// Same suite lineage as Driftproof (agentskills.io evals.json). Runs each eval
+// with_skill and without_skill; an LLM judge grades BINARY per assertion. The
+// converter reads the rolled-up benchmark artifact:
+//   { skill_name, version?, target, judge, timestamp?, evals: [
+//       { id, with_skill: { pass, assertions?: [{assertion, pass, reasoning?}] },
+//         without_skill: { ... } } ] }
+// Per (eval, mode): mean = fraction of assertions passed (else 1/0 from the
+// eval-level pass); samples = [mean] — the ONE grade their judge produced,
+// never resampled; stddev 0; outcome from their pass verdict.
+function modeScore(grading) {
+  const asserts = Array.isArray(grading.assertions) ? grading.assertions : [];
+  if (asserts.length) return round(asserts.filter((a) => a.pass === true).length / asserts.length);
+  return grading.pass === true ? 1 : 0;
+}
+function importAgentSkillsEval(data, { importedAt } = {}) {
+  if (!data || !Array.isArray(data.evals)) throw new Error('agent-skills-eval import: expected { skill_name, target, judge, evals: [...] } (see docs/interop.md)');
+  const judgeModel = data.judge || 'unknown';
+  const cases = [];
+  for (const ev of data.evals) {
+    for (const [theirMode, ourMode] of [['with_skill', 'with_skill'], ['without_skill', 'baseline']]) {
+      const grading = ev[theirMode];
+      if (!grading) continue;
+      const m = modeScore(grading);
+      const failing = (grading.assertions || []).find((a) => a.pass === false);
+      const c = {
+        id: String(ev.id),
+        mode: ourMode,
+        outcome: grading.pass === true ? 'pass' : 'fail',
+        score: m, mean: m, stddev: 0,
+        samples: [m],
+        threshold: null,
+        judge: { model_id: judgeModel, rubric_hash: null },
+      };
+      if (failing && failing.reasoning) c.reason = String(failing.reasoning).slice(0, 300);
+      cases.push(c);
+    }
+  }
+  const withMeans = cases.filter((c) => c.mode === 'with_skill').map((c) => c.mean);
+  const baseMeans = cases.filter((c) => c.mode === 'baseline').map((c) => c.mean);
+  const hasBaseline = baseMeans.length > 0;
+  const wAgg = aggregateBands(cases.filter((c) => c.mode === 'with_skill').map((c) => ({ mean: c.mean, stddev: 0, n: 1 })));
+  const bAgg = aggregateBands(cases.filter((c) => c.mode === 'baseline').map((c) => ({ mean: c.mean, stddev: 0, n: 1 })));
+  const comparison = {
+    with_skill_score: round(mean(withMeans)),
+    baseline_score: hasBaseline ? round(mean(baseMeans)) : null,
+    delta: hasBaseline ? round(mean(withMeans) - mean(baseMeans)) : null,
+    delta_uncertainty: hasBaseline ? combineUncertainty(wAgg.stddev, bAgg.stddev) : null,
+  };
+  return importedReceipt({
+    tool: 'agent-skills-eval',
+    skillName: data.skill_name || 'unknown',
+    skillVersion: data.version,
+    suiteFormat: SUITE_FORMAT, // agentskills.io/evals — same suite lineage
+    caseCount: data.evals.length,
+    modelId: data.target || 'unknown',
+    dateUtc: data.timestamp || importedAt || new Date().toISOString(),
+    judgeBlock: { samples: 1, temperature: null, sampling: 'external', surface: 'external' },
+    cases,
+    comparison,
+  });
+}
+
+// ── skillgrade (mgechev/skillgrade) ──────────────────────────────────────────
+// "Unit tests for your agent skills": N trials per task, each trial's reward =
+// weighted grader scores (0..1), compared against a threshold. NO baseline mode
+// — so the receipt carries an empty baseline aggregate and a null comparison.
+// Trials are real repeated runs, so per-trial rewards map onto samples[]
+// honestly (a genuine cross-trial band). Expected shape:
+//   { skill, agent, grader_model?, threshold?, timestamp?, tasks: [
+//       { name, threshold?, trials: [ { reward }, ... ] } ] }
+function importSkillgrade(data, { importedAt } = {}) {
+  if (!data || !Array.isArray(data.tasks)) throw new Error('skillgrade import: expected { skill, agent, tasks: [...] } (see docs/interop.md)');
+  const graderModel = data.grader_model || 'unknown';
+  const defaultThreshold = typeof data.threshold === 'number' ? data.threshold : 0.8;
+  const cases = [];
+  let maxTrials = 1;
+  for (const task of data.tasks) {
+    const rewards = (task.trials || []).map((t) => (typeof t === 'number' ? t : t.reward)).filter((r) => typeof r === 'number');
+    if (!rewards.length) continue;
+    maxTrials = Math.max(maxTrials, rewards.length);
+    const m = round(mean(rewards));
+    const sd = round(stddev(rewards));
+    const threshold = typeof task.threshold === 'number' ? task.threshold : defaultThreshold;
+    cases.push({
+      id: String(task.name),
+      mode: 'with_skill',
+      // Same outcome rule as a Driftproof run (borderline when the threshold
+      // sits inside mean ± stddev) — a deterministic READING of their numbers.
+      outcome: outcomeFor(m, sd, threshold),
+      score: m, mean: m, stddev: sd,
+      samples: rewards.map((r) => round(r)),
+      threshold,
+      judge: { model_id: graderModel, rubric_hash: null },
+    });
+  }
+  return importedReceipt({
+    tool: 'skillgrade',
+    skillName: data.skill || 'unknown',
+    skillVersion: data.version,
+    suiteFormat: 'skillgrade/eval.yaml',
+    caseCount: data.tasks.length,
+    // skillgrade names an AGENT CLI (claude/gemini/codex), not a model id —
+    // imported verbatim (or the results' model field when present).
+    modelId: data.model || data.agent || 'unknown',
+    dateUtc: data.timestamp || importedAt || new Date().toISOString(),
+    judgeBlock: { samples: maxTrials, temperature: null, sampling: 'external', surface: 'external' },
+    cases,
+    // No baseline mode exists in skillgrade — nulls, never a fabricated 0.
+    comparison: {
+      with_skill_score: round(mean(cases.map((c) => c.mean))),
+      baseline_score: null,
+      delta: null,
+      delta_uncertainty: null,
+    },
+  });
+}
+
+// Dispatch. `from` must name a supported tool.
+function importResults(data, { from, importedAt } = {}) {
+  switch (from) {
+    case 'agent-skills-eval': return importAgentSkillsEval(data, { importedAt });
+    case 'skillgrade': return importSkillgrade(data, { importedAt });
+    default: throw new Error(`unknown import source "${from}" — supported: ${IMPORT_TOOLS.join(', ')}`);
+  }
+}
+
+module.exports = { importResults, importAgentSkillsEval, importSkillgrade, IMPORT_TOOLS };

package/lib/judge.js

@@ -1,10 +1,11 @@
 // SPDX-License-Identifier: Apache-2.0
 'use strict';
 
-const { complete, surfaceLabel } = require('./provider');
+const { complete, surfaceForModel } = require('./provider');
 const { extractJsonObject } = require('./json');
 const { sha256 } = require('./canonical');
 const { mean, stddev } = require('./stats');
+const { sumUsage } = require('./usage');
 
 // Rubric-based LLM judge.
 //
@@ -63,13 +64,15 @@ function clamp01(n) {
   return Math.max(0, Math.min(1, x));
 }
 
-// Judge settings for the CURRENT surface. Determinism where the surface allows:
-// on the api surface we pin temperature 0 for judge calls (and record it); on
-// the cli surface sampling params are surface-controlled and cannot be set, so
-// temperature is null and `sampling` says so. Recorded into every receipt.
-function judgeSettings(samples) {
-  const surface = surfaceLabel();
-  if (surface === 'api') {
+// Judge settings for the JUDGE model's surface. Determinism where the surface
+// allows: on an api surface (Anthropic `api` or `openai-api`) we pin temperature 0
+// for judge calls (and record it); on a cli/subscription surface sampling params
+// are surface-controlled and cannot be set, so temperature is null and `sampling`
+// says so. `judgeModel` defaults to the fixed Haiku judge, whose surface is the
+// Anthropic axis. Recorded into every receipt.
+function judgeSettings(samples, judgeModel) {
+  const surface = surfaceForModel(judgeModel || 'claude-haiku-4-5');
+  if (surface === 'api' || surface === 'openai-api') {
     return { samples, temperature: 0, sampling: 'api-temperature-0', surface };
   }
   return { samples, temperature: null, sampling: 'surface-controlled', surface };
@@ -80,16 +83,16 @@ function judgeSettings(samples) {
 // transcript auditability, and optionally retained under --keep-transcripts).
 async function gradeOnce({ task, response, rubric, model, timeoutMs, temperature }) {
   const prompt = buildJudgePrompt({ task, response, rubric });
-  const { text } = await complete({ system: JUDGE_SYSTEM, prompt, model, maxTokens: 400, timeoutMs, temperature });
+  const { text, attempts, usage } = await complete({ system: JUDGE_SYSTEM, prompt, model, maxTokens: 400, timeoutMs, temperature });
   let parsed;
   try {
     parsed = extractJsonObject(text);
   } catch (_e) {
     // Unsalvageable judge output → conservative 0 (a judge that can't be parsed
     // must never silently "pass"), tagged so the caller can see it happened.
-    return { score: 0, reason: 'judge output unparseable', unparsed: true, raw: String(text || '') };
+    return { score: 0, reason: 'judge output unparseable', unparsed: true, raw: String(text || ''), attempts: attempts || 1, usage };
   }
-  return { score: clamp01(parsed.score), reason: String(parsed.reason || '').slice(0, 300), raw: String(text || '') };
+  return { score: clamp01(parsed.score), reason: String(parsed.reason || '').slice(0, 300), raw: String(text || ''), attempts: attempts || 1, usage };
 }
 
 // Grade a response N times and return the sampled distribution:
@@ -97,12 +100,25 @@ async function gradeOnce({ task, response, rubric, model, timeoutMs, temperature
 // `mean` ± `stddev` is the per-case confidence band (raw spread of the N scores)
 // used by the borderline-outcome rule and per-case drift band-overlap logic.
 async function gradeSamples({ task, response, rubric, model, samples = 5, timeoutMs = 120000 }) {
-  const settings = judgeSettings(samples);
+  const settings = judgeSettings(samples, model);
   const scores = [];
   const reasons = [];
   const rawTexts = [];
+  const usages = [];
+  let attemptsTotal = 0;
   for (let i = 0; i < samples; i++) {
-    const r = await gradeOnce({ task, response, rubric, model, timeoutMs, temperature: settings.temperature === null ? undefined : settings.temperature });
+    let r;
+    try {
+      r = await gradeOnce({ task, response, rubric, model, timeoutMs, temperature: settings.temperature === null ? undefined : settings.temperature });
+    } catch (e) {
+      // A judge sample that persistently failed (e.g. timed out after retries):
+      // tag the error so the runner can charge for the spend and mark the whole
+      // case failed_timeout (a partial sample set must never become a band).
+      if (e && typeof e === 'object') { e.phase = 'judge'; e.judgeAttempts = attemptsTotal + (e.attempts || 1); }
+      throw e;
+    }
+    attemptsTotal += r.attempts || 1;
+    usages.push(r.usage || null);
     scores.push(r.score);
     reasons.push(r.reason);
     rawTexts.push(r.raw || '');
@@ -120,6 +136,12 @@ async function gradeSamples({ task, response, rubric, model, samples = 5, timeou
     judge_settings: settings,
     model_id: model,
     rubric_hash: rubricHash(rubric),
+    attempts: attemptsTotal,
+    // v0.4: the measurement overhead of grading this one case — the SUM over all
+    // N judge calls. Recorded in the receipt as the case's `judge_usage` and
+    // EXCLUDED from every skill-value figure (lib/value.js): it is a cost we
+    // impose to measure, not a cost of running the skill.
+    usage: sumUsage(usages),
   };
 }
 

package/lib/models.js

@@ -37,10 +37,24 @@ function loadRegistry(force) {
 }
 
 // Infer the model family from an id. Used to find a same-family predecessor when
-// the release trigger discovers a new model.
+// the release trigger discovers a new model. Covers both providers:
+//   anthropic — claude-<family>-…      → the family word (opus/sonnet/haiku/…)
+//   openai    — gpt-<major.minor>-…    → gpt-<major.minor> (so sol/terra/luna variants group)
 function familyOf(id) {
-  const m = String(id).match(/claude-(fable|mythos|opus|sonnet|haiku)/i);
-  return m ? m[1].toLowerCase() : 'unknown';
+  const s = String(id);
+  const c = s.match(/claude-(fable|mythos|opus|sonnet|haiku)/i);
+  if (c) return c[1].toLowerCase();
+  const g = s.match(/^(gpt-\d+(?:\.\d+)?)/i);
+  if (g) return g[1].toLowerCase();
+  const o = s.match(/^(o[1-9]+)/i);
+  if (o) return o[1].toLowerCase();
+  return 'unknown';
+}
+
+// Infer the provider from an id when the registry does not carry it (prefix-based,
+// mirrors lib/provider.inferProvider — kept here too so models.js has no cycle).
+function inferProviderFromId(id) {
+  return /^(gpt-|o[1-9]|chatgpt|codex|text-|davinci|omni)/i.test(String(id)) ? 'openai' : 'anthropic';
 }
 
 // Resolve a (possibly aliased) model id against the registry.
@@ -72,6 +86,36 @@ function isJudgeEligible(modelId) {
   return !!(entry && entry.judge_eligible);
 }
 
+// Enforce the fixed-judge policy at a published-run boundary: throw unless the
+// chosen judge is judge_eligible in the registry. This is what makes "attempting
+// an OpenAI judge errors" true — no OpenAI model is judge_eligible, so a cross-
+// provider report can only ever grade with the pinned Haiku judge. Callers on the
+// published/report path (prepare-report, prepare-report-002) invoke this; the dev
+// `run` command's self-judge is intentionally not gated by it.
+function assertJudgeEligible(modelId) {
+  if (!isJudgeEligible(modelId)) {
+    const { id } = resolveRegistry(modelId);
+    const e = new Error(`judge model "${id}" is not judge_eligible: the judge is fixed to a judge_eligible model (claude-haiku-*); no OpenAI model may serve as the judge (see docs/judge-policy.html).`);
+    e.code = 'JUDGE_INELIGIBLE';
+    throw e;
+  }
+  return true;
+}
+
+// The provider for a model: the registry `provider` when registered, else inferred
+// from the id prefix. Stamped into receipts as run.provider.
+function providerForModel(modelId) {
+  const { entry, id } = resolveRegistry(modelId);
+  return (entry && entry.provider) || inferProviderFromId(id);
+}
+
+// Per-provider config block from the registry (config/models.json `providers`):
+// { base_url, api_key_env, surfaces }. Null when the registry omits the provider.
+function providerConfig(provider) {
+  const reg = loadRegistry();
+  return (reg.providers && reg.providers[provider]) || null;
+}
+
 // The servable same-family predecessor of a model: the registered model in the
 // same family with the most recent `released` date strictly before this one
 // (falling back to any other same-family model). Null when the family has no
@@ -103,13 +147,18 @@ function addDiscoveredModel(id, { firstSeen = null, registryPath = REGISTRY_PATH
   const raw = JSON.parse(fs.readFileSync(registryPath, 'utf8'));
   if (raw.models.some((m) => m.id === id)) return null;
   const family = familyOf(id);
+  const provider = inferProviderFromId(id);
   // Best-effort tier from family; unknown families default to frontier pricing.
-  const tier = family === 'haiku' ? 'cheap' : family === 'sonnet' ? 'standard' : 'frontier';
-  const price = family === 'haiku' ? { input: 1.0, output: 5.0 }
-    : family === 'sonnet' ? { input: 3.0, output: 15.0 }
-      : { input: 5.0, output: 25.0 };
+  // A newly-discovered id gets a CONSERVATIVE (upper-bound) price so the budget
+  // guard never under-projects an unknown model — exact rates are set by hand
+  // when the model is reviewed for a published run.
+  const tier = /haiku|luna|mini|nano/.test(family) || /luna|mini|nano/.test(String(id)) ? 'cheap'
+    : /sonnet|terra/.test(family) ? 'standard' : 'frontier';
+  const price = provider === 'openai'
+    ? (tier === 'cheap' ? { input: 1.0, output: 6.0 } : tier === 'standard' ? { input: 2.5, output: 15.0 } : { input: 5.0, output: 30.0 })
+    : (family === 'haiku' ? { input: 1.0, output: 5.0 } : family === 'sonnet' ? { input: 3.0, output: 15.0 } : { input: 5.0, output: 25.0 });
   const entry = {
-    id, family, provider: 'anthropic', released: firstSeen,
+    id, family, provider, released: firstSeen,
     input_price: price.input, output_price: price.output, tier,
     judge_eligible: false, auto_added: true,
   };
@@ -121,5 +170,6 @@ function addDiscoveredModel(id, { firstSeen = null, registryPath = REGISTRY_PATH
 
 module.exports = {
   loadRegistry, resolveRegistry, registryStatus, priceForModel, isJudgeEligible,
+  assertJudgeEligible, providerForModel, providerConfig, inferProviderFromId,
   familyOf, familyPredecessor, addDiscoveredModel, DEFAULT_PRICE, REGISTRY_PATH,
 };