driftproof 0.3.0

package/bin/driftproof

@@ -0,0 +1,325 @@
+#!/usr/bin/env node
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { PROJECT_NAME, RUNNER_VERSION, DEFAULT_JUDGE_SAMPLES, DEV_MAX_USD } = require('../config');
+const { loadSkill } = require('../lib/skill');
+const { runSkillOnModel, summarizeReceipt, projectCalls } = require('../lib/run');
+const { validateReceipt, verifyReceiptHash } = require('../lib/receipt');
+const { buildDriftReport } = require('../lib/diff');
+const { surfaceLabel, providerName, resolveModel } = require('../lib/provider');
+const { estimateRunCostUSD, BudgetTracker } = require('../lib/cost');
+const { registryStatus } = require('../lib/models');
+const { verdictFromReceipt, badgeEndpoint, githubOutputLines } = require('../lib/verdict');
+const { scaffoldInit } = require('../lib/init');
+
+// Output dirs default to the USER's current directory, not the package dir, so a
+// global/npx install writes receipts/transcripts into the project being tested
+// (never into a read-only global install). --out overrides the receipts dir.
+const TRANSCRIPTS_DIR = path.join(process.cwd(), 'transcripts');
+
+// Write retained transcripts for one receipt to transcripts/<receipt_hash>/.
+// One JSON file per (case, mode); gitignored by default. Called only when
+// --keep-transcripts is set (run.transcripts === 'retained-local').
+function writeTranscripts(receipt, transcripts) {
+  const dir = path.join(TRANSCRIPTS_DIR, receipt.receipt_hash);
+  fs.mkdirSync(dir, { recursive: true });
+  const manifest = { receipt_hash: receipt.receipt_hash, model_id: receipt.run.model_id, date_utc: receipt.run.date_utc, entries: [] };
+  for (const t of transcripts) {
+    if (!t) continue;
+    const base = `${slug(t.id)}-${t.mode}`;
+    fs.writeFileSync(path.join(dir, `${base}.json`), JSON.stringify({ id: t.id, mode: t.mode, generation: t.generation, judge_outputs: t.judge_outputs }, null, 2));
+    manifest.entries.push(`${base}.json`);
+  }
+  fs.writeFileSync(path.join(dir, 'index.json'), JSON.stringify(manifest, null, 2));
+  return dir;
+}
+
+const RECEIPTS_DIR = path.join(process.cwd(), 'receipts');
+
+// Read an optional .driftproofrc (JSON) for per-project run defaults. Looked up
+// in the skill dir first (where `driftproof init` writes it), then the CWD. CLI
+// flags always win over the rc; the rc wins over built-in defaults.
+function loadRc(skillDir) {
+  const merged = {};
+  for (const dir of [process.cwd(), skillDir].filter(Boolean)) {
+    const p = path.join(dir, '.driftproofrc');
+    try {
+      if (fs.existsSync(p)) Object.assign(merged, JSON.parse(fs.readFileSync(p, 'utf8')));
+    } catch (_e) { /* a malformed rc is ignored, not fatal */ }
+  }
+  return merged;
+}
+
+function parseArgs(argv) {
+  const positional = [];
+  const flags = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a.startsWith('--')) {
+      const key = a.slice(2);
+      const next = argv[i + 1];
+      if (next === undefined || next.startsWith('--')) { flags[key] = true; }
+      else { flags[key] = next; i++; }
+    } else positional.push(a);
+  }
+  return { positional, flags };
+}
+
+function usage() {
+  console.log(`${PROJECT_NAME} v${RUNNER_VERSION} — continuous verification of agent skills
+
+USAGE
+  ${PROJECT_NAME} init <dir>                          scaffold SKILL.md + evals/evals.json + .driftproofrc
+  ${PROJECT_NAME} run <skill-dir> [--models a,b] [--samples N] [--max-cases N] [--max-calls N]
+                        [--judge-model M] [--concurrency N] [--max-usd N]
+                        [--keep-transcripts] [--out DIR]
+  ${PROJECT_NAME} diff <receiptA.json> <receiptB.json> [--out FILE]
+  ${PROJECT_NAME} validate <receipt.json>
+  ${PROJECT_NAME} badge <receipt.json> [--out FILE] [--github-output]
+
+ENV
+  CLAUDE_PROVIDER   api | cli   (default: cli — spawns \`claude -p\`, strips ANTHROPIC_API_KEY)
+  ANTHROPIC_API_KEY required when CLAUDE_PROVIDER=api
+  DRIFTPROOF_REGISTRY  path to an alternate model registry (default: the packaged config/models.json)
+  DRIFTPROOF_STUB=1    offline stub surface — canned receipts, zero model calls (for CI)
+
+NOTES
+  - Default models list is 'haiku' only (cheap dev default).
+  - Sampled judging: each case does 2 generations + 2×samples judge calls.
+    Default --samples 5 → 12 calls/case; --samples 1 disables bands.
+  - --max-calls is a hard per-run call cap (default 200). The whole run is
+    projected up front and REFUSED before any call if it would exceed the cap.
+  - --max-usd is a hard dollar budget (default ${DEV_MAX_USD} for dev runs). The projected
+    cost is printed up front and the run is REFUSED on BOTH surfaces if it would
+    exceed the budget; on claude-cli the metered spend is $0 but the estimated-
+    equivalent api cost is counted against the cap identically. The run also
+    hard-stops mid-flight at 1.25× the cap.
+  - --keep-transcripts writes the raw generations + judge outputs to
+    transcripts/<receipt-hash>/ (gitignored) and records transcripts:"retained-
+    local" in the receipt. Default is "hashes-only" (only the sha256 hashes).
+  - Prices and registry status come from config/models.json; an unregistered
+    model still runs but is marked registry:"unregistered" and costed at the
+    conservative default price.
+  - --concurrency runs that many (case,mode) tasks at once (default 1). Higher
+    values cut wall-clock on the cli surface (cold-start dominated).`);
+}
+
+function slug(s) { return String(s).toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, ''); }
+function dateStamp(iso) { return iso.slice(0, 10); }
+
+async function cmdRun(positional, flags) {
+  const skillDir = positional[0];
+  if (!skillDir) { usage(); process.exit(2); }
+
+  // Per-project defaults from .driftproofrc (if any); CLI flags override these.
+  const rc = loadRc(skillDir);
+  const models = String(flags.models || rc.models || 'haiku').split(',').map((s) => s.trim()).filter(Boolean);
+  const maxCases = flags['max-cases'] ? parseInt(flags['max-cases'], 10) : (rc.max_cases != null ? parseInt(rc.max_cases, 10) : null);
+  const maxCalls = flags['max-calls'] ? parseInt(flags['max-calls'], 10) : (rc.max_calls != null ? parseInt(rc.max_calls, 10) : 200);
+  const samples = flags.samples ? parseInt(flags.samples, 10) : (rc.samples != null ? parseInt(rc.samples, 10) : DEFAULT_JUDGE_SAMPLES);
+  const judgeModel = flags['judge-model'] || rc.judge_model || null;
+  const concurrency = flags.concurrency ? parseInt(flags.concurrency, 10) : (rc.concurrency != null ? parseInt(rc.concurrency, 10) : 1);
+  const maxUsd = flags['max-usd'] ? parseFloat(flags['max-usd']) : (rc.max_usd != null ? parseFloat(rc.max_usd) : DEV_MAX_USD);
+  const keepTranscripts = !!flags['keep-transcripts'];
+  const outDir = flags.out ? path.resolve(flags.out) : RECEIPTS_DIR;
+  fs.mkdirSync(outDir, { recursive: true });
+
+  const skill = loadSkill(skillDir);
+  const nCases = maxCases ? Math.min(maxCases, skill.suite.caseCount) : skill.suite.caseCount;
+  const perModelCalls = projectCalls(nCases, samples);
+  const totalProjected = perModelCalls * models.length;
+
+  // Dollar cost guard (Week 3): project the METERED USD cost up front. On the
+  // `api` surface this is real spend, so we refuse if it would exceed --max-usd.
+  // On `claude-cli` the metered spend is $0 (subscription); the figure is the
+  // hypothetical "if run on the metered API" cost — printed, never blocks.
+  const cost = estimateRunCostUSD({ caseCount: nCases, samples, models: models.map((m) => require('../lib/provider').resolveModel(m)), judgeModel: judgeModel || 'haiku' });
+  const surface = surfaceLabel();
+
+  const regStatuses = models.map((m) => `${m}:${registryStatus(m)}`);
+  console.log(`\n${PROJECT_NAME} run — skill "${skill.name}" v${skill.version}`);
+  console.log(`  content_hash: ${skill.contentHash.slice(0, 16)}…  suite: ${skill.suite.caseCount} cases (${skill.suite.suiteHash.slice(0, 12)}…)`);
+  console.log(`  surface: ${surface}   models: ${models.join(', ')}   samples/case: ${samples}   concurrency: ${concurrency}`);
+  console.log(`  registry: ${regStatuses.join('  ')}${keepTranscripts ? '   transcripts: retained-local' : ''}`);
+  console.log(`  projected calls: ${perModelCalls}/model × ${models.length} model(s) = ${totalProjected}   per-model cap: ${maxCalls}`);
+  console.log(`  projected cost: ~$${cost.totalUSD.toFixed(2)} (rough upper bound; budget $${maxUsd.toFixed(2)}, hard-stop $${(maxUsd * 1.25).toFixed(2)})`);
+  if (surface !== 'api') console.log(`  actual metered spend on ${surface}: $0.00 (subscription; the $ figure is the estimated-equivalent API cost, counted against the cap identically)`);
+
+  // Call-count guard: refuse before spending anything if a single model run
+  // would exceed the per-run call cap.
+  if (perModelCalls > maxCalls) {
+    console.error(`\n  ✗ ABORT (cost guard): projected ${perModelCalls} calls/model exceeds --max-calls ${maxCalls}.`);
+    console.error(`    Lower --samples/--max-cases or raise --max-calls.`);
+    process.exit(3);
+  }
+  // Dollar guard: the projection is refused on BOTH surfaces. Subscription (cli)
+  // attention is not free, so its estimated-equivalent cost is counted against
+  // --max-usd identically to metered api spend.
+  if (cost.totalUSD > maxUsd) {
+    console.error(`\n  ✗ ABORT (cost guard): projected cost ~$${cost.totalUSD.toFixed(2)} exceeds --max-usd $${maxUsd.toFixed(2)}.`);
+    console.error(`    Trim skills/cases or raise --max-usd (do NOT lower --samples for a published run).`);
+    process.exit(3);
+  }
+  console.log('');
+
+  // One live budget tracker for the whole run: accumulates estimated per-call
+  // cost across all models and hard-stops at 1.25× the cap.
+  const budget = new BudgetTracker(maxUsd);
+
+  const emitted = [];
+  for (const model of models) {
+    console.log(`── model: ${model} ──`);
+    let result;
+    try {
+      result = await runSkillOnModel({
+        skill,
+        model,
+        opts: {
+          maxCases, maxCalls, samples, judgeModel, concurrency, budget, keepTranscripts,
+          onProgress: (p) => {
+            if (p.phase === 'done') console.log(`  ${p.case} / ${p.mode}: ${p.outcome} (${p.score.toFixed(2)} ± ${(p.stddev || 0).toFixed(2)})`);
+          },
+        },
+      });
+    } catch (e) {
+      if (e && e.code === 'BUDGET_HARDSTOP') {
+        console.error(`\n  ✗ ABORT (budget hard-stop): ${e.message}`);
+        console.error(`    ${emitted.length} receipt(s) already written to ${path.relative(process.cwd(), outDir)}/.`);
+        process.exit(3);
+      }
+      throw e;
+    }
+    const { receipt, calls, transcripts } = result;
+
+    const { valid, errors } = validateReceipt(receipt);
+    if (!valid) {
+      console.error('  ✗ emitted receipt FAILED schema validation:', JSON.stringify(errors, null, 2));
+      process.exitCode = 1;
+    }
+    if (!verifyReceiptHash(receipt)) {
+      console.error('  ✗ receipt_hash does not verify');
+      process.exitCode = 1;
+    }
+
+    const base = `${slug(skill.name)}-${slug(receipt.run.model_id)}-${dateStamp(receipt.run.date_utc)}`;
+    const jsonPath = path.join(outDir, `${base}.json`);
+    const mdPath = path.join(outDir, `${base}.summary.md`);
+    fs.writeFileSync(jsonPath, JSON.stringify(receipt, null, 2));
+    fs.writeFileSync(mdPath, summarizeReceipt(receipt));
+    emitted.push(jsonPath);
+
+    // v0.3: retained transcripts are written keyed by the sealed receipt_hash,
+    // so a reader can verify every generation_hash / judge_sample_hash.
+    if (keepTranscripts && transcripts) {
+      const tdir = writeTranscripts(receipt, transcripts);
+      console.log(`  → transcripts (retained-local): ${path.relative(process.cwd(), tdir)}/`);
+    }
+
+    const cmp = receipt.comparison;
+    const aw = receipt.results.aggregates.with_skill;
+    const ab = receipt.results.aggregates.baseline;
+    console.log(`  → ${path.relative(process.cwd(), jsonPath)}  (${calls} calls)`);
+    console.log(`  → skill lift ${cmp.delta >= 0 ? '+' : ''}${cmp.delta.toFixed(3)} ± ${cmp.delta_uncertainty.toFixed(3)}  (with ${aw.mean_score.toFixed(3)} ± ${aw.stddev.toFixed(3)} vs base ${ab.mean_score.toFixed(3)} ± ${ab.stddev.toFixed(3)})\n`);
+  }
+
+  console.log(`Done. ${emitted.length} receipt(s) emitted to ${path.relative(process.cwd(), outDir)}/`);
+}
+
+function cmdDiff(positional, flags) {
+  const [aPath, bPath] = positional;
+  if (!aPath || !bPath) { usage(); process.exit(2); }
+  const a = JSON.parse(fs.readFileSync(aPath, 'utf8'));
+  const b = JSON.parse(fs.readFileSync(bPath, 'utf8'));
+
+  // Warn (do not block) if either receipt fails self-verification.
+  for (const [p, r] of [[aPath, a], [bPath, b]]) {
+    if (!verifyReceiptHash(r)) console.error(`  ⚠ ${path.basename(p)}: receipt_hash does not verify (tampered or hand-edited)`);
+  }
+
+  const labelA = dateStamp(a.run.date_utc);
+  const labelB = dateStamp(b.run.date_utc);
+  const { markdown } = buildDriftReport(a, b, { labelA, labelB });
+
+  if (flags.out) {
+    fs.writeFileSync(path.resolve(flags.out), markdown);
+    console.log(`Drift report written to ${flags.out}`);
+  } else {
+    console.log(markdown);
+  }
+}
+
+function cmdValidate(positional) {
+  const p = positional[0];
+  if (!p) { usage(); process.exit(2); }
+  const receipt = JSON.parse(fs.readFileSync(p, 'utf8'));
+  const { valid, errors } = validateReceipt(receipt);
+  const hashOk = verifyReceiptHash(receipt);
+  console.log(`schema: ${valid ? 'VALID' : 'INVALID'}   receipt_hash: ${hashOk ? 'VERIFIED' : 'MISMATCH'}`);
+  if (!valid) console.log(JSON.stringify(errors, null, 2));
+  process.exit(valid && hashOk ? 0 : 1);
+}
+
+function cmdInit(positional) {
+  const target = positional[0];
+  if (!target) {
+    console.error('usage: driftproof init <dir>\n');
+    console.error('Scaffolds SKILL.md (stub), evals/evals.json (3 example cases), and .driftproofrc.');
+    process.exit(2);
+  }
+  const { dir, created, skipped } = scaffoldInit(target);
+  const rel = (p) => path.relative(process.cwd(), p) || '.';
+  console.log(`driftproof init — ${rel(dir)}/`);
+  for (const f of created) console.log(`  + ${rel(f)}`);
+  for (const f of skipped) console.log(`  · ${rel(f)} (exists — left untouched)`);
+  if (!created.length) {
+    console.log('\nNothing to create; every file already existed. init never overwrites.');
+    return;
+  }
+  console.log(`\nNext:
+  1. Edit ${rel(path.join(dir, 'SKILL.md'))} with your skill's instructions.
+  2. Edit ${rel(path.join(dir, 'evals', 'evals.json'))} — replace the 3 example cases (rubrics anchored at 0.80).
+  3. driftproof run ${rel(dir)} --models claude-haiku-4-5 --max-usd 2
+See AUTHORING.md for how to write a fair suite.`);
+}
+
+// Emit a shields.io endpoint badge for a single receipt, or GitHub Actions
+// output lines. Default: print the shields JSON to stdout (commit it and point
+// the shields endpoint URL at it). --out FILE writes the JSON to a file.
+// --github-output prints verdict/delta/message/color as key=value lines.
+function cmdBadge(positional, flags) {
+  const p = positional[0];
+  if (!p) { console.error('usage: driftproof badge <receipt.json> [--out FILE] [--github-output]'); process.exit(2); }
+  const receipt = JSON.parse(fs.readFileSync(p, 'utf8'));
+  if (flags['github-output']) { console.log(githubOutputLines(receipt)); return; }
+  const badge = badgeEndpoint(receipt);
+  const json = JSON.stringify(badge, null, 2);
+  if (flags.out) {
+    const out = path.resolve(flags.out);
+    fs.mkdirSync(path.dirname(out), { recursive: true });
+    fs.writeFileSync(out, json + '\n');
+    const v = verdictFromReceipt(receipt);
+    console.log(`badge written to ${flags.out}  (${v.verdict}: ${badge.message}, ${badge.color})`);
+  } else {
+    console.log(json);
+  }
+}
+
+async function main() {
+  const [, , cmd, ...rest] = process.argv;
+  const { positional, flags } = parseArgs(rest);
+  switch (cmd) {
+    case 'init': return cmdInit(positional);
+    case 'run': return cmdRun(positional, flags);
+    case 'diff': return cmdDiff(positional, flags);
+    case 'validate': return cmdValidate(positional);
+    case 'badge': return cmdBadge(positional, flags);
+    case 'version': case '--version': case '-v': console.log(RUNNER_VERSION); return;
+    case 'help': case '--help': case '-h': case undefined: return usage();
+    default: console.error(`unknown command: ${cmd}\n`); usage(); process.exit(2);
+  }
+}
+
+main().catch((e) => { console.error('FATAL', e && (e.stack || e.message || e)); process.exit(2); });

package/config/models.json

@@ -0,0 +1,19 @@
+{
+  "_comment": "Driftproof model registry. The runner resolves --models ids against this list; an unknown id still runs but its receipt is marked registry:\"unregistered\" and its cost is estimated with the conservative default price in lib/models.js. Prices are STANDARD first-party USD per 1,000,000 tokens (input/output) from platform.claude.com (July 2026); Sonnet 5's introductory rate is deliberately NOT used so projections stay an upper bound. `released` is best-effort (spec RECEIPT.md open question #4): dates derived from a dated model id or Anthropic launch posts, else null. `tier` drives nothing but reporting; `judge_eligible` encodes the fixed-judge policy (see docs/judge-policy.html) — only the cheap Haiku judge is eligible. The release trigger (scripts/release-watch.js) appends auto-discovered ids here with auto_added:true.",
+  "registry_version": "1.0",
+  "provider": "anthropic",
+  "default_price_note": "Unregistered ids are costed at the most expensive known tier (Fable, $10/$50) — a budget guard must never under-estimate. See lib/models.js DEFAULT_PRICE.",
+  "models": [
+    { "id": "claude-fable-5",              "family": "fable",  "provider": "anthropic", "released": null,         "input_price": 10.0, "output_price": 50.0, "tier": "frontier", "judge_eligible": false },
+    { "id": "claude-opus-5",               "family": "opus",   "provider": "anthropic", "released": null,         "input_price": 5.0,  "output_price": 25.0, "tier": "frontier", "judge_eligible": false },
+    { "id": "claude-opus-4-8",             "family": "opus",   "provider": "anthropic", "released": null,         "input_price": 5.0,  "output_price": 25.0, "tier": "frontier", "judge_eligible": false },
+    { "id": "claude-opus-4-7",             "family": "opus",   "provider": "anthropic", "released": null,         "input_price": 5.0,  "output_price": 25.0, "tier": "frontier", "judge_eligible": false },
+    { "id": "claude-opus-4-6",             "family": "opus",   "provider": "anthropic", "released": null,         "input_price": 5.0,  "output_price": 25.0, "tier": "frontier", "judge_eligible": false },
+    { "id": "claude-opus-4-5-20251101",    "family": "opus",   "provider": "anthropic", "released": "2025-11-01", "input_price": 5.0,  "output_price": 25.0, "tier": "frontier", "judge_eligible": false },
+    { "id": "claude-sonnet-5",             "family": "sonnet", "provider": "anthropic", "released": "2026-06-30", "input_price": 3.0,  "output_price": 15.0, "tier": "standard", "judge_eligible": false },
+    { "id": "claude-sonnet-4-6",           "family": "sonnet", "provider": "anthropic", "released": "2026-02-17", "input_price": 3.0,  "output_price": 15.0, "tier": "standard", "judge_eligible": false },
+    { "id": "claude-sonnet-4-5-20250929",  "family": "sonnet", "provider": "anthropic", "released": "2025-09-29", "input_price": 3.0,  "output_price": 15.0, "tier": "standard", "judge_eligible": false },
+    { "id": "claude-haiku-4-5",            "family": "haiku",  "provider": "anthropic", "released": "2025-10-01", "input_price": 1.0,  "output_price": 5.0,  "tier": "cheap",    "judge_eligible": true },
+    { "id": "claude-haiku-4-5-20251001",   "family": "haiku",  "provider": "anthropic", "released": "2025-10-01", "input_price": 1.0,  "output_price": 5.0,  "tier": "cheap",    "judge_eligible": true }
+  ]
+}

package/config.js

@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+// The project name lives in exactly ONE place so a future rename touches one
+// constant. Everything user-facing (CLI banner, receipt runner id, file names)
+// derives from PROJECT_NAME.
+const PROJECT_NAME = 'driftproof';
+
+// Bumped whenever the runner's behaviour or receipt-generation semantics change
+// in a way that could affect results. Recorded into every receipt as
+// run.runner_version so a receipt is reproducible against a known engine.
+const RUNNER_VERSION = '0.3.0';
+
+// The eval format we CONSUME (we deliberately do not invent our own).
+const SUITE_FORMAT = 'agentskills.io/evals';
+
+// Receipt schema version this runner emits. Loader/validator accept older
+// versions too (see lib/receipt.js), but new receipts are stamped current.
+// v0.3 adds transcript-auditability: per-case generation_hash + judge_sample_
+// hashes[], plus run.registry and run.transcripts. Additive; v0.1/v0.2 still load.
+const RECEIPT_SCHEMA_VERSION = '0.3';
+
+// Hard USD budget defaults per entry point (Week 4). --max-usd overrides any of
+// these. The projection is refused before any call if it exceeds the cap, on
+// BOTH surfaces; a run also hard-stops mid-flight at 1.25× (see lib/cost.js).
+//   dev     — an interactive `driftproof run`
+//   report  — a full Report-#001-style suite run (scripts/run-report-001.js)
+//   trigger — a release-trigger-initiated prepare-report run
+const DEV_MAX_USD = 2;
+const REPORT_MAX_USD = 40;
+const TRIGGER_MAX_USD = 25;
+
+// Default number of judge samples per case in sampled mode. Sampling is what
+// turns a single noisy grade into a mean ± band; --samples overrides it.
+const DEFAULT_JUDGE_SAMPLES = 5;
+
+// Minimum practical-effect floor for a drift verdict. Band separation alone is
+// necessary but NOT sufficient to claim a regression/improvement: the mean must
+// ALSO move by at least this much. Rationale — the Haiku judge quantizes scores
+// to a coarse ~0.05–0.1 grid, so a confident grade often has stddev 0 (a "point
+// band"). Two point bands that differ by a single quantum (e.g. 0.60 vs 0.64)
+// are technically non-overlapping yet represent no meaningful behaviour change.
+// The floor turns such statistically-separated-but-trivial moves into
+// "within noise (below effect floor)". 0.05 = one judge quantum. Documented in
+// spec/RECEIPT.md § "Drift verdict rule" and the report methodology.
+const EFFECT_FLOOR = 0.05;
+
+module.exports = { PROJECT_NAME, RUNNER_VERSION, SUITE_FORMAT, RECEIPT_SCHEMA_VERSION, DEFAULT_JUDGE_SAMPLES, EFFECT_FLOOR, DEV_MAX_USD, REPORT_MAX_USD, TRIGGER_MAX_USD };

package/lib/canonical.js

@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+const crypto = require('crypto');
+
+// Deterministic JSON serialization: object keys sorted lexicographically at
+// every level, arrays kept in order, no insignificant whitespace. Two
+// structurally-equal values always produce the same string, which is what makes
+// content/suite/receipt hashes stable and comparable across machines.
+function canonicalize(value) {
+  if (value === null || typeof value !== 'object') return JSON.stringify(value);
+  if (Array.isArray(value)) return '[' + value.map(canonicalize).join(',') + ']';
+  const keys = Object.keys(value).filter((k) => value[k] !== undefined).sort();
+  return '{' + keys.map((k) => JSON.stringify(k) + ':' + canonicalize(value[k])).join(',') + '}';
+}
+
+function sha256(buf) {
+  return crypto.createHash('sha256').update(buf).digest('hex');
+}
+
+// sha256 over the canonical form of a JSON-able value.
+function sha256Canonical(value) {
+  return sha256(canonicalize(value));
+}
+
+// Hash an ordered list of { path, bytes } file entries. The list is sorted by
+// path so ordering on disk never changes the digest. Each entry contributes its
+// path and a hash of its bytes, giving a single content hash over a file set.
+function sha256Files(files) {
+  const sorted = [...files].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+  const h = crypto.createHash('sha256');
+  for (const f of sorted) {
+    h.update(f.path, 'utf8');
+    h.update('\0');
+    h.update(sha256(f.bytes));
+    h.update('\n');
+  }
+  return h.digest('hex');
+}
+
+module.exports = { canonicalize, sha256, sha256Canonical, sha256Files };

package/lib/cost.js

@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+const { priceForModel } = require('./models');
+
+// Dollar cost estimation and live budget tracking for a run.
+//
+// Week 3 added a HARD USD budget on top of the Week-2 call-count cap. Week 4
+// moves prices into the model registry (config/models.json, via lib/models.js)
+// and makes the budget guard apply on EVERY entry point and BOTH surfaces:
+//   - the projection is printed and, if it exceeds --max-usd, the run is refused
+//     BEFORE any model call (on cli too — subscription attention is not free);
+//   - a BudgetTracker accumulates the estimated spend AS THE RUN PROCEEDS and
+//     hard-stops at 1.25× the cap.
+//
+// IMPORTANT (honesty): the numbers below are a DELIBERATELY ROUGH upper bound.
+// Token counts per call are estimated from a small fixed table (not measured
+// with count_tokens), and prices are the standard first-party per-MTok rates
+// from the registry. The estimate keeps a run bounded and states the order of
+// magnitude — it is not an invoice. On the `claude-cli` surface the ACTUAL
+// metered spend is $0 (subscription); the dollar figure is the hypothetical
+// "if this had run on the metered API" cost, and it is counted against the caps
+// identically so subscription usage is never treated as free.
+
+// Rough per-call token estimates. Chosen to over- rather than under-estimate.
+//   - a generation call with the skill prepended carries the SKILL.md as a
+//     system prompt (our skills run ~0.4k–8k tokens; 3000 is a generous mean),
+//   - a baseline generation carries only the short case prompt,
+//   - a judge call carries task + response + rubric + judge system, and emits a
+//     short JSON grade.
+const TOKENS = {
+  gen_with_skill: { input: 3000, output: 900 },
+  gen_baseline: { input: 250, output: 900 },
+  judge: { input: 1300, output: 200 },
+};
+
+// Price per MTok for a model, from the registry (conservative default for an
+// unregistered id — see lib/models.js).
+function priceFor(modelId) {
+  const p = priceForModel(modelId);
+  return { input: p.input, output: p.output };
+}
+
+function callCostUSD(price, tokens) {
+  return (tokens.input / 1e6) * price.input + (tokens.output / 1e6) * price.output;
+}
+
+// Estimated USD cost of ONE call of a given kind on a given model. Used by the
+// live BudgetTracker so the running spend estimate is built from the same
+// per-call pieces as the up-front projection.
+function perCallCostUSD(modelId, kind) {
+  const tokens = TOKENS[kind];
+  if (!tokens) throw new Error(`unknown call kind: ${kind}`);
+  return callCostUSD(priceFor(modelId), tokens);
+}
+
+// Estimate the metered USD cost of a full run: `caseCount` cases × 2 modes,
+// generated on each target model, judged `samples` times each on `judgeModel`.
+//
+//   returns { totalUSD, perModel: [{ model, usd }], judgeUSD, genUSD, assumptions }
+function estimateRunCostUSD({ caseCount, samples, models, judgeModel }) {
+  const judgePrice = priceFor(judgeModel);
+  const perModel = [];
+  let judgeUSD = 0;
+  let genUSD = 0;
+
+  for (const model of models) {
+    const price = priceFor(model);
+    // Two generations per case (with_skill + baseline).
+    const gen = caseCount * (callCostUSD(price, TOKENS.gen_with_skill) + callCostUSD(price, TOKENS.gen_baseline));
+    // Judge every generation `samples` times → caseCount × 2 modes × samples.
+    const judge = caseCount * 2 * samples * callCostUSD(judgePrice, TOKENS.judge);
+    perModel.push({ model, usd: round4(gen + judge) });
+    genUSD += gen;
+    judgeUSD += judge;
+  }
+
+  return {
+    totalUSD: round4(genUSD + judgeUSD),
+    perModel,
+    judgeUSD: round4(judgeUSD),
+    genUSD: round4(genUSD),
+    assumptions: { tokens: TOKENS, judgeModel, note: 'rough upper-bound estimate; registry per-MTok pricing; not measured with count_tokens' },
+  };
+}
+
+// Live budget tracker. The runner adds the estimated cost of each call as it is
+// made; if the accumulated estimate crosses 1.25× the cap the tracker throws a
+// BUDGET_HARDSTOP, aborting the run mid-flight. The 1.25× headroom exists because
+// the up-front projection already guards the cap; the hard-stop is a backstop
+// against a projection that turned out low (e.g. an unregistered model, longer
+// outputs). On both surfaces the estimate is counted identically.
+class BudgetTracker {
+  constructor(capUsd) {
+    this.capUsd = capUsd;
+    this.hardCap = round4(capUsd * 1.25);
+    this.spent = 0;
+  }
+
+  // Add estimated USD; throws BUDGET_HARDSTOP if the running total crosses 1.25×.
+  add(usd) {
+    this.spent = round4(this.spent + usd);
+    if (this.spent > this.hardCap) {
+      const e = new Error(
+        `budget hard-stop: estimated spend $${this.spent.toFixed(4)} exceeded 1.25× cap `
+        + `($${this.hardCap.toFixed(4)}; --max-usd $${this.capUsd.toFixed(2)})`);
+      e.code = 'BUDGET_HARDSTOP';
+      throw e;
+    }
+    return this.spent;
+  }
+
+  remaining() { return round4(this.hardCap - this.spent); }
+}
+
+function round4(n) { return Math.round(n * 1e4) / 1e4; }
+
+module.exports = { estimateRunCostUSD, perCallCostUSD, priceFor, BudgetTracker, TOKENS };