openmerit 0.1.6 → 0.1.7-preview.2

package/dist/terminal/src/cli.mjs

@@ -9,6 +9,7 @@ import { openBrowser } from './browser.mjs';
 const help = `OpenMerit · Inference Terminal
 
   openmerit dash [path] [options]
+  openmerit eval --help              Evaluate CLI changes across coding agents
 
   --port <number>      Require a specific local port (default: 4310 or a free port)
   --interval <seconds> Collection interval, 10–30 (default: 15)
@@ -26,6 +27,7 @@ Coding-harness usage is outside this terminal's scope.
 
 async function main() {
   const args = process.argv.slice(2);
+  if (args[0] === 'eval') { const { main } = await import('../../eval/src/cli.mjs'); await main(args.slice(1)); return; }
   if (!args.length || args.includes('--help') || args.includes('-h')) { console.log(help); return; }
   if (args.shift() !== 'dash') throw new Error('Unknown command. Run openmerit --help.');
   let root = process.cwd(), port = 4310, interval = 15, demo = false, json = false, pathSeen = false, discovery = true, noOpen = false, explicitPort = false, diagnostics = false;

package/dist/terminal/src/demo-policy.mjs

@@ -0,0 +1,205 @@
+// A reviewed model/service-offer catalog for the hosted fictional comparison only.
+// Scores are publisher-reported point estimates, not insurance-workload evaluations.
+export const EVIDENCE_DATE = '2026-10-10';
+export const PREFERENCES = [
+  { value: 'default', label: 'Default · within your stack', note: 'Search all configured services; choose the lowest modeled cost among qualified replacements.' },
+  { value: 'zdr', label: 'Bias ZDR models', note: 'Prefer confirmed ZDR endpoints. Eligibility alone does not establish account approval or enabled settings.' },
+  { value: 'open', label: 'Bias open-weights models', note: 'Prefer models with open weights.' },
+  { value: 'closed', label: 'No open-weights models', note: 'Require a closed-weight assignment for every workload.' },
+  { value: 'usa', label: 'Bias USA models', note: 'Prefer models from US developers. This does not specify hosting location or data residency.' },
+  { value: 'china', label: 'Bias Chinese models', note: 'Prefer models from Chinese developers. This does not specify hosting location or data residency.' },
+];
+export const preferenceFor = value => PREFERENCES.find(item => item.value === value) ?? PREFERENCES[0];
+const oa = 'https://openai.com/index/gpt-4-1/';
+const claude = 'https://www-cdn.anthropic.com/bbd8ef16d70b7a1665f14f306ee88b53f686aa75/Claude%20Sonnet%204.6%20System%20Card.pdf';
+const gemini = 'https://storage.googleapis.com/deepmind-media/gemini/gemini_3_flash_model_evaluation.pdf';
+const metric = (axis, name, value, suite, source, settings) => ({ axis, name, value, suite, source, settings, unit: '%', direction: 'higher' });
+const oaScores = (tool, capability, reasoning) => [
+  metric('tools', 'τ-bench retail', tool, 'openai-4.1-tau-retail-gpt4o-user', oa, '5-run average; original GPT-4o user simulator; no custom tools or prompting.'),
+  metric('capabilities', 'MMLU', capability, 'openai-4.1-mmlu', oa, 'OpenAI GPT-4.1 release comparison; GPT-4o snapshot 2024-11-20.'),
+  metric('reasoning', 'AIME 2024', reasoning, 'openai-4.1-aime24', oa, 'OpenAI GPT-4.1 release comparison, AIME 2024.'),
+];
+const claudeScores = (tool, capability, reasoning) => [
+  metric('tools', 'τ²-bench retail', tool, 'anthropic-sonnet46-tau2-retail', claude, 'Sonnet 4.6 system card §2.5; max effort, 10 trials for Sonnet; published Opus configuration.'),
+  metric('capabilities', 'MMMU-Pro · no tools', capability, 'anthropic-sonnet46-mmmupro-notools', claude, 'Sonnet 4.6 system card Table 2.1.A; publisher model configurations.'),
+  metric('reasoning', 'GPQA Diamond', reasoning, 'anthropic-sonnet46-gpqa', claude, 'Sonnet 4.6 system card §2.8; adaptive thinking, max effort for Sonnet; published Opus configuration.'),
+];
+const googleScores = (tool, capability, reasoning) => [
+  metric('tools', 'τ²-bench', tool, 'google-3flash-tau2-fixed-airline', gemini, 'Gemini 3 Pro user simulator; fixed Airline domain; mean of Retail, Airline and Telecom.'),
+  metric('capabilities', 'MMMU-Pro', capability, 'google-3flash-mmmupro', gemini, 'Thinking; mean of Standard and Vision settings; no tools.'),
+  metric('reasoning', 'GPQA Diamond', reasoning, 'google-3flash-gpqa', gemini, 'Thinking; no tools; Google December 2025 comparison.'),
+];
+const ossCard = 'https://arxiv.org/html/2508.10925v1';
+const ossScores = (tool, capability, reasoning) => [
+  metric('tools', 'τ-bench retail', tool, 'openai-oss-high-retail', ossCard, 'Table 3; high reasoning effort; developer function calling.'),
+  metric('capabilities', 'MMLU', capability, 'openai-oss-high-mmlu', ossCard, 'Table 3; high reasoning effort, default system prompt, pass@1.'),
+  metric('reasoning', 'GPQA Diamond · no tools', reasoning, 'openai-oss-high-gpqa', ossCard, 'Table 3; high reasoning effort, no tools, pass@1.'),
+];
+const dsCard = 'https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash';
+const dsScores = (tool, capability, reasoning) => [
+  metric('tools', 'AutomationBench', tool, 'deepseek41-automation', dsCard, 'Frontier instruct-model comparison; official scaffold, maximum reasoning effort, temperature 1, top-p .95.'),
+  metric('capabilities', "Agent’s Last Exam", capability, 'deepseek41-ale', dsCard, 'Frontier instruct-model comparison; broad agent tasks, official scaffold, maximum reasoning effort.'),
+  metric('reasoning', 'GPQA Diamond', reasoning, 'deepseek41-gpqa', dsCard, 'Frontier instruct-model comparison; pass@1, maximum reasoning effort. Published model scores, not endpoint measurements.'),
+];
+const qwenCard = 'https://huggingface.co/Qwen/Qwen3.6-27B';
+const qwenScores = (tool, pass3, capability, reasoning) => [
+  metric('tools', 'ClawEval · average', tool, 'qwen36-claw-avg', qwenCard, 'Qwen 3.6 27B release comparison; published model configurations.'),
+  metric('tools', 'ClawEval · pass³', pass3, 'qwen36-claw-pass3', qwenCard, 'Same comparison, stricter repeated-task success. Both tool metrics must pass.'),
+  metric('capabilities', 'MMMU-Pro', capability, 'qwen36-mmmupro', qwenCard, 'Vision benchmark; published Qwen 3.6 comparison configurations.'),
+  metric('reasoning', 'GPQA Diamond', reasoning, 'qwen36-gpqa', qwenCard, 'Published Qwen 3.6 comparison configurations.'),
+];
+const cohortScores = (source, suite, names, values, settings) => ['tools', 'capabilities', 'reasoning'].map((axis, index) => metric(axis, names[index], values[index], `${suite}-${axis}`, source, settings));
+const flashCard = 'https://huggingface.co/zai-org/GLM-5.3-Flash';
+const flashScores = values => cohortScores(flashCard, 'glm53flash-release', ['AutomationBench v1.0.6', 'GDPval-AA v2', 'HLE · with tools'], values, 'Published release chart and footnotes; maximum effort; GDPval-AA rating, not percent.').map(item => item.axis === 'capabilities' ? { ...item, unit: ' rating' } : item);
+const qwen38Card = 'https://huggingface.co/Qwen/Qwen3.8-27B';
+const qwen38Scores = values => cohortScores(qwen38Card, 'qwen38-release', ['ClawEval-MM · average', 'IFBench', 'GPQA Diamond'], values, 'Qwen 3.8 model card comparison; published thinking configurations. ClawEval-MM is distinct from ClawEval.');
+// The application-code profile screens generated SQL/Python used by the
+// insurance product. These are application workers, not developer-harness usage.
+// Keep publisher cohorts separate: LiveCodeBench v6 % is not LCB Pro Elo.
+const glm47Card = 'https://huggingface.co/zai-org/GLM-4.7';
+const glmCodeScores = values => cohortScores(glm47Card, 'glm47-application-code', ['τ²-bench', 'LiveCodeBench v6', 'GPQA Diamond'], values, 'GLM-4.7 release table; thinking enabled, preserved thinking for multi-turn tools; adjusted user prompts and fixed Airline domain. Published comparator configurations; not endpoint parity.');
+const googleCodeScores = values => cohortScores(gemini, 'google-3flash-application-code', ['τ²-bench', 'LiveCodeBench Pro', 'GPQA Diamond'], values, 'December 2025 comparison: Gemini thinking, Sonnet 4.5 high/best available thinking. LCB Pro official leaderboard Elo; τ² fixed Airline, Gemini user simulator for Gemini and publisher-reported Sonnet score. Public comparison, not a controlled endpoint evaluation.').map(item => item.axis === 'capabilities' ? { ...item, unit: ' Elo' } : item);
+const basetenPolicy = { confirmedZdr: true, policySource: 'https://www.baseten.co/security-practices/', endpointMode: 'Synchronous Model API; request/response logging off' };
+const model = (id, name, provider, input, output, source, benchmarks = [], extra = {}) => ({ id, modelId: id, name, provider, input, output, source, benchmarks, creator: provider, origin: 'US', openWeights: false, outputType: 'text', confirmedZdr: false, ...extra });
+const openai = name => `https://developers.openai.com/api/docs/models/${name}`;
+const anthropic = 'https://platform.claude.com/docs/en/about-claude/pricing';
+const google = 'https://ai.google.dev/gemini-api/docs/pricing';
+const groq = 'https://console.groq.com/docs/models';
+const baseten = 'https://www.baseten.co/pricing/';
+const mistral = 'https://docs.mistral.ai/inference/pricing';
+export const CATALOG = [
+  model('gpt4o', 'GPT-4o', 'OpenAI', 2.5, 10, openai('gpt-4o'), oaScores(60.3, 85.7, 13.1)),
+  model('gpt41', 'GPT-4.1', 'OpenAI', 2, 8, openai('gpt-4.1'), oaScores(68, 90.2, 48.1)),
+  model('gpt41mini', 'GPT-4.1 mini', 'OpenAI', .4, 1.6, openai('gpt-4.1-mini'), oaScores(55.8, 87.5, 49.6)),
+  model('gpt54', 'GPT-5.4', 'OpenAI', 2.5, 15, openai('gpt-5.4')),
+  model('gpt54mini', 'GPT-5.4 mini', 'OpenAI', .75, 4.5, openai('gpt-5.4-mini')),
+  model('opus45', 'Claude Opus 4.5', 'Anthropic', 5, 25, anthropic, [...claudeScores(88.9, 70.6, 87), ...qwenScores(76.6, 59.6, 70.6, 87)]),
+  model('sonnet46', 'Claude Sonnet 4.6', 'Anthropic', 3, 15, anthropic, claudeScores(91.7, 74.5, 89.9)),
+  model('sonnet45', 'Claude Sonnet 4.5', 'Anthropic', 3, 15, anthropic, [...glmCodeScores([87.2, 64.0, 83.4]), ...googleCodeScores([87.2, 1418, 83.4])], { configuration: 'Thinking enabled; published comparison configurations' }),
+  model('haiku45', 'Claude Haiku 4.5', 'Anthropic', 1, 5, anthropic),
+  model('gemini25pro', 'Gemini 2.5 Pro', 'Google', 1.25, 10, google, googleScores(77.8, 68, 86.4)),
+  model('gemini25flash', 'Gemini 2.5 Flash', 'Google', .3, 2.5, google, googleScores(79.5, 66.7, 82.8)),
+  model('gemini3flash', 'Gemini 3 Flash Preview', 'Google', .5, 3, google, [...googleScores(90.2, 81.2, 90.4), ...googleCodeScores([90.2, 2316, 90.4])]),
+  model('oss120', 'gpt-oss-120b', 'Groq', .15, .6, groq, ossScores(67.8, 90, 80.1), { creator: 'OpenAI', openWeights: true, configuration: 'High reasoning effort' }),
+  model('oss20', 'gpt-oss-20b', 'Groq', .075, .3, groq, ossScores(54.8, 85.3, 71.5), { creator: 'OpenAI', openWeights: true, configuration: 'High reasoning effort' }),
+  model('glm53', 'GLM-5.3', 'Baseten', 1.4, 4.4, baseten, dsScores(48.8, 28.5, 88.1), { creator: 'Z.ai', origin: 'CN', openWeights: true, ...basetenPolicy, configuration: 'Maximum reasoning effort' }),
+  model('glm53flash', 'GLM-5.3-Flash', 'Baseten', .15, .5, baseten, flashScores([48.8, 1773, 55.3]), { creator: 'Z.ai', origin: 'CN', openWeights: true, ...basetenPolicy }),
+  model('mistral35', 'Mistral Medium 3.5', 'Mistral', 1.5, 7.5, mistral, [], { origin: 'FR', openWeights: true, metadataSource: 'https://docs.mistral.ai/models/mistral-medium-3-5-26-04' }),
+  model('mistral4', 'Mistral Small 4', 'Mistral', .15, .6, mistral, [], { origin: 'FR', openWeights: true }),
+  model('oss120baseten', 'gpt-oss-120b', 'Baseten', .1, .5, baseten, ossScores(67.8, 90, 80.1), { modelId: 'oss120', creator: 'OpenAI', openWeights: true, ...basetenPolicy, configuration: 'High reasoning effort; same published checkpoint' }),
+  model('deepseek41flash', 'DeepSeek V4.1 Flash', 'Baseten', .3, 1.2, baseten, dsScores(54.8, 31.8, 90.9), { creator: 'DeepSeek', origin: 'CN', openWeights: true, ...basetenPolicy, configuration: 'Maximum reasoning effort (100)' }),
+  model('deepseek4pro0813', 'DeepSeek V4 Pro 0813', 'Baseten', 1.32, 3.96, baseten, [], { creator: 'DeepSeek', origin: 'CN', openWeights: true, ...basetenPolicy, metadataSource: 'https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813' }),
+  model('nemotron3ultra', 'Nemotron 3 Ultra', 'Baseten', .6, 2.4, baseten, cohortScores('https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16', 'nvidia-ultra-nemo', ['TauBench V3 · average', 'MMLU-Pro', 'GPQA · no tools'], [70.9, 86.8, 87.0], 'NVIDIA NeMo evaluation SDK; model-card configurations. TauBench V3 is not τ-bench or τ²-bench.'), { creator: 'NVIDIA', origin: 'US', openWeights: true, ...basetenPolicy, metadataSource: 'https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16' }),
+  model('glm52', 'GLM-5.2', 'Baseten', 1.4, 4.4, baseten, flashScores([26.2, 1504, 54.7]), { creator: 'Z.ai', origin: 'CN', openWeights: true, ...basetenPolicy, metadataSource: 'https://huggingface.co/zai-org/GLM-5.2' }),
+  model('glm47', 'GLM-4.7', 'Baseten', .6, 2.2, 'https://www.baseten.co/library/glm-4-7/', [...cohortScores('https://huggingface.co/zai-org/GLM-4.7', 'glm47-release', ['τ²-bench', 'MMLU-Pro', 'GPQA Diamond'], [87.4, 84.3, 85.7], 'GLM-4.7 release configurations; preserved thinking, adjusted user prompts and fixed Airline domain for τ²-bench.'), ...glmCodeScores([87.4, 84.9, 85.7])], { creator: 'Z.ai', origin: 'CN', openWeights: true, ...basetenPolicy, configuration: 'Thinking enabled', metadataSource: 'https://huggingface.co/zai-org/GLM-4.7' }),
+  model('qwen36', 'Qwen 3.6 · 27B', 'Groq', .6, 3, groq, [...qwenScores(72.4, 60.6, 75.8, 87.8), ...qwen38Scores([50.4, 69.1, 87.8])], { creator: 'Alibaba', origin: 'CN', openWeights: true, availability: 'Preview', metadataSource: qwenCard }),
+  model('qwen38', 'Qwen 3.8 · 27B', 'Groq', .8, 4, groq, qwen38Scores([56.9, 79.5, 89.2]), { creator: 'Alibaba', origin: 'CN', openWeights: true, availability: 'Preview', metadataSource: 'https://huggingface.co/Qwen/Qwen3.8-27B' }),
+  model('jev', 'Jev 1.13', 'TypeSafe', .042, 0, 'https://docs.typesafe.ai/models', [], { outputType: 'decision' }),
+];
+export const CONFIGURED_SERVICES = ['OpenAI', 'Anthropic', 'Google', 'Groq', 'Baseten', 'Mistral'];
+const byId = id => CATALOG.find(item => item.id === id);
+// Every workload is fictional application work. Names are plausible declarations,
+// never represented as discoveries from a visitor's repository.
+const APPLICATION_CODE_PROFILES = {
+  'actuarial.queries': { input: 14000, output: 700 },
+  'portfolio.reporting': { input: 18000, output: 900 },
+  'claims.reconciliation': { input: 5000, output: 2000 },
+  'reserves.analysis': { input: 6000, output: 2400 },
+};
+export const WORKLOADS = [
+  ['claims.intake', 'Claim intake', 'gpt4o', ['parse_notice', 'extract_loss_date', 'normalize_address', 'link_policy', 'check_attachments', 'summarize_notice', 'prepare_case']],
+  ['policy.extraction', 'Policy document extraction', 'gpt54', ['read_schedule', 'extract_limits', 'extract_deductibles', 'identify_endorsements', 'normalize_terms', 'link_sections', 'check_completeness']],
+  ['coverage.review', 'Coverage review support', 'opus45', ['retrieve_wording', 'compare_endorsements', 'locate_exclusions', 'cite_conditions', 'summarize_scope', 'flag_ambiguity', 'prepare_review']],
+  ['claims.correspondence', 'Claim correspondence', 'sonnet46', ['draft_acknowledgment', 'request_documents', 'summarize_progress', 'draft_follow_up', 'check_tone', 'cite_case_notes', 'prepare_handoff']],
+  ['documents.verification', 'Document verification', 'gemini25pro', ['read_estimate', 'extract_line_items', 'match_invoices', 'check_totals', 'compare_versions', 'flag_missing_pages', 'prepare_evidence']],
+  ['renewals.summary', 'Renewal preparation', 'gemini25flash', ['summarize_history', 'compare_schedules', 'list_changes', 'check_dates', 'extract_terms', 'prepare_brief', 'draft_reminder']],
+  ['claims.tagging', 'Claim tagging', 'oss120', ['tag_document', 'label_channel', 'extract_reference', 'normalize_category', 'link_attachment', 'label_language', 'index_summary']],
+  ['fraud.review', 'Fraud review support', 'glm53', ['summarize_signals', 'link_evidence', 'compare_statements', 'flag_inconsistency', 'build_timeline', 'prepare_case_note', 'route_to_reviewer']],
+  ['broker.translation', 'Broker correspondence', 'mistral35', ['detect_language', 'translate_email', 'preserve_policy_terms', 'check_names', 'summarize_request', 'draft_reply', 'prepare_review']],
+  ['claims.routing', 'Decision routing', 'gpt4o', ['route_notice', 'rank_queue', 'classify_document', 'score_completeness', 'flag_duplicates', 'choose_workflow', 'route_exception']],
+  ['actuarial.queries', 'Actuarial query generation', 'sonnet45', ['read_schema', 'select_tables', 'generate_query', 'check_join_keys', 'validate_query', 'explain_result', 'prepare_review']],
+  ['claims.reconciliation', 'Claim reconciliation scripts', 'sonnet45', ['read_line_items', 'normalize_columns', 'generate_transform', 'compare_totals', 'build_checks', 'explain_variance', 'prepare_review']],
+  ['portfolio.reporting', 'Portfolio reporting queries', 'sonnet45', ['read_report_spec', 'map_dimensions', 'generate_query', 'check_aggregates', 'validate_periods', 'explain_result', 'prepare_report']],
+  ['reserves.analysis', 'Reserve analysis scripts', 'sonnet45', ['read_triangle', 'normalize_periods', 'generate_calculation', 'check_assumptions', 'build_sensitivity', 'explain_projection', 'prepare_review']],
+].map(([id, purpose, before, tasks]) => ({
+  id, purpose, before: byId(before), configuredServices: CONFIGURED_SERVICES, tasks,
+  benchmarkProfile: APPLICATION_CODE_PROFILES[id] ? 'Application code · tools, code generation, reasoning' : 'General application · tools, capabilities, reasoning',
+  tokenProfile: APPLICATION_CODE_PROFILES[id] ?? null,
+  requiredAxes: ['tools', 'capabilities', 'reasoning', ...(id === 'claims.routing' ? ['decisions'] : [])],
+}));
+
+export function modeledCost(model, usage) {
+  return model && [model.input, model.output, usage.inputTokens, usage.outputTokens].every(value => Number.isFinite(value) && value >= 0)
+    ? (model.input * usage.inputTokens + model.output * usage.outputTokens) / 1e6 : null;
+}
+export function assessReplacement(workload, candidate, usage) {
+  const before = workload.before;
+  const checks = workload.requiredAxes.flatMap(axis => axis === 'decisions' ? ['decisionAccuracy', 'calibration'] : [axis]).flatMap(axis => {
+    const baselines = before.benchmarks.filter(item => item.axis === axis);
+    const pairs = baselines.flatMap(a => candidate.benchmarks.filter(b => b.axis === axis && a.suite === b.suite && a.name === b.name && a.unit === b.unit && a.direction === b.direction).map(b => ({ a, b })));
+    if (!pairs.length) return [{ axis, before: baselines[0] ?? null, after: candidate.benchmarks.find(item => item.axis === axis) ?? null, status: 'missing' }];
+    // Check every matched metric, not the first passing score in a cohort.
+    return pairs.map(({ a, b }) => {
+      const comparable = Boolean(a.source && b.source && Number.isFinite(a.value) && Number.isFinite(b.value));
+      const pass = comparable && (a.direction === 'lower' ? b.value <= a.value : b.value >= a.value);
+      return { axis, before: a, after: b, status: !comparable ? 'missing' : pass ? 'pass' : 'regression' };
+    });
+  });
+  const beforeCost = modeledCost(before, usage), afterCost = modeledCost(candidate, usage);
+  const cheaper = beforeCost !== null && afterCost !== null && afterCost < beforeCost;
+  // Only paired latency measurements with the same workload/configuration qualify.
+  // No latency data is fabricated or inferred from token generation speed in this demo.
+  const a = before.latency, b = candidate.latency;
+  const faster = Boolean(a && b && a.suite && a.metric && a.suite === b.suite && a.metric === b.metric && a.source && b.source && Number.isFinite(a.value) && Number.isFinite(b.value) && b.value > 0 && b.value < a.value);
+  const compatible = (workload.configuredServices ?? [before.provider]).includes(candidate.provider) && candidate.outputType === before.outputType && afterCost !== null;
+  const issues = [];
+  if (!(workload.configuredServices ?? [before.provider]).includes(candidate.provider)) issues.push('Service is not configured in this environment');
+  if (candidate.outputType !== before.outputType) issues.push('Different output contract; application adaptation required');
+  if (checks.some(check => check.status === 'missing')) issues.push('Comparable benchmark evidence is incomplete');
+  if (checks.some(check => check.status === 'regression')) issues.push('A required benchmark regresses');
+  if (afterCost === null) issues.push('Token price is unavailable');
+  if (!cheaper && !faster) issues.push('No verified cost or latency improvement');
+  return { candidate, checks, cheaper, faster, beforeCost, afterCost, compatible, issues, eligible: issues.length === 0 };
+}
+export function matchesPreference(model, preference) {
+  return preference === 'zdr' ? model.confirmedZdr === true : preference === 'open' ? model.openWeights === true : preference === 'closed' ? model.openWeights === false : preference === 'usa' ? model.origin === 'US' : preference === 'china' ? model.origin === 'CN' : true;
+}
+export function selectReplacement(workload, usage, preference = 'default') {
+  preference = preferenceFor(preference).value;
+  const assessments = (workload.candidates ?? CATALOG).filter(candidate => candidate.id !== workload.before.id).map(candidate => assessReplacement(workload, candidate, usage));
+  const allowed = item => item.eligible && (preference !== 'closed' || item.candidate.openWeights === false);
+  const byCost = (a, b) => (a.afterCost ?? Infinity) - (b.afterCost ?? Infinity) || a.candidate.id.localeCompare(b.candidate.id);
+  const qualified = assessments.filter(allowed).sort(byCost);
+  const defaultChoice = assessments.filter(item => item.eligible).sort(byCost)[0]?.candidate ?? workload.before;
+  // A preference first seeks a fully qualified match. Retaining an already
+  // matching baseline needs no exception. Only then relax benchmark/efficiency
+  // criteria, never the configured service, output contract or known pricing.
+  let selected = preference === 'default' ? qualified[0] : qualified.find(item => matchesPreference(item.candidate, preference));
+  if (!selected && preference !== 'default' && !matchesPreference(workload.before, preference)) {
+    const axes = (item, status) => new Set(item.checks.filter(check => check.status === status).map(check => check.axis)).size;
+    selected = assessments.filter(item => item.compatible && matchesPreference(item.candidate, preference)).sort((a, b) =>
+      axes(a, 'regression') - axes(b, 'regression') || axes(a, 'missing') - axes(b, 'missing') ||
+      Number(!a.cheaper && !a.faster) - Number(!b.cheaper && !b.faster) || byCost(a, b))[0];
+    if (!selected && preference !== 'closed') selected = qualified[0];
+  }
+  const proposed = selected?.candidate ?? workload.before;
+  const unresolved = preference === 'closed' && proposed.openWeights !== false;
+  const after = unresolved ? null : proposed;
+  const override = Boolean(after && selected && !selected.eligible);
+  const overrideReasons = override ? selected.issues : [];
+  const serviceChanged = Boolean(after && after.provider !== workload.before.provider);
+  const modelChanged = Boolean(after && after.modelId !== workload.before.modelId);
+  return {
+    id: workload.id, purpose: workload.purpose, benchmarkProfile: workload.benchmarkProfile, tokenProfile: workload.tokenProfile, before: workload.before, after,
+    changed: Boolean(after && selected), serviceChanged, modelChanged,
+    checks: selected?.checks ?? [], assessments, qualifiedCount: qualified.length,
+    preferenceMatched: Boolean(after && matchesPreference(after, preference)),
+    preferenceChanged: Boolean(after && after.id !== defaultChoice.id),
+    defaultChoice: defaultChoice.id, unresolved, override, overrideReasons,
+    reason: unresolved ? 'No compatible closed-weight assignment in the reviewed catalog. Service, output contract and known pricing remain required.'
+      : override ? `Selected for ${preferenceFor(preference).label}. No preferred replacement passed every primary check.`
+      : selected ? `${modelChanged ? 'Selected benchmark scores match or improve' : 'Same model checkpoint and published benchmark scores; service changes'}; ${selected.cheaper ? 'modeled token cost is lower' : 'paired latency is lower'}.`
+      : preference !== 'default' && matchesPreference(workload.before, preference) ? 'Baseline retained: already matches this preference; no qualifying preferred replacement.'
+      : 'Baseline retained. No reviewed candidate passes every required check.',
+  };
+}

package/dist/terminal/src/enterprise-sample.mjs

@@ -0,0 +1,151 @@
+import { sampleSnapshot } from './sample.mjs';
+import { WORKLOADS, CATALOG, CONFIGURED_SERVICES, PREFERENCES, preferenceFor, selectReplacement } from './demo-policy.mjs';
+
+// Published standard, uncached text-token rates in USD per million tokens.
+// This catalog belongs to the fictional hosted demonstration, never live billing.
+export const PRICE_DATE = '2026-10-10';
+const openai = model => `https://developers.openai.com/api/docs/models/${model}`;
+const anthropic = 'https://platform.claude.com/docs/en/about-claude/pricing';
+const google = 'https://ai.google.dev/gemini-api/docs/pricing';
+const groq = 'https://console.groq.com/docs/models';
+const baseten = 'https://www.baseten.co/pricing/';
+const mistral = 'https://docs.mistral.ai/inference/pricing';
+const rate = (name, provider, input, output, source, creator = provider) => ({ name, provider, input, output, source, creator });
+export const PRICES = [
+  rate('GPT-5.4', 'OpenAI', 2.5, 15, openai('gpt-5.4')),
+  rate('GPT-5.4 mini', 'OpenAI', .75, 4.5, openai('gpt-5.4-mini')),
+  rate('GPT-5.4 nano', 'OpenAI', .2, 1.25, openai('gpt-5.4-nano')),
+  rate('Claude Opus 4.6', 'Anthropic', 5, 25, anthropic),
+  rate('Claude Sonnet 4.6', 'Anthropic', 3, 15, anthropic),
+  rate('Claude Haiku 4.5', 'Anthropic', 1, 5, anthropic),
+  rate('Gemini 2.5 Pro', 'Google', 1.25, 10, google),
+  rate('Gemini 2.5 Flash', 'Google', .3, 2.5, google),
+  rate('Gemini 2.5 Flash-Lite', 'Google', .1, .4, google),
+  rate('gpt-oss-120b', 'Groq', .15, .6, groq, 'OpenAI'),
+  rate('gpt-oss-20b', 'Groq', .075, .3, groq, 'OpenAI'),
+  rate('GLM-5.3', 'Baseten', 1.4, 4.4, baseten, 'Z.ai'),
+  rate('GLM-5.3-Flash', 'Baseten', .15, .5, baseten, 'Z.ai'),
+  rate('Mistral Medium 3.5', 'Mistral', 1.5, 7.5, mistral),
+  rate('Mistral Small 4', 'Mistral', .15, .6, mistral),
+];
+export function tokenCost(price, input, output) {
+  if (!price || !Number.isFinite(input) || !Number.isFinite(output)) return null;
+  return (input * price.input + output * price.output) / 1_000_000;
+}
+const priceFor = (provider, model) => PRICES.find(price => price.provider === provider && price.name === model);
+const REQUESTS_PER_GROUP = 1000;
+
+// A bounded set of explicit synthetic groups, not millions of duplicated rows.
+// Tokens and cost are group totals. The request count is sample-only metadata.
+export function enterpriseSnapshot(now = Date.now()) {
+  const data = sampleSnapshot(now);
+  data.simulation = { kind: 'enterprise-groups', days: 30, requestsPerGroup: REQUESTS_PER_GROUP, priceDate: PRICE_DATE, prices: PRICES };
+  data.project = { name: 'enterprise-sample', scope: 'Fictional enterprise environment' };
+  const replacements = new Map([
+    ['GPT-5', priceFor('OpenAI', 'GPT-5.4')], ['GPT-5 mini', priceFor('OpenAI', 'GPT-5.4 mini')], ['GPT-5 nano', priceFor('OpenAI', 'GPT-5.4 nano')],
+    ['Claude Sonnet 4.5', priceFor('Anthropic', 'Claude Sonnet 4.6')], ['Claude Opus 4.1', priceFor('Anthropic', 'Claude Opus 4.6')], ['Claude Haiku 3.5', priceFor('Anthropic', 'Claude Haiku 4.5')],
+    ['DeepSeek R1', priceFor('Baseten', 'GLM-5.3')], ['Mistral Medium 3', priceFor('Mistral', 'Mistral Medium 3.5')],
+  ]);
+  const rename = (item, field) => { const price = replacements.get(item[field]); if (price) { item[field] = price.name; item.modelCreator = price.creator; } };
+  data.models.forEach(item => rename(item, 'name'));
+  data.deployments.forEach(item => { rename(item, 'model'); if (item.provider === 'Baseten') item.servingMode = 'shared'; });
+  data.records.forEach((record, index) => {
+    rename(record, 'model');
+    record.sampleRequests = REQUESTS_PER_GROUP;
+    record.occurredAt = new Date(now - (now - Date.parse(record.occurredAt)) * 30).toISOString();
+    const shard = index % 5 + 1;
+    record.agentId += `.worker-${shard}`;
+    if (record.agent) record.agent += `.worker-${shard}`;
+    record.taskId += `-partition-${shard}`;
+    record.inputTokens *= REQUESTS_PER_GROUP;
+    if (record.status === 'running') { record.status = 'completed'; record.outputTokens = 800; }
+    if (record.outputTokens != null) record.outputTokens *= REQUESTS_PER_GROUP;
+    record.cacheReadTokens = 0;
+    record.cost = tokenCost(priceFor(record.provider, record.model), record.inputTokens, record.outputTokens);
+    record.costKind = 'estimated';
+    record.requestId = null;
+  });
+  for (const source of data.sources) {
+    if (source.id === 'sample-activity') {
+      source.records = 9_600_000;
+      source.detail = 'Fictional request groups · 30 days';
+      source.note = '9.6 million fictional requests represented by 9,600 groups of 1,000. Tokens and spend are group totals. Published uncached text-token prices are applied where available; other costs are unknown. No application or provider was contacted.';
+    }
+  }
+  return data;
+}
+
+export function demoSnapshot(now = Date.now(), stage = 'before', preference = 'default') {
+  if (!['before', 'after'].includes(stage)) throw new Error('Unknown comparison stage');
+  preference = preferenceFor(preference).value;
+  const data = enterpriseSnapshot(now);
+  data.project = { name: 'insurance-tech-demo', scope: 'Fictional insurance operations application' };
+  const usage = WORKLOADS.map(() => ({ inputTokens: 0, outputTokens: 0 }));
+  data.records.forEach((record, index) => {
+    record.assignmentId = index % WORKLOADS.length;
+    const profile = WORKLOADS[record.assignmentId].tokenProfile;
+    if (profile) {
+      // Fixed fictional application profiles, independent of stage/preference.
+      // Schema-heavy SQL prompts and longer reconciliation scripts have distinct
+      // input/output mixes. Both include any billed reasoning output.
+      const variation = .85 + (index * 17 % 31) / 100;
+      record.inputTokens = Math.round(profile.input * variation) * REQUESTS_PER_GROUP;
+      record.outputTokens = Math.round(profile.output * variation) * REQUESTS_PER_GROUP;
+    }
+    if (!record.outputTokens) record.outputTokens = (80 + index * 37 % 1600) * REQUESTS_PER_GROUP;
+    usage[record.assignmentId].inputTokens += record.inputTokens;
+    usage[record.assignmentId].outputTokens += record.outputTokens;
+  });
+  const plans = Object.fromEntries(PREFERENCES.map(({ value }) => [value, WORKLOADS.map((workload, index) => selectReplacement(workload, usage[index], value))]));
+  const assignments = plans[preference];
+  const signature = pairs => JSON.stringify(pairs.map(pair => pair.after?.id ?? null));
+  const equivalentPreferences = Object.keys(plans).filter(value => value !== preference && signature(plans[value]) === signature(assignments));
+  const feasible = assignments.every(pair => !pair.unresolved);
+  data.simulation = { ...data.simulation, kind: 'comparison-groups', stage, preference, feasible, equivalentPreferences, catalogSize: CATALOG.filter(model => CONFIGURED_SERVICES.includes(model.provider)).length, configuredServices: CONFIGURED_SERVICES, assignments, prices: [...new Map(assignments.flatMap(pair => [pair.before, pair.after]).filter(Boolean).map(model => [model.id, model])).values()] };
+  const models = new Map(), deployments = new Map();
+  data.records.forEach((record, index) => {
+    const assignmentId = record.assignmentId, assignment = assignments[assignmentId], workload = WORKLOADS[assignmentId];
+    const price = stage === 'after' ? assignment.after ?? assignment.before : assignment.before, sequence = Math.floor(index / WORKLOADS.length);
+    const worker = String(sequence % 20 + 1).padStart(2, '0');
+    record.workload = `services/${workload.id.replaceAll('.', '/')}`;
+    record.agentId = record.agent = `${workload.id}.worker-${worker}`;
+    record.task = workload.tasks[sequence % workload.tasks.length];
+    record.taskId = `${workload.id}.${record.task}.batch-${Math.floor(sequence / workload.tasks.length) % 10 + 1}`;
+    record.model = price.name; record.modelCreator = price.creator; record.provider = price.provider;
+    record.location = 'cloud'; record.status = 'completed'; record.durationMs = null;
+    record.cost = tokenCost(price, record.inputTokens, record.outputTokens);
+    record.beforeCost = tokenCost(assignment.before, record.inputTokens, record.outputTokens);
+    record.afterCost = tokenCost(assignment.after, record.inputTokens, record.outputTokens);
+    record.routeKind = ['direct', 'router', 'gateway'][index % 3];
+    record.route = record.routeKind === 'direct' ? `${price.provider} API` : record.routeKind === 'router' ? 'insurance.router' : 'insurance-gateway';
+    record.path = [{ name: record.route, kind: record.routeKind }];
+    if (record.routeKind !== 'direct') record.path.push({ name: `${price.provider} API`, kind: 'direct' });
+    record.deploymentId = `demo-deployment-${assignmentId}`;
+    if (stage === 'after' && !assignment.after) return;
+    models.set(price.id, { id: `demo-model-${price.id}`, name: price.name, provider: price.provider, modelCreator: price.creator, sourceId: 'sample-activity', location: 'cloud', state: 'observed' });
+    deployments.set(assignmentId, { id: record.deploymentId, name: workload.id, model: price.name, modelCreator: price.creator, provider: price.provider, location: 'cloud', servingMode: 'shared', state: 'configured', sourceId: 'inventory', poolIds: [`pool-${price.provider.toLowerCase()}`] });
+  });
+  data.models = [...models.values()]; data.deployments = [...deployments.values()];
+  const providers = CONFIGURED_SERVICES;
+  data.inventory = [...providers.map(name => ({ name, kind: 'provider', location: 'cloud', state: 'configured', sourceId: 'inventory' })), { name: 'insurance.router', kind: 'router', location: 'cloud', state: 'configured' }, { name: 'insurance-gateway', kind: 'gateway', location: 'cloud', state: 'configured' }];
+  data.capacityPools = data.capacityPools.filter(pool => providers.some(name => pool.id === `pool-${name.toLowerCase()}`));
+  data.hosts = []; data.runtimeStats = []; delete data.machine;
+  data.sources = data.sources.filter(source => ['sample-activity', 'sample-capacity', 'inventory'].includes(source.id));
+  data.sources.find(source => source.id === 'sample-capacity').note = 'Fictional provider quota readings, held identical in both stages. They do not establish a change in latency, throughput or available capacity.';
+  data.sources.find(source => source.id === 'sample-activity').note += ' Insurance workflow and worker names are fictional declarations. Assignments distinguish qualified replacements, retained baselines and explicit preference overrides. Overrides do not pass all primary criteria. Application quality and latency have not been measured.';
+  data.simulation.windows = Object.fromEntries([0, 1, 6, 24, 168, 720].map(hours => {
+    const rows = data.records.filter(record => !hours || Date.parse(record.occurredAt) >= now - hours * 3600000);
+    const totals = selected => ({ requests: selected.length * REQUESTS_PER_GROUP, assignedRequests: selected.filter(record => assignments[record.assignmentId].after).length * REQUESTS_PER_GROUP, qualifiedRequests: selected.filter(record => assignments[record.assignmentId].after && !assignments[record.assignmentId].override).length * REQUESTS_PER_GROUP, overrideRequests: selected.filter(record => assignments[record.assignmentId].override).length * REQUESTS_PER_GROUP, beforeCost: selected.reduce((n, r) => n + r.beforeCost, 0), afterCost: feasible ? selected.reduce((n, r) => n + r.afterCost, 0) : null });
+    return [hours, { ...totals(rows), workloads: assignments.map((pair, index) => {
+      const selected = rows.filter(record => record.assignmentId === index);
+      return { ...totals(selected), afterCost: pair.after ? selected.reduce((n, r) => n + tokenCost(pair.after, r.inputTokens, r.outputTokens), 0) : null };
+    }) }];
+  }));
+  if (stage === 'after' && !feasible) {
+    data.records = data.records.filter(record => assignments[record.assignmentId].after);
+    const source = data.sources.find(source => source.id === 'sample-activity');
+    source.records = data.records.length * REQUESTS_PER_GROUP;
+    source.note += ' Incomplete After plan: only workloads with an assignment are included; preference overrides are identified in comparison metadata. Unresolved workloads remain in comparison metadata, without an After assignment or cost.';
+  }
+  return data;
+}

package/docs/adapter-guide.md

@@ -67,7 +67,7 @@ const coordinator = new OpenMeritCoordinator(
 
 The confirmed task profile identifies the application LLM call under evaluation. Every metric, assessment, candidate, frontier artifact, and automation signal must carry that target ID; the harness's own model usage is orchestration data and must not be reported as product evidence. Application-provider cost, token, and latency telemetry belongs on the application signal or metric record.
 
-In the unreleased source checkout, setup may return `preliminaryModelLeads`: model IDs with sourced price estimates and reputation summaries. The field is optional under protocol 0.1 for older adapters. Core stores the leads and includes both the confirmed task profile and these leads in later discovery intent constraints, so a headless harness run does not depend on prior chat context. Leads are research priors, never measured candidate assessments.
+Setup may return `preliminaryModelLeads`: model IDs with sourced price estimates and reputation summaries. The field is optional under protocol 0.1 for older adapters. Core stores the leads and includes both the confirmed task profile and these leads in later discovery intent constraints, so a headless harness run does not depend on prior chat context. Leads are research priors, never measured candidate assessments.
 
 ## External telemetry export
 
@@ -143,7 +143,7 @@ await coordinator.recordTaskObservation({
   modelId: "vendor/model",
 });
 
-// In the unreleased source checkout, core has already checked baseline
+// Core has already checked baseline
 // sufficiency when the confirmed cadence became due. Inspect durable state
 // and dispatch only a pending downstream intent, if any.
 ```

package/docs/agent-evaluations.md

@@ -0,0 +1,145 @@
+# Evaluate CLI changes across coding agents
+
+**npm preview `0.1.7-preview.2`.** Install from the `preview` tag; stable `openmerit@0.1.6` does not include these commands. This evaluates the coding agent with your CLI and documentation. The [Pi workflow](getting-started.md) still evaluates application inference; the [Inference Terminal](inference-terminal.md) remains read-only.
+
+Compared with `0.1.7-preview.1`, this version adds [corrected cost accounting and blocked-execution handling](#accounting-and-blocked-executions), plus [budget selection after model scope](#budget-selection). It does not claim reconciled provider billing or a guaranteed exact model order.
+
+One local runner supports Claude Code, Codex and OpenCode. Scout supplies candidates; local task checks measure each model/agent/CLI combination. Installing npm alone does not install or authenticate agents, modify project instructions, or start paid evaluations.
+
+## Set up
+
+Install the preview and initialize it inside the project you want to evaluate:
+
+```sh
+npm install -g openmerit@preview
+openmerit eval init
+openmerit eval doctor
+```
+
+Init creates `.openmerit/eval.json`, a tiny extraction fixture, a tool and a standalone grader. It appends marked instructions to `AGENTS.md` and `CLAUDE.md`, preserving existing contents. Codex/OpenCode can read `AGENTS.md`; Claude Code reads `CLAUDE.md`. The instructions ask the agent to configure representative tasks, choose scope, obtain the user's budget, and rerun after CLI changes. These are instructions the host must follow, not guaranteed hooks. There are no install-time hooks or background services. A second init refuses to overwrite the suite.
+
+Replace the starter before drawing product conclusions. Have your agent author representative fixtures and graders with you, and reserve fresh cases for final validation.
+
+## Choose scope and connections
+
+```sh
+openmerit eval plan
+```
+
+The first interactive plan asks: **existing project connections or all models Scout indexes?** It then asks for a **$10, custom or no-fixed-cap budget**. Both choices are saved. Noninteractive agents must ask the user and pass `--scope existing` or `--scope all` together with `--budget <USD|unlimited>` on first setup; missing input never chooses scope or spending automatically.
+
+Add explicit routes to `connections` in the config, for example:
+
+```json
+{
+  "id": "claude-project",
+  "agent": "claude",
+  "scoutId": "anthropic/claude-haiku-4.5",
+  "model": "claude-haiku-4-5-20251001",
+  "env": ["ANTHROPIC_API_KEY"]
+}
+```
+
+`scoutId` is Scout's catalogue ID. `model` is the native identifier your authenticated agent accepts. `command` optionally selects another executable path. `env` lists environment variable **names**, never values. Only basic OS variables and explicitly listed names pass to child agents. Saved authentication remains available through the normal home directory. No dotenv files are loaded automatically. Configure endpoint details through the native agent or named environment variables.
+
+An SDK dependency or key-variable name does not prove access. The agent should inspect the project's intended connections and declare the models they can serve. This preview does not automatically enumerate endpoint catalogues. Multiple connections can test one Scout model through different agents/routes. Prefer pinned model versions; aliases can change.
+
+Existing scope sends IDs and requires Scout to acknowledge filtering **before ranking five**. Fewer than five eligible models remain fewer. All scope searches the index, but missing authenticated routes block execution until configured and replanned. Inaccessible candidates are not silently replaced. Unranked Hugging Face specialists are not executed automatically.
+
+Scout receives only each task's `description`, plus IDs for existing scope. Scout sends descriptions to OpenAI Decisions. Fixtures, execution prompts, endpoint URLs and credentials stay out of this request. The shortlist, warnings and ranking context are frozen locally. Scout scores are screening evidence, not measured success with your harness.
+
+## Tasks and variants
+
+Each task needs `id`, `description` (for Scout, at most 4,000 characters), `prompt` (for execution), `fixture` (a project-relative directory) and `grader` (a standalone Node `.mjs` file).
+
+Each variant needs `id`, `files` and `instructions`. The starter compares `bare` against `cli`. Bare omits your CLI/docs; native agent tools remain available. Across agents you are comparing different harnesses, with different permission systems. A no-native-tools experiment needs a separately controlled configuration.
+
+Selected files, including uncommitted changes, are snapshotted before execution. Variant paths must not overwrite fixture inputs and retain their project-relative locations; fixture contents go at the workspace root. Include self-contained CLI artifacts and lean docs explicitly. Dependencies are not installed automatically; tools must be bundled or have their runtime requirements already available. Symlinks, parent-directory paths, `.env*`, `.git`, `.openmerit` and `node_modules` are rejected. Each selected snapshot is limited to 25 MB. Do not select credentials stored under other filenames.
+
+The grader reads `{ "workspace": "/temporary/path", "output": "final response" }` from stdin and prints exactly `{ "quality": 0.0 }`, with a finite score from 0 to 1. It can check generated files using executable assertions. Keep it self-contained: imports beside it are not copied automatically. Graders are copied outside agent workspaces; never also include them or answer keys in fixtures/variant files. Test graders against correct and incorrect answers. A broken grader stops the experiment with unknown quality. Ordinary agent failures/timeouts count as quality zero; recognized budget, environment and connection blocks instead stop with unknown quality. See [failure classification](#accounting-and-blocked-executions).
+
+## Run after changes
+
+The starter's `budget.maxUsd` is zero. After the user's scope and budget choices, for example:
+
+```sh
+openmerit eval plan --scope existing --budget 10
+openmerit eval run
+# Edit your CLI or its documentation.
+openmerit eval run
+```
+
+For repeated local development:
+
+```sh
+openmerit eval watch
+```
+
+Watch runs initially, then after selected source/docs/fixture/grader/config bytes change and remain stable for two polls. Result writes do not trigger it. Runs never overlap; edits during execution become the next revision. Ctrl+C stops the watcher and active child process. No OS schedule or daemon is installed. `OPENMERIT_EVAL=1` tells cooperating agent instructions not to recurse into another evaluation.
+
+Every attempt uses a new workspace and agent session. Each task runs balanced rounds across all selected connections and variants; order rotates and reverses. The default is 3–10 repetitions. After three, rounds continue while any group's quality range exceeds 5 percentage points, an attempt failed, or cost, input-token or latency variation has a coefficient of variation above 0.25. Ten is the maximum. Missing metrics cannot establish stability. Set `repeats.min` equal to `repeats.max` for fixed repetitions, useful for holdouts. This heuristic is not a significance test, and repetition is not task diversity.
+
+The frozen plan survives CLI changes. Changes to descriptions, scope or connections require replanning. Previous-revision comparisons require matching prompts, fixtures, graders, routes, repeat settings, selected candidates, variant IDs, timeouts, agent versions, runner versions, invocation settings and reported model identities. CLI/docs bytes and variant instructions are the interventions. Keep external services and native agent settings stable too; they are not hermetically captured.
+
+## Metrics
+
+```sh
+openmerit eval report
+openmerit eval report --json
+```
+
+Reports show quality mean/range, failures, mean input/output tokens, mean reported dollars and median full-process seconds per task/connection/variant. They retain Scout ranks but do not force a weighted winner. Lower cost with lower quality is a tradeoff. Matching complete revisions show deltas; partial/invalid experiments do not.
+
+Claude provides aggregate and per-model usage and cost estimates, reconciled without counting the same usage twice. Codex JSON events provide tokens but not billed dollars, so cost stays unknown. OpenCode per-step usage/cost is summed across the episode, including tools; missing or zero step pricing leaves full cost unknown and preserves partial estimates. Input totals include separately reported cache reads/writes; cached tokens are retained separately without counting them twice. OpenCode output totals include separately reported reasoning tokens. Agent accounting and subscriptions differ. Reported costs are not reconciled provider bills; zero pricing is treated as unknown. Catalogue prices never replace unknown measured costs.
+
+Preflight requires supported noninteractive JSON interfaces: Claude print JSON, Codex `exec --json --ephemeral --sandbox workspace-write`, and OpenCode `run --format json --auto`. OpenCode's auto option approves tool requests that are not explicitly denied, avoiding an interactive approval stall. Claude uses an explicit local tool allowlist without inherited MCP configuration; Codex and OpenCode retain native global settings. Recognized permission failures stop as connection problems, not model-quality evidence. OpenMerit does not add a sandbox around Claude/OpenCode. The old 2025 Codex CLI is unsupported. Native references: [Claude Code](https://code.claude.com/docs/en/headless), [Codex](https://developers.openai.com/codex/noninteractive), [OpenCode](https://opencode.ai/docs/cli/).
+
+## Budget and data
+
+`budget.maxUsd` applies cumulatively across runs and restarts. Before each attempt the runner persists a reservation. It retains the greater of that reservation and the available full or partial cost estimate; estimates never release reserved allowance. Partial events never count as a complete bill. Insufficient remaining allowance stops a partial run. Raising maxUsd extends the allowance, not resets it. Do not delete unknown spend from the ledger. This is admission control, not a billing cutoff: one attempt may exceed its reservation. Finite budgets also use Claude's native maximum-budget flag. Use provider-side limits for hard monetary caps. Each attempt also has a timeout and 4 MB process-output limit. See [budget selection](#budget-selection) for the three choices and automatic reservations.
+
+`.openmerit/eval/` holds `plan.json`, `budget.json`, `latest.json`, `runs/*.json` and `lock/pid`. Init ignores it in Git. Reports contain identities, metrics, hashes and status, not full transcripts or final outputs. Temporary files are removed afterward. Native agents/providers may retain their own data; Claude/Codex receive ephemeral flags, while OpenCode uses native storage. `.openmerit/eval.json` can be versioned without credentials.
+
+Fresh workspaces are not security sandboxes. Trusted agents and graders execute with native permissions and user-level resources. Hard kills may leave temporary directories. After a crash, inspect the lock's PID and remove it only if that process is gone. This preview does not change Pi approvals, swap production models or upload results into Scout rankings.
+
+
+## VM acceptance limits
+
+The published `0.1.7-preview.1` package was tested on an Ubuntu ARM64 VM with Node 22.23.1 on 15 October 2026. Claude Code 2.1.300 and OpenCode 1.19.0 executed a small CSV CLI task before and after a parser change. A separate OpenCode watcher completed three attempts per revision and emitted comparison deltas. Codex 0.162.1 passed interface preflight but could not execute filesystem tools because that VM denied the required Linux user namespaces. CLI flag checks do not establish that the native sandbox works on a host.
+
+In that earlier `0.1.7-preview.1` test, a native budget stop could appear as a generic agent failure, and a successful final response explaining a host permission problem could receive zero from the task grader. These did not establish poor model quality; the corrections below address recognized native failures. The combined VM run also stopped before its final repetition because conservative reservations and agent cost estimates exhausted its local allowance; partial reports correctly omitted automatic comparison. Agent-reported cost differed substantially from OpenRouter account usage. This was one integration fixture, not validation of general task quality, exact model order, or billing accuracy.
+
+## Accounting and blocked executions
+
+Since `0.1.7-preview.2`, recognized budget, sandbox, permission, authentication and model-access blocks are separate from task failures. A blocked attempt has `quality: null`, stops the run and cannot produce a revision comparison. Reports distinguish scored attempts, blocked attempts and task-check failures. Ordinary wrong outputs still receive their grader score; ordinary agent failures and configured timeouts still count as failures. Cancellation, a missing executable and output-limit interruption are unscored. Error classification uses native failure fields and failed-tool diagnostics, not arbitrary task text or model commentary. Unrecognized host failures can still require manual review.
+
+On Linux, Codex preflight also runs a local sandbox probe before any model request or budget reservation. It uses the configured executable and the workspace-write policy; it does not weaken sandbox permissions. A failed probe produces `environment_blocked` with no task score. Doctor performs this same probe. Other operating systems still have interface checks only.
+
+Claude's per-model totals recover usage omitted from aggregate budget-stop responses, without adding the same aggregate and model counters twice. Costs carry `costBasis` and remain native estimates. Zero or missing pricing is unknown; incomplete OpenCode step pricing stays partial. Partial costs survive into the row, summary and ledger, including failed attempts. No provider bill is reconstructed from these estimates, and no account-wide spend delta is assigned to individual attempts.
+
+Budget admission now retains the greater of each reservation and its available full or partial estimate, including old ledger entries. Estimates never release a reservation. This may exhaust an existing allowance earlier; increasing it requires an explicit budget decision. The displayed reserved allowance is not billed spend, and a provider-side limit is still needed for a hard cap. Report version 2 prevents comparisons with the earlier accounting rules. There is no new provider integration, proxy, background service or model-specific rule.
+
+## Budget selection
+
+After choosing existing connections or all indexed models, the first interactive `eval plan` asks:
+
+1. **$10** — a cumulative evaluation allowance for this project.
+2. **Custom amount** — asks how much in USD.
+3. **No fixed cap** — run the configured suite without a monetary admission limit.
+
+Nothing is preselected. Blank, invalid or cancelled input cannot enable spending. Init still starts at zero. Newly generated project instructions tell coding agents to ask the same follow-up and pass the answer explicitly, for example:
+
+```sh
+openmerit eval plan --scope existing --budget 10
+# Alternatives after the user's choice:
+openmerit eval plan --budget 25.50
+openmerit eval plan --budget unlimited
+```
+
+`--budget` belongs to `plan`, accepts positive USD amounts with at most two decimal places or the exact word `unlimited`, and never starts evaluation by itself. Noninteractive planning requires explicit flags when scope or budget has not been chosen. Existing positive budgets are reused. The choices are saved even if Scout subsequently fails, so a retry does not ask again. Init does not rewrite instructions in already-initialized projects; an agent in such a project can use these flags directly.
+
+The allowance is cumulative across runs, watched edits, restarts and replanning. Changing from $10 to $20 raises the total allowance to $20; it does not add $20 or erase earlier reservations. Changing to or from no fixed cap also preserves the ledger. A normal `eval plan` reuses the saved choice. To change it, pass a new `--budget` value; setting `budget.maxUsd` back to zero in the config disables paid execution and prompts again on the next interactive plan.
+
+Reservations are internal. New suites and budget choices use `reservationUsd: "auto"`: the smaller of $0.25 and the total allowance divided across the minimum planned attempts (tasks × selected connections × variants × minimum repetitions). No fixed cap uses $0.25 for ledger bookkeeping. This is a conservative starting allowance, not a prediction of an attempt's price or a guarantee the suite will finish. Higher reported charges still consume more allowance. Existing explicitly configured numeric reservations remain supported. Finite budgets retain Claude's native per-attempt budget flag; as before, an attempt can exceed its reservation, and provider limits are needed for a hard billing cap.
+
+No fixed cap is stored explicitly as `budget.maxUsd: "unlimited"`; missing, null or invalid values never mean unlimited. It also removes Claude's per-attempt dollar limit, while keeping the same task count, 3–10 repetition bounds, timeouts and native permissions. Provider/account limits still apply. Usage and cost estimates remain recorded. `eval watch` can incur further charges after each selected change until stopped; Ctrl+C stops it and the active attempt. No background service or automatic production model change is added.

package/docs/architecture.md

@@ -38,7 +38,7 @@ Contains deterministic behavior: choosing the next lifecycle intent according to
 
 `OpenMeritCoordinator` is the reusable adapter boundary. It constructs canonical intents, negotiates capabilities, records lifecycle progress and observations, validates results, and advances durable state. `OpenMeritStore` makes persistence replaceable; `ProjectStore` is the bundled filesystem backend.
 
-The coordinator also accepts idempotent application-target automation signals and evaluates the user-confirmed `AutomationCheckPolicy`. `buildAutomationPlan` tells an adapter which signals and wakeups are required and reports unsupported persistent scheduling honestly. OpenMerit never installs a scheduler or creates a paid cloud worker itself.
+The coordinator also accepts idempotent application-target automation signals and evaluates the user-confirmed `AutomationCheckPolicy`. `buildAutomationPlan` tells an adapter which signals and wakeups are required and reports unsupported persistent scheduling honestly. Provisioning belongs to the adapter: Pi can install an authorized macOS user LaunchAgent; the coordinator itself does not create operating-system schedulers or paid cloud workers.
 
 ### `openmerit/pi`
 
@@ -46,9 +46,9 @@ Exposes the command surface, supplies private typed intent context, triggers Pi,
 
 The Pi package implements `HarnessAdapter`. It does not define canonical intents, per-intent result contracts, stage transitions, or validation rules.
 
-### Inference Terminal (0.1.6)
+### Inference Terminal
 
-`packages/terminal` supplies a standalone `openmerit dash` executable, passive collectors, a loopback read-only HTTP API, and bundled dark-only browser assets. It does not invoke the coordinator or harness. One collector cycle reads existing metadata and runtime inventory; browser clients receive the same cached snapshot. Imported activity, runtime counters, and sampled whole-machine resources retain their separate scopes. Missing measurements remain unknown, and fictional sample data is isolated from the real snapshot. See the [terminal guide](inference-terminal.md) for the supported sources and JSON contract.
+`packages/terminal` supplies a standalone `openmerit dash` executable, passive collectors, a loopback read-only HTTP API, and bundled browser assets. Stable npm 0.1.6 includes the terminal. The source interface and hosted dashboard follow the device’s light/dark preference; that appearance change is not included in npm 0.1.6. It does not invoke the coordinator or harness. One collector cycle reads existing metadata and runtime inventory; browser clients receive the same cached snapshot. Imported activity, runtime counters, and sampled whole-machine resources retain their separate scopes. Missing measurements remain unknown, and fictional sample data is isolated from the real snapshot. See the [terminal guide](inference-terminal.md) for the supported sources and JSON contract.
 
 ## Intelligence resources