openmerit 0.1.6 → 0.1.7-preview.1

package/dist/terminal/src/cli.mjs

@@ -9,6 +9,7 @@ import { openBrowser } from './browser.mjs';
 const help = `OpenMerit · Inference Terminal
 
   openmerit dash [path] [options]
+  openmerit eval --help              Evaluate CLI changes across coding agents
 
   --port <number>      Require a specific local port (default: 4310 or a free port)
   --interval <seconds> Collection interval, 10–30 (default: 15)
@@ -26,6 +27,7 @@ Coding-harness usage is outside this terminal's scope.
 
 async function main() {
   const args = process.argv.slice(2);
+  if (args[0] === 'eval') { const { main } = await import('../../eval/src/cli.mjs'); await main(args.slice(1)); return; }
   if (!args.length || args.includes('--help') || args.includes('-h')) { console.log(help); return; }
   if (args.shift() !== 'dash') throw new Error('Unknown command. Run openmerit --help.');
   let root = process.cwd(), port = 4310, interval = 15, demo = false, json = false, pathSeen = false, discovery = true, noOpen = false, explicitPort = false, diagnostics = false;

package/dist/terminal/src/demo-policy.mjs

@@ -0,0 +1,205 @@
+// A reviewed model/service-offer catalog for the hosted fictional comparison only.
+// Scores are publisher-reported point estimates, not insurance-workload evaluations.
+export const EVIDENCE_DATE = '2026-10-10';
+export const PREFERENCES = [
+  { value: 'default', label: 'Default · within your stack', note: 'Search all configured services; choose the lowest modeled cost among qualified replacements.' },
+  { value: 'zdr', label: 'Bias ZDR models', note: 'Prefer confirmed ZDR endpoints. Eligibility alone does not establish account approval or enabled settings.' },
+  { value: 'open', label: 'Bias open-weights models', note: 'Prefer models with open weights.' },
+  { value: 'closed', label: 'No open-weights models', note: 'Require a closed-weight assignment for every workload.' },
+  { value: 'usa', label: 'Bias USA models', note: 'Prefer models from US developers. This does not specify hosting location or data residency.' },
+  { value: 'china', label: 'Bias Chinese models', note: 'Prefer models from Chinese developers. This does not specify hosting location or data residency.' },
+];
+export const preferenceFor = value => PREFERENCES.find(item => item.value === value) ?? PREFERENCES[0];
+const oa = 'https://openai.com/index/gpt-4-1/';
+const claude = 'https://www-cdn.anthropic.com/bbd8ef16d70b7a1665f14f306ee88b53f686aa75/Claude%20Sonnet%204.6%20System%20Card.pdf';
+const gemini = 'https://storage.googleapis.com/deepmind-media/gemini/gemini_3_flash_model_evaluation.pdf';
+const metric = (axis, name, value, suite, source, settings) => ({ axis, name, value, suite, source, settings, unit: '%', direction: 'higher' });
+const oaScores = (tool, capability, reasoning) => [
+  metric('tools', 'τ-bench retail', tool, 'openai-4.1-tau-retail-gpt4o-user', oa, '5-run average; original GPT-4o user simulator; no custom tools or prompting.'),
+  metric('capabilities', 'MMLU', capability, 'openai-4.1-mmlu', oa, 'OpenAI GPT-4.1 release comparison; GPT-4o snapshot 2024-11-20.'),
+  metric('reasoning', 'AIME 2024', reasoning, 'openai-4.1-aime24', oa, 'OpenAI GPT-4.1 release comparison, AIME 2024.'),
+];
+const claudeScores = (tool, capability, reasoning) => [
+  metric('tools', 'τ²-bench retail', tool, 'anthropic-sonnet46-tau2-retail', claude, 'Sonnet 4.6 system card §2.5; max effort, 10 trials for Sonnet; published Opus configuration.'),
+  metric('capabilities', 'MMMU-Pro · no tools', capability, 'anthropic-sonnet46-mmmupro-notools', claude, 'Sonnet 4.6 system card Table 2.1.A; publisher model configurations.'),
+  metric('reasoning', 'GPQA Diamond', reasoning, 'anthropic-sonnet46-gpqa', claude, 'Sonnet 4.6 system card §2.8; adaptive thinking, max effort for Sonnet; published Opus configuration.'),
+];
+const googleScores = (tool, capability, reasoning) => [
+  metric('tools', 'τ²-bench', tool, 'google-3flash-tau2-fixed-airline', gemini, 'Gemini 3 Pro user simulator; fixed Airline domain; mean of Retail, Airline and Telecom.'),
+  metric('capabilities', 'MMMU-Pro', capability, 'google-3flash-mmmupro', gemini, 'Thinking; mean of Standard and Vision settings; no tools.'),
+  metric('reasoning', 'GPQA Diamond', reasoning, 'google-3flash-gpqa', gemini, 'Thinking; no tools; Google December 2025 comparison.'),
+];
+const ossCard = 'https://arxiv.org/html/2508.10925v1';
+const ossScores = (tool, capability, reasoning) => [
+  metric('tools', 'τ-bench retail', tool, 'openai-oss-high-retail', ossCard, 'Table 3; high reasoning effort; developer function calling.'),
+  metric('capabilities', 'MMLU', capability, 'openai-oss-high-mmlu', ossCard, 'Table 3; high reasoning effort, default system prompt, pass@1.'),
+  metric('reasoning', 'GPQA Diamond · no tools', reasoning, 'openai-oss-high-gpqa', ossCard, 'Table 3; high reasoning effort, no tools, pass@1.'),
+];
+const dsCard = 'https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash';
+const dsScores = (tool, capability, reasoning) => [
+  metric('tools', 'AutomationBench', tool, 'deepseek41-automation', dsCard, 'Frontier instruct-model comparison; official scaffold, maximum reasoning effort, temperature 1, top-p .95.'),
+  metric('capabilities', "Agent’s Last Exam", capability, 'deepseek41-ale', dsCard, 'Frontier instruct-model comparison; broad agent tasks, official scaffold, maximum reasoning effort.'),
+  metric('reasoning', 'GPQA Diamond', reasoning, 'deepseek41-gpqa', dsCard, 'Frontier instruct-model comparison; pass@1, maximum reasoning effort. Published model scores, not endpoint measurements.'),
+];
+const qwenCard = 'https://huggingface.co/Qwen/Qwen3.6-27B';
+const qwenScores = (tool, pass3, capability, reasoning) => [
+  metric('tools', 'ClawEval · average', tool, 'qwen36-claw-avg', qwenCard, 'Qwen 3.6 27B release comparison; published model configurations.'),
+  metric('tools', 'ClawEval · pass³', pass3, 'qwen36-claw-pass3', qwenCard, 'Same comparison, stricter repeated-task success. Both tool metrics must pass.'),
+  metric('capabilities', 'MMMU-Pro', capability, 'qwen36-mmmupro', qwenCard, 'Vision benchmark; published Qwen 3.6 comparison configurations.'),
+  metric('reasoning', 'GPQA Diamond', reasoning, 'qwen36-gpqa', qwenCard, 'Published Qwen 3.6 comparison configurations.'),
+];
+const cohortScores = (source, suite, names, values, settings) => ['tools', 'capabilities', 'reasoning'].map((axis, index) => metric(axis, names[index], values[index], `${suite}-${axis}`, source, settings));
+const flashCard = 'https://huggingface.co/zai-org/GLM-5.3-Flash';
+const flashScores = values => cohortScores(flashCard, 'glm53flash-release', ['AutomationBench v1.0.6', 'GDPval-AA v2', 'HLE · with tools'], values, 'Published release chart and footnotes; maximum effort; GDPval-AA rating, not percent.').map(item => item.axis === 'capabilities' ? { ...item, unit: ' rating' } : item);
+const qwen38Card = 'https://huggingface.co/Qwen/Qwen3.8-27B';
+const qwen38Scores = values => cohortScores(qwen38Card, 'qwen38-release', ['ClawEval-MM · average', 'IFBench', 'GPQA Diamond'], values, 'Qwen 3.8 model card comparison; published thinking configurations. ClawEval-MM is distinct from ClawEval.');
+// The application-code profile screens generated SQL/Python used by the
+// insurance product. These are application workers, not developer-harness usage.
+// Keep publisher cohorts separate: LiveCodeBench v6 % is not LCB Pro Elo.
+const glm47Card = 'https://huggingface.co/zai-org/GLM-4.7';
+const glmCodeScores = values => cohortScores(glm47Card, 'glm47-application-code', ['τ²-bench', 'LiveCodeBench v6', 'GPQA Diamond'], values, 'GLM-4.7 release table; thinking enabled, preserved thinking for multi-turn tools; adjusted user prompts and fixed Airline domain. Published comparator configurations; not endpoint parity.');
+const googleCodeScores = values => cohortScores(gemini, 'google-3flash-application-code', ['τ²-bench', 'LiveCodeBench Pro', 'GPQA Diamond'], values, 'December 2025 comparison: Gemini thinking, Sonnet 4.5 high/best available thinking. LCB Pro official leaderboard Elo; τ² fixed Airline, Gemini user simulator for Gemini and publisher-reported Sonnet score. Public comparison, not a controlled endpoint evaluation.').map(item => item.axis === 'capabilities' ? { ...item, unit: ' Elo' } : item);
+const basetenPolicy = { confirmedZdr: true, policySource: 'https://www.baseten.co/security-practices/', endpointMode: 'Synchronous Model API; request/response logging off' };
+const model = (id, name, provider, input, output, source, benchmarks = [], extra = {}) => ({ id, modelId: id, name, provider, input, output, source, benchmarks, creator: provider, origin: 'US', openWeights: false, outputType: 'text', confirmedZdr: false, ...extra });
+const openai = name => `https://developers.openai.com/api/docs/models/${name}`;
+const anthropic = 'https://platform.claude.com/docs/en/about-claude/pricing';
+const google = 'https://ai.google.dev/gemini-api/docs/pricing';
+const groq = 'https://console.groq.com/docs/models';
+const baseten = 'https://www.baseten.co/pricing/';
+const mistral = 'https://docs.mistral.ai/inference/pricing';
+export const CATALOG = [
+  model('gpt4o', 'GPT-4o', 'OpenAI', 2.5, 10, openai('gpt-4o'), oaScores(60.3, 85.7, 13.1)),
+  model('gpt41', 'GPT-4.1', 'OpenAI', 2, 8, openai('gpt-4.1'), oaScores(68, 90.2, 48.1)),
+  model('gpt41mini', 'GPT-4.1 mini', 'OpenAI', .4, 1.6, openai('gpt-4.1-mini'), oaScores(55.8, 87.5, 49.6)),
+  model('gpt54', 'GPT-5.4', 'OpenAI', 2.5, 15, openai('gpt-5.4')),
+  model('gpt54mini', 'GPT-5.4 mini', 'OpenAI', .75, 4.5, openai('gpt-5.4-mini')),
+  model('opus45', 'Claude Opus 4.5', 'Anthropic', 5, 25, anthropic, [...claudeScores(88.9, 70.6, 87), ...qwenScores(76.6, 59.6, 70.6, 87)]),
+  model('sonnet46', 'Claude Sonnet 4.6', 'Anthropic', 3, 15, anthropic, claudeScores(91.7, 74.5, 89.9)),
+  model('sonnet45', 'Claude Sonnet 4.5', 'Anthropic', 3, 15, anthropic, [...glmCodeScores([87.2, 64.0, 83.4]), ...googleCodeScores([87.2, 1418, 83.4])], { configuration: 'Thinking enabled; published comparison configurations' }),
+  model('haiku45', 'Claude Haiku 4.5', 'Anthropic', 1, 5, anthropic),
+  model('gemini25pro', 'Gemini 2.5 Pro', 'Google', 1.25, 10, google, googleScores(77.8, 68, 86.4)),
+  model('gemini25flash', 'Gemini 2.5 Flash', 'Google', .3, 2.5, google, googleScores(79.5, 66.7, 82.8)),
+  model('gemini3flash', 'Gemini 3 Flash Preview', 'Google', .5, 3, google, [...googleScores(90.2, 81.2, 90.4), ...googleCodeScores([90.2, 2316, 90.4])]),
+  model('oss120', 'gpt-oss-120b', 'Groq', .15, .6, groq, ossScores(67.8, 90, 80.1), { creator: 'OpenAI', openWeights: true, configuration: 'High reasoning effort' }),
+  model('oss20', 'gpt-oss-20b', 'Groq', .075, .3, groq, ossScores(54.8, 85.3, 71.5), { creator: 'OpenAI', openWeights: true, configuration: 'High reasoning effort' }),
+  model('glm53', 'GLM-5.3', 'Baseten', 1.4, 4.4, baseten, dsScores(48.8, 28.5, 88.1), { creator: 'Z.ai', origin: 'CN', openWeights: true, ...basetenPolicy, configuration: 'Maximum reasoning effort' }),
+  model('glm53flash', 'GLM-5.3-Flash', 'Baseten', .15, .5, baseten, flashScores([48.8, 1773, 55.3]), { creator: 'Z.ai', origin: 'CN', openWeights: true, ...basetenPolicy }),
+  model('mistral35', 'Mistral Medium 3.5', 'Mistral', 1.5, 7.5, mistral, [], { origin: 'FR', openWeights: true, metadataSource: 'https://docs.mistral.ai/models/mistral-medium-3-5-26-04' }),
+  model('mistral4', 'Mistral Small 4', 'Mistral', .15, .6, mistral, [], { origin: 'FR', openWeights: true }),
+  model('oss120baseten', 'gpt-oss-120b', 'Baseten', .1, .5, baseten, ossScores(67.8, 90, 80.1), { modelId: 'oss120', creator: 'OpenAI', openWeights: true, ...basetenPolicy, configuration: 'High reasoning effort; same published checkpoint' }),
+  model('deepseek41flash', 'DeepSeek V4.1 Flash', 'Baseten', .3, 1.2, baseten, dsScores(54.8, 31.8, 90.9), { creator: 'DeepSeek', origin: 'CN', openWeights: true, ...basetenPolicy, configuration: 'Maximum reasoning effort (100)' }),
+  model('deepseek4pro0813', 'DeepSeek V4 Pro 0813', 'Baseten', 1.32, 3.96, baseten, [], { creator: 'DeepSeek', origin: 'CN', openWeights: true, ...basetenPolicy, metadataSource: 'https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813' }),
+  model('nemotron3ultra', 'Nemotron 3 Ultra', 'Baseten', .6, 2.4, baseten, cohortScores('https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16', 'nvidia-ultra-nemo', ['TauBench V3 · average', 'MMLU-Pro', 'GPQA · no tools'], [70.9, 86.8, 87.0], 'NVIDIA NeMo evaluation SDK; model-card configurations. TauBench V3 is not τ-bench or τ²-bench.'), { creator: 'NVIDIA', origin: 'US', openWeights: true, ...basetenPolicy, metadataSource: 'https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16' }),
+  model('glm52', 'GLM-5.2', 'Baseten', 1.4, 4.4, baseten, flashScores([26.2, 1504, 54.7]), { creator: 'Z.ai', origin: 'CN', openWeights: true, ...basetenPolicy, metadataSource: 'https://huggingface.co/zai-org/GLM-5.2' }),
+  model('glm47', 'GLM-4.7', 'Baseten', .6, 2.2, 'https://www.baseten.co/library/glm-4-7/', [...cohortScores('https://huggingface.co/zai-org/GLM-4.7', 'glm47-release', ['τ²-bench', 'MMLU-Pro', 'GPQA Diamond'], [87.4, 84.3, 85.7], 'GLM-4.7 release configurations; preserved thinking, adjusted user prompts and fixed Airline domain for τ²-bench.'), ...glmCodeScores([87.4, 84.9, 85.7])], { creator: 'Z.ai', origin: 'CN', openWeights: true, ...basetenPolicy, configuration: 'Thinking enabled', metadataSource: 'https://huggingface.co/zai-org/GLM-4.7' }),
+  model('qwen36', 'Qwen 3.6 · 27B', 'Groq', .6, 3, groq, [...qwenScores(72.4, 60.6, 75.8, 87.8), ...qwen38Scores([50.4, 69.1, 87.8])], { creator: 'Alibaba', origin: 'CN', openWeights: true, availability: 'Preview', metadataSource: qwenCard }),
+  model('qwen38', 'Qwen 3.8 · 27B', 'Groq', .8, 4, groq, qwen38Scores([56.9, 79.5, 89.2]), { creator: 'Alibaba', origin: 'CN', openWeights: true, availability: 'Preview', metadataSource: 'https://huggingface.co/Qwen/Qwen3.8-27B' }),
+  model('jev', 'Jev 1.13', 'TypeSafe', .042, 0, 'https://docs.typesafe.ai/models', [], { outputType: 'decision' }),
+];
+export const CONFIGURED_SERVICES = ['OpenAI', 'Anthropic', 'Google', 'Groq', 'Baseten', 'Mistral'];
+const byId = id => CATALOG.find(item => item.id === id);
+// Every workload is fictional application work. Names are plausible declarations,
+// never represented as discoveries from a visitor's repository.
+const APPLICATION_CODE_PROFILES = {
+  'actuarial.queries': { input: 14000, output: 700 },
+  'portfolio.reporting': { input: 18000, output: 900 },
+  'claims.reconciliation': { input: 5000, output: 2000 },
+  'reserves.analysis': { input: 6000, output: 2400 },
+};
+export const WORKLOADS = [
+  ['claims.intake', 'Claim intake', 'gpt4o', ['parse_notice', 'extract_loss_date', 'normalize_address', 'link_policy', 'check_attachments', 'summarize_notice', 'prepare_case']],
+  ['policy.extraction', 'Policy document extraction', 'gpt54', ['read_schedule', 'extract_limits', 'extract_deductibles', 'identify_endorsements', 'normalize_terms', 'link_sections', 'check_completeness']],
+  ['coverage.review', 'Coverage review support', 'opus45', ['retrieve_wording', 'compare_endorsements', 'locate_exclusions', 'cite_conditions', 'summarize_scope', 'flag_ambiguity', 'prepare_review']],
+  ['claims.correspondence', 'Claim correspondence', 'sonnet46', ['draft_acknowledgment', 'request_documents', 'summarize_progress', 'draft_follow_up', 'check_tone', 'cite_case_notes', 'prepare_handoff']],
+  ['documents.verification', 'Document verification', 'gemini25pro', ['read_estimate', 'extract_line_items', 'match_invoices', 'check_totals', 'compare_versions', 'flag_missing_pages', 'prepare_evidence']],
+  ['renewals.summary', 'Renewal preparation', 'gemini25flash', ['summarize_history', 'compare_schedules', 'list_changes', 'check_dates', 'extract_terms', 'prepare_brief', 'draft_reminder']],
+  ['claims.tagging', 'Claim tagging', 'oss120', ['tag_document', 'label_channel', 'extract_reference', 'normalize_category', 'link_attachment', 'label_language', 'index_summary']],
+  ['fraud.review', 'Fraud review support', 'glm53', ['summarize_signals', 'link_evidence', 'compare_statements', 'flag_inconsistency', 'build_timeline', 'prepare_case_note', 'route_to_reviewer']],
+  ['broker.translation', 'Broker correspondence', 'mistral35', ['detect_language', 'translate_email', 'preserve_policy_terms', 'check_names', 'summarize_request', 'draft_reply', 'prepare_review']],
+  ['claims.routing', 'Decision routing', 'gpt4o', ['route_notice', 'rank_queue', 'classify_document', 'score_completeness', 'flag_duplicates', 'choose_workflow', 'route_exception']],
+  ['actuarial.queries', 'Actuarial query generation', 'sonnet45', ['read_schema', 'select_tables', 'generate_query', 'check_join_keys', 'validate_query', 'explain_result', 'prepare_review']],
+  ['claims.reconciliation', 'Claim reconciliation scripts', 'sonnet45', ['read_line_items', 'normalize_columns', 'generate_transform', 'compare_totals', 'build_checks', 'explain_variance', 'prepare_review']],
+  ['portfolio.reporting', 'Portfolio reporting queries', 'sonnet45', ['read_report_spec', 'map_dimensions', 'generate_query', 'check_aggregates', 'validate_periods', 'explain_result', 'prepare_report']],
+  ['reserves.analysis', 'Reserve analysis scripts', 'sonnet45', ['read_triangle', 'normalize_periods', 'generate_calculation', 'check_assumptions', 'build_sensitivity', 'explain_projection', 'prepare_review']],
+].map(([id, purpose, before, tasks]) => ({
+  id, purpose, before: byId(before), configuredServices: CONFIGURED_SERVICES, tasks,
+  benchmarkProfile: APPLICATION_CODE_PROFILES[id] ? 'Application code · tools, code generation, reasoning' : 'General application · tools, capabilities, reasoning',
+  tokenProfile: APPLICATION_CODE_PROFILES[id] ?? null,
+  requiredAxes: ['tools', 'capabilities', 'reasoning', ...(id === 'claims.routing' ? ['decisions'] : [])],
+}));
+
+export function modeledCost(model, usage) {
+  return model && [model.input, model.output, usage.inputTokens, usage.outputTokens].every(value => Number.isFinite(value) && value >= 0)
+    ? (model.input * usage.inputTokens + model.output * usage.outputTokens) / 1e6 : null;
+}
+export function assessReplacement(workload, candidate, usage) {
+  const before = workload.before;
+  const checks = workload.requiredAxes.flatMap(axis => axis === 'decisions' ? ['decisionAccuracy', 'calibration'] : [axis]).flatMap(axis => {
+    const baselines = before.benchmarks.filter(item => item.axis === axis);
+    const pairs = baselines.flatMap(a => candidate.benchmarks.filter(b => b.axis === axis && a.suite === b.suite && a.name === b.name && a.unit === b.unit && a.direction === b.direction).map(b => ({ a, b })));
+    if (!pairs.length) return [{ axis, before: baselines[0] ?? null, after: candidate.benchmarks.find(item => item.axis === axis) ?? null, status: 'missing' }];
+    // Check every matched metric, not the first passing score in a cohort.
+    return pairs.map(({ a, b }) => {
+      const comparable = Boolean(a.source && b.source && Number.isFinite(a.value) && Number.isFinite(b.value));
+      const pass = comparable && (a.direction === 'lower' ? b.value <= a.value : b.value >= a.value);
+      return { axis, before: a, after: b, status: !comparable ? 'missing' : pass ? 'pass' : 'regression' };
+    });
+  });
+  const beforeCost = modeledCost(before, usage), afterCost = modeledCost(candidate, usage);
+  const cheaper = beforeCost !== null && afterCost !== null && afterCost < beforeCost;
+  // Only paired latency measurements with the same workload/configuration qualify.
+  // No latency data is fabricated or inferred from token generation speed in this demo.
+  const a = before.latency, b = candidate.latency;
+  const faster = Boolean(a && b && a.suite && a.metric && a.suite === b.suite && a.metric === b.metric && a.source && b.source && Number.isFinite(a.value) && Number.isFinite(b.value) && b.value > 0 && b.value < a.value);
+  const compatible = (workload.configuredServices ?? [before.provider]).includes(candidate.provider) && candidate.outputType === before.outputType && afterCost !== null;
+  const issues = [];
+  if (!(workload.configuredServices ?? [before.provider]).includes(candidate.provider)) issues.push('Service is not configured in this environment');
+  if (candidate.outputType !== before.outputType) issues.push('Different output contract; application adaptation required');
+  if (checks.some(check => check.status === 'missing')) issues.push('Comparable benchmark evidence is incomplete');
+  if (checks.some(check => check.status === 'regression')) issues.push('A required benchmark regresses');
+  if (afterCost === null) issues.push('Token price is unavailable');
+  if (!cheaper && !faster) issues.push('No verified cost or latency improvement');
+  return { candidate, checks, cheaper, faster, beforeCost, afterCost, compatible, issues, eligible: issues.length === 0 };
+}
+export function matchesPreference(model, preference) {
+  return preference === 'zdr' ? model.confirmedZdr === true : preference === 'open' ? model.openWeights === true : preference === 'closed' ? model.openWeights === false : preference === 'usa' ? model.origin === 'US' : preference === 'china' ? model.origin === 'CN' : true;
+}
+export function selectReplacement(workload, usage, preference = 'default') {
+  preference = preferenceFor(preference).value;
+  const assessments = (workload.candidates ?? CATALOG).filter(candidate => candidate.id !== workload.before.id).map(candidate => assessReplacement(workload, candidate, usage));
+  const allowed = item => item.eligible && (preference !== 'closed' || item.candidate.openWeights === false);
+  const byCost = (a, b) => (a.afterCost ?? Infinity) - (b.afterCost ?? Infinity) || a.candidate.id.localeCompare(b.candidate.id);
+  const qualified = assessments.filter(allowed).sort(byCost);
+  const defaultChoice = assessments.filter(item => item.eligible).sort(byCost)[0]?.candidate ?? workload.before;
+  // A preference first seeks a fully qualified match. Retaining an already
+  // matching baseline needs no exception. Only then relax benchmark/efficiency
+  // criteria, never the configured service, output contract or known pricing.
+  let selected = preference === 'default' ? qualified[0] : qualified.find(item => matchesPreference(item.candidate, preference));
+  if (!selected && preference !== 'default' && !matchesPreference(workload.before, preference)) {
+    const axes = (item, status) => new Set(item.checks.filter(check => check.status === status).map(check => check.axis)).size;
+    selected = assessments.filter(item => item.compatible && matchesPreference(item.candidate, preference)).sort((a, b) =>
+      axes(a, 'regression') - axes(b, 'regression') || axes(a, 'missing') - axes(b, 'missing') ||
+      Number(!a.cheaper && !a.faster) - Number(!b.cheaper && !b.faster) || byCost(a, b))[0];
+    if (!selected && preference !== 'closed') selected = qualified[0];
+  }
+  const proposed = selected?.candidate ?? workload.before;
+  const unresolved = preference === 'closed' && proposed.openWeights !== false;
+  const after = unresolved ? null : proposed;
+  const override = Boolean(after && selected && !selected.eligible);
+  const overrideReasons = override ? selected.issues : [];
+  const serviceChanged = Boolean(after && after.provider !== workload.before.provider);
+  const modelChanged = Boolean(after && after.modelId !== workload.before.modelId);
+  return {
+    id: workload.id, purpose: workload.purpose, benchmarkProfile: workload.benchmarkProfile, tokenProfile: workload.tokenProfile, before: workload.before, after,
+    changed: Boolean(after && selected), serviceChanged, modelChanged,
+    checks: selected?.checks ?? [], assessments, qualifiedCount: qualified.length,
+    preferenceMatched: Boolean(after && matchesPreference(after, preference)),
+    preferenceChanged: Boolean(after && after.id !== defaultChoice.id),
+    defaultChoice: defaultChoice.id, unresolved, override, overrideReasons,
+    reason: unresolved ? 'No compatible closed-weight assignment in the reviewed catalog. Service, output contract and known pricing remain required.'
+      : override ? `Selected for ${preferenceFor(preference).label}. No preferred replacement passed every primary check.`
+      : selected ? `${modelChanged ? 'Selected benchmark scores match or improve' : 'Same model checkpoint and published benchmark scores; service changes'}; ${selected.cheaper ? 'modeled token cost is lower' : 'paired latency is lower'}.`
+      : preference !== 'default' && matchesPreference(workload.before, preference) ? 'Baseline retained: already matches this preference; no qualifying preferred replacement.'
+      : 'Baseline retained. No reviewed candidate passes every required check.',
+  };
+}

package/dist/terminal/src/enterprise-sample.mjs

@@ -0,0 +1,151 @@
+import { sampleSnapshot } from './sample.mjs';
+import { WORKLOADS, CATALOG, CONFIGURED_SERVICES, PREFERENCES, preferenceFor, selectReplacement } from './demo-policy.mjs';
+
+// Published standard, uncached text-token rates in USD per million tokens.
+// This catalog belongs to the fictional hosted demonstration, never live billing.
+export const PRICE_DATE = '2026-10-10';
+const openai = model => `https://developers.openai.com/api/docs/models/${model}`;
+const anthropic = 'https://platform.claude.com/docs/en/about-claude/pricing';
+const google = 'https://ai.google.dev/gemini-api/docs/pricing';
+const groq = 'https://console.groq.com/docs/models';
+const baseten = 'https://www.baseten.co/pricing/';
+const mistral = 'https://docs.mistral.ai/inference/pricing';
+const rate = (name, provider, input, output, source, creator = provider) => ({ name, provider, input, output, source, creator });
+export const PRICES = [
+  rate('GPT-5.4', 'OpenAI', 2.5, 15, openai('gpt-5.4')),
+  rate('GPT-5.4 mini', 'OpenAI', .75, 4.5, openai('gpt-5.4-mini')),
+  rate('GPT-5.4 nano', 'OpenAI', .2, 1.25, openai('gpt-5.4-nano')),
+  rate('Claude Opus 4.6', 'Anthropic', 5, 25, anthropic),
+  rate('Claude Sonnet 4.6', 'Anthropic', 3, 15, anthropic),
+  rate('Claude Haiku 4.5', 'Anthropic', 1, 5, anthropic),
+  rate('Gemini 2.5 Pro', 'Google', 1.25, 10, google),
+  rate('Gemini 2.5 Flash', 'Google', .3, 2.5, google),
+  rate('Gemini 2.5 Flash-Lite', 'Google', .1, .4, google),
+  rate('gpt-oss-120b', 'Groq', .15, .6, groq, 'OpenAI'),
+  rate('gpt-oss-20b', 'Groq', .075, .3, groq, 'OpenAI'),
+  rate('GLM-5.3', 'Baseten', 1.4, 4.4, baseten, 'Z.ai'),
+  rate('GLM-5.3-Flash', 'Baseten', .15, .5, baseten, 'Z.ai'),
+  rate('Mistral Medium 3.5', 'Mistral', 1.5, 7.5, mistral),
+  rate('Mistral Small 4', 'Mistral', .15, .6, mistral),
+];
+export function tokenCost(price, input, output) {
+  if (!price || !Number.isFinite(input) || !Number.isFinite(output)) return null;
+  return (input * price.input + output * price.output) / 1_000_000;
+}
+const priceFor = (provider, model) => PRICES.find(price => price.provider === provider && price.name === model);
+const REQUESTS_PER_GROUP = 1000;
+
+// A bounded set of explicit synthetic groups, not millions of duplicated rows.
+// Tokens and cost are group totals. The request count is sample-only metadata.
+export function enterpriseSnapshot(now = Date.now()) {
+  const data = sampleSnapshot(now);
+  data.simulation = { kind: 'enterprise-groups', days: 30, requestsPerGroup: REQUESTS_PER_GROUP, priceDate: PRICE_DATE, prices: PRICES };
+  data.project = { name: 'enterprise-sample', scope: 'Fictional enterprise environment' };
+  const replacements = new Map([
+    ['GPT-5', priceFor('OpenAI', 'GPT-5.4')], ['GPT-5 mini', priceFor('OpenAI', 'GPT-5.4 mini')], ['GPT-5 nano', priceFor('OpenAI', 'GPT-5.4 nano')],
+    ['Claude Sonnet 4.5', priceFor('Anthropic', 'Claude Sonnet 4.6')], ['Claude Opus 4.1', priceFor('Anthropic', 'Claude Opus 4.6')], ['Claude Haiku 3.5', priceFor('Anthropic', 'Claude Haiku 4.5')],
+    ['DeepSeek R1', priceFor('Baseten', 'GLM-5.3')], ['Mistral Medium 3', priceFor('Mistral', 'Mistral Medium 3.5')],
+  ]);
+  const rename = (item, field) => { const price = replacements.get(item[field]); if (price) { item[field] = price.name; item.modelCreator = price.creator; } };
+  data.models.forEach(item => rename(item, 'name'));
+  data.deployments.forEach(item => { rename(item, 'model'); if (item.provider === 'Baseten') item.servingMode = 'shared'; });
+  data.records.forEach((record, index) => {
+    rename(record, 'model');
+    record.sampleRequests = REQUESTS_PER_GROUP;
+    record.occurredAt = new Date(now - (now - Date.parse(record.occurredAt)) * 30).toISOString();
+    const shard = index % 5 + 1;
+    record.agentId += `.worker-${shard}`;
+    if (record.agent) record.agent += `.worker-${shard}`;
+    record.taskId += `-partition-${shard}`;
+    record.inputTokens *= REQUESTS_PER_GROUP;
+    if (record.status === 'running') { record.status = 'completed'; record.outputTokens = 800; }
+    if (record.outputTokens != null) record.outputTokens *= REQUESTS_PER_GROUP;
+    record.cacheReadTokens = 0;
+    record.cost = tokenCost(priceFor(record.provider, record.model), record.inputTokens, record.outputTokens);
+    record.costKind = 'estimated';
+    record.requestId = null;
+  });
+  for (const source of data.sources) {
+    if (source.id === 'sample-activity') {
+      source.records = 9_600_000;
+      source.detail = 'Fictional request groups · 30 days';
+      source.note = '9.6 million fictional requests represented by 9,600 groups of 1,000. Tokens and spend are group totals. Published uncached text-token prices are applied where available; other costs are unknown. No application or provider was contacted.';
+    }
+  }
+  return data;
+}
+
+export function demoSnapshot(now = Date.now(), stage = 'before', preference = 'default') {
+  if (!['before', 'after'].includes(stage)) throw new Error('Unknown comparison stage');
+  preference = preferenceFor(preference).value;
+  const data = enterpriseSnapshot(now);
+  data.project = { name: 'insurance-tech-demo', scope: 'Fictional insurance operations application' };
+  const usage = WORKLOADS.map(() => ({ inputTokens: 0, outputTokens: 0 }));
+  data.records.forEach((record, index) => {
+    record.assignmentId = index % WORKLOADS.length;
+    const profile = WORKLOADS[record.assignmentId].tokenProfile;
+    if (profile) {
+      // Fixed fictional application profiles, independent of stage/preference.
+      // Schema-heavy SQL prompts and longer reconciliation scripts have distinct
+      // input/output mixes. Both include any billed reasoning output.
+      const variation = .85 + (index * 17 % 31) / 100;
+      record.inputTokens = Math.round(profile.input * variation) * REQUESTS_PER_GROUP;
+      record.outputTokens = Math.round(profile.output * variation) * REQUESTS_PER_GROUP;
+    }
+    if (!record.outputTokens) record.outputTokens = (80 + index * 37 % 1600) * REQUESTS_PER_GROUP;
+    usage[record.assignmentId].inputTokens += record.inputTokens;
+    usage[record.assignmentId].outputTokens += record.outputTokens;
+  });
+  const plans = Object.fromEntries(PREFERENCES.map(({ value }) => [value, WORKLOADS.map((workload, index) => selectReplacement(workload, usage[index], value))]));
+  const assignments = plans[preference];
+  const signature = pairs => JSON.stringify(pairs.map(pair => pair.after?.id ?? null));
+  const equivalentPreferences = Object.keys(plans).filter(value => value !== preference && signature(plans[value]) === signature(assignments));
+  const feasible = assignments.every(pair => !pair.unresolved);
+  data.simulation = { ...data.simulation, kind: 'comparison-groups', stage, preference, feasible, equivalentPreferences, catalogSize: CATALOG.filter(model => CONFIGURED_SERVICES.includes(model.provider)).length, configuredServices: CONFIGURED_SERVICES, assignments, prices: [...new Map(assignments.flatMap(pair => [pair.before, pair.after]).filter(Boolean).map(model => [model.id, model])).values()] };
+  const models = new Map(), deployments = new Map();
+  data.records.forEach((record, index) => {
+    const assignmentId = record.assignmentId, assignment = assignments[assignmentId], workload = WORKLOADS[assignmentId];
+    const price = stage === 'after' ? assignment.after ?? assignment.before : assignment.before, sequence = Math.floor(index / WORKLOADS.length);
+    const worker = String(sequence % 20 + 1).padStart(2, '0');
+    record.workload = `services/${workload.id.replaceAll('.', '/')}`;
+    record.agentId = record.agent = `${workload.id}.worker-${worker}`;
+    record.task = workload.tasks[sequence % workload.tasks.length];
+    record.taskId = `${workload.id}.${record.task}.batch-${Math.floor(sequence / workload.tasks.length) % 10 + 1}`;
+    record.model = price.name; record.modelCreator = price.creator; record.provider = price.provider;
+    record.location = 'cloud'; record.status = 'completed'; record.durationMs = null;
+    record.cost = tokenCost(price, record.inputTokens, record.outputTokens);
+    record.beforeCost = tokenCost(assignment.before, record.inputTokens, record.outputTokens);
+    record.afterCost = tokenCost(assignment.after, record.inputTokens, record.outputTokens);
+    record.routeKind = ['direct', 'router', 'gateway'][index % 3];
+    record.route = record.routeKind === 'direct' ? `${price.provider} API` : record.routeKind === 'router' ? 'insurance.router' : 'insurance-gateway';
+    record.path = [{ name: record.route, kind: record.routeKind }];
+    if (record.routeKind !== 'direct') record.path.push({ name: `${price.provider} API`, kind: 'direct' });
+    record.deploymentId = `demo-deployment-${assignmentId}`;
+    if (stage === 'after' && !assignment.after) return;
+    models.set(price.id, { id: `demo-model-${price.id}`, name: price.name, provider: price.provider, modelCreator: price.creator, sourceId: 'sample-activity', location: 'cloud', state: 'observed' });
+    deployments.set(assignmentId, { id: record.deploymentId, name: workload.id, model: price.name, modelCreator: price.creator, provider: price.provider, location: 'cloud', servingMode: 'shared', state: 'configured', sourceId: 'inventory', poolIds: [`pool-${price.provider.toLowerCase()}`] });
+  });
+  data.models = [...models.values()]; data.deployments = [...deployments.values()];
+  const providers = CONFIGURED_SERVICES;
+  data.inventory = [...providers.map(name => ({ name, kind: 'provider', location: 'cloud', state: 'configured', sourceId: 'inventory' })), { name: 'insurance.router', kind: 'router', location: 'cloud', state: 'configured' }, { name: 'insurance-gateway', kind: 'gateway', location: 'cloud', state: 'configured' }];
+  data.capacityPools = data.capacityPools.filter(pool => providers.some(name => pool.id === `pool-${name.toLowerCase()}`));
+  data.hosts = []; data.runtimeStats = []; delete data.machine;
+  data.sources = data.sources.filter(source => ['sample-activity', 'sample-capacity', 'inventory'].includes(source.id));
+  data.sources.find(source => source.id === 'sample-capacity').note = 'Fictional provider quota readings, held identical in both stages. They do not establish a change in latency, throughput or available capacity.';
+  data.sources.find(source => source.id === 'sample-activity').note += ' Insurance workflow and worker names are fictional declarations. Assignments distinguish qualified replacements, retained baselines and explicit preference overrides. Overrides do not pass all primary criteria. Application quality and latency have not been measured.';
+  data.simulation.windows = Object.fromEntries([0, 1, 6, 24, 168, 720].map(hours => {
+    const rows = data.records.filter(record => !hours || Date.parse(record.occurredAt) >= now - hours * 3600000);
+    const totals = selected => ({ requests: selected.length * REQUESTS_PER_GROUP, assignedRequests: selected.filter(record => assignments[record.assignmentId].after).length * REQUESTS_PER_GROUP, qualifiedRequests: selected.filter(record => assignments[record.assignmentId].after && !assignments[record.assignmentId].override).length * REQUESTS_PER_GROUP, overrideRequests: selected.filter(record => assignments[record.assignmentId].override).length * REQUESTS_PER_GROUP, beforeCost: selected.reduce((n, r) => n + r.beforeCost, 0), afterCost: feasible ? selected.reduce((n, r) => n + r.afterCost, 0) : null });
+    return [hours, { ...totals(rows), workloads: assignments.map((pair, index) => {
+      const selected = rows.filter(record => record.assignmentId === index);
+      return { ...totals(selected), afterCost: pair.after ? selected.reduce((n, r) => n + tokenCost(pair.after, r.inputTokens, r.outputTokens), 0) : null };
+    }) }];
+  }));
+  if (stage === 'after' && !feasible) {
+    data.records = data.records.filter(record => assignments[record.assignmentId].after);
+    const source = data.sources.find(source => source.id === 'sample-activity');
+    source.records = data.records.length * REQUESTS_PER_GROUP;
+    source.note += ' Incomplete After plan: only workloads with an assignment are included; preference overrides are identified in comparison metadata. Unresolved workloads remain in comparison metadata, without an After assignment or cost.';
+  }
+  return data;
+}

package/docs/adapter-guide.md

@@ -67,7 +67,7 @@ const coordinator = new OpenMeritCoordinator(
 
 The confirmed task profile identifies the application LLM call under evaluation. Every metric, assessment, candidate, frontier artifact, and automation signal must carry that target ID; the harness's own model usage is orchestration data and must not be reported as product evidence. Application-provider cost, token, and latency telemetry belongs on the application signal or metric record.
 
-In the unreleased source checkout, setup may return `preliminaryModelLeads`: model IDs with sourced price estimates and reputation summaries. The field is optional under protocol 0.1 for older adapters. Core stores the leads and includes both the confirmed task profile and these leads in later discovery intent constraints, so a headless harness run does not depend on prior chat context. Leads are research priors, never measured candidate assessments.
+Setup may return `preliminaryModelLeads`: model IDs with sourced price estimates and reputation summaries. The field is optional under protocol 0.1 for older adapters. Core stores the leads and includes both the confirmed task profile and these leads in later discovery intent constraints, so a headless harness run does not depend on prior chat context. Leads are research priors, never measured candidate assessments.
 
 ## External telemetry export
 
@@ -143,7 +143,7 @@ await coordinator.recordTaskObservation({
   modelId: "vendor/model",
 });
 
-// In the unreleased source checkout, core has already checked baseline
+// Core has already checked baseline
 // sufficiency when the confirmed cadence became due. Inspect durable state
 // and dispatch only a pending downstream intent, if any.
 ```

package/docs/agent-evaluations.md

@@ -0,0 +1,101 @@
+# Evaluate CLI changes across coding agents
+
+**npm preview `0.1.7-preview.1`.** Available under the `preview` tag; stable `openmerit@0.1.6` does not include these commands. This evaluates the coding agent with your CLI and documentation. The [Pi workflow](getting-started.md) still evaluates application inference; the [Inference Terminal](inference-terminal.md) remains read-only.
+
+One local runner supports Claude Code, Codex and OpenCode. Scout supplies candidates; local task checks measure each model/agent/CLI combination. Installing npm alone does not install or authenticate agents, modify project instructions, or start paid evaluations.
+
+## Set up
+
+Install the preview and initialize it inside the project you want to evaluate:
+
+```sh
+npm install -g openmerit@preview
+openmerit eval init
+openmerit eval doctor
+```
+
+Init creates `.openmerit/eval.json`, a tiny extraction fixture, a tool and a standalone grader. It appends marked instructions to `AGENTS.md` and `CLAUDE.md`, preserving existing contents. Codex/OpenCode can read `AGENTS.md`; Claude Code reads `CLAUDE.md`. The instructions ask the agent to configure representative tasks, choose scope, obtain the user's budget, and rerun after CLI changes. These are instructions the host must follow, not guaranteed hooks. There are no install-time hooks or background services. A second init refuses to overwrite the suite.
+
+Replace the starter before drawing product conclusions. Have your agent author representative fixtures and graders with you, and reserve fresh cases for final validation.
+
+## Choose scope and connections
+
+```sh
+openmerit eval plan
+```
+
+The first interactive plan asks: **existing project connections or all models Scout indexes?** The choice is saved. Noninteractive agents must ask the user and pass `--scope existing` or `--scope all`; missing input never defaults to all.
+
+Add explicit routes to `connections` in the config, for example:
+
+```json
+{
+  "id": "claude-project",
+  "agent": "claude",
+  "scoutId": "anthropic/claude-haiku-4.5",
+  "model": "claude-haiku-4-5-20251001",
+  "env": ["ANTHROPIC_API_KEY"]
+}
+```
+
+`scoutId` is Scout's catalogue ID. `model` is the native identifier your authenticated agent accepts. `command` optionally selects another executable path. `env` lists environment variable **names**, never values. Only basic OS variables and explicitly listed names pass to child agents. Saved authentication remains available through the normal home directory. No dotenv files are loaded automatically. Configure endpoint details through the native agent or named environment variables.
+
+An SDK dependency or key-variable name does not prove access. The agent should inspect the project's intended connections and declare the models they can serve. This preview does not automatically enumerate endpoint catalogues. Multiple connections can test one Scout model through different agents/routes. Prefer pinned model versions; aliases can change.
+
+Existing scope sends IDs and requires Scout to acknowledge filtering **before ranking five**. Fewer than five eligible models remain fewer. All scope searches the index, but missing authenticated routes block execution until configured and replanned. Inaccessible candidates are not silently replaced. Unranked Hugging Face specialists are not executed automatically.
+
+Scout receives only each task's `description`, plus IDs for existing scope. Scout sends descriptions to OpenAI Decisions. Fixtures, execution prompts, endpoint URLs and credentials stay out of this request. The shortlist, warnings and ranking context are frozen locally. Scout scores are screening evidence, not measured success with your harness.
+
+## Tasks and variants
+
+Each task needs `id`, `description` (for Scout, at most 4,000 characters), `prompt` (for execution), `fixture` (a project-relative directory) and `grader` (a standalone Node `.mjs` file).
+
+Each variant needs `id`, `files` and `instructions`. The starter compares `bare` against `cli`. Bare omits your CLI/docs; native agent tools remain available. Across agents you are comparing different harnesses, with different permission systems. A no-native-tools experiment needs a separately controlled configuration.
+
+Selected files, including uncommitted changes, are snapshotted before execution. Variant paths must not overwrite fixture inputs and retain their project-relative locations; fixture contents go at the workspace root. Include self-contained CLI artifacts and lean docs explicitly. Dependencies are not installed automatically; tools must be bundled or have their runtime requirements already available. Symlinks, parent-directory paths, `.env*`, `.git`, `.openmerit` and `node_modules` are rejected. Each selected snapshot is limited to 25 MB. Do not select credentials stored under other filenames.
+
+The grader reads `{ "workspace": "/temporary/path", "output": "final response" }` from stdin and prints exactly `{ "quality": 0.0 }`, with a finite score from 0 to 1. It can check generated files using executable assertions. Keep it self-contained: imports beside it are not copied automatically. Graders are copied outside agent workspaces; never also include them or answer keys in fixtures/variant files. Test graders against correct and incorrect answers. A broken grader stops the experiment with unknown quality; agent failures/timeouts count as quality zero. Recognized authentication or model-access errors instead stop after one attempt with unknown quality, so an unavailable connection is not rated as a weak model.
+
+## Run after changes
+
+The starter's `budget.maxUsd` is zero. Set an approved cumulative budget, then:
+
+```sh
+openmerit eval plan --scope existing
+openmerit eval run
+# Edit your CLI or its documentation.
+openmerit eval run
+```
+
+For repeated local development:
+
+```sh
+openmerit eval watch
+```
+
+Watch runs initially, then after selected source/docs/fixture/grader/config bytes change and remain stable for two polls. Result writes do not trigger it. Runs never overlap; edits during execution become the next revision. Ctrl+C stops the watcher and active child process. No OS schedule or daemon is installed. `OPENMERIT_EVAL=1` tells cooperating agent instructions not to recurse into another evaluation.
+
+Every attempt uses a new workspace and agent session. Each task runs balanced rounds across all selected connections and variants; order rotates and reverses. The default is 3–10 repetitions. After three, rounds continue while any group's quality range exceeds 5 percentage points, an attempt failed, or cost, input-token or latency variation has a coefficient of variation above 0.25. Ten is the maximum. Missing metrics cannot establish stability. Set `repeats.min` equal to `repeats.max` for fixed repetitions, useful for holdouts. This heuristic is not a significance test, and repetition is not task diversity.
+
+The frozen plan survives CLI changes. Changes to descriptions, scope or connections require replanning. Previous-revision comparisons require matching prompts, fixtures, graders, routes, repeat settings, selected candidates, variant IDs, timeouts, agent versions, runner versions, invocation settings and reported model identities. CLI/docs bytes and variant instructions are the interventions. Keep external services and native agent settings stable too; they are not hermetically captured.
+
+## Metrics
+
+```sh
+openmerit eval report
+openmerit eval report --json
+```
+
+Reports show quality mean/range, failures, mean input/output tokens, mean reported dollars and median full-process seconds per task/connection/variant. They retain Scout ranks but do not force a weighted winner. Lower cost with lower quality is a tradeoff. Matching complete revisions show deltas; partial/invalid experiments do not.
+
+Claude provides aggregate usage and reported cost. Codex JSON events provide tokens but not billed dollars, so cost stays unknown. OpenCode per-step usage/cost is summed across the episode, including tools. Input totals include separately reported cache reads/writes; cached tokens are retained separately without counting them twice. OpenCode output totals include separately reported reasoning tokens. Agent accounting and subscriptions differ. Reported costs are not reconciled provider bills; reported zero may mean missing pricing metadata. Catalogue prices never replace unknown measured costs.
+
+Preflight requires supported noninteractive JSON interfaces: Claude print JSON, Codex `exec --json --ephemeral --sandbox workspace-write`, and OpenCode `run --format json --auto`. OpenCode's auto option approves tool requests that are not explicitly denied, avoiding an interactive approval stall. Claude uses an explicit local tool allowlist without inherited MCP configuration; Codex and OpenCode retain native global settings. Recognized permission failures stop as connection problems, not model-quality evidence. OpenMerit does not add a sandbox around Claude/OpenCode. The old 2025 Codex CLI is unsupported. Native references: [Claude Code](https://code.claude.com/docs/en/headless), [Codex](https://developers.openai.com/codex/noninteractive), [OpenCode](https://opencode.ai/docs/cli/).
+
+## Budget and data
+
+`budget.maxUsd` applies cumulatively across runs and restarts. Before each attempt the runner persists `reservationUsd`. A known reported cost replaces it; unknown/interrupted charges keep at least the full reservation. Partial cost events from an unsuccessful process never count as a complete bill. Insufficient remaining allowance stops a partial run. Raising maxUsd extends the allowance, not resets it. Do not delete unknown spend from the ledger. This is admission control, not a billing cutoff: one attempt may exceed its reservation. Claude also receives its native maximum-budget flag. Use provider-side limits for hard monetary caps. Each attempt also has a timeout and 4 MB process-output limit.
+
+`.openmerit/eval/` holds `plan.json`, `budget.json`, `latest.json`, `runs/*.json` and `lock/pid`. Init ignores it in Git. Reports contain identities, metrics, hashes and status, not full transcripts or final outputs. Temporary files are removed afterward. Native agents/providers may retain their own data; Claude/Codex receive ephemeral flags, while OpenCode uses native storage. `.openmerit/eval.json` can be versioned without credentials.
+
+Fresh workspaces are not security sandboxes. Trusted agents and graders execute with native permissions and user-level resources. Hard kills may leave temporary directories. After a crash, inspect the lock's PID and remove it only if that process is gone. This preview does not change Pi approvals, swap production models or upload results into Scout rankings.

package/docs/architecture.md

@@ -38,7 +38,7 @@ Contains deterministic behavior: choosing the next lifecycle intent according to
 
 `OpenMeritCoordinator` is the reusable adapter boundary. It constructs canonical intents, negotiates capabilities, records lifecycle progress and observations, validates results, and advances durable state. `OpenMeritStore` makes persistence replaceable; `ProjectStore` is the bundled filesystem backend.
 
-The coordinator also accepts idempotent application-target automation signals and evaluates the user-confirmed `AutomationCheckPolicy`. `buildAutomationPlan` tells an adapter which signals and wakeups are required and reports unsupported persistent scheduling honestly. OpenMerit never installs a scheduler or creates a paid cloud worker itself.
+The coordinator also accepts idempotent application-target automation signals and evaluates the user-confirmed `AutomationCheckPolicy`. `buildAutomationPlan` tells an adapter which signals and wakeups are required and reports unsupported persistent scheduling honestly. Provisioning belongs to the adapter: Pi can install an authorized macOS user LaunchAgent; the coordinator itself does not create operating-system schedulers or paid cloud workers.
 
 ### `openmerit/pi`
 
@@ -46,9 +46,9 @@ Exposes the command surface, supplies private typed intent context, triggers Pi,
 
 The Pi package implements `HarnessAdapter`. It does not define canonical intents, per-intent result contracts, stage transitions, or validation rules.
 
-### Inference Terminal (0.1.6)
+### Inference Terminal
 
-`packages/terminal` supplies a standalone `openmerit dash` executable, passive collectors, a loopback read-only HTTP API, and bundled dark-only browser assets. It does not invoke the coordinator or harness. One collector cycle reads existing metadata and runtime inventory; browser clients receive the same cached snapshot. Imported activity, runtime counters, and sampled whole-machine resources retain their separate scopes. Missing measurements remain unknown, and fictional sample data is isolated from the real snapshot. See the [terminal guide](inference-terminal.md) for the supported sources and JSON contract.
+`packages/terminal` supplies a standalone `openmerit dash` executable, passive collectors, a loopback read-only HTTP API, and bundled browser assets. Stable npm 0.1.6 includes the terminal. The source interface and hosted dashboard follow the device’s light/dark preference; that appearance change is not included in npm 0.1.6. It does not invoke the coordinator or harness. One collector cycle reads existing metadata and runtime inventory; browser clients receive the same cached snapshot. Imported activity, runtime counters, and sampled whole-machine resources retain their separate scopes. Missing measurements remain unknown, and fictional sample data is isolated from the real snapshot. See the [terminal guide](inference-terminal.md) for the supported sources and JSON contract.
 
 ## Intelligence resources
 

package/docs/automation.md

@@ -28,7 +28,7 @@ No work due, or one typed OpenMerit intent
 
 Task-count and elapsed-time triggers use OR semantics. A policy with `afterCompletedTasks: 100` and `afterElapsedSeconds: 604800` becomes due at the first threshold reached. A scheduled wakeup does not directly launch expensive work: it submits `scheduled_tick`, and OpenMerit evaluates policy and durable state before issuing an intent.
 
-In the unreleased source checkout, the baseline elapsed-time clock starts when setup or required instrumentation finishes. Ordinary task and metric signals do not restart that clock. OpenMerit retains the latest reported aggregate window for each application metric, including sample count and evidence references. It never sums possibly overlapping windows. At a due baseline check, core requires ready instrumentation and a finite measured value, enough samples, and an evidence reference for every required objective. An insufficient check records the gaps and waits for more data; sufficient evidence advances to candidate discovery.
+The baseline elapsed-time clock starts when setup or required instrumentation finishes. Ordinary task and metric signals do not restart that clock. OpenMerit retains the latest reported aggregate window for each application metric, including sample count and evidence references. It never sums possibly overlapping windows. At a due baseline check, core requires ready instrumentation and a finite measured value, enough samples, and an evidence reference for every required objective. An insufficient check records the gaps and waits for more data; sufficient evidence advances to candidate discovery.
 
 ## Signals
 
@@ -65,10 +65,10 @@ Subsequent lifecycle transitions continue through candidate trials, frontier cal
 
 ## Pi behavior
 
-Pi reports catalogue and scheduled-tick signals on session start. Application instrumentation submits product-task, metric-window, and post-swap verification signals through its typed `openmerit_report_signal` tool; Pi's own settled-turn telemetry is not treated as product evidence. Published `openmerit@0.1.5` cannot provision persistent background wakeups and reports an external-scheduler gap.
+Pi reports catalogue and scheduled-tick signals on session start. Application instrumentation submits product-task, metric-window, and post-swap verification signals through its typed `openmerit_report_signal` tool; Pi's own settled-turn telemetry is not treated as product evidence.
 
-In the unreleased source checkout on macOS, a confirmed policy with `execution.mode: "persistent"` and `infrastructureChangesAllowed: true` allows the Pi adapter to install a per-project user LaunchAgent. It uses the model selected in the interactive Pi session and checks that Pi authentication works without shell-only credentials. The agent checks local state every five minutes without calling a model. When a baseline check is due, the runner submits `scheduled_tick` directly to core: an insufficient result exits without starting Pi; a sufficient result starts Pi for candidate discovery. The Mac must be logged in; this is not a cloud service, an exact-time alarm, or a source of product metrics. Task-count and metric-window checks still depend on application signals.
+On macOS, a confirmed policy with `execution.mode: "persistent"` and `infrastructureChangesAllowed: true` allows the Pi adapter to install a per-project user LaunchAgent. It uses the model selected in the interactive Pi session and checks that Pi authentication works without shell-only credentials. The agent checks local state every five minutes without calling a model. When a baseline check is due, the runner submits `scheduled_tick` directly to core: an insufficient result exits without starting Pi; a sufficient result starts Pi for candidate discovery. The Mac must be logged in; this is not a cloud service, an exact-time alarm, or a source of product metrics. Task-count and metric-window checks still depend on application signals.
 
-This background path is experimental in the source checkout. A local runner test verifies that an insufficient baseline check needs no Pi process. An earlier live OpenRouter/Pi test reached the former assessment intent and made tool calls, but did not complete it within the bounded window. Candidate discovery, comparison, and swapping from a background Pi run are not yet product-validated. A background Pi run is capped at 15 minutes and writes its exit code and any still-active intent ID to `.openmerit/scheduler-status.json`; an unfinished intent requires inspection and explicit recovery in Pi.
+Background candidate work remains experimental. A local runner test verifies that an insufficient baseline check needs no Pi process. Candidate discovery, comparison, and swapping from a background Pi run are not yet product-validated. A background Pi run is capped at 15 minutes and writes its exit code and any still-active intent ID to `.openmerit/scheduler-status.json`; an unfinished intent requires inspection and explicit recovery in Pi.
 
-`/openmerit pause` in this source checkout persists the pause, unloads the project's LaunchAgent, and prevents automatic due decisions while still recording incoming signals. `/openmerit resume` clears the pause and restores an authorized schedule. Neither command cancels already-active work. A declined or failed scheduler installation is reported as a gap, not success. On other platforms, persistent policies still require an external scheduler.
+`/openmerit pause` persists the pause, unloads the project's LaunchAgent, and prevents automatic due decisions while still recording incoming signals. `/openmerit resume` clears the pause and restores an authorized schedule. Neither command cancels already-active work. A declined or failed scheduler installation is reported as a gap, not success. On other platforms, persistent policies still require an external scheduler.

package/docs/budgets.md

@@ -16,7 +16,7 @@ OpenMerit has no subscription fee. Model-provider charges still apply to setup,
 | `maximumRunsPerCandidate` | Optional limit on runs per candidate. |
 | `expiresAt` | Optional expiration for the evaluation budget. |
 
-In the unreleased development version, OpenMerit keeps a separate ledger for controlled challenger trials. Each challenger intent receives the amount already spent and the remaining allowance. Challenger results that exceed the approved limit are rejected, and another challenger intent cannot start after the limit is exhausted. Provider-side spending controls remain the strongest protection against a request that exceeds its estimate while it is running.
+OpenMerit keeps a separate ledger for controlled challenger trials. Each challenger intent receives the amount already spent and the remaining allowance. Challenger results that exceed the approved limit are rejected, and another challenger intent cannot start after the limit is exhausted. Provider-side spending controls remain the strongest protection against a request that exceeds its estimate while it is running.
 
 Setup, baseline product work, assessments, and swap verification remain visible costs, but do not consume `evaluationBudget.maximumSpend`. This separation prevents normal product usage from silently exhausting the allowance reserved for comparing challengers.
 
@@ -30,7 +30,7 @@ Post-swap evaluation remains part of the workflow. The harness performs the chan
 
 Persistent scheduling is a separate permission. An adapter that provisions infrastructure must declare that capability, and the confirmed check policy must allow infrastructure changes before OpenMerit asks it to provision a wakeup.
 
-The current Pi adapter does not provide persistent scheduling or background execution. An external scheduler is needed to start Pi for unattended checks. See [automatic checks](automation.md).
+On macOS, the Pi adapter can install a per-project user LaunchAgent when persistent execution and infrastructure changes are both authorized. The user must remain logged in and Pi authentication must work in the background environment. Other platforms require an external scheduler. Background candidate work still needs representative product validation. See [automatic checks](automation.md).
 
 ## What a policy can establish