openmerit 0.1.7-preview.1 → 0.1.7-preview.2

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.1.7-preview.2 — 2026-10-15
+
+- Ask for a $10, custom or no-fixed-cap evaluation budget after model scope; save the explicit choice, handle reservations internally, and preserve cumulative usage across reruns and budget changes. Add `eval plan --budget <USD|unlimited>` for coding agents and noninteractive setup.
+- Separate native budget and environment blocks from task scores in agent evaluations; probe the Codex Linux sandbox before making model requests.
+- Recover partial Claude usage, retain partial cost estimates, keep missing native pricing unknown, and preserve budget reservations independently of estimated charges. Report version 2 separates these accounting rules from earlier comparisons.
+
 ## 0.1.7-preview.1 — 2026-10-15
 
 - Add `openmerit eval init/doctor/plan/run/watch/report`: Scout shortlists, explicit model scope, Claude Code/Codex/OpenCode adapters, fresh task attempts, local graders, balanced adaptive 3–10 repetitions, revision comparisons and cumulative budget accounting.

package/README.md

@@ -22,7 +22,7 @@ Explore the [hosted sample](https://dash.openmerit.ai/) or [before/after dashboa
 
 ## Compare models with Pi
 
-The **npm preview `0.1.7-preview.1`** adds `openmerit eval` for evaluating Claude Code, Codex and OpenCode with your CLI. Choose existing connections or Scout's full index, freeze a shortlist, and compare 3–10 fresh attempts after source changes. Install with `npm install -g openmerit@preview`. See [agent evaluations](https://unpkg.com/openmerit@0.1.7-preview.1/docs/agent-evaluations.md) for setup, graders, budgets and availability. This is separate from the Pi workflow below.
+The **npm preview `0.1.7-preview.2`** provides `openmerit eval` for evaluating Claude Code, Codex and OpenCode with your CLI. Choose model scope and a $10, custom or no-fixed-cap budget, freeze Scout's shortlist, and compare 3–10 fresh attempts after source changes. Budget and sandbox blocks stay separate from task scores; unknown costs stay unknown. Install with `npm install -g openmerit@preview`. See [agent evaluations](https://unpkg.com/openmerit@0.1.7-preview.2/docs/agent-evaluations.md) for setup, graders, budgets and availability. This is separate from the Pi workflow below.
 
 With Pi 0.87 and a model provider configured:
 

package/dist/eval/src/adapters.mjs

@@ -2,18 +2,28 @@ import { execute, agentEnvironment } from './process.mjs';
 
 const number = value => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null;
 const sum = values => values.length && values.every(v => v !== null) ? values.reduce((a, b) => a + b, 0) : null;
+const maximum = values => values.some(v => v !== null) ? Math.max(...values.filter(v => v !== null)) : null;
 export const agents = ['claude', 'codex', 'opencode'];
 
 export function connectionFailure(text) {
-  if (/401|unauthorized|authentication[_ -]?(?:error|required)|invalid api key|not logged in|please log in/i.test(text)) return 'authentication';
+  if (/budget_exhausted|error_max_budget_usd|(?:maximum|exceeded|exhausted|insufficient).{0,25}(?:budget|credits)|(?:budget|credits).{0,25}(?:exceeded|exhausted)/i.test(text)) return 'budget_exhausted';
+  if (/(?:bwrap|bubblewrap|sandbox|landlock|seccomp|unshare|user namespace|uid_map).{0,160}(?:failed|not permitted|unavailable|denied)|failed to (?:start|initialize|create).{0,40}sandbox/i.test(text)) return 'sandbox_unavailable';
+  if (/\b(?:HTTP\s*)?401\b.{0,30}(?:unauthorized|authentication)|unauthorized|authentication[_ -]?(?:error|required)|invalid api key|not logged in|please log in/i.test(text)) return 'authentication';
   if (/model.{0,100}(?:not supported|not found|not available|does not exist)|(?:unknown|invalid|unsupported) model|ProviderModelNotFoundError/i.test(text)) return 'model_unavailable';
   if (/rejected permission|permission.{0,30}(?:denied|rejected)|not allowed to use.{0,30}tool/i.test(text)) return 'permission_denied';
   return null;
 }
 
-export function invocation(connection, reservationUsd) {
+function toolFailure(text) {
+  const reason = connectionFailure(text);
+  // A task's own HTTP/auth/budget errors are task outcomes, not evidence that
+  // the coding agent's provider connection failed.
+  return ['sandbox_unavailable', 'permission_denied'].includes(reason) ? reason : null;
+}
+
+export function invocation(connection, reservationUsd, unlimitedBudget = false) {
   const model = connection.model;
-  if (connection.agent === 'claude') return ['-p', '--output-format', 'json', '--model', model, '--no-session-persistence', '--permission-mode', 'dontAsk', '--allowedTools', 'Read,Glob,Grep,Bash,Edit,Write', '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}', '--setting-sources', '', '--max-budget-usd', String(reservationUsd)];
+  if (connection.agent === 'claude') return ['-p', '--output-format', 'json', '--model', model, '--no-session-persistence', '--permission-mode', 'dontAsk', '--allowedTools', 'Read,Glob,Grep,Bash,Edit,Write', '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}', '--setting-sources', '', ...(unlimitedBudget ? [] : ['--max-budget-usd', String(reservationUsd)])];
   if (connection.agent === 'codex') return ['exec', '--json', '--model', model, '--sandbox', 'workspace-write', '--skip-git-repo-check', '--ephemeral', '-'];
   if (connection.agent === 'opencode') return ['run', '--format', 'json', '--model', model, '--auto'];
   throw new Error('Unknown agent. Use claude, codex or opencode.');
@@ -23,7 +33,11 @@ export function parseOutput(agent, stdout) {
   let events;
   try { const parsed = JSON.parse(stdout); events = Array.isArray(parsed) ? parsed : [parsed]; }
   catch { events = stdout.split('\n').flatMap(line => { try { return [JSON.parse(line)]; } catch { return []; } }); }
-  let output = '', usage = null, costUsd = null, completed = false, failed = false;
+  let output = '', usage = null, costUsd = null, partialCostUsd = null, completed = false, failed = false, failureReason = null;
+  // Only native failure fields are evidence of a blocked execution. Task text,
+  // successful tool output and model commentary can legitimately discuss errors.
+  const errors = events.filter(e => ['error', 'turn.failed'].includes(e.type));
+  for (const event of errors) failureReason ??= connectionFailure(JSON.stringify(event));
   const observedModels = new Set();
   if (agent === 'claude') {
     const result = events.findLast(e => e.type === 'result');
@@ -32,8 +46,16 @@ export function parseOutput(agent, stdout) {
       failed = !completed; output = result.result ?? '';
       const u = result.usage;
       if (u) usage = { input: sum([number(u.input_tokens), number(u.cache_read_input_tokens ?? 0), number(u.cache_creation_input_tokens ?? 0)]), output: number(u.output_tokens), cached: number(u.cache_read_input_tokens ?? 0) };
-      costUsd = number(result.total_cost_usd);
+      const models = Object.values(result.modelUsage ?? {});
+      if (models.length) {
+        const modelUsage = { input: sum(models.map(m => sum([number(m.inputTokens), number(m.cacheReadInputTokens ?? 0), number(m.cacheCreationInputTokens ?? 0)]))), output: sum(models.map(m => number(m.outputTokens))), cached: sum(models.map(m => number(m.cacheReadInputTokens ?? 0))) };
+        // A budget stop can emit zero aggregate usage but nonzero per-model usage.
+        usage = Object.fromEntries(Object.keys(modelUsage).map(k => [k, maximum([usage?.[k] ?? null, modelUsage[k]])]));
+      }
+      costUsd = maximum([number(result.total_cost_usd), sum(models.map(m => number(m.costUSD)))]);
       for (const model of Object.keys(result.modelUsage ?? {})) observedModels.add(model);
+      if (failed) failureReason ??= connectionFailure(JSON.stringify({ subtype: result.subtype, terminal_reason: result.terminal_reason, errors: result.errors }));
+      if (result.permission_denials?.length) failureReason ??= 'permission_denied';
     }
   } else if (agent === 'codex') {
     const turns = events.filter(e => e.type === 'turn.completed');
@@ -41,6 +63,9 @@ export function parseOutput(agent, stdout) {
     failed = events.some(e => ['turn.failed', 'error'].includes(e.type));
     output = events.findLast(e => e.type === 'item.completed' && e.item?.type === 'agent_message')?.item.text ?? '';
     if (turns.length) usage = { input: sum(turns.map(e => number(e.usage?.input_tokens))), output: sum(turns.map(e => number(e.usage?.output_tokens))), cached: sum(turns.map(e => number(e.usage?.cached_input_tokens))) };
+    for (const event of events.filter(e => e.type === 'item.completed' && e.item?.type === 'command_execution' && (e.item.exit_code > 0 || e.item.status === 'failed'))) {
+      failureReason ??= toolFailure(event.item.aggregated_output ?? '');
+    }
   } else if (agent === 'opencode') {
     const steps = events.filter(e => e.type === 'step_finish');
     const finalStep = steps.findLast(e => ['stop', 'end_turn'].includes(e.part?.reason));
@@ -50,24 +75,38 @@ export function parseOutput(agent, stdout) {
     output = finalStep?.part?.messageID ? texts.filter(e => e.part?.messageID === finalStep.part.messageID).map(e => e.part?.text ?? '').join('\n') : texts.at(-1)?.part?.text ?? '';
     if (steps.length) {
       usage = { input: sum(steps.map(e => sum([number(e.part?.tokens?.input), number(e.part?.tokens?.cache?.read ?? 0), number(e.part?.tokens?.cache?.write ?? 0)]))), output: sum(steps.map(e => sum([number(e.part?.tokens?.output), number(e.part?.tokens?.reasoning ?? 0)]))), cached: sum(steps.map(e => number(e.part?.tokens?.cache?.read ?? 0))) };
-      costUsd = sum(steps.map(e => number(e.part?.cost)));
+      const costs = steps.map(e => number(e.part?.cost));
+      costUsd = costs.every(c => c !== null && c > 0) ? sum(costs) : null;
+      if (costUsd === null) partialCostUsd = sum(costs.filter(c => c !== null && c > 0));
     }
+    for (const event of events.filter(e => e.type === 'tool_use' && e.part?.state?.status === 'error')) failureReason ??= toolFailure(event.part.state.error ?? '');
   }
-  return { output, usage, costUsd, completed: completed && !failed, observedModels: [...observedModels] };
+  // Native zero prices can mean missing pricing metadata, not free inference.
+  return { output, usage, costUsd: costUsd > 0 ? costUsd : null, partialCostUsd, completed: completed && !failed, failureReason, observedModels: [...observedModels] };
 }
 
-export async function preflight(connection) {
+export async function preflight(connection, { cwd, executeCommand = execute, platform = process.platform } = {}) {
   const command = connection.command ?? connection.agent;
-  const version = await execute(command, ['--version'], { timeoutMs: 10_000, env: agentEnvironment(connection.env) });
-  const help = await execute(command, connection.agent === 'codex' ? ['exec', '--help'] : connection.agent === 'opencode' ? ['run', '--help'] : ['--help'], { timeoutMs: 10_000, env: agentEnvironment(connection.env) });
+  const options = { cwd, timeoutMs: 10_000, env: agentEnvironment(connection.env) };
+  const version = await executeCommand(command, ['--version'], options);
+  const help = await executeCommand(command, connection.agent === 'codex' ? ['exec', '--help'] : connection.agent === 'opencode' ? ['run', '--help'] : ['--help'], options);
   const flags = connection.agent === 'codex' ? ['--json', '--ephemeral', '--sandbox'] : connection.agent === 'claude' ? ['--output-format', '--no-session-persistence', '--max-budget-usd'] : ['--format', '--model', '--auto'];
   if (version.code !== 0 || help.code !== 0 || !flags.every(f => (help.stdout + help.stderr).includes(f))) throw new Error(`${connection.agent} lacks the required non-interactive JSON interface. Update it or set this connection's command to a compatible executable.`);
+  if (connection.agent === 'codex' && platform === 'linux') {
+    const sandboxHelp = await executeCommand(command, ['sandbox', '--help'], options);
+    const legacy = /^\s+linux\s/m.test(sandboxHelp.stdout);
+    const probe = await executeCommand(command, ['-c', 'sandbox_mode="workspace-write"', 'sandbox', ...(legacy ? ['linux'] : []), '--', process.execPath, '-e', 'process.stdout.write("openmerit-sandbox-ok")'], options);
+    if (probe.code !== 0 || probe.reason || probe.stdout !== 'openmerit-sandbox-ok') {
+      throw Object.assign(new Error('Codex sandbox could not run a local probe. Fix host sandbox support before evaluating; no model request was made.'), { failureReason: 'sandbox_unavailable' });
+    }
+  }
   return version.stdout.trim().slice(0, 160);
 }
 
 export async function runAgent(connection, prompt, options) {
-  const result = await execute(connection.command ?? connection.agent, invocation(connection, options.reservationUsd), { ...options, input: prompt, env: agentEnvironment(connection.env) });
+  const result = await execute(connection.command ?? connection.agent, invocation(connection, options.reservationUsd, options.unlimitedBudget), { ...options, input: prompt, env: agentEnvironment(connection.env) });
   const parsed = parseOutput(connection.agent, result.stdout);
-  const status = result.reason ?? (result.code !== 0 || !parsed.completed ? 'agent_error' : 'ok');
-  return { ...parsed, costUsd: status === 'ok' ? parsed.costUsd : null, partialCostUsd: status === 'ok' ? null : parsed.costUsd, seconds: result.seconds, status, failureReason: status === 'agent_error' ? connectionFailure(result.stdout + result.stderr) : null };
+  const failureReason = parsed.failureReason ?? (result.code !== 0 ? connectionFailure(result.stderr) : null) ?? (['spawn_failed', 'cancelled', 'output_limit'].includes(result.reason) ? result.reason : null);
+  const status = failureReason === 'budget_exhausted' ? 'budget_exhausted' : ['sandbox_unavailable', 'permission_denied', 'spawn_failed'].includes(failureReason) ? 'environment_error' : result.reason ?? (failureReason || result.code !== 0 || !parsed.completed ? 'agent_error' : 'ok');
+  return { ...parsed, costUsd: status === 'ok' ? parsed.costUsd : null, partialCostUsd: status === 'ok' ? parsed.partialCostUsd : maximum([parsed.costUsd, parsed.partialCostUsd]), costBasis: parsed.costUsd === null && parsed.partialCostUsd === null ? 'unknown' : 'agent_estimate', seconds: result.seconds, status, failureReason };
 }

package/dist/eval/src/cli.mjs

@@ -11,8 +11,8 @@ export const help = `OpenMerit · Agent evaluations (source preview)
 
   openmerit eval init                 Add a starter suite and project agent instructions
   openmerit eval doctor               Check installed Claude Code, Codex and OpenCode CLIs
-  openmerit eval plan [--scope existing|all]
-                                     Ask scope, then freeze Scout's shortlist per task
+  openmerit eval plan [--scope existing|all] [--budget <USD|unlimited>]
+                                     Choose scope and budget, then freeze Scout's shortlist
   openmerit eval run [--json]         Evaluate the frozen shortlist in fresh workspaces
   openmerit eval watch [--json]       Repeat after selected CLI/docs/fixture files change
   openmerit eval report [--json]      Read the latest result without running models
@@ -21,6 +21,9 @@ export const help = `OpenMerit · Agent evaluations (source preview)
 
 Configure .openmerit/eval.json: explicit agent connections, task fixtures, local
 graders, CLI variants, 3–10 repeats, and a cumulative budget. Init starts at $0.
+Plan asks for $10, a custom amount, or no fixed cap; nothing is preselected.
+Saved choices and prior budget usage persist across reruns. --budget changes
+the total allowance, not the amount remaining. Reservations are automatic.
 Scout receives task descriptions and, for existing scope, model IDs only.
 Run/watch execute local agents and may incur charges. No background daemon.
 `;
@@ -34,9 +37,10 @@ When building or changing this project's CLI, tools or harness documentation, us
 
 - Read .openmerit/eval.json and run \`openmerit eval doctor\`.
 - Ask once: "Models available through this project's existing connections, or all models Scout indexes?" Use \`eval plan --scope existing\` or \`--scope all\`. Reuse the saved choice. Never infer endpoint access from a model name or expose credentials.
+- Then ask once: "What budget should OpenMerit use for this project's evaluations? 1. $10; 2. Custom amount (how much?); 3. No fixed cap." Pass the explicit answer as \`eval plan --budget 10\`, \`--budget <amount>\` or \`--budget unlimited\`, together with scope on first setup. Never choose a default or infer unlimited spending. Reuse an existing positive or unlimited budget unless the user changes it. The allowance is cumulative across reruns, not reset by edits or replanning. Reservations are internal; do not ask the user to calculate them. No fixed cap still has bounded tasks, 3–10 repetitions, timeouts and user-controlled watch; subsequent watched edits can incur further charges until stopped.
 - Declare real connections with Scout model IDs, agent-native model IDs, and environment variable NAMES. Existing scope uses this project-declared inventory. All scope may require new authenticated connections before execution.
 - Replace starter cases with representative tasks, fresh fixtures and deterministic standalone Node graders. A grader reads {workspace, output} JSON on stdin and returns {quality: 0..1}. Keep answer keys outside agent workspaces. Include a bare variant and a CLI/docs variant where useful.
-- Confirm the suite, variants and cumulative budget with the user before enabling paid runs. Then plan once, run a baseline, make the requested CLI changes, and run again. Use \`eval watch\` only when the user wants continuous local reruns; it exits with the terminal and retains budget usage across restarts.
+- Confirm the suite and variants before enabling paid runs, using the budget already selected above. Do not ask again for an unchanged authorized budget. Then plan once, run a baseline, make the requested CLI changes, and run again. Use \`eval watch\` only when the user wants continuous local reruns; it exits with the terminal and retains budget usage across restarts.
 - Report quality, failures, tokens, reported costs and elapsed time per task/model/variant. A shortlist is a prior, not measured success. Do not claim missing costs are zero or that 3–10 repeats prove an exact ordering. Keep fresh holdouts for final validation.
 - If a compatible watcher is already active, let it run; never start overlapping loops. Do not overwrite the frozen plan merely because the CLI changed.
 <!-- openmerit-eval:end -->`;
@@ -61,16 +65,57 @@ export async function initialize(root) {
   if (!ignore.split('\n').includes('/.openmerit/eval/')) await writeFile(join(root, '.gitignore'), `${ignore}${ignore.endsWith('\n') || !ignore ? '' : '\n'}/.openmerit/eval/\n`);
 }
 
+export function parseBudget(value) {
+  const input = value.trim();
+  if (input === 'unlimited') return input;
+  const amount = input.replace(/^\$/, '');
+  if (!/^(?:\d+(?:\.\d{1,2})?|\.\d{1,2})$/.test(amount) || Number(amount) <= 0 || Number(amount) > Number.MAX_SAFE_INTEGER / 100) throw new Error('Enter a positive USD amount with at most two decimal places, or "unlimited" for no fixed cap.');
+  return Number(amount);
+}
+
+// ask is supplied only for an interactive terminal. Coding agents pass explicit
+// flags after asking the user; EOF, blank input and missing flags never consent.
+export async function choosePlanOptions(config, { scope, budget, ask, tell = () => {} } = {}) {
+  const next = structuredClone(config);
+  if (scope) next.scope = scope;
+  while (!['existing', 'all'].includes(next.scope)) {
+    if (!ask) throw new Error('Choose a scope with eval plan --scope existing or --scope all, then choose --budget <USD|unlimited>.');
+    const answer = (await ask('Use existing project connections or all models Scout indexes? [existing/all] ')).trim();
+    if (['existing', 'all'].includes(answer)) next.scope = answer;
+    else tell('Choose existing or all.');
+  }
+  if (budget !== undefined) next.budget = { maxUsd: parseBudget(budget), reservationUsd: 'auto' };
+  while (next.budget.maxUsd === 0) {
+    if (!ask) throw new Error('Choose an evaluation budget after model scope: --budget 10, --budget <USD>, or --budget unlimited. No amount is selected automatically.');
+    const choice = (await ask('What budget should OpenMerit use for this project’s evaluations?\n  1. $10\n  2. Custom amount\n  3. No fixed cap (watched changes can incur further charges until stopped)\nChoose [1/2/3]: ')).trim();
+    if (choice === '1') next.budget = { maxUsd: 10, reservationUsd: 'auto' };
+    else if (choice === '3') next.budget = { maxUsd: 'unlimited', reservationUsd: 'auto' };
+    else if (choice === '2') {
+      while (next.budget.maxUsd === 0) {
+        const amount = await ask('How much in USD? ');
+        try {
+          const parsed = parseBudget(amount);
+          if (parsed === 'unlimited') throw new Error('Enter a custom USD amount.');
+          next.budget = { maxUsd: parsed, reservationUsd: 'auto' };
+        } catch { tell('Enter a positive USD amount with at most two decimal places.'); }
+      }
+    } else tell('Choose 1, 2 or 3. Nothing is selected automatically.');
+  }
+  return next;
+}
+
 export async function main(args) {
   if (!args.length || args.includes('--help') || args.includes('-h')) { console.log(help); return; }
-  const action = args.shift(); let root = process.cwd(), scope, json = false;
+  const action = args.shift(); let root = process.cwd(), scope, budget, json = false;
   for (let i = 0; i < args.length; i++) {
     if (args[i] === '--path' && args[i + 1]) root = resolve(args[++i]);
     else if (args[i] === '--scope' && ['existing', 'all'].includes(args[i + 1])) scope = args[++i];
+    else if (args[i] === '--budget' && args[i + 1] && budget === undefined) { budget = args[++i]; parseBudget(budget); }
     else if (args[i] === '--json') json = true;
     else throw new Error(`Unknown eval argument: ${args[i]}`);
   }
   if (scope && action !== 'plan') throw new Error('--scope belongs to eval plan.');
+  if (budget !== undefined && action !== 'plan') throw new Error('--budget belongs to eval plan.');
   if (action === 'init') { await initialize(root); console.log('Created .openmerit/eval.json, starter fixtures, and AGENTS.md / CLAUDE.md instructions. Declare connections, replace the starter task, and set an approved budget before running.'); return; }
   if (action === 'doctor') {
     const results = [];
@@ -84,14 +129,17 @@ export async function main(args) {
   process.once('SIGINT', stop); process.once('SIGTERM', stop);
   try {
     if (action === 'plan') {
-      const config = await loadConfig(root);
-      if (scope) config.scope = scope;
-      if (!config.scope && process.stdin.isTTY && !json) {
-        const rl = createInterface({ input: process.stdin, output: process.stdout });
-        try { const answer = await rl.question('Use existing project connections or all models Scout indexes? [existing/all] ', { signal: controller.signal }); if (['existing', 'all'].includes(answer.trim())) config.scope = answer.trim(); } finally { rl.close(); }
-      }
+      const saved = await loadConfig(root);
+      const rl = process.stdin.isTTY && !json ? createInterface({ input: process.stdin, output: process.stdout }) : null;
+      const inputClosed = new AbortController();
+      rl?.once('close', () => inputClosed.abort());
+      let config;
+      try { config = await choosePlanOptions(saved, { scope, budget, ask: rl ? question => rl.question(question, { signal: AbortSignal.any([controller.signal, inputClosed.signal]) }) : undefined, tell: message => console.log(message) }); }
+      finally { rl?.close(); }
+      // Preserve explicit choices even if Scout is temporarily unavailable.
+      await saveJSON(join(root, configPath), config);
       const plan = await makePlan(config);
-      await saveJSON(join(root, configPath), config); await savePlan(root, plan);
+      await savePlan(root, plan);
       console.log(JSON.stringify(plan, null, 2));
       return;
     }

package/dist/eval/src/config.mjs

@@ -43,7 +43,7 @@ export function validateConfig(config) {
     v.files.forEach(safePath);
   }
   requireThat(Number.isInteger(config.repeats?.min) && Number.isInteger(config.repeats?.max) && config.repeats.min >= 3 && config.repeats.max <= 10 && config.repeats.min <= config.repeats.max, 'repeats must be between 3 and 10.');
-  requireThat(Number.isFinite(config.budget?.maxUsd) && config.budget.maxUsd >= 0 && Number.isFinite(config.budget.reservationUsd) && config.budget.reservationUsd > 0, 'set nonnegative maxUsd and positive reservationUsd.');
+  requireThat((config.budget?.maxUsd === 'unlimited' || Number.isFinite(config.budget?.maxUsd) && config.budget.maxUsd >= 0) && (config.budget.reservationUsd === 'auto' || Number.isFinite(config.budget.reservationUsd) && config.budget.reservationUsd > 0), 'set nonnegative maxUsd or explicit "unlimited", and reservationUsd to "auto" or a positive amount.');
   requireThat(Number.isInteger(config.timeoutSeconds) && config.timeoutSeconds >= 1 && config.timeoutSeconds <= 1800, 'timeoutSeconds must be 1–1800.');
   if (config.scoutURL !== undefined) {
     const url = new URL(config.scoutURL);
@@ -53,6 +53,12 @@ export function validateConfig(config) {
 }
 export async function loadConfig(root) { return validateConfig(await readJSON(join(root, configPath))); }
 
+export function reservationFor(config, plan) {
+  if (config.budget.reservationUsd !== 'auto') return config.budget.reservationUsd;
+  const attempts = plan.tasks.reduce((total, task) => total + task.candidates.reduce((n, model) => n + model.connections.length, 0), 0) * config.variants.length * config.repeats.min;
+  return config.budget.maxUsd === 'unlimited' || config.budget.maxUsd === 0 ? .25 : Math.min(.25, config.budget.maxUsd / Math.max(1, attempts));
+}
+
 // Snapshot exact bytes before executing; edits during a run belong to the next revision.
 export async function filesAt(root, paths) {
   const files = Object.create(null); let bytes = 0; const canonicalRoot = await realpath(root);
@@ -98,5 +104,5 @@ export const starter = {
   version: 1, scope: null, connections: [],
   tasks: [{ id: 'json', description: 'Extract a customer name and item count from text into a strict JSON object.', prompt: 'Read request.txt. Return only JSON with keys name (string) and count (number).', fixture: 'eval/fixtures/json', grader: 'eval/grade.mjs' }],
   variants: [{ id: 'bare', files: [], instructions: '' }, { id: 'cli', files: ['eval/tool.mjs'], instructions: 'You can use node eval/tool.mjs request.txt to extract the fields.' }],
-  repeats: { min: 3, max: 10 }, budget: { maxUsd: 0, reservationUsd: 0.25 }, timeoutSeconds: 180,
+  repeats: { min: 3, max: 10 }, budget: { maxUsd: 0, reservationUsd: 'auto' }, timeoutSeconds: 180,
 };

package/dist/eval/src/report.mjs

@@ -9,10 +9,13 @@ export function summarize(rows) {
   }
   return [...groups].map(([key, runs]) => ({
     key, task: runs[0].task, connection: runs[0].connection, variant: runs[0].variant, runs: runs.length,
-    quality: completeMean(runs.map(r => r.quality)), failures: runs.filter(r => r.status !== 'ok').length,
+    quality: completeMean(runs.map(r => r.quality)), failures: runs.filter(r => r.status !== 'ok' && r.quality !== null).length,
+    blockedAttempts: runs.filter(r => r.failureReason).length, scoredRuns: runs.filter(r => r.quality !== null).length,
+    failedTaskChecks: runs.filter(r => r.status === 'ok' && r.quality === 0).length,
     qualityRange: runs.every(r => r.quality !== null) ? [Math.min(...runs.map(r => r.quality)), Math.max(...runs.map(r => r.quality))] : null,
     inputTokens: completeMean(runs.map(r => r.usage?.input ?? null)), outputTokens: completeMean(runs.map(r => r.usage?.output ?? null)),
-    reportedCostUsd: runs.filter(r => r.costUsd !== null).reduce((s, r) => s + r.costUsd, 0), unknownCosts: runs.filter(r => r.costUsd === null).length,
+    reportedCostUsd: runs.some(r => r.costUsd !== null) ? runs.filter(r => r.costUsd !== null).reduce((s, r) => s + r.costUsd, 0) : null, unknownCosts: runs.filter(r => r.costUsd === null).length,
+    partialReportedCostUsd: runs.reduce((s, r) => s + (r.partialCostUsd ?? 0), 0),
     meanCostUsd: completeMean(runs.map(r => r.costUsd)), medianSeconds: median(runs.map(r => r.seconds)),
   }));
 }
@@ -39,13 +42,18 @@ export function compareReports(current, previous) {
 }
 const fmt = (v, digits = 2) => v === null || v === undefined ? '?' : v.toFixed(digits);
 export function formatReport(report) {
-  const lines = [`OpenMerit eval · ${report.status} · ${report.revision.slice(0, 12)}`, '', 'Task / connection / variant | runs | quality | input / output tokens | reported $/run | seconds (median)'];
-  for (const row of report.summary) lines.push(`${row.key} | ${row.runs} | ${fmt(row.quality === null ? null : row.quality * 100, 1)}% | ${fmt(row.inputTokens, 0)} / ${fmt(row.outputTokens, 0)} | ${fmt(row.meanCostUsd, 5)} | ${fmt(row.medianSeconds)}`);
+  const lines = [`OpenMerit eval · ${report.status} · ${report.revision.slice(0, 12)}`, '', 'Task / connection / variant | scored / attempts | blocked | quality | input / output tokens | estimated $/run | seconds (median)'];
+  for (const row of report.summary) {
+    lines.push(`${row.key} | ${row.scoredRuns ?? row.runs} / ${row.runs} | ${row.blockedAttempts ?? 0} | ${row.quality === null ? '?' : fmt(row.quality * 100, 1) + '%'} | ${fmt(row.inputTokens, 0)} / ${fmt(row.outputTokens, 0)} | ${fmt(row.meanCostUsd, 5)} | ${fmt(row.medianSeconds)}`);
+    if (row.partialReportedCostUsd > 0) lines.push(`  Partial cost estimate: $${fmt(row.partialReportedCostUsd, 5)}; full attempt cost unknown.`);
+  }
   if (report.comparison) {
     lines.push('', 'Change from the previous compatible revision (negative tokens/cost/time is lower):');
     for (const row of report.comparison) lines.push(`${row.key}: quality ${fmt(row.qualityDelta === null ? null : row.qualityDelta * 100, 1)} pp; tokens ${fmt(row.inputTokensDelta, 0)} / ${fmt(row.outputTokensDelta, 0)}; $ ${fmt(row.meanCostUsdDelta, 5)}; seconds ${fmt(row.medianSecondsDelta)}`);
   } else lines.push('', 'No complete, compatible previous revision for comparison.');
   lines.push('', '3–10 repetitions measure variability, not statistical certainty. ? means unavailable.', 'Costs are agent-reported estimates, not reconciled bills. Compare agents as different harnesses.');
+  if (report.budgetLimitUsd !== undefined) lines.push(report.budgetLimitUsd === 'unlimited' ? 'Budget: no fixed cap. Task/repetition limits and timeouts still apply; watched changes can incur further charges until stopped.' : `Budget: $${fmt(report.budgetLimitUsd)} cumulative across reruns.`);
+  lines.push(`Reserved allowance used: $${fmt(report.budgetUsedUsd, 5)} (not billed spend). Blocked attempts are unscored.`);
   if (report.reason) lines.push(`Stopped: ${report.reason}`);
   return lines.join('\n');
 }

package/dist/eval/src/runner.mjs

@@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, readFile, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { tmpdir } from 'node:os';
 import { randomUUID } from 'node:crypto';
-import { loadConfig, snapshot, materialize, readJSON, saveJSON, statePath, digest } from './config.mjs';
+import { loadConfig, snapshot, materialize, readJSON, saveJSON, statePath, digest, reservationFor } from './config.mjs';
 import { validatePlan } from './scout.mjs';
 import { preflight, runAgent, invocation } from './adapters.mjs';
 import { execute, agentEnvironment } from './process.mjs';
@@ -17,7 +17,9 @@ export async function acquireLock(root) {
 }
 export function budgetUsed(ledger) {
   if (ledger?.version !== 1 || !Array.isArray(ledger.entries) || ledger.entries.some(e => !Number.isFinite(e.reservationUsd) || e.reservationUsd <= 0 || e.costUsd !== null && (!Number.isFinite(e.costUsd) || e.costUsd < 0))) throw new Error('Invalid budget ledger. Restore it before running; never discard unknown charges.');
-  return ledger.entries.reduce((sum, e) => sum + (e.costUsd ?? e.reservationUsd), 0);
+  // Agent prices are estimates, not provider settlement. Never release a
+  // reservation on that evidence, including entries written by older previews.
+  return ledger.entries.reduce((sum, e) => sum + Math.max(e.costUsd ?? 0, e.reservationUsd), 0);
 }
 
 export async function evaluate(root, { signal, onProgress = () => {}, agentRunner = runAgent, checkAgent = preflight } = {}) {
@@ -28,17 +30,22 @@ export async function evaluate(root, { signal, onProgress = () => {}, agentRunne
   const versions = {};
   const ledgerPath = join(root, statePath, 'budget.json');
   const ledger = await optionalJSON(ledgerPath) ?? { version: 1, entries: [] };
-  if (budgetUsed(ledger) + config.budget.reservationUsd > config.budget.maxUsd + 1e-10) throw new Error('Evaluation budget is exhausted or disabled. Set budget.maxUsd in .openmerit/eval.json; prior reservations still count.');
-  for (const connection of config.connections.filter(c => selected.has(c.id))) versions[connection.id] = await checkAgent(connection);
+  const reservationUsd = reservationFor(config, plan), unlimitedBudget = config.budget.maxUsd === 'unlimited';
+  const exhausted = () => { const used = budgetUsed(ledger); return !unlimitedBudget && used + reservationUsd > config.budget.maxUsd + 1e-10; };
+  if (exhausted()) throw new Error('Evaluation budget is exhausted or disabled. Choose a budget with eval plan --budget <USD|unlimited>; prior reservations still count.');
   const runnerVersion = (await readJSON(new URL('../../../package.json', import.meta.url))).version;
-  const commands = config.connections.filter(c => selected.has(c.id)).map(c => ({ id: c.id, args: invocation(c, config.budget.reservationUsd) }));
-  const report = { version: 1, runnerVersion, id: randomUUID(), startedAt: new Date().toISOString(), revision: snap.revision, compatibility: digest({ snapshot: snap.compatibility, versions, runnerVersion, commands }), planHash: digest(plan.tasks), agentVersions: versions, status: 'running', rows: [], summary: [] };
+  const commands = config.connections.filter(c => selected.has(c.id)).map(c => ({ id: c.id, args: invocation(c, reservationUsd, unlimitedBudget) }));
+  const report = { version: 2, runnerVersion, budgetLimitUsd: config.budget.maxUsd, reservationUsd, id: randomUUID(), startedAt: new Date().toISOString(), revision: snap.revision, compatibility: null, planHash: digest(plan.tasks), agentVersions: versions, status: 'running', rows: [], summary: [] };
   const reportPath = join(root, statePath, 'runs', `${report.startedAt.replace(/[:.]/g, '-')}-${report.id}.json`);
   const previous = await optionalJSON(join(root, statePath, 'latest.json'));
   const temp = await mkdtemp(join(tmpdir(), 'openmerit-eval-'));
   const checkpoint = async () => { report.summary = summarize(report.rows); await saveJSON(reportPath, report); };
   try {
     await checkpoint();
+    for (const connection of config.connections.filter(c => selected.has(c.id))) {
+      try { versions[connection.id] = await checkAgent(connection, { cwd: temp }); }
+      catch (error) { error.connection = connection.id; throw error; }
+    }
     // Graders are copied outside all agent workspaces, before any trial runs.
     for (const task of snap.tasks) await materialize(task.graderFiles, join(temp, 'graders', task.id));
     outer: for (const task of snap.tasks) {
@@ -50,15 +57,16 @@ export async function evaluate(root, { signal, onProgress = () => {}, agentRunne
         if (round % 2) ordered.reverse();
         for (const { connection, variant, candidate } of ordered) {
           if (signal?.aborted) { report.status = 'cancelled'; report.reason = 'Interrupted'; break outer; }
-          if (budgetUsed(ledger) + config.budget.reservationUsd > config.budget.maxUsd + 1e-10) { report.status = 'budget_stopped'; report.reason = 'Cumulative budget cannot admit another reserved attempt.'; break outer; }
-          const entry = { id: randomUUID(), run: report.id, reservationUsd: config.budget.reservationUsd, costUsd: null, settled: false };
+          if (exhausted()) { report.status = 'budget_stopped'; report.reason = 'Cumulative budget cannot admit another reserved attempt.'; break outer; }
+          const entry = { id: randomUUID(), run: report.id, reservationUsd, costUsd: null, settled: false };
           ledger.entries.push(entry); await saveJSON(ledgerPath, ledger);
           const workspace = join(temp, entry.id); await mkdir(workspace);
           await materialize(task.fixtureFiles, workspace, task.fixture);
           await materialize(variant.sourceFiles, workspace);
           const prompt = `${task.prompt}\n\n${variant.instructions}\n\nThis is an isolated evaluation attempt. Do not run OpenMerit evaluation commands or inspect paths outside this workspace.`;
-          const result = await agentRunner(connection, prompt, { cwd: workspace, timeoutMs: config.timeoutSeconds * 1000, signal, reservationUsd: config.budget.reservationUsd });
-          entry.costUsd = result.costUsd; entry.reservationUsd = Math.max(entry.reservationUsd, result.partialCostUsd ?? 0); entry.settled = true; await saveJSON(ledgerPath, ledger);
+          const result = await agentRunner(connection, prompt, { cwd: workspace, timeoutMs: config.timeoutSeconds * 1000, signal, reservationUsd, unlimitedBudget });
+          entry.costUsd = result.costUsd; entry.costBasis = result.costBasis ?? (result.costUsd === null ? 'unknown' : 'agent_estimate');
+          entry.partialCostUsd = result.partialCostUsd ?? null; entry.reservationUsd = Math.max(entry.reservationUsd, result.partialCostUsd ?? 0); entry.settled = true; await saveJSON(ledgerPath, ledger);
           let quality = result.status === 'ok' || result.failureReason ? null : 0, status = result.status;
           if (result.status === 'ok') {
             const grader = await execute(process.execPath, [join(temp, 'graders', task.id, task.grader)], { cwd: temp, env: agentEnvironment(), input: JSON.stringify({ workspace, output: result.output }), timeoutMs: 30_000, signal });
@@ -68,10 +76,14 @@ export async function evaluate(root, { signal, onProgress = () => {}, agentRunne
               quality = value.quality;
             } catch { status = 'grader_error'; }
           }
-          const row = { task: task.id, connection: connection.id, agent: connection.agent, requestedModel: connection.model, observedModels: result.observedModels, scoutId: candidate.scoutId, scoutRank: candidate.rank, variant: variant.id, round: round + 1, quality, status, failureReason: result.failureReason ?? null, usage: result.usage, costUsd: result.costUsd, seconds: result.seconds };
+          const row = { task: task.id, connection: connection.id, agent: connection.agent, requestedModel: connection.model, observedModels: result.observedModels, scoutId: candidate.scoutId, scoutRank: candidate.rank, variant: variant.id, round: round + 1, quality, status, failureReason: result.failureReason ?? null, usage: result.usage, costUsd: result.costUsd, partialCostUsd: result.partialCostUsd ?? null, costBasis: entry.costBasis, seconds: result.seconds };
           report.rows.push(row); await checkpoint(); await rm(workspace, { recursive: true, force: true });
           onProgress(row);
-          if (result.failureReason) { report.status = 'connection_blocked'; report.reason = `${connection.id}: ${result.failureReason}. Fix the native agent connection before retrying.`; break outer; }
+          if (result.failureReason) {
+            report.status = result.failureReason === 'budget_exhausted' ? 'budget_stopped' : result.failureReason === 'cancelled' ? 'cancelled' : ['sandbox_unavailable', 'permission_denied', 'spawn_failed'].includes(result.failureReason) ? 'environment_blocked' : 'connection_blocked';
+            report.reason = `${connection.id}: ${result.failureReason}. This attempt is unscored; resolve the execution block before retrying.`;
+            break outer;
+          }
           if (status === 'grader_error') { report.status = 'invalid'; report.reason = `Grader failed for ${task.id}. Fix it before collecting more trials.`; break outer; }
         }
         if (round + 1 >= config.repeats.min && !isVolatile(report.rows.filter(r => r.task === task.id))) break;
@@ -79,11 +91,16 @@ export async function evaluate(root, { signal, onProgress = () => {}, agentRunne
     }
     if (report.status === 'running') report.status = 'complete';
   } catch (error) {
-    report.status = 'error'; report.reason = 'Execution failed; unsettled reservations remain charged. Check configuration and agent authentication.';
-    throw error;
+    if (error.failureReason) {
+      report.status = 'environment_blocked'; report.reason = `${error.connection}: ${error.failureReason}. ${error.message}`;
+      report.blockers = [{ connection: error.connection, failureReason: error.failureReason }];
+    } else {
+      report.status = 'error'; report.reason = 'Execution failed; unsettled reservations remain charged. Check configuration and agent authentication.';
+      throw error;
+    }
   } finally {
     report.finishedAt = new Date().toISOString(); report.budgetUsedUsd = budgetUsed(ledger);
-    report.compatibility = digest({ base: report.compatibility, observedModels: Object.fromEntries([...selected].sort().map(id => [id, [...new Set(report.rows.filter(r => r.connection === id).flatMap(r => r.observedModels ?? []))].sort()])) });
+    report.compatibility = digest({ accountingVersion: report.version, snapshot: snap.compatibility, versions, runnerVersion, commands, observedModels: Object.fromEntries([...selected].sort().map(id => [id, [...new Set(report.rows.filter(r => r.connection === id).flatMap(r => r.observedModels ?? []))].sort()])) });
     report.summary = summarize(report.rows); report.comparison = compareReports(report, previous);
     await saveJSON(reportPath, report); await saveJSON(join(root, statePath, 'latest.json'), report);
     await rm(temp, { recursive: true, force: true });

package/docs/agent-evaluations.md

@@ -1,6 +1,8 @@
 # Evaluate CLI changes across coding agents
 
-**npm preview `0.1.7-preview.1`.** Available under the `preview` tag; stable `openmerit@0.1.6` does not include these commands. This evaluates the coding agent with your CLI and documentation. The [Pi workflow](getting-started.md) still evaluates application inference; the [Inference Terminal](inference-terminal.md) remains read-only.
+**npm preview `0.1.7-preview.2`.** Install from the `preview` tag; stable `openmerit@0.1.6` does not include these commands. This evaluates the coding agent with your CLI and documentation. The [Pi workflow](getting-started.md) still evaluates application inference; the [Inference Terminal](inference-terminal.md) remains read-only.
+
+Compared with `0.1.7-preview.1`, this version adds [corrected cost accounting and blocked-execution handling](#accounting-and-blocked-executions), plus [budget selection after model scope](#budget-selection). It does not claim reconciled provider billing or a guaranteed exact model order.
 
 One local runner supports Claude Code, Codex and OpenCode. Scout supplies candidates; local task checks measure each model/agent/CLI combination. Installing npm alone does not install or authenticate agents, modify project instructions, or start paid evaluations.
 
@@ -24,7 +26,7 @@ Replace the starter before drawing product conclusions. Have your agent author r
 openmerit eval plan
 ```
 
-The first interactive plan asks: **existing project connections or all models Scout indexes?** The choice is saved. Noninteractive agents must ask the user and pass `--scope existing` or `--scope all`; missing input never defaults to all.
+The first interactive plan asks: **existing project connections or all models Scout indexes?** It then asks for a **$10, custom or no-fixed-cap budget**. Both choices are saved. Noninteractive agents must ask the user and pass `--scope existing` or `--scope all` together with `--budget <USD|unlimited>` on first setup; missing input never chooses scope or spending automatically.
 
 Add explicit routes to `connections` in the config, for example:
 
@@ -54,14 +56,14 @@ Each variant needs `id`, `files` and `instructions`. The starter compares `bare`
 
 Selected files, including uncommitted changes, are snapshotted before execution. Variant paths must not overwrite fixture inputs and retain their project-relative locations; fixture contents go at the workspace root. Include self-contained CLI artifacts and lean docs explicitly. Dependencies are not installed automatically; tools must be bundled or have their runtime requirements already available. Symlinks, parent-directory paths, `.env*`, `.git`, `.openmerit` and `node_modules` are rejected. Each selected snapshot is limited to 25 MB. Do not select credentials stored under other filenames.
 
-The grader reads `{ "workspace": "/temporary/path", "output": "final response" }` from stdin and prints exactly `{ "quality": 0.0 }`, with a finite score from 0 to 1. It can check generated files using executable assertions. Keep it self-contained: imports beside it are not copied automatically. Graders are copied outside agent workspaces; never also include them or answer keys in fixtures/variant files. Test graders against correct and incorrect answers. A broken grader stops the experiment with unknown quality; agent failures/timeouts count as quality zero. Recognized authentication or model-access errors instead stop after one attempt with unknown quality, so an unavailable connection is not rated as a weak model.
+The grader reads `{ "workspace": "/temporary/path", "output": "final response" }` from stdin and prints exactly `{ "quality": 0.0 }`, with a finite score from 0 to 1. It can check generated files using executable assertions. Keep it self-contained: imports beside it are not copied automatically. Graders are copied outside agent workspaces; never also include them or answer keys in fixtures/variant files. Test graders against correct and incorrect answers. A broken grader stops the experiment with unknown quality. Ordinary agent failures/timeouts count as quality zero; recognized budget, environment and connection blocks instead stop with unknown quality. See [failure classification](#accounting-and-blocked-executions).
 
 ## Run after changes
 
-The starter's `budget.maxUsd` is zero. Set an approved cumulative budget, then:
+The starter's `budget.maxUsd` is zero. After the user's scope and budget choices, for example:
 
 ```sh
-openmerit eval plan --scope existing
+openmerit eval plan --scope existing --budget 10
 openmerit eval run
 # Edit your CLI or its documentation.
 openmerit eval run
@@ -88,14 +90,56 @@ openmerit eval report --json
 
 Reports show quality mean/range, failures, mean input/output tokens, mean reported dollars and median full-process seconds per task/connection/variant. They retain Scout ranks but do not force a weighted winner. Lower cost with lower quality is a tradeoff. Matching complete revisions show deltas; partial/invalid experiments do not.
 
-Claude provides aggregate usage and reported cost. Codex JSON events provide tokens but not billed dollars, so cost stays unknown. OpenCode per-step usage/cost is summed across the episode, including tools. Input totals include separately reported cache reads/writes; cached tokens are retained separately without counting them twice. OpenCode output totals include separately reported reasoning tokens. Agent accounting and subscriptions differ. Reported costs are not reconciled provider bills; reported zero may mean missing pricing metadata. Catalogue prices never replace unknown measured costs.
+Claude provides aggregate and per-model usage and cost estimates, reconciled without counting the same usage twice. Codex JSON events provide tokens but not billed dollars, so cost stays unknown. OpenCode per-step usage/cost is summed across the episode, including tools; missing or zero step pricing leaves full cost unknown and preserves partial estimates. Input totals include separately reported cache reads/writes; cached tokens are retained separately without counting them twice. OpenCode output totals include separately reported reasoning tokens. Agent accounting and subscriptions differ. Reported costs are not reconciled provider bills; zero pricing is treated as unknown. Catalogue prices never replace unknown measured costs.
 
 Preflight requires supported noninteractive JSON interfaces: Claude print JSON, Codex `exec --json --ephemeral --sandbox workspace-write`, and OpenCode `run --format json --auto`. OpenCode's auto option approves tool requests that are not explicitly denied, avoiding an interactive approval stall. Claude uses an explicit local tool allowlist without inherited MCP configuration; Codex and OpenCode retain native global settings. Recognized permission failures stop as connection problems, not model-quality evidence. OpenMerit does not add a sandbox around Claude/OpenCode. The old 2025 Codex CLI is unsupported. Native references: [Claude Code](https://code.claude.com/docs/en/headless), [Codex](https://developers.openai.com/codex/noninteractive), [OpenCode](https://opencode.ai/docs/cli/).
 
 ## Budget and data
 
-`budget.maxUsd` applies cumulatively across runs and restarts. Before each attempt the runner persists `reservationUsd`. A known reported cost replaces it; unknown/interrupted charges keep at least the full reservation. Partial cost events from an unsuccessful process never count as a complete bill. Insufficient remaining allowance stops a partial run. Raising maxUsd extends the allowance, not resets it. Do not delete unknown spend from the ledger. This is admission control, not a billing cutoff: one attempt may exceed its reservation. Claude also receives its native maximum-budget flag. Use provider-side limits for hard monetary caps. Each attempt also has a timeout and 4 MB process-output limit.
+`budget.maxUsd` applies cumulatively across runs and restarts. Before each attempt the runner persists a reservation. It retains the greater of that reservation and the available full or partial cost estimate; estimates never release reserved allowance. Partial events never count as a complete bill. Insufficient remaining allowance stops a partial run. Raising maxUsd extends the allowance, not resets it. Do not delete unknown spend from the ledger. This is admission control, not a billing cutoff: one attempt may exceed its reservation. Finite budgets also use Claude's native maximum-budget flag. Use provider-side limits for hard monetary caps. Each attempt also has a timeout and 4 MB process-output limit. See [budget selection](#budget-selection) for the three choices and automatic reservations.
 
 `.openmerit/eval/` holds `plan.json`, `budget.json`, `latest.json`, `runs/*.json` and `lock/pid`. Init ignores it in Git. Reports contain identities, metrics, hashes and status, not full transcripts or final outputs. Temporary files are removed afterward. Native agents/providers may retain their own data; Claude/Codex receive ephemeral flags, while OpenCode uses native storage. `.openmerit/eval.json` can be versioned without credentials.
 
 Fresh workspaces are not security sandboxes. Trusted agents and graders execute with native permissions and user-level resources. Hard kills may leave temporary directories. After a crash, inspect the lock's PID and remove it only if that process is gone. This preview does not change Pi approvals, swap production models or upload results into Scout rankings.
+
+
+## VM acceptance limits
+
+The published `0.1.7-preview.1` package was tested on an Ubuntu ARM64 VM with Node 22.23.1 on 15 October 2026. Claude Code 2.1.300 and OpenCode 1.19.0 executed a small CSV CLI task before and after a parser change. A separate OpenCode watcher completed three attempts per revision and emitted comparison deltas. Codex 0.162.1 passed interface preflight but could not execute filesystem tools because that VM denied the required Linux user namespaces. CLI flag checks do not establish that the native sandbox works on a host.
+
+In that earlier `0.1.7-preview.1` test, a native budget stop could appear as a generic agent failure, and a successful final response explaining a host permission problem could receive zero from the task grader. These did not establish poor model quality; the corrections below address recognized native failures. The combined VM run also stopped before its final repetition because conservative reservations and agent cost estimates exhausted its local allowance; partial reports correctly omitted automatic comparison. Agent-reported cost differed substantially from OpenRouter account usage. This was one integration fixture, not validation of general task quality, exact model order, or billing accuracy.
+
+## Accounting and blocked executions
+
+Since `0.1.7-preview.2`, recognized budget, sandbox, permission, authentication and model-access blocks are separate from task failures. A blocked attempt has `quality: null`, stops the run and cannot produce a revision comparison. Reports distinguish scored attempts, blocked attempts and task-check failures. Ordinary wrong outputs still receive their grader score; ordinary agent failures and configured timeouts still count as failures. Cancellation, a missing executable and output-limit interruption are unscored. Error classification uses native failure fields and failed-tool diagnostics, not arbitrary task text or model commentary. Unrecognized host failures can still require manual review.
+
+On Linux, Codex preflight also runs a local sandbox probe before any model request or budget reservation. It uses the configured executable and the workspace-write policy; it does not weaken sandbox permissions. A failed probe produces `environment_blocked` with no task score. Doctor performs this same probe. Other operating systems still have interface checks only.
+
+Claude's per-model totals recover usage omitted from aggregate budget-stop responses, without adding the same aggregate and model counters twice. Costs carry `costBasis` and remain native estimates. Zero or missing pricing is unknown; incomplete OpenCode step pricing stays partial. Partial costs survive into the row, summary and ledger, including failed attempts. No provider bill is reconstructed from these estimates, and no account-wide spend delta is assigned to individual attempts.
+
+Budget admission now retains the greater of each reservation and its available full or partial estimate, including old ledger entries. Estimates never release a reservation. This may exhaust an existing allowance earlier; increasing it requires an explicit budget decision. The displayed reserved allowance is not billed spend, and a provider-side limit is still needed for a hard cap. Report version 2 prevents comparisons with the earlier accounting rules. There is no new provider integration, proxy, background service or model-specific rule.
+
+## Budget selection
+
+After choosing existing connections or all indexed models, the first interactive `eval plan` asks:
+
+1. **$10** — a cumulative evaluation allowance for this project.
+2. **Custom amount** — asks how much in USD.
+3. **No fixed cap** — run the configured suite without a monetary admission limit.
+
+Nothing is preselected. Blank, invalid or cancelled input cannot enable spending. Init still starts at zero. Newly generated project instructions tell coding agents to ask the same follow-up and pass the answer explicitly, for example:
+
+```sh
+openmerit eval plan --scope existing --budget 10
+# Alternatives after the user's choice:
+openmerit eval plan --budget 25.50
+openmerit eval plan --budget unlimited
+```
+
+`--budget` belongs to `plan`, accepts positive USD amounts with at most two decimal places or the exact word `unlimited`, and never starts evaluation by itself. Noninteractive planning requires explicit flags when scope or budget has not been chosen. Existing positive budgets are reused. The choices are saved even if Scout subsequently fails, so a retry does not ask again. Init does not rewrite instructions in already-initialized projects; an agent in such a project can use these flags directly.
+
+The allowance is cumulative across runs, watched edits, restarts and replanning. Changing from $10 to $20 raises the total allowance to $20; it does not add $20 or erase earlier reservations. Changing to or from no fixed cap also preserves the ledger. A normal `eval plan` reuses the saved choice. To change it, pass a new `--budget` value; setting `budget.maxUsd` back to zero in the config disables paid execution and prompts again on the next interactive plan.
+
+Reservations are internal. New suites and budget choices use `reservationUsd: "auto"`: the smaller of $0.25 and the total allowance divided across the minimum planned attempts (tasks × selected connections × variants × minimum repetitions). No fixed cap uses $0.25 for ledger bookkeeping. This is a conservative starting allowance, not a prediction of an attempt's price or a guarantee the suite will finish. Higher reported charges still consume more allowance. Existing explicitly configured numeric reservations remain supported. Finite budgets retain Claude's native per-attempt budget flag; as before, an attempt can exceed its reservation, and provider limits are needed for a hard billing cap.
+
+No fixed cap is stored explicitly as `budget.maxUsd: "unlimited"`; missing, null or invalid values never mean unlimited. It also removes Claude's per-attempt dollar limit, while keeping the same task count, 3–10 repetition bounds, timeouts and native permissions. Provider/account limits still apply. Usage and cost estimates remain recorded. `eval watch` can incur further charges after each selected change until stopped; Ctrl+C stops it and the active attempt. No background service or automatic production model change is added.

package/docs/commands.md

@@ -2,7 +2,9 @@
 
 The standalone `openmerit dash` command runs in your shell. The `/openmerit` slash commands run inside Pi with the extension loaded.
 
-The npm `0.1.7-preview.1` release (`openmerit@preview`) also provides `openmerit eval init`, `doctor`, `plan`, `run`, `watch` and `report`. See [agent evaluations](agent-evaluations.md) for scope selection, repeated task checks, budgets and release status. These are shell commands, separate from Pi commands.
+The npm `0.1.7-preview.2` preview (`openmerit@preview`) provides `openmerit eval init`, `doctor`, `plan`, `run`, `watch` and `report`. See [agent evaluations](agent-evaluations.md) for scope selection, repeated task checks and budgets. These are shell commands, separate from Pi commands.
+
+Since `0.1.7-preview.2`, `openmerit eval plan [--scope existing|all] [--budget <USD|unlimited>]` asks for a budget after scope. Choose $10, a custom amount, or no fixed cap; saved choices and cumulative usage persist. See [budget selection](agent-evaluations.md#budget-selection).
 
 ## Inference Terminal
 

package/docs/security.md

@@ -2,6 +2,8 @@
 
 The [preview agent-evaluation runner](agent-evaluations.md) is a separate execution path: it runs trusted local agents/graders, passes only basic OS variables and explicitly named environment variables, and preserves agent-owned authentication. It sends task descriptions and optional model IDs to Scout. Local metric reports and budget reservations persist; temporary task outputs are removed. Fresh directories are not security sandboxes. Its budget admission controls do not enforce provider billing limits.
 
+The [budget selector](agent-evaluations.md#budget-selection) in `0.1.7-preview.2` requires an explicit $10, custom or no-fixed-cap choice after model scope. No fixed cap removes monetary admission limits, including Claude's per-attempt dollar flag, while retaining tasks, repetition bounds, timeouts, native permissions and accounting. Saved choices apply across reruns; a foreground watcher can incur further charges on each change until stopped. Provider limits remain independent.
+
 ## Credential ownership
 
 OpenMerit accepts no model-provider or Jev credential. Credentials belong to the coding harness, its environment, or an external secret store.

package/docs/site-faq-maintenance.md

@@ -303,3 +303,15 @@ Reviewed evidence and comparison claims after removing the prose summary from ev
 ## Agent evaluation npm preview — 15 October 2026
 
 Prepared the `0.1.7-preview.1` npm preview with explicit installation and setup instructions. The stable `latest` tag remains `0.1.6`. Reviewed homepage readiness, evaluation, budget and data claims: they describe stable terminal/Pi behavior and require no copy changes. The preview retains explicit initialization, connection scope, local execution and budget limitations; no install-time automation is claimed.
+
+## Agent evaluation accounting — unreleased source, 15 October 2026
+
+Reviewed evidence, comparisons, budgets, data and readiness against the source corrections. Native cost estimates and partial charges remain distinct from provider billing; reservations are not released by estimates. Recognized budget/environment blocks are unscored and stop comparisons. The Linux Codex sandbox probe is a local preflight action with no inference call or permission bypass. There is no provider-account ingestion in the product, production model swap or new background process. The guide distinguishes these changes from npm `0.1.7-preview.1`; stable homepage FAQ wording remains unchanged.
+
+## Agent evaluation budget selection — unreleased source, 15 October 2026
+
+Reviewed budget, approval, scheduling and readiness claims. Source planning now asks for an explicit $10, custom or no-fixed-cap allowance after model scope; no option is selected automatically. New project instructions reuse the saved choice, and reservations stay internal. Cumulative accounting survives replanning and limit changes. No fixed cap retains task/repetition/time limits and native permissions, but a foreground watcher can incur further charges until stopped. Provider billing limits remain independent. These changes are documented separately from published `0.1.7-preview.1`; the stable homepage FAQ still describes terminal/Pi behavior and needs no wording change.
+
+## Agent evaluation preview.2 release preparation — 15 October 2026
+
+Prepared `0.1.7-preview.2` for the existing npm preview channel, combining corrected accounting and execution-block handling with explicit budget selection. The public guide now describes these versioned behaviors together and keeps earlier VM findings historical. Stable `latest` remains `0.1.6`; homepage FAQ claims still describe terminal/Pi behavior and require no copy changes. Registry publication and fresh-install VM acceptance are recorded separately; a successful local build alone does not establish either.

package/docs/testing.md

@@ -75,3 +75,22 @@ An isolated installation of the actual npm tarball completed init, real hosted S
 Real macOS smoke runs used Claude Code 2.1.7, Codex CLI 0.162.0-alpha.17.2 and OpenCode 1.18.30. On one tiny JavaScript normalization fixture, with three fresh attempts per route, Claude/Haiku 4.5 and Codex/GPT-6 Luna passed all executable checks. OpenCode/Gemini 2.5 Flash Lite passed one of three. Costs remained unknown for Codex. These different model/harness combinations validate execution and grading paths; they are not an unbiased comparison of agents or a general model ranking. Earlier smoke runs deliberately exposed unsupported authentication/model access, native permission failures, a timeout and invalid JSON. Those cases informed connection-error handling and conservative partial-cost accounting. No model-specific ranking rule was added.
 
 Scout Worker `7ddeacd3-1310-48bd-b546-60fcab4fa4ef` passed live HTTP/MCP checks for pre-ranking model eligibility, empty inventory and duplicate-ID rejection. npm publication is separate from the successful tarball installation; the registry credential returned HTTP 401 during release preparation.
+
+
+### Published preview on Linux — 15 October 2026
+
+Installed `openmerit@0.1.7-preview.1` from npm into a separate prefix in the existing Ubuntu ARM64 VM. The live Scout API returned a frozen shortlist within the declared OpenRouter connections. A deterministic CSV grader scored quoted commas, escaped quotes, leading-zero identifiers and field types; the uncorrected and corrected CLI passed 8/12 and 12/12 checks respectively in grader sanity checks.
+
+Claude Code/Haiku 4.5 completed three attempts per revision at 66.7% then 100% mean field accuracy. The combined watcher stopped on budget admission before OpenCode’s last post-change attempt, so its report has no automatic comparison. A separate OpenCode/Gemini 2.5 Flash Lite watcher completed three attempts per revision at 66.7% then 100%, detected the source edit, emitted token/cost/time deltas, and exited cleanly. Earlier partial trials remain separate. Codex/GPT-4.1 Mini could not execute filesystem tools under this VM’s Linux sandbox; no passing Codex task result is claimed. Total account usage was approximately $0.26 including diagnostics, while native reported estimates differed. One authored fixture and three repetitions do not establish general performance or a cost improvement. Source and diagnostic artifacts are retained in the task’s `openmerit-vm-eval` directory; a scan found no temporary key in the 66 test-project files.
+
+### Accounting corrections on Linux — unreleased source, 15 October 2026
+
+The source package was packed and installed into a separate VM prefix; the registry preview and existing VM installations were left unchanged. All 400 repository tests pass, including 22 eval-runner tests. The added cases cover partial usage/cost recovery, missing prices, conservative old/new ledgers, scored wrong answers, native budget and permission blocks, zero-exit sandbox failures, and the free Linux sandbox probe. Replaying the previously captured Claude budget response recovers 30,820 input tokens and 196 output tokens instead of zero.
+
+The real Codex connection now stops during sandbox preflight with `environment_blocked`, zero attempts, no reserved allowance and no inference spend. A deliberately small Claude budget stops after one attempt with unknown quality, 21,532 input tokens, 190 output tokens and a retained $0.110705 partial native estimate. A second normal-run attempt also hit its native cap; its unscored record and $0.1505768 reservation were preserved. With sufficient per-attempt allowance, Claude completed three scored CSV attempts at 100%, a 12.82-second median, and a $0.06029 mean native estimate. This is not a billed-dollar measurement or a before/after Claude comparison.
+
+OpenCode's real watcher completed three attempts per revision. The earlier CLI averaged 44.4% field accuracy (one actual zero-score task output remained included); the corrected CLI averaged 100%. Median times were 5.05 and 5.79 seconds. One earlier and two later attempts had incomplete native step pricing, so the cost means and cost delta correctly remained unknown while partial estimates were retained. The cumulative reservation stayed at $0.03. These checks validate accounting and failure separation on the same integration fixture, not general task quality, model ordering or savings. Sanitized reports, package and verification output are retained in the task's `openmerit-eval-accounting` directory.
+
+### Budget setup — unreleased source, 15 October 2026
+
+The eval suite now has 29 tests. New coverage checks the three budget choices after scope, custom amount validation, explicit unlimited configuration, reuse of saved choices, automatic reservations, retained cumulative usage, bounded repetitions and cancellation. A real CLI subprocess uses a loopback Scout fixture to verify persistence across replans and failed requests, rejection of missing or invalid noninteractive choices, and lock cleanup. An interactive terminal smoke test also verifies scope-before-budget, custom input, no-repeat questions, no fixed cap and EOF cancellation. These are offline workflow tests; no model calls, provider charges or npm publication were required.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openmerit",
-  "version": "0.1.7-preview.1",
+  "version": "0.1.7-preview.2",
   "description": "Evidence-driven model decisions for coding harnesses.",
   "type": "module",
   "bin": {