openmerit 0.1.12 → 0.3.0

package/dist/core/src/store.d.ts

@@ -1,35 +0,0 @@
-import type { EvidenceManifest, OpenMeritAuditEvent, OpenMeritProjectConfig, OpenMeritProjectState } from "../../protocol/src/index.ts";
-export declare const OPENMERIT_DIRECTORY = ".openmerit";
-export interface OpenMeritStore {
-    initialize(config: OpenMeritProjectConfig, state: OpenMeritProjectState): Promise<void>;
-    readConfig(): Promise<OpenMeritProjectConfig | undefined>;
-    writeConfig(config: OpenMeritProjectConfig): Promise<void>;
-    readState(): Promise<OpenMeritProjectState | undefined>;
-    writeState(state: OpenMeritProjectState): Promise<void>;
-    appendEvent(event: OpenMeritAuditEvent): Promise<void>;
-    writeEvidenceManifest(manifest: EvidenceManifest): Promise<string>;
-}
-export interface AuditEventExporter {
-    readonly id: string;
-    export(event: OpenMeritAuditEvent): Promise<void>;
-}
-export interface ProjectStoreOptions {
-    readonly eventExporters?: readonly AuditEventExporter[];
-}
-export declare class ProjectStore implements OpenMeritStore {
-    readonly directory: string;
-    readonly configPath: string;
-    readonly statePath: string;
-    readonly eventsPath: string;
-    readonly exportErrorsPath: string;
-    readonly evidenceDirectory: string;
-    private readonly eventExporters;
-    constructor(projectRoot: string, options?: ProjectStoreOptions);
-    initialize(config: OpenMeritProjectConfig, state: OpenMeritProjectState): Promise<void>;
-    readConfig(): Promise<OpenMeritProjectConfig | undefined>;
-    writeConfig(config: OpenMeritProjectConfig): Promise<void>;
-    readState(): Promise<OpenMeritProjectState | undefined>;
-    writeState(state: OpenMeritProjectState): Promise<void>;
-    appendEvent(event: OpenMeritAuditEvent): Promise<void>;
-    writeEvidenceManifest(manifest: EvidenceManifest): Promise<string>;
-}

package/dist/core/src/store.js

@@ -1,102 +0,0 @@
-import { appendFile, mkdir, readFile, rename, writeFile } from "node:fs/promises";
-import { dirname, join } from "node:path";
-export const OPENMERIT_DIRECTORY = ".openmerit";
-export class ProjectStore {
-    directory;
-    configPath;
-    statePath;
-    eventsPath;
-    exportErrorsPath;
-    evidenceDirectory;
-    eventExporters;
-    constructor(projectRoot, options = {}) {
-        this.directory = join(projectRoot, OPENMERIT_DIRECTORY);
-        this.configPath = join(this.directory, "config.json");
-        this.statePath = join(this.directory, "state.json");
-        this.eventsPath = join(this.directory, "events.jsonl");
-        this.exportErrorsPath = join(this.directory, "audit-export-errors.jsonl");
-        this.evidenceDirectory = join(this.directory, "evidence");
-        this.eventExporters = options.eventExporters ?? [];
-    }
-    async initialize(config, state) {
-        await mkdir(this.evidenceDirectory, { recursive: true });
-        await this.writeConfig(config);
-        await this.writeState(state);
-    }
-    async readConfig() {
-        return readJsonIfPresent(this.configPath);
-    }
-    async writeConfig(config) {
-        await atomicWriteJson(this.configPath, config);
-    }
-    async readState() {
-        return readJsonIfPresent(this.statePath);
-    }
-    async writeState(state) {
-        await atomicWriteJson(this.statePath, state);
-    }
-    async appendEvent(event) {
-        await mkdir(dirname(this.eventsPath), { recursive: true });
-        const safeEvent = redactSensitive(event);
-        await appendFile(this.eventsPath, `${JSON.stringify(safeEvent)}\n`, { encoding: "utf8", mode: 0o600 });
-        const results = await Promise.allSettled(this.eventExporters.map((exporter) => exporter.export(safeEvent)));
-        for (const [index, result] of results.entries()) {
-            if (result.status === "fulfilled")
-                continue;
-            await appendFile(this.exportErrorsPath, `${JSON.stringify({
-                occurredAt: new Date().toISOString(),
-                eventId: safeEvent.id,
-                exporterId: this.eventExporters[index]?.id ?? "unknown",
-                error: safeErrorMessage(result.reason),
-            })}\n`, { encoding: "utf8", mode: 0o600 });
-        }
-    }
-    async writeEvidenceManifest(manifest) {
-        const path = join(this.evidenceDirectory, `${safeFileComponent(manifest.id)}.json`);
-        await atomicWriteJson(path, redactSensitive(manifest));
-        return path;
-    }
-}
-const SENSITIVE_KEY = /(?:authorization|api[_-]?key|token|secret|password|cookie)/i;
-const SECRET_VALUE = /\b(?:sk-[a-zA-Z0-9_-]{12,}|apikey_[a-zA-Z0-9_-]{12,})\b/g;
-function redactSensitive(value, key) {
-    if (key && SENSITIVE_KEY.test(key))
-        return "[REDACTED]";
-    if (typeof value === "string")
-        return value.replaceAll(SECRET_VALUE, "[REDACTED]");
-    if (Array.isArray(value))
-        return value.map((item) => redactSensitive(item));
-    if (value && typeof value === "object") {
-        return Object.fromEntries(Object.entries(value).map(([childKey, child]) => [
-            childKey,
-            redactSensitive(child, childKey),
-        ]));
-    }
-    return value;
-}
-function safeErrorMessage(reason) {
-    const message = reason instanceof Error ? reason.message : String(reason);
-    return redactSensitive(message);
-}
-function safeFileComponent(value) {
-    const safe = value.replaceAll(/[^a-zA-Z0-9._-]/g, "-");
-    if (!safe || safe === "." || safe === "..")
-        throw new Error("Evidence manifest ID is not file-safe.");
-    return safe;
-}
-async function readJsonIfPresent(path) {
-    try {
-        return JSON.parse(await readFile(path, "utf8"));
-    }
-    catch (error) {
-        if (error.code === "ENOENT")
-            return undefined;
-        throw error;
-    }
-}
-async function atomicWriteJson(path, value) {
-    await mkdir(dirname(path), { recursive: true });
-    const temporaryPath = `${path}.${process.pid}.${crypto.randomUUID()}.tmp`;
-    await writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
-    await rename(temporaryPath, path);
-}

package/dist/eval/src/adapters.mjs

@@ -1,112 +0,0 @@
-import { execute, agentEnvironment } from './process.mjs';
-
-const number = value => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null;
-const sum = values => values.length && values.every(v => v !== null) ? values.reduce((a, b) => a + b, 0) : null;
-const maximum = values => values.some(v => v !== null) ? Math.max(...values.filter(v => v !== null)) : null;
-export const agents = ['claude', 'codex', 'opencode'];
-
-export function connectionFailure(text) {
-  if (/budget_exhausted|error_max_budget_usd|(?:maximum|exceeded|exhausted|insufficient).{0,25}(?:budget|credits)|(?:budget|credits).{0,25}(?:exceeded|exhausted)/i.test(text)) return 'budget_exhausted';
-  if (/(?:bwrap|bubblewrap|sandbox|landlock|seccomp|unshare|user namespace|uid_map).{0,160}(?:failed|not permitted|unavailable|denied)|failed to (?:start|initialize|create).{0,40}sandbox/i.test(text)) return 'sandbox_unavailable';
-  if (/\b(?:HTTP\s*)?401\b.{0,30}(?:unauthorized|authentication)|unauthorized|authentication[_ -]?(?:error|required)|invalid api key|not logged in|please log in/i.test(text)) return 'authentication';
-  if (/model.{0,100}(?:not supported|not found|not available|does not exist)|(?:unknown|invalid|unsupported) model|ProviderModelNotFoundError/i.test(text)) return 'model_unavailable';
-  if (/rejected permission|permission.{0,30}(?:denied|rejected)|not allowed to use.{0,30}tool/i.test(text)) return 'permission_denied';
-  return null;
-}
-
-function toolFailure(text) {
-  const reason = connectionFailure(text);
-  // A task's own HTTP/auth/budget errors are task outcomes, not evidence that
-  // the coding agent's provider connection failed.
-  return ['sandbox_unavailable', 'permission_denied'].includes(reason) ? reason : null;
-}
-
-export function invocation(connection, reservationUsd, unlimitedBudget = false) {
-  const model = connection.model;
-  if (connection.agent === 'claude') return ['-p', '--output-format', 'json', '--model', model, '--no-session-persistence', '--permission-mode', 'dontAsk', '--allowedTools', 'Read,Glob,Grep,Bash,Edit,Write', '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}', '--setting-sources', '', ...(unlimitedBudget ? [] : ['--max-budget-usd', String(reservationUsd)])];
-  if (connection.agent === 'codex') return ['exec', '--json', '--model', model, '--sandbox', 'workspace-write', '--skip-git-repo-check', '--ephemeral', '-'];
-  if (connection.agent === 'opencode') return ['run', '--format', 'json', '--model', model, '--auto'];
-  throw new Error('Unknown agent. Use claude, codex or opencode.');
-}
-
-export function parseOutput(agent, stdout) {
-  let events;
-  try { const parsed = JSON.parse(stdout); events = Array.isArray(parsed) ? parsed : [parsed]; }
-  catch { events = stdout.split('\n').flatMap(line => { try { return [JSON.parse(line)]; } catch { return []; } }); }
-  let output = '', usage = null, costUsd = null, partialCostUsd = null, completed = false, failed = false, failureReason = null;
-  // Only native failure fields are evidence of a blocked execution. Task text,
-  // successful tool output and model commentary can legitimately discuss errors.
-  const errors = events.filter(e => ['error', 'turn.failed'].includes(e.type));
-  for (const event of errors) failureReason ??= connectionFailure(JSON.stringify(event));
-  const observedModels = new Set();
-  if (agent === 'claude') {
-    const result = events.findLast(e => e.type === 'result');
-    if (result) {
-      completed = result.subtype === 'success' && !result.is_error;
-      failed = !completed; output = result.result ?? '';
-      const u = result.usage;
-      if (u) usage = { input: sum([number(u.input_tokens), number(u.cache_read_input_tokens ?? 0), number(u.cache_creation_input_tokens ?? 0)]), output: number(u.output_tokens), cached: number(u.cache_read_input_tokens ?? 0) };
-      const models = Object.values(result.modelUsage ?? {});
-      if (models.length) {
-        const modelUsage = { input: sum(models.map(m => sum([number(m.inputTokens), number(m.cacheReadInputTokens ?? 0), number(m.cacheCreationInputTokens ?? 0)]))), output: sum(models.map(m => number(m.outputTokens))), cached: sum(models.map(m => number(m.cacheReadInputTokens ?? 0))) };
-        // A budget stop can emit zero aggregate usage but nonzero per-model usage.
-        usage = Object.fromEntries(Object.keys(modelUsage).map(k => [k, maximum([usage?.[k] ?? null, modelUsage[k]])]));
-      }
-      costUsd = maximum([number(result.total_cost_usd), sum(models.map(m => number(m.costUSD)))]);
-      for (const model of Object.keys(result.modelUsage ?? {})) observedModels.add(model);
-      if (failed) failureReason ??= connectionFailure(JSON.stringify({ subtype: result.subtype, terminal_reason: result.terminal_reason, errors: result.errors }));
-      if (result.permission_denials?.length) failureReason ??= 'permission_denied';
-    }
-  } else if (agent === 'codex') {
-    const turns = events.filter(e => e.type === 'turn.completed');
-    completed = turns.length > 0;
-    failed = events.some(e => ['turn.failed', 'error'].includes(e.type));
-    output = events.findLast(e => e.type === 'item.completed' && e.item?.type === 'agent_message')?.item.text ?? '';
-    if (turns.length) usage = { input: sum(turns.map(e => number(e.usage?.input_tokens))), output: sum(turns.map(e => number(e.usage?.output_tokens))), cached: sum(turns.map(e => number(e.usage?.cached_input_tokens))) };
-    for (const event of events.filter(e => e.type === 'item.completed' && e.item?.type === 'command_execution' && (e.item.exit_code > 0 || e.item.status === 'failed'))) {
-      failureReason ??= toolFailure(event.item.aggregated_output ?? '');
-    }
-  } else if (agent === 'opencode') {
-    const steps = events.filter(e => e.type === 'step_finish');
-    const finalStep = steps.findLast(e => ['stop', 'end_turn'].includes(e.part?.reason));
-    completed = !!finalStep;
-    failed = events.some(e => e.type === 'error');
-    const texts = events.filter(e => e.type === 'text');
-    output = finalStep?.part?.messageID ? texts.filter(e => e.part?.messageID === finalStep.part.messageID).map(e => e.part?.text ?? '').join('\n') : texts.at(-1)?.part?.text ?? '';
-    if (steps.length) {
-      usage = { input: sum(steps.map(e => sum([number(e.part?.tokens?.input), number(e.part?.tokens?.cache?.read ?? 0), number(e.part?.tokens?.cache?.write ?? 0)]))), output: sum(steps.map(e => sum([number(e.part?.tokens?.output), number(e.part?.tokens?.reasoning ?? 0)]))), cached: sum(steps.map(e => number(e.part?.tokens?.cache?.read ?? 0))) };
-      const costs = steps.map(e => number(e.part?.cost));
-      costUsd = costs.every(c => c !== null && c > 0) ? sum(costs) : null;
-      if (costUsd === null) partialCostUsd = sum(costs.filter(c => c !== null && c > 0));
-    }
-    for (const event of events.filter(e => e.type === 'tool_use' && e.part?.state?.status === 'error')) failureReason ??= toolFailure(event.part.state.error ?? '');
-  }
-  // Native zero prices can mean missing pricing metadata, not free inference.
-  return { output, usage, costUsd: costUsd > 0 ? costUsd : null, partialCostUsd, completed: completed && !failed, failureReason, observedModels: [...observedModels] };
-}
-
-export async function preflight(connection, { cwd, executeCommand = execute, platform = process.platform } = {}) {
-  const command = connection.command ?? connection.agent;
-  const options = { cwd, timeoutMs: 10_000, env: agentEnvironment(connection.env) };
-  const version = await executeCommand(command, ['--version'], options);
-  const help = await executeCommand(command, connection.agent === 'codex' ? ['exec', '--help'] : connection.agent === 'opencode' ? ['run', '--help'] : ['--help'], options);
-  const flags = connection.agent === 'codex' ? ['--json', '--ephemeral', '--sandbox'] : connection.agent === 'claude' ? ['--output-format', '--no-session-persistence', '--max-budget-usd'] : ['--format', '--model', '--auto'];
-  if (version.code !== 0 || help.code !== 0 || !flags.every(f => (help.stdout + help.stderr).includes(f))) throw new Error(`${connection.agent} lacks the required non-interactive JSON interface. Update it or set this connection's command to a compatible executable.`);
-  if (connection.agent === 'codex' && platform === 'linux') {
-    const sandboxHelp = await executeCommand(command, ['sandbox', '--help'], options);
-    const legacy = /^\s+linux\s/m.test(sandboxHelp.stdout);
-    const probe = await executeCommand(command, ['-c', 'sandbox_mode="workspace-write"', 'sandbox', ...(legacy ? ['linux'] : []), '--', process.execPath, '-e', 'process.stdout.write("openmerit-sandbox-ok")'], options);
-    if (probe.code !== 0 || probe.reason || probe.stdout !== 'openmerit-sandbox-ok') {
-      throw Object.assign(new Error('Codex sandbox could not run a local probe. Fix host sandbox support before evaluating; no model request was made.'), { failureReason: 'sandbox_unavailable' });
-    }
-  }
-  return version.stdout.trim().slice(0, 160);
-}
-
-export async function runAgent(connection, prompt, options) {
-  const result = await execute(connection.command ?? connection.agent, invocation(connection, options.reservationUsd, options.unlimitedBudget), { ...options, input: prompt, env: agentEnvironment(connection.env) });
-  const parsed = parseOutput(connection.agent, result.stdout);
-  const failureReason = parsed.failureReason ?? (result.code !== 0 ? connectionFailure(result.stderr) : null) ?? (['spawn_failed', 'cancelled', 'output_limit'].includes(result.reason) ? result.reason : null);
-  const status = failureReason === 'budget_exhausted' ? 'budget_exhausted' : ['sandbox_unavailable', 'permission_denied', 'spawn_failed'].includes(failureReason) ? 'environment_error' : result.reason ?? (failureReason || result.code !== 0 || !parsed.completed ? 'agent_error' : 'ok');
-  return { ...parsed, costUsd: status === 'ok' ? parsed.costUsd : null, partialCostUsd: status === 'ok' ? parsed.partialCostUsd : maximum([parsed.costUsd, parsed.partialCostUsd]), costBasis: parsed.costUsd === null && parsed.partialCostUsd === null ? 'unknown' : 'agent_estimate', seconds: result.seconds, status, failureReason };
-}

package/dist/eval/src/cli.mjs

@@ -1,151 +0,0 @@
-import { mkdir, readFile, writeFile } from 'node:fs/promises';
-import { join, resolve } from 'node:path';
-import { createInterface } from 'node:readline/promises';
-import { agents, preflight } from './adapters.mjs';
-import { configPath, statePath, starter, loadConfig, readJSON, saveJSON } from './config.mjs';
-import { makePlan, savePlan } from './scout.mjs';
-import { acquireLock, evaluate, watch } from './runner.mjs';
-import { formatReport } from './report.mjs';
-
-export const help = `OpenMerit · Agent evaluations (source preview)
-
-  openmerit eval init                 Add a starter suite and project agent instructions
-  openmerit eval doctor               Check installed Claude Code, Codex and OpenCode CLIs
-  openmerit eval plan [--scope existing|all] [--budget <USD|unlimited>]
-                                     Choose scope and budget, then freeze Scout's shortlist
-  openmerit eval run [--json]         Evaluate the frozen shortlist in fresh workspaces
-  openmerit eval watch [--json]       Repeat after selected CLI/docs/fixture files change
-  openmerit eval report [--json]      Read the latest result without running models
-
-  --path <directory>                 Project directory (default: current directory)
-
-Configure .openmerit/eval.json: explicit agent connections, task fixtures, local
-graders, CLI variants, 3–10 repeats, and a cumulative budget. Init starts at $0.
-Plan asks for $10, a custom amount, or no fixed cap; nothing is preselected.
-Saved choices and prior budget usage persist across reruns. --budget changes
-the total allowance, not the amount remaining. Reservations are automatic.
-Scout receives task descriptions and, for existing scope, model IDs only.
-Run/watch execute local agents and may incur charges. No background daemon.
-`;
-
-const instructions = `<!-- openmerit-eval:start -->
-## OpenMerit agent evaluations
-
-When building or changing this project's CLI, tools or harness documentation, use
-\`openmerit eval\` to compare task performance. Skip this workflow when
-\`OPENMERIT_EVAL=1\` to avoid recursive evaluations.
-
-- Read .openmerit/eval.json and run \`openmerit eval doctor\`.
-- Ask once: "Models available through this project's existing connections, or all models Scout indexes?" Use \`eval plan --scope existing\` or \`--scope all\`. Reuse the saved choice. Never infer endpoint access from a model name or expose credentials.
-- Then ask once: "What budget should OpenMerit use for this project's evaluations? 1. $10; 2. Custom amount (how much?); 3. No fixed cap." Pass the explicit answer as \`eval plan --budget 10\`, \`--budget <amount>\` or \`--budget unlimited\`, together with scope on first setup. Never choose a default or infer unlimited spending. Reuse an existing positive or unlimited budget unless the user changes it. The allowance is cumulative across reruns, not reset by edits or replanning. Reservations are internal; do not ask the user to calculate them. No fixed cap still has bounded tasks, 3–10 repetitions, timeouts and user-controlled watch; subsequent watched edits can incur further charges until stopped.
-- Declare real connections with Scout model IDs, agent-native model IDs, and environment variable NAMES. Existing scope uses this project-declared inventory. All scope may require new authenticated connections before execution.
-- Replace starter cases with representative tasks, fresh fixtures and deterministic standalone Node graders. A grader reads {workspace, output} JSON on stdin and returns {quality: 0..1}. Keep answer keys outside agent workspaces. Include a bare variant and a CLI/docs variant where useful.
-- Confirm the suite and variants before enabling paid runs, using the budget already selected above. Do not ask again for an unchanged authorized budget. Then plan once, run a baseline, make the requested CLI changes, and run again. Use \`eval watch\` only when the user wants continuous local reruns; it exits with the terminal and retains budget usage across restarts.
-- Report quality, failures, tokens, reported costs and elapsed time per task/model/variant. A shortlist is a prior, not measured success. Do not claim missing costs are zero or that 3–10 repeats prove an exact ordering. Keep fresh holdouts for final validation.
-- If a compatible watcher is already active, let it run; never start overlapping loops. Do not overwrite the frozen plan merely because the CLI changed.
-<!-- openmerit-eval:end -->`;
-
-export async function initialize(root) {
-  // Refuse partial overwrite of an existing suite or someone's fixtures.
-  const files = {
-    [configPath]: JSON.stringify(starter, null, 2) + '\n',
-    'eval/fixtures/json/request.txt': 'Customer: Mira Chen\nItems: 7\n',
-    'eval/tool.mjs': `import {readFileSync} from 'node:fs';\nconst s=readFileSync(process.argv[2],'utf8');\nconsole.log(JSON.stringify({name:s.match(/Customer: (.*)/)?.[1],count:Number(s.match(/Items: (\\d+)/)?.[1])}));\n`,
-    'eval/grade.mjs': `let input=''; for await (const chunk of process.stdin) input+=chunk;\nconst {output}=JSON.parse(input); let quality=0;\ntry {const v=JSON.parse(output); quality=(Number(v.name==='Mira Chen')+Number(v.count===7)+Number(Object.keys(v).sort().join(',')==='count,name'))/3;} catch {}\nconsole.log(JSON.stringify({quality}));\n`,
-  };
-  for (const file of Object.keys(files)) {
-    try { await readFile(join(root, file)); throw new Error(`Already exists: ${file}. Init does not overwrite a suite.`); } catch (error) { if (error.code !== 'ENOENT') throw error; }
-  }
-  for (const [file, body] of Object.entries(files)) { await mkdir(join(root, file, '..'), { recursive: true }); await writeFile(join(root, file), body, { flag: 'wx' }); }
-  for (const name of ['AGENTS.md', 'CLAUDE.md']) {
-    let body = ''; try { body = await readFile(join(root, name), 'utf8'); } catch (e) { if (e.code !== 'ENOENT') throw e; }
-    if (!body.includes('<!-- openmerit-eval:start -->')) await writeFile(join(root, name), `${body}${body ? '\n\n' : ''}${instructions}\n`);
-  }
-  let ignore = ''; try { ignore = await readFile(join(root, '.gitignore'), 'utf8'); } catch (e) { if (e.code !== 'ENOENT') throw e; }
-  if (!ignore.split('\n').includes('/.openmerit/eval/')) await writeFile(join(root, '.gitignore'), `${ignore}${ignore.endsWith('\n') || !ignore ? '' : '\n'}/.openmerit/eval/\n`);
-}
-
-export function parseBudget(value) {
-  const input = value.trim();
-  if (input === 'unlimited') return input;
-  const amount = input.replace(/^\$/, '');
-  if (!/^(?:\d+(?:\.\d{1,2})?|\.\d{1,2})$/.test(amount) || Number(amount) <= 0 || Number(amount) > Number.MAX_SAFE_INTEGER / 100) throw new Error('Enter a positive USD amount with at most two decimal places, or "unlimited" for no fixed cap.');
-  return Number(amount);
-}
-
-// ask is supplied only for an interactive terminal. Coding agents pass explicit
-// flags after asking the user; EOF, blank input and missing flags never consent.
-export async function choosePlanOptions(config, { scope, budget, ask, tell = () => {} } = {}) {
-  const next = structuredClone(config);
-  if (scope) next.scope = scope;
-  while (!['existing', 'all'].includes(next.scope)) {
-    if (!ask) throw new Error('Choose a scope with eval plan --scope existing or --scope all, then choose --budget <USD|unlimited>.');
-    const answer = (await ask('Use existing project connections or all models Scout indexes? [existing/all] ')).trim();
-    if (['existing', 'all'].includes(answer)) next.scope = answer;
-    else tell('Choose existing or all.');
-  }
-  if (budget !== undefined) next.budget = { maxUsd: parseBudget(budget), reservationUsd: 'auto' };
-  while (next.budget.maxUsd === 0) {
-    if (!ask) throw new Error('Choose an evaluation budget after model scope: --budget 10, --budget <USD>, or --budget unlimited. No amount is selected automatically.');
-    const choice = (await ask('What budget should OpenMerit use for this project’s evaluations?\n  1. $10\n  2. Custom amount\n  3. No fixed cap (watched changes can incur further charges until stopped)\nChoose [1/2/3]: ')).trim();
-    if (choice === '1') next.budget = { maxUsd: 10, reservationUsd: 'auto' };
-    else if (choice === '3') next.budget = { maxUsd: 'unlimited', reservationUsd: 'auto' };
-    else if (choice === '2') {
-      while (next.budget.maxUsd === 0) {
-        const amount = await ask('How much in USD? ');
-        try {
-          const parsed = parseBudget(amount);
-          if (parsed === 'unlimited') throw new Error('Enter a custom USD amount.');
-          next.budget = { maxUsd: parsed, reservationUsd: 'auto' };
-        } catch { tell('Enter a positive USD amount with at most two decimal places.'); }
-      }
-    } else tell('Choose 1, 2 or 3. Nothing is selected automatically.');
-  }
-  return next;
-}
-
-export async function main(args) {
-  if (!args.length || args.includes('--help') || args.includes('-h')) { console.log(help); return; }
-  const action = args.shift(); let root = process.cwd(), scope, budget, json = false;
-  for (let i = 0; i < args.length; i++) {
-    if (args[i] === '--path' && args[i + 1]) root = resolve(args[++i]);
-    else if (args[i] === '--scope' && ['existing', 'all'].includes(args[i + 1])) scope = args[++i];
-    else if (args[i] === '--budget' && args[i + 1] && budget === undefined) { budget = args[++i]; parseBudget(budget); }
-    else if (args[i] === '--json') json = true;
-    else throw new Error(`Unknown eval argument: ${args[i]}`);
-  }
-  if (scope && action !== 'plan') throw new Error('--scope belongs to eval plan.');
-  if (budget !== undefined && action !== 'plan') throw new Error('--budget belongs to eval plan.');
-  if (action === 'init') { await initialize(root); console.log('Created .openmerit/eval.json, starter fixtures, and AGENTS.md / CLAUDE.md instructions. Declare connections, replace the starter task, and set an approved budget before running.'); return; }
-  if (action === 'doctor') {
-    const results = [];
-    for (const agent of agents) { try { results.push({ agent, version: await preflight({ agent }), ready: true }); } catch (error) { results.push({ agent, ready: false, reason: error.message }); } }
-    console.log(JSON.stringify(results, null, 2)); return;
-  }
-  if (action === 'report') { const report = await readJSON(join(root, statePath, 'latest.json')); console.log(json ? JSON.stringify(report, null, 2) : formatReport(report)); return; }
-  if (!['plan', 'run', 'watch'].includes(action)) throw new Error('Unknown eval command. Run openmerit eval --help.');
-  const unlock = await acquireLock(root);
-  const controller = new AbortController(), stop = () => controller.abort();
-  process.once('SIGINT', stop); process.once('SIGTERM', stop);
-  try {
-    if (action === 'plan') {
-      const saved = await loadConfig(root);
-      const rl = process.stdin.isTTY && !json ? createInterface({ input: process.stdin, output: process.stdout }) : null;
-      const inputClosed = new AbortController();
-      rl?.once('close', () => inputClosed.abort());
-      let config;
-      try { config = await choosePlanOptions(saved, { scope, budget, ask: rl ? question => rl.question(question, { signal: AbortSignal.any([controller.signal, inputClosed.signal]) }) : undefined, tell: message => console.log(message) }); }
-      finally { rl?.close(); }
-      // Preserve explicit choices even if Scout is temporarily unavailable.
-      await saveJSON(join(root, configPath), config);
-      const plan = await makePlan(config);
-      await savePlan(root, plan);
-      console.log(JSON.stringify(plan, null, 2));
-      return;
-    }
-    const onReport = report => { console.log(json ? JSON.stringify(report) : formatReport(report)); if (report.status !== 'complete') process.exitCode = 1; };
-    const onProgress = json ? () => {} : row => process.stderr.write(`${row.task} · ${row.connection} · ${row.variant} · ${row.round}: ${row.status}\n`);
-    if (action === 'run') onReport(await evaluate(root, { signal: controller.signal, onProgress }));
-    else { if (!json) console.log('Watching selected files. Ctrl+C stops; budget usage persists.'); await watch(root, { signal: controller.signal, onReport, onProgress }); }
-  } finally { process.removeListener('SIGINT', stop); process.removeListener('SIGTERM', stop); await unlock(); }
-}

package/dist/eval/src/config.mjs

@@ -1,108 +0,0 @@
-import { readFile, writeFile, mkdir, rename, readdir, lstat, realpath } from 'node:fs/promises';
-import { resolve, relative, join, dirname, isAbsolute } from 'node:path';
-import { createHash } from 'node:crypto';
-import { agents } from './adapters.mjs';
-
-export const configPath = '.openmerit/eval.json';
-export const statePath = '.openmerit/eval';
-export const digest = value => createHash('sha256').update(JSON.stringify(value)).digest('hex');
-export async function readJSON(path) { return JSON.parse(await readFile(path, 'utf8')); }
-export async function saveJSON(path, data) {
-  await mkdir(dirname(path), { recursive: true });
-  const temp = `${path}.${process.pid}.tmp`;
-  await writeFile(temp, JSON.stringify(data, null, 2) + '\n', { mode: 0o600 }); await rename(temp, path);
-}
-const text = v => typeof v === 'string' && v.trim().length > 0;
-const id = v => text(v) && /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,79}$/.test(v);
-function requireThat(ok, message) { if (!ok) throw new Error(`Eval config: ${message}`); }
-export function safePath(path) {
-  requireThat(text(path) && !isAbsolute(path) && !path.split(/[\\/]/).some(p => ['..', '.', '.git', '.openmerit', 'node_modules'].includes(p) || p.startsWith('.env')), `unsafe file path ${path}. Use explicit project-relative source/fixture paths, without secrets or symlinks.`);
-  return path;
-}
-export function validateConfig(config) {
-  requireThat(config.version === 1, 'version must be 1.');
-  requireThat([null, 'existing', 'all'].includes(config.scope), 'scope must be null, existing or all.');
-  requireThat(Array.isArray(config.connections) && config.connections.length <= 100, 'connections must be a list of up to 100 routes.');
-  requireThat(new Set(config.connections.map(c => c.id)).size === config.connections.length, 'connection IDs must be unique.');
-  for (const c of config.connections) {
-    requireThat(id(c.id) && agents.includes(c.agent) && text(c.scoutId) && text(c.model), 'each connection needs id, scoutId, agent and native model.');
-    requireThat(!c.model.startsWith('-') && c.model.length <= 200, 'invalid native model.');
-    requireThat(c.env === undefined || Array.isArray(c.env) && c.env.every(k => /^[A-Z][A-Z0-9_]*$/.test(k)), 'env contains variable names, never credential values.');
-    requireThat(c.command === undefined || text(c.command), 'command must be an executable path.');
-  }
-  requireThat(Array.isArray(config.tasks) && config.tasks.length > 0 && config.tasks.length <= 30, 'provide 1–30 tasks.');
-  requireThat(new Set(config.tasks.map(t => t.id)).size === config.tasks.length, 'task IDs must be unique.');
-  for (const t of config.tasks) {
-    requireThat(id(t.id) && text(t.description) && t.description.length <= 4000 && text(t.prompt), 'each task needs id, description for Scout and a local execution prompt.');
-    safePath(t.fixture); safePath(t.grader);
-  }
-  requireThat(Array.isArray(config.variants) && config.variants.length > 0 && config.variants.length <= 5, 'provide 1–5 variants.');
-  requireThat(new Set(config.variants.map(v => v.id)).size === config.variants.length, 'variant IDs must be unique.');
-  for (const v of config.variants) {
-    requireThat(id(v.id) && Array.isArray(v.files) && typeof v.instructions === 'string', 'each variant needs id, files and instructions.');
-    v.files.forEach(safePath);
-  }
-  requireThat(Number.isInteger(config.repeats?.min) && Number.isInteger(config.repeats?.max) && config.repeats.min >= 3 && config.repeats.max <= 10 && config.repeats.min <= config.repeats.max, 'repeats must be between 3 and 10.');
-  requireThat((config.budget?.maxUsd === 'unlimited' || Number.isFinite(config.budget?.maxUsd) && config.budget.maxUsd >= 0) && (config.budget.reservationUsd === 'auto' || Number.isFinite(config.budget.reservationUsd) && config.budget.reservationUsd > 0), 'set nonnegative maxUsd or explicit "unlimited", and reservationUsd to "auto" or a positive amount.');
-  requireThat(Number.isInteger(config.timeoutSeconds) && config.timeoutSeconds >= 1 && config.timeoutSeconds <= 1800, 'timeoutSeconds must be 1–1800.');
-  if (config.scoutURL !== undefined) {
-    const url = new URL(config.scoutURL);
-    requireThat(!url.username && !url.password && (url.protocol === 'https:' || url.protocol === 'http:' && ['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname)), 'scoutURL requires HTTPS, or HTTP on loopback for development.');
-  }
-  return config;
-}
-export async function loadConfig(root) { return validateConfig(await readJSON(join(root, configPath))); }
-
-export function reservationFor(config, plan) {
-  if (config.budget.reservationUsd !== 'auto') return config.budget.reservationUsd;
-  const attempts = plan.tasks.reduce((total, task) => total + task.candidates.reduce((n, model) => n + model.connections.length, 0), 0) * config.variants.length * config.repeats.min;
-  return config.budget.maxUsd === 'unlimited' || config.budget.maxUsd === 0 ? .25 : Math.min(.25, config.budget.maxUsd / Math.max(1, attempts));
-}
-
-// Snapshot exact bytes before executing; edits during a run belong to the next revision.
-export async function filesAt(root, paths) {
-  const files = Object.create(null); let bytes = 0; const canonicalRoot = await realpath(root);
-  async function visit(path) {
-    safePath(path);
-    const full = resolve(root, path), info = await lstat(full);
-    if (info.isSymbolicLink() || relative(canonicalRoot, await realpath(full)).startsWith('..')) throw new Error(`Eval refuses symlink/outside path: ${path}`);
-    if (info.isDirectory()) {
-      for (const name of (await readdir(full)).sort()) await visit(join(path, name));
-    } else if (info.isFile()) {
-      bytes += info.size;
-      if (bytes > 25_000_000) throw new Error('Eval fixture/source snapshot exceeds 25 MB. Select smaller paths.');
-      files[path] = { data: (await readFile(full)).toString('base64'), executable: !!(info.mode & 0o111) };
-    } else throw new Error(`Eval requires regular files: ${path}`);
-  }
-  for (const path of paths) await visit(path);
-  return files;
-}
-export async function snapshot(root, config) {
-  const tasks = [];
-  for (const task of config.tasks) {
-    if (!(await lstat(join(root, task.fixture))).isDirectory() || !(await lstat(join(root, task.grader))).isFile()) throw new Error(`${task.id}: fixture must be a directory and grader a standalone file.`);
-    tasks.push({ ...task, fixtureFiles: await filesAt(root, [task.fixture]), graderFiles: await filesAt(root, [task.grader]) });
-  }
-  const variants = [];
-  for (const variant of config.variants) variants.push({ ...variant, sourceFiles: await filesAt(root, variant.files) });
-  for (const task of tasks) for (const variant of variants) {
-    const fixtureNames = new Set(Object.keys(task.fixtureFiles).map(path => relative(task.fixture, path)));
-    if (Object.keys(variant.sourceFiles).some(path => fixtureNames.has(path))) throw new Error(`${variant.id} overwrites a fixture file for ${task.id}. Keep CLI files and task inputs separate.`);
-  }
-  const compatibility = digest({ tasks, connections: config.connections, repeats: config.repeats, variantIds: variants.map(v => v.id), timeoutSeconds: config.timeoutSeconds });
-  return { tasks, variants, compatibility, revision: digest({ tasks, variants, config }) };
-}
-export async function materialize(files, dest, strip = '') {
-  for (const [path, file] of Object.entries(files)) {
-    const target = join(dest, strip ? relative(strip, path) : path);
-    await mkdir(dirname(target), { recursive: true });
-    await writeFile(target, Buffer.from(file.data, 'base64'), { mode: file.executable ? 0o700 : 0o600 });
-  }
-}
-
-export const starter = {
-  version: 1, scope: null, connections: [],
-  tasks: [{ id: 'json', description: 'Extract a customer name and item count from text into a strict JSON object.', prompt: 'Read request.txt. Return only JSON with keys name (string) and count (number).', fixture: 'eval/fixtures/json', grader: 'eval/grade.mjs' }],
-  variants: [{ id: 'bare', files: [], instructions: '' }, { id: 'cli', files: ['eval/tool.mjs'], instructions: 'You can use node eval/tool.mjs request.txt to extract the fields.' }],
-  repeats: { min: 3, max: 10 }, budget: { maxUsd: 0, reservationUsd: 'auto' }, timeoutSeconds: 180,
-};

package/dist/eval/src/process.mjs

@@ -1,44 +0,0 @@
-import { spawn } from 'node:child_process';
-
-// No shell interpolation. Bound both time and output, and stop descendants too.
-export function execute(command, args, { cwd, env = process.env, input = '', timeoutMs = 180_000, signal } = {}) {
-  return new Promise(resolve => {
-    const start = performance.now();
-    let stdout = '', stderr = '', bytes = 0, reason = null, escalation;
-    const child = spawn(command, args, { cwd, env, detached: process.platform !== 'win32', stdio: ['pipe', 'pipe', 'pipe'] });
-    const kill = sig => {
-      try {
-        if (process.platform === 'win32') { const killer = spawn('taskkill', ['/pid', String(child.pid), '/T', '/F'], { stdio: 'ignore' }); killer.on('error', () => child.kill(sig)); }
-        else process.kill(-child.pid, sig);
-      } catch {}
-    };
-    const stop = why => { if (reason) return; reason = why; kill('SIGTERM'); escalation = setTimeout(() => kill('SIGKILL'), 500); };
-    const abort = () => stop('cancelled');
-    signal?.addEventListener('abort', abort, { once: true });
-    if (signal?.aborted) abort();
-    const timer = setTimeout(() => stop('timeout'), timeoutMs);
-    const collect = (chunk, which) => {
-      bytes += chunk.length;
-      if (bytes > 4_000_000) return stop('output_limit');
-      if (which === 'out') stdout += chunk; else stderr += chunk;
-    };
-    child.stdout.on('data', chunk => collect(chunk, 'out'));
-    child.stderr.on('data', chunk => collect(chunk, 'err'));
-    child.on('error', () => { reason = 'spawn_failed'; });
-    child.stdin.on('error', () => {});
-    child.stdin.end(input);
-    child.on('close', code => {
-      clearTimeout(timer); clearTimeout(escalation); signal?.removeEventListener('abort', abort);
-      resolve({ code, reason, stdout, stderr, seconds: (performance.now() - start) / 1000 });
-    });
-  });
-}
-
-export function agentEnvironment(names = []) {
-  const result = {};
-  for (const key of ['PATH', 'HOME', 'USERPROFILE', 'SystemRoot', 'TMPDIR', 'TMP', 'TEMP', 'LANG', 'TERM', ...names]) {
-    if (process.env[key] !== undefined) result[key] = process.env[key];
-  }
-  result.OPENMERIT_EVAL = '1';
-  return result;
-}

package/dist/eval/src/report.mjs

@@ -1,59 +0,0 @@
-const mean = values => values.length ? values.reduce((a, b) => a + b, 0) / values.length : null;
-const median = values => { const sorted = [...values].sort((a, b) => a - b); return sorted.length ? (sorted[Math.floor((sorted.length - 1) / 2)] + sorted[Math.floor(sorted.length / 2)]) / 2 : null; };
-const completeMean = values => values.every(v => v !== null) ? mean(values) : null;
-export function summarize(rows) {
-  const groups = new Map();
-  for (const row of rows) {
-    const key = [row.task, row.connection, row.variant].join('/');
-    if (!groups.has(key)) groups.set(key, []); groups.get(key).push(row);
-  }
-  return [...groups].map(([key, runs]) => ({
-    key, task: runs[0].task, connection: runs[0].connection, variant: runs[0].variant, runs: runs.length,
-    quality: completeMean(runs.map(r => r.quality)), failures: runs.filter(r => r.status !== 'ok' && r.quality !== null).length,
-    blockedAttempts: runs.filter(r => r.failureReason).length, scoredRuns: runs.filter(r => r.quality !== null).length,
-    failedTaskChecks: runs.filter(r => r.status === 'ok' && r.quality === 0).length,
-    qualityRange: runs.every(r => r.quality !== null) ? [Math.min(...runs.map(r => r.quality)), Math.max(...runs.map(r => r.quality))] : null,
-    inputTokens: completeMean(runs.map(r => r.usage?.input ?? null)), outputTokens: completeMean(runs.map(r => r.usage?.output ?? null)),
-    reportedCostUsd: runs.some(r => r.costUsd !== null) ? runs.filter(r => r.costUsd !== null).reduce((s, r) => s + r.costUsd, 0) : null, unknownCosts: runs.filter(r => r.costUsd === null).length,
-    partialReportedCostUsd: runs.reduce((s, r) => s + (r.partialCostUsd ?? 0), 0),
-    meanCostUsd: completeMean(runs.map(r => r.costUsd)), medianSeconds: median(runs.map(r => r.seconds)),
-  }));
-}
-export function isVolatile(rows) {
-  const spread = values => {
-    if (values.length < 3 || values.some(v => v === null)) return false;
-    const avg = mean(values);
-    return avg > 0 && Math.sqrt(mean(values.map(v => (v - avg) ** 2))) / avg > .25;
-  };
-  for (const group of summarize(rows)) {
-    const runs = rows.filter(r => `${r.task}/${r.connection}/${r.variant}` === group.key);
-    if (group.qualityRange && group.qualityRange[1] - group.qualityRange[0] > .05 || runs.some(r => r.status !== 'ok') || spread(runs.map(r => r.seconds)) || spread(runs.map(r => r.costUsd)) || spread(runs.map(r => r.usage?.input ?? null))) return true;
-  }
-  return false;
-}
-export function compareReports(current, previous) {
-  if (!previous || previous.compatibility !== current.compatibility || previous.planHash !== current.planHash || previous.status !== 'complete' || current.status !== 'complete') return null;
-  return current.summary.map(now => {
-    const before = previous.summary.find(row => row.key === now.key);
-    if (!before) return null;
-    const delta = key => now[key] === null || before[key] === null ? null : now[key] - before[key];
-    return { key: now.key, previousRuns: before.runs, runs: now.runs, qualityDelta: delta('quality'), inputTokensDelta: delta('inputTokens'), outputTokensDelta: delta('outputTokens'), meanCostUsdDelta: delta('meanCostUsd'), medianSecondsDelta: delta('medianSeconds') };
-  }).filter(Boolean);
-}
-const fmt = (v, digits = 2) => v === null || v === undefined ? '?' : v.toFixed(digits);
-export function formatReport(report) {
-  const lines = [`OpenMerit eval · ${report.status} · ${report.revision.slice(0, 12)}`, '', 'Task / connection / variant | scored / attempts | blocked | quality | input / output tokens | estimated $/run | seconds (median)'];
-  for (const row of report.summary) {
-    lines.push(`${row.key} | ${row.scoredRuns ?? row.runs} / ${row.runs} | ${row.blockedAttempts ?? 0} | ${row.quality === null ? '?' : fmt(row.quality * 100, 1) + '%'} | ${fmt(row.inputTokens, 0)} / ${fmt(row.outputTokens, 0)} | ${fmt(row.meanCostUsd, 5)} | ${fmt(row.medianSeconds)}`);
-    if (row.partialReportedCostUsd > 0) lines.push(`  Partial cost estimate: $${fmt(row.partialReportedCostUsd, 5)}; full attempt cost unknown.`);
-  }
-  if (report.comparison) {
-    lines.push('', 'Change from the previous compatible revision (negative tokens/cost/time is lower):');
-    for (const row of report.comparison) lines.push(`${row.key}: quality ${fmt(row.qualityDelta === null ? null : row.qualityDelta * 100, 1)} pp; tokens ${fmt(row.inputTokensDelta, 0)} / ${fmt(row.outputTokensDelta, 0)}; $ ${fmt(row.meanCostUsdDelta, 5)}; seconds ${fmt(row.medianSecondsDelta)}`);
-  } else lines.push('', 'No complete, compatible previous revision for comparison.');
-  lines.push('', '3–10 repetitions measure variability, not statistical certainty. ? means unavailable.', 'Costs are agent-reported estimates, not reconciled bills. Compare agents as different harnesses.');
-  if (report.budgetLimitUsd !== undefined) lines.push(report.budgetLimitUsd === 'unlimited' ? 'Budget: no fixed cap. Task/repetition limits and timeouts still apply; watched changes can incur further charges until stopped.' : `Budget: $${fmt(report.budgetLimitUsd)} cumulative across reruns.`);
-  lines.push(`Reserved allowance used: $${fmt(report.budgetUsedUsd, 5)} (not billed spend). Blocked attempts are unscored.`);
-  if (report.reason) lines.push(`Stopped: ${report.reason}`);
-  return lines.join('\n');
-}