openmerit 0.2.0 → 0.3.0

package/dist/eval/src/config.mjs

@@ -1,108 +0,0 @@
-import { readFile, writeFile, mkdir, rename, readdir, lstat, realpath } from 'node:fs/promises';
-import { resolve, relative, join, dirname, isAbsolute } from 'node:path';
-import { createHash } from 'node:crypto';
-import { agents } from './adapters.mjs';
-
-export const configPath = '.openmerit/eval.json';
-export const statePath = '.openmerit/eval';
-export const digest = value => createHash('sha256').update(JSON.stringify(value)).digest('hex');
-export async function readJSON(path) { return JSON.parse(await readFile(path, 'utf8')); }
-export async function saveJSON(path, data) {
-  await mkdir(dirname(path), { recursive: true });
-  const temp = `${path}.${process.pid}.tmp`;
-  await writeFile(temp, JSON.stringify(data, null, 2) + '\n', { mode: 0o600 }); await rename(temp, path);
-}
-const text = v => typeof v === 'string' && v.trim().length > 0;
-const id = v => text(v) && /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,79}$/.test(v);
-function requireThat(ok, message) { if (!ok) throw new Error(`Eval config: ${message}`); }
-export function safePath(path) {
-  requireThat(text(path) && !isAbsolute(path) && !path.split(/[\\/]/).some(p => ['..', '.', '.git', '.openmerit', 'node_modules'].includes(p) || p.startsWith('.env')), `unsafe file path ${path}. Use explicit project-relative source/fixture paths, without secrets or symlinks.`);
-  return path;
-}
-export function validateConfig(config) {
-  requireThat(config.version === 1, 'version must be 1.');
-  requireThat([null, 'existing', 'all'].includes(config.scope), 'scope must be null, existing or all.');
-  requireThat(Array.isArray(config.connections) && config.connections.length <= 100, 'connections must be a list of up to 100 routes.');
-  requireThat(new Set(config.connections.map(c => c.id)).size === config.connections.length, 'connection IDs must be unique.');
-  for (const c of config.connections) {
-    requireThat(id(c.id) && agents.includes(c.agent) && text(c.scoutId) && text(c.model), 'each connection needs id, scoutId, agent and native model.');
-    requireThat(!c.model.startsWith('-') && c.model.length <= 200, 'invalid native model.');
-    requireThat(c.env === undefined || Array.isArray(c.env) && c.env.every(k => /^[A-Z][A-Z0-9_]*$/.test(k)), 'env contains variable names, never credential values.');
-    requireThat(c.command === undefined || text(c.command), 'command must be an executable path.');
-  }
-  requireThat(Array.isArray(config.tasks) && config.tasks.length > 0 && config.tasks.length <= 30, 'provide 1–30 tasks.');
-  requireThat(new Set(config.tasks.map(t => t.id)).size === config.tasks.length, 'task IDs must be unique.');
-  for (const t of config.tasks) {
-    requireThat(id(t.id) && text(t.description) && t.description.length <= 4000 && text(t.prompt), 'each task needs id, description for Scout and a local execution prompt.');
-    safePath(t.fixture); safePath(t.grader);
-  }
-  requireThat(Array.isArray(config.variants) && config.variants.length > 0 && config.variants.length <= 5, 'provide 1–5 variants.');
-  requireThat(new Set(config.variants.map(v => v.id)).size === config.variants.length, 'variant IDs must be unique.');
-  for (const v of config.variants) {
-    requireThat(id(v.id) && Array.isArray(v.files) && typeof v.instructions === 'string', 'each variant needs id, files and instructions.');
-    v.files.forEach(safePath);
-  }
-  requireThat(Number.isInteger(config.repeats?.min) && Number.isInteger(config.repeats?.max) && config.repeats.min >= 3 && config.repeats.max <= 10 && config.repeats.min <= config.repeats.max, 'repeats must be between 3 and 10.');
-  requireThat((config.budget?.maxUsd === 'unlimited' || Number.isFinite(config.budget?.maxUsd) && config.budget.maxUsd >= 0) && (config.budget.reservationUsd === 'auto' || Number.isFinite(config.budget.reservationUsd) && config.budget.reservationUsd > 0), 'set nonnegative maxUsd or explicit "unlimited", and reservationUsd to "auto" or a positive amount.');
-  requireThat(Number.isInteger(config.timeoutSeconds) && config.timeoutSeconds >= 1 && config.timeoutSeconds <= 1800, 'timeoutSeconds must be 1–1800.');
-  if (config.scoutURL !== undefined) {
-    const url = new URL(config.scoutURL);
-    requireThat(!url.username && !url.password && (url.protocol === 'https:' || url.protocol === 'http:' && ['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname)), 'scoutURL requires HTTPS, or HTTP on loopback for development.');
-  }
-  return config;
-}
-export async function loadConfig(root) { return validateConfig(await readJSON(join(root, configPath))); }
-
-export function reservationFor(config, plan) {
-  if (config.budget.reservationUsd !== 'auto') return config.budget.reservationUsd;
-  const attempts = plan.tasks.reduce((total, task) => total + task.candidates.reduce((n, model) => n + model.connections.length, 0), 0) * config.variants.length * config.repeats.min;
-  return config.budget.maxUsd === 'unlimited' || config.budget.maxUsd === 0 ? .25 : Math.min(.25, config.budget.maxUsd / Math.max(1, attempts));
-}
-
-// Snapshot exact bytes before executing; edits during a run belong to the next revision.
-export async function filesAt(root, paths) {
-  const files = Object.create(null); let bytes = 0; const canonicalRoot = await realpath(root);
-  async function visit(path) {
-    safePath(path);
-    const full = resolve(root, path), info = await lstat(full);
-    if (info.isSymbolicLink() || relative(canonicalRoot, await realpath(full)).startsWith('..')) throw new Error(`Eval refuses symlink/outside path: ${path}`);
-    if (info.isDirectory()) {
-      for (const name of (await readdir(full)).sort()) await visit(join(path, name));
-    } else if (info.isFile()) {
-      bytes += info.size;
-      if (bytes > 25_000_000) throw new Error('Eval fixture/source snapshot exceeds 25 MB. Select smaller paths.');
-      files[path] = { data: (await readFile(full)).toString('base64'), executable: !!(info.mode & 0o111) };
-    } else throw new Error(`Eval requires regular files: ${path}`);
-  }
-  for (const path of paths) await visit(path);
-  return files;
-}
-export async function snapshot(root, config) {
-  const tasks = [];
-  for (const task of config.tasks) {
-    if (!(await lstat(join(root, task.fixture))).isDirectory() || !(await lstat(join(root, task.grader))).isFile()) throw new Error(`${task.id}: fixture must be a directory and grader a standalone file.`);
-    tasks.push({ ...task, fixtureFiles: await filesAt(root, [task.fixture]), graderFiles: await filesAt(root, [task.grader]) });
-  }
-  const variants = [];
-  for (const variant of config.variants) variants.push({ ...variant, sourceFiles: await filesAt(root, variant.files) });
-  for (const task of tasks) for (const variant of variants) {
-    const fixtureNames = new Set(Object.keys(task.fixtureFiles).map(path => relative(task.fixture, path)));
-    if (Object.keys(variant.sourceFiles).some(path => fixtureNames.has(path))) throw new Error(`${variant.id} overwrites a fixture file for ${task.id}. Keep CLI files and task inputs separate.`);
-  }
-  const compatibility = digest({ tasks, connections: config.connections, repeats: config.repeats, variantIds: variants.map(v => v.id), timeoutSeconds: config.timeoutSeconds });
-  return { tasks, variants, compatibility, revision: digest({ tasks, variants, config }) };
-}
-export async function materialize(files, dest, strip = '') {
-  for (const [path, file] of Object.entries(files)) {
-    const target = join(dest, strip ? relative(strip, path) : path);
-    await mkdir(dirname(target), { recursive: true });
-    await writeFile(target, Buffer.from(file.data, 'base64'), { mode: file.executable ? 0o700 : 0o600 });
-  }
-}
-
-export const starter = {
-  version: 1, scope: null, connections: [],
-  tasks: [{ id: 'json', description: 'Extract a customer name and item count from text into a strict JSON object.', prompt: 'Read request.txt. Return only JSON with keys name (string) and count (number).', fixture: 'eval/fixtures/json', grader: 'eval/grade.mjs' }],
-  variants: [{ id: 'bare', files: [], instructions: '' }, { id: 'cli', files: ['eval/tool.mjs'], instructions: 'You can use node eval/tool.mjs request.txt to extract the fields.' }],
-  repeats: { min: 3, max: 10 }, budget: { maxUsd: 0, reservationUsd: 'auto' }, timeoutSeconds: 180,
-};

package/dist/eval/src/process.mjs

@@ -1,44 +0,0 @@
-import { spawn } from 'node:child_process';
-
-// No shell interpolation. Bound both time and output, and stop descendants too.
-export function execute(command, args, { cwd, env = process.env, input = '', timeoutMs = 180_000, signal } = {}) {
-  return new Promise(resolve => {
-    const start = performance.now();
-    let stdout = '', stderr = '', bytes = 0, reason = null, escalation;
-    const child = spawn(command, args, { cwd, env, detached: process.platform !== 'win32', stdio: ['pipe', 'pipe', 'pipe'] });
-    const kill = sig => {
-      try {
-        if (process.platform === 'win32') { const killer = spawn('taskkill', ['/pid', String(child.pid), '/T', '/F'], { stdio: 'ignore' }); killer.on('error', () => child.kill(sig)); }
-        else process.kill(-child.pid, sig);
-      } catch {}
-    };
-    const stop = why => { if (reason) return; reason = why; kill('SIGTERM'); escalation = setTimeout(() => kill('SIGKILL'), 500); };
-    const abort = () => stop('cancelled');
-    signal?.addEventListener('abort', abort, { once: true });
-    if (signal?.aborted) abort();
-    const timer = setTimeout(() => stop('timeout'), timeoutMs);
-    const collect = (chunk, which) => {
-      bytes += chunk.length;
-      if (bytes > 4_000_000) return stop('output_limit');
-      if (which === 'out') stdout += chunk; else stderr += chunk;
-    };
-    child.stdout.on('data', chunk => collect(chunk, 'out'));
-    child.stderr.on('data', chunk => collect(chunk, 'err'));
-    child.on('error', () => { reason = 'spawn_failed'; });
-    child.stdin.on('error', () => {});
-    child.stdin.end(input);
-    child.on('close', code => {
-      clearTimeout(timer); clearTimeout(escalation); signal?.removeEventListener('abort', abort);
-      resolve({ code, reason, stdout, stderr, seconds: (performance.now() - start) / 1000 });
-    });
-  });
-}
-
-export function agentEnvironment(names = []) {
-  const result = {};
-  for (const key of ['PATH', 'HOME', 'USERPROFILE', 'SystemRoot', 'TMPDIR', 'TMP', 'TEMP', 'LANG', 'TERM', ...names]) {
-    if (process.env[key] !== undefined) result[key] = process.env[key];
-  }
-  result.OPENMERIT_EVAL = '1';
-  return result;
-}

package/dist/eval/src/report.mjs

@@ -1,59 +0,0 @@
-const mean = values => values.length ? values.reduce((a, b) => a + b, 0) / values.length : null;
-const median = values => { const sorted = [...values].sort((a, b) => a - b); return sorted.length ? (sorted[Math.floor((sorted.length - 1) / 2)] + sorted[Math.floor(sorted.length / 2)]) / 2 : null; };
-const completeMean = values => values.every(v => v !== null) ? mean(values) : null;
-export function summarize(rows) {
-  const groups = new Map();
-  for (const row of rows) {
-    const key = [row.task, row.connection, row.variant].join('/');
-    if (!groups.has(key)) groups.set(key, []); groups.get(key).push(row);
-  }
-  return [...groups].map(([key, runs]) => ({
-    key, task: runs[0].task, connection: runs[0].connection, variant: runs[0].variant, runs: runs.length,
-    quality: completeMean(runs.map(r => r.quality)), failures: runs.filter(r => r.status !== 'ok' && r.quality !== null).length,
-    blockedAttempts: runs.filter(r => r.failureReason).length, scoredRuns: runs.filter(r => r.quality !== null).length,
-    failedTaskChecks: runs.filter(r => r.status === 'ok' && r.quality === 0).length,
-    qualityRange: runs.every(r => r.quality !== null) ? [Math.min(...runs.map(r => r.quality)), Math.max(...runs.map(r => r.quality))] : null,
-    inputTokens: completeMean(runs.map(r => r.usage?.input ?? null)), outputTokens: completeMean(runs.map(r => r.usage?.output ?? null)),
-    reportedCostUsd: runs.some(r => r.costUsd !== null) ? runs.filter(r => r.costUsd !== null).reduce((s, r) => s + r.costUsd, 0) : null, unknownCosts: runs.filter(r => r.costUsd === null).length,
-    partialReportedCostUsd: runs.reduce((s, r) => s + (r.partialCostUsd ?? 0), 0),
-    meanCostUsd: completeMean(runs.map(r => r.costUsd)), medianSeconds: median(runs.map(r => r.seconds)),
-  }));
-}
-export function isVolatile(rows) {
-  const spread = values => {
-    if (values.length < 3 || values.some(v => v === null)) return false;
-    const avg = mean(values);
-    return avg > 0 && Math.sqrt(mean(values.map(v => (v - avg) ** 2))) / avg > .25;
-  };
-  for (const group of summarize(rows)) {
-    const runs = rows.filter(r => `${r.task}/${r.connection}/${r.variant}` === group.key);
-    if (group.qualityRange && group.qualityRange[1] - group.qualityRange[0] > .05 || runs.some(r => r.status !== 'ok') || spread(runs.map(r => r.seconds)) || spread(runs.map(r => r.costUsd)) || spread(runs.map(r => r.usage?.input ?? null))) return true;
-  }
-  return false;
-}
-export function compareReports(current, previous) {
-  if (!previous || previous.compatibility !== current.compatibility || previous.planHash !== current.planHash || previous.status !== 'complete' || current.status !== 'complete') return null;
-  return current.summary.map(now => {
-    const before = previous.summary.find(row => row.key === now.key);
-    if (!before) return null;
-    const delta = key => now[key] === null || before[key] === null ? null : now[key] - before[key];
-    return { key: now.key, previousRuns: before.runs, runs: now.runs, qualityDelta: delta('quality'), inputTokensDelta: delta('inputTokens'), outputTokensDelta: delta('outputTokens'), meanCostUsdDelta: delta('meanCostUsd'), medianSecondsDelta: delta('medianSeconds') };
-  }).filter(Boolean);
-}
-const fmt = (v, digits = 2) => v === null || v === undefined ? '?' : v.toFixed(digits);
-export function formatReport(report) {
-  const lines = [`OpenMerit eval · ${report.status} · ${report.revision.slice(0, 12)}`, '', 'Task / connection / variant | scored / attempts | blocked | quality | input / output tokens | estimated $/run | seconds (median)'];
-  for (const row of report.summary) {
-    lines.push(`${row.key} | ${row.scoredRuns ?? row.runs} / ${row.runs} | ${row.blockedAttempts ?? 0} | ${row.quality === null ? '?' : fmt(row.quality * 100, 1) + '%'} | ${fmt(row.inputTokens, 0)} / ${fmt(row.outputTokens, 0)} | ${fmt(row.meanCostUsd, 5)} | ${fmt(row.medianSeconds)}`);
-    if (row.partialReportedCostUsd > 0) lines.push(`  Partial cost estimate: $${fmt(row.partialReportedCostUsd, 5)}; full attempt cost unknown.`);
-  }
-  if (report.comparison) {
-    lines.push('', 'Change from the previous compatible revision (negative tokens/cost/time is lower):');
-    for (const row of report.comparison) lines.push(`${row.key}: quality ${fmt(row.qualityDelta === null ? null : row.qualityDelta * 100, 1)} pp; tokens ${fmt(row.inputTokensDelta, 0)} / ${fmt(row.outputTokensDelta, 0)}; $ ${fmt(row.meanCostUsdDelta, 5)}; seconds ${fmt(row.medianSecondsDelta)}`);
-  } else lines.push('', 'No complete, compatible previous revision for comparison.');
-  lines.push('', '3–10 repetitions measure variability, not statistical certainty. ? means unavailable.', 'Costs are agent-reported estimates, not reconciled bills. Compare agents as different harnesses.');
-  if (report.budgetLimitUsd !== undefined) lines.push(report.budgetLimitUsd === 'unlimited' ? 'Budget: no fixed cap. Task/repetition limits and timeouts still apply; watched changes can incur further charges until stopped.' : `Budget: $${fmt(report.budgetLimitUsd)} cumulative across reruns.`);
-  lines.push(`Reserved allowance used: $${fmt(report.budgetUsedUsd, 5)} (not billed spend). Blocked attempts are unscored.`);
-  if (report.reason) lines.push(`Stopped: ${report.reason}`);
-  return lines.join('\n');
-}

package/dist/eval/src/runner.mjs

@@ -1,123 +0,0 @@
-import { mkdir, mkdtemp, rm, readFile, writeFile } from 'node:fs/promises';
-import { join } from 'node:path';
-import { tmpdir } from 'node:os';
-import { randomUUID } from 'node:crypto';
-import { loadConfig, snapshot, materialize, readJSON, saveJSON, statePath, digest, reservationFor } from './config.mjs';
-import { validatePlan } from './scout.mjs';
-import { preflight, runAgent, invocation } from './adapters.mjs';
-import { execute, agentEnvironment } from './process.mjs';
-import { summarize, isVolatile, compareReports } from './report.mjs';
-
-async function optionalJSON(path) { try { return await readJSON(path); } catch (e) { if (e.code === 'ENOENT') return null; throw e; } }
-export async function acquireLock(root) {
-  const lock = join(root, statePath, 'lock'); await mkdir(join(root, statePath), { recursive: true });
-  try { await mkdir(lock); } catch (error) { if (error.code === 'EEXIST') throw new Error('An eval run/watch owns this project. Stop it first. After a crash, verify the PID in .openmerit/eval/lock/pid before removing that lock directory.'); throw error; }
-  await writeFile(join(lock, 'pid'), String(process.pid));
-  return () => rm(lock, { recursive: true, force: true });
-}
-export function budgetUsed(ledger) {
-  if (ledger?.version !== 1 || !Array.isArray(ledger.entries) || ledger.entries.some(e => !Number.isFinite(e.reservationUsd) || e.reservationUsd <= 0 || e.costUsd !== null && (!Number.isFinite(e.costUsd) || e.costUsd < 0))) throw new Error('Invalid budget ledger. Restore it before running; never discard unknown charges.');
-  // Agent prices are estimates, not provider settlement. Never release a
-  // reservation on that evidence, including entries written by older previews.
-  return ledger.entries.reduce((sum, e) => sum + Math.max(e.costUsd ?? 0, e.reservationUsd), 0);
-}
-
-export async function evaluate(root, { signal, onProgress = () => {}, agentRunner = runAgent, checkAgent = preflight } = {}) {
-  const config = await loadConfig(root), plan = await readJSON(join(root, statePath, 'plan.json'));
-  validatePlan(config, plan);
-  const snap = await snapshot(root, config);
-  const selected = new Set(plan.tasks.flatMap(t => t.candidates.flatMap(c => c.connections)));
-  const versions = {};
-  const ledgerPath = join(root, statePath, 'budget.json');
-  const ledger = await optionalJSON(ledgerPath) ?? { version: 1, entries: [] };
-  const reservationUsd = reservationFor(config, plan), unlimitedBudget = config.budget.maxUsd === 'unlimited';
-  const exhausted = () => { const used = budgetUsed(ledger); return !unlimitedBudget && used + reservationUsd > config.budget.maxUsd + 1e-10; };
-  if (exhausted()) throw new Error('Evaluation budget is exhausted or disabled. Choose a budget with eval plan --budget <USD|unlimited>; prior reservations still count.');
-  const runnerVersion = (await readJSON(new URL('../../../package.json', import.meta.url))).version;
-  const commands = config.connections.filter(c => selected.has(c.id)).map(c => ({ id: c.id, args: invocation(c, reservationUsd, unlimitedBudget) }));
-  const report = { version: 2, runnerVersion, budgetLimitUsd: config.budget.maxUsd, reservationUsd, id: randomUUID(), startedAt: new Date().toISOString(), revision: snap.revision, compatibility: null, planHash: digest(plan.tasks), agentVersions: versions, status: 'running', rows: [], summary: [] };
-  const reportPath = join(root, statePath, 'runs', `${report.startedAt.replace(/[:.]/g, '-')}-${report.id}.json`);
-  const previous = await optionalJSON(join(root, statePath, 'latest.json'));
-  const temp = await mkdtemp(join(tmpdir(), 'openmerit-eval-'));
-  const checkpoint = async () => { report.summary = summarize(report.rows); await saveJSON(reportPath, report); };
-  try {
-    await checkpoint();
-    for (const connection of config.connections.filter(c => selected.has(c.id))) {
-      try { versions[connection.id] = await checkAgent(connection, { cwd: temp }); }
-      catch (error) { error.connection = connection.id; throw error; }
-    }
-    // Graders are copied outside all agent workspaces, before any trial runs.
-    for (const task of snap.tasks) await materialize(task.graderFiles, join(temp, 'graders', task.id));
-    outer: for (const task of snap.tasks) {
-      const taskPlan = plan.tasks.find(t => t.id === task.id);
-      const cells = taskPlan.candidates.flatMap(candidate => candidate.connections.flatMap(connection => snap.variants.map(variant => ({ connection: config.connections.find(c => c.id === connection), variant, candidate }))));
-      for (let round = 0; round < config.repeats.max; round++) {
-        // Rotate and reverse the balanced order so one candidate is not always warm/last.
-        const ordered = [...cells.slice(round % cells.length), ...cells.slice(0, round % cells.length)];
-        if (round % 2) ordered.reverse();
-        for (const { connection, variant, candidate } of ordered) {
-          if (signal?.aborted) { report.status = 'cancelled'; report.reason = 'Interrupted'; break outer; }
-          if (exhausted()) { report.status = 'budget_stopped'; report.reason = 'Cumulative budget cannot admit another reserved attempt.'; break outer; }
-          const entry = { id: randomUUID(), run: report.id, reservationUsd, costUsd: null, settled: false };
-          ledger.entries.push(entry); await saveJSON(ledgerPath, ledger);
-          const workspace = join(temp, entry.id); await mkdir(workspace);
-          await materialize(task.fixtureFiles, workspace, task.fixture);
-          await materialize(variant.sourceFiles, workspace);
-          const prompt = `${task.prompt}\n\n${variant.instructions}\n\nThis is an isolated evaluation attempt. Do not run OpenMerit evaluation commands or inspect paths outside this workspace.`;
-          const result = await agentRunner(connection, prompt, { cwd: workspace, timeoutMs: config.timeoutSeconds * 1000, signal, reservationUsd, unlimitedBudget });
-          entry.costUsd = result.costUsd; entry.costBasis = result.costBasis ?? (result.costUsd === null ? 'unknown' : 'agent_estimate');
-          entry.partialCostUsd = result.partialCostUsd ?? null; entry.reservationUsd = Math.max(entry.reservationUsd, result.partialCostUsd ?? 0); entry.settled = true; await saveJSON(ledgerPath, ledger);
-          let quality = result.status === 'ok' || result.failureReason ? null : 0, status = result.status;
-          if (result.status === 'ok') {
-            const grader = await execute(process.execPath, [join(temp, 'graders', task.id, task.grader)], { cwd: temp, env: agentEnvironment(), input: JSON.stringify({ workspace, output: result.output }), timeoutMs: 30_000, signal });
-            try {
-              const value = JSON.parse(grader.stdout);
-              if (grader.code !== 0 || grader.reason || typeof value.quality !== 'number' || !Number.isFinite(value.quality) || value.quality < 0 || value.quality > 1) throw new Error('Invalid grade');
-              quality = value.quality;
-            } catch { status = 'grader_error'; }
-          }
-          const row = { task: task.id, connection: connection.id, agent: connection.agent, requestedModel: connection.model, observedModels: result.observedModels, scoutId: candidate.scoutId, scoutRank: candidate.rank, variant: variant.id, round: round + 1, quality, status, failureReason: result.failureReason ?? null, usage: result.usage, costUsd: result.costUsd, partialCostUsd: result.partialCostUsd ?? null, costBasis: entry.costBasis, seconds: result.seconds };
-          report.rows.push(row); await checkpoint(); await rm(workspace, { recursive: true, force: true });
-          onProgress(row);
-          if (result.failureReason) {
-            report.status = result.failureReason === 'budget_exhausted' ? 'budget_stopped' : result.failureReason === 'cancelled' ? 'cancelled' : ['sandbox_unavailable', 'permission_denied', 'spawn_failed'].includes(result.failureReason) ? 'environment_blocked' : 'connection_blocked';
-            report.reason = `${connection.id}: ${result.failureReason}. This attempt is unscored; resolve the execution block before retrying.`;
-            break outer;
-          }
-          if (status === 'grader_error') { report.status = 'invalid'; report.reason = `Grader failed for ${task.id}. Fix it before collecting more trials.`; break outer; }
-        }
-        if (round + 1 >= config.repeats.min && !isVolatile(report.rows.filter(r => r.task === task.id))) break;
-      }
-    }
-    if (report.status === 'running') report.status = 'complete';
-  } catch (error) {
-    if (error.failureReason) {
-      report.status = 'environment_blocked'; report.reason = `${error.connection}: ${error.failureReason}. ${error.message}`;
-      report.blockers = [{ connection: error.connection, failureReason: error.failureReason }];
-    } else {
-      report.status = 'error'; report.reason = 'Execution failed; unsettled reservations remain charged. Check configuration and agent authentication.';
-      throw error;
-    }
-  } finally {
-    report.finishedAt = new Date().toISOString(); report.budgetUsedUsd = budgetUsed(ledger);
-    report.compatibility = digest({ accountingVersion: report.version, snapshot: snap.compatibility, versions, runnerVersion, commands, observedModels: Object.fromEntries([...selected].sort().map(id => [id, [...new Set(report.rows.filter(r => r.connection === id).flatMap(r => r.observedModels ?? []))].sort()])) });
-    report.summary = summarize(report.rows); report.comparison = compareReports(report, previous);
-    await saveJSON(reportPath, report); await saveJSON(join(root, statePath, 'latest.json'), report);
-    await rm(temp, { recursive: true, force: true });
-  }
-  return report;
-}
-
-export async function watch(root, { signal, onReport, onProgress, intervalMs = 1500, ...options } = {}) {
-  let last = null, pending = null;
-  while (!signal?.aborted) {
-    const config = await loadConfig(root), current = (await snapshot(root, config)).revision;
-    if (current !== last && current === pending) {
-      const result = await evaluate(root, { signal, onProgress, ...options });
-      onReport?.(result); last = result.revision;
-      if (result.status !== 'complete') return result;
-    }
-    pending = current;
-    await new Promise(resolve => { const done = () => { clearTimeout(timer); signal?.removeEventListener('abort', done); resolve(); }; const timer = setTimeout(done, intervalMs); signal?.addEventListener('abort', done, { once: true }); });
-  }
-}

package/dist/eval/src/scout.mjs

@@ -1,48 +0,0 @@
-import { digest, saveJSON, statePath } from './config.mjs';
-import { join } from 'node:path';
-import { scoutDiagnostic } from '../../terminal/src/telemetry.mjs';
-
-export const selectionHash = config => digest({ scope: config.scope, connections: config.connections, tasks: config.tasks.map(t => ({ id: t.id, description: t.description })), scoutURL: config.scoutURL });
-export async function makePlan(config, fetcher = fetch) {
-  if (!['existing', 'all'].includes(config.scope)) throw new Error('Choose a scope with eval plan --scope existing or --scope all. Existing means the connections declared for this project.');
-  const modelIds = [...new Set(config.connections.map(c => c.scoutId))];
-  if (config.scope === 'existing' && !modelIds.length) throw new Error('Declare project connections in .openmerit/eval.json first. No credentials are sent to Scout.');
-  const tasks = [];
-  for (const task of config.tasks) {
-    let trace, code = 'scout_network_error';
-    try {
-    const response = await fetcher(config.scoutURL ?? 'https://openmerit.ai/scout/api/recommend', { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Scout-Client': 'cli', 'X-Scout-Analytics-Content': 'off', 'X-Scout-Analytics': process.env.OPENMERIT_TELEMETRY === '1' && process.env.DO_NOT_TRACK !== '1' ? 'on' : 'off' }, body: JSON.stringify({ prompt: task.description, ...(config.scope === 'existing' ? { modelIds } : {}) }), signal: AbortSignal.timeout(15_000), redirect: 'error' });
-    trace = response.headers?.get('X-Scout-Trace-Id');
-    code = 'scout_http_error';
-    if (!response.ok) throw new Error(`Scout returned HTTP ${response.status}; no evaluation was started.`);
-    code = 'scout_response_error';
-    const body = await response.json();
-    if (!Array.isArray(body.recommendations) || body.recommendations.length > 5) throw new Error('Invalid Scout response.');
-    if (config.scope === 'existing' && (!Array.isArray(body.selection?.modelIds) || digest([...body.selection.modelIds].sort()) !== digest([...modelIds].sort()))) throw new Error('Scout did not acknowledge filtering before ranking. Update the Scout endpoint; no unscoped fallback is allowed.');
-    const seen = new Set();
-    const candidates = body.recommendations.map(m => {
-      if (typeof m.id !== 'string' || seen.has(m.id) || config.scope === 'existing' && !modelIds.includes(m.id)) throw new Error('Invalid Scout model selection.');
-      seen.add(m.id);
-      return { scoutId: m.id, name: m.name, rank: m.rank, connections: config.connections.filter(c => c.scoutId === m.id).map(c => c.id) };
-    });
-    tasks.push({ id: task.id, candidates, ranking: body.ranking, warnings: body.warnings ?? [] });
-    } catch (error) {
-      try { scoutDiagnostic(error, trace, error?.name === 'TimeoutError' || error?.name === 'AbortError' ? 'scout_timeout' : code); }
-      catch { /* Diagnostics must not replace the original Scout failure. */ }
-      throw error;
-    }
-  }
-  return { version: 1, createdAt: new Date().toISOString(), scope: config.scope, selectionHash: selectionHash(config), tasks };
-}
-export async function savePlan(root, plan) { await saveJSON(join(root, statePath, 'plan.json'), plan); }
-export function validatePlan(config, plan) {
-  if (plan.selectionHash !== selectionHash(config)) throw new Error('Task descriptions, scope or connections changed. Run eval plan again.');
-  if (!Array.isArray(plan.tasks) || plan.tasks.length !== config.tasks.length) throw new Error('Incomplete evaluation plan. Run eval plan again.');
-  for (const task of config.tasks) {
-    const entry = plan.tasks.find(t => t.id === task.id);
-    if (!entry?.candidates?.length) throw new Error(`Scout has no supported candidates for ${task.id}. Broaden scope or revise the task description.`);
-    for (const model of entry.candidates) {
-      if (!model.connections.length || model.connections.some(id => !config.connections.some(c => c.id === id && c.scoutId === model.scoutId))) throw new Error(`Configure an authenticated agent connection for ${model.scoutId}, then plan again. All-model discovery does not grant endpoint access.`);
-    }
-  }
-}