openmerit 0.1.6 → 0.1.7-preview.2

package/dist/eval/src/report.mjs

@@ -0,0 +1,59 @@
+const mean = values => values.length ? values.reduce((a, b) => a + b, 0) / values.length : null;
+const median = values => { const sorted = [...values].sort((a, b) => a - b); return sorted.length ? (sorted[Math.floor((sorted.length - 1) / 2)] + sorted[Math.floor(sorted.length / 2)]) / 2 : null; };
+const completeMean = values => values.every(v => v !== null) ? mean(values) : null;
+export function summarize(rows) {
+  const groups = new Map();
+  for (const row of rows) {
+    const key = [row.task, row.connection, row.variant].join('/');
+    if (!groups.has(key)) groups.set(key, []); groups.get(key).push(row);
+  }
+  return [...groups].map(([key, runs]) => ({
+    key, task: runs[0].task, connection: runs[0].connection, variant: runs[0].variant, runs: runs.length,
+    quality: completeMean(runs.map(r => r.quality)), failures: runs.filter(r => r.status !== 'ok' && r.quality !== null).length,
+    blockedAttempts: runs.filter(r => r.failureReason).length, scoredRuns: runs.filter(r => r.quality !== null).length,
+    failedTaskChecks: runs.filter(r => r.status === 'ok' && r.quality === 0).length,
+    qualityRange: runs.every(r => r.quality !== null) ? [Math.min(...runs.map(r => r.quality)), Math.max(...runs.map(r => r.quality))] : null,
+    inputTokens: completeMean(runs.map(r => r.usage?.input ?? null)), outputTokens: completeMean(runs.map(r => r.usage?.output ?? null)),
+    reportedCostUsd: runs.some(r => r.costUsd !== null) ? runs.filter(r => r.costUsd !== null).reduce((s, r) => s + r.costUsd, 0) : null, unknownCosts: runs.filter(r => r.costUsd === null).length,
+    partialReportedCostUsd: runs.reduce((s, r) => s + (r.partialCostUsd ?? 0), 0),
+    meanCostUsd: completeMean(runs.map(r => r.costUsd)), medianSeconds: median(runs.map(r => r.seconds)),
+  }));
+}
+export function isVolatile(rows) {
+  const spread = values => {
+    if (values.length < 3 || values.some(v => v === null)) return false;
+    const avg = mean(values);
+    return avg > 0 && Math.sqrt(mean(values.map(v => (v - avg) ** 2))) / avg > .25;
+  };
+  for (const group of summarize(rows)) {
+    const runs = rows.filter(r => `${r.task}/${r.connection}/${r.variant}` === group.key);
+    if (group.qualityRange && group.qualityRange[1] - group.qualityRange[0] > .05 || runs.some(r => r.status !== 'ok') || spread(runs.map(r => r.seconds)) || spread(runs.map(r => r.costUsd)) || spread(runs.map(r => r.usage?.input ?? null))) return true;
+  }
+  return false;
+}
+export function compareReports(current, previous) {
+  if (!previous || previous.compatibility !== current.compatibility || previous.planHash !== current.planHash || previous.status !== 'complete' || current.status !== 'complete') return null;
+  return current.summary.map(now => {
+    const before = previous.summary.find(row => row.key === now.key);
+    if (!before) return null;
+    const delta = key => now[key] === null || before[key] === null ? null : now[key] - before[key];
+    return { key: now.key, previousRuns: before.runs, runs: now.runs, qualityDelta: delta('quality'), inputTokensDelta: delta('inputTokens'), outputTokensDelta: delta('outputTokens'), meanCostUsdDelta: delta('meanCostUsd'), medianSecondsDelta: delta('medianSeconds') };
+  }).filter(Boolean);
+}
+const fmt = (v, digits = 2) => v === null || v === undefined ? '?' : v.toFixed(digits);
+export function formatReport(report) {
+  const lines = [`OpenMerit eval · ${report.status} · ${report.revision.slice(0, 12)}`, '', 'Task / connection / variant | scored / attempts | blocked | quality | input / output tokens | estimated $/run | seconds (median)'];
+  for (const row of report.summary) {
+    lines.push(`${row.key} | ${row.scoredRuns ?? row.runs} / ${row.runs} | ${row.blockedAttempts ?? 0} | ${row.quality === null ? '?' : fmt(row.quality * 100, 1) + '%'} | ${fmt(row.inputTokens, 0)} / ${fmt(row.outputTokens, 0)} | ${fmt(row.meanCostUsd, 5)} | ${fmt(row.medianSeconds)}`);
+    if (row.partialReportedCostUsd > 0) lines.push(`  Partial cost estimate: $${fmt(row.partialReportedCostUsd, 5)}; full attempt cost unknown.`);
+  }
+  if (report.comparison) {
+    lines.push('', 'Change from the previous compatible revision (negative tokens/cost/time is lower):');
+    for (const row of report.comparison) lines.push(`${row.key}: quality ${fmt(row.qualityDelta === null ? null : row.qualityDelta * 100, 1)} pp; tokens ${fmt(row.inputTokensDelta, 0)} / ${fmt(row.outputTokensDelta, 0)}; $ ${fmt(row.meanCostUsdDelta, 5)}; seconds ${fmt(row.medianSecondsDelta)}`);
+  } else lines.push('', 'No complete, compatible previous revision for comparison.');
+  lines.push('', '3–10 repetitions measure variability, not statistical certainty. ? means unavailable.', 'Costs are agent-reported estimates, not reconciled bills. Compare agents as different harnesses.');
+  if (report.budgetLimitUsd !== undefined) lines.push(report.budgetLimitUsd === 'unlimited' ? 'Budget: no fixed cap. Task/repetition limits and timeouts still apply; watched changes can incur further charges until stopped.' : `Budget: $${fmt(report.budgetLimitUsd)} cumulative across reruns.`);
+  lines.push(`Reserved allowance used: $${fmt(report.budgetUsedUsd, 5)} (not billed spend). Blocked attempts are unscored.`);
+  if (report.reason) lines.push(`Stopped: ${report.reason}`);
+  return lines.join('\n');
+}

package/dist/eval/src/runner.mjs

@@ -0,0 +1,123 @@
+import { mkdir, mkdtemp, rm, readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { randomUUID } from 'node:crypto';
+import { loadConfig, snapshot, materialize, readJSON, saveJSON, statePath, digest, reservationFor } from './config.mjs';
+import { validatePlan } from './scout.mjs';
+import { preflight, runAgent, invocation } from './adapters.mjs';
+import { execute, agentEnvironment } from './process.mjs';
+import { summarize, isVolatile, compareReports } from './report.mjs';
+
+async function optionalJSON(path) { try { return await readJSON(path); } catch (e) { if (e.code === 'ENOENT') return null; throw e; } }
+export async function acquireLock(root) {
+  const lock = join(root, statePath, 'lock'); await mkdir(join(root, statePath), { recursive: true });
+  try { await mkdir(lock); } catch (error) { if (error.code === 'EEXIST') throw new Error('An eval run/watch owns this project. Stop it first. After a crash, verify the PID in .openmerit/eval/lock/pid before removing that lock directory.'); throw error; }
+  await writeFile(join(lock, 'pid'), String(process.pid));
+  return () => rm(lock, { recursive: true, force: true });
+}
+export function budgetUsed(ledger) {
+  if (ledger?.version !== 1 || !Array.isArray(ledger.entries) || ledger.entries.some(e => !Number.isFinite(e.reservationUsd) || e.reservationUsd <= 0 || e.costUsd !== null && (!Number.isFinite(e.costUsd) || e.costUsd < 0))) throw new Error('Invalid budget ledger. Restore it before running; never discard unknown charges.');
+  // Agent prices are estimates, not provider settlement. Never release a
+  // reservation on that evidence, including entries written by older previews.
+  return ledger.entries.reduce((sum, e) => sum + Math.max(e.costUsd ?? 0, e.reservationUsd), 0);
+}
+
+export async function evaluate(root, { signal, onProgress = () => {}, agentRunner = runAgent, checkAgent = preflight } = {}) {
+  const config = await loadConfig(root), plan = await readJSON(join(root, statePath, 'plan.json'));
+  validatePlan(config, plan);
+  const snap = await snapshot(root, config);
+  const selected = new Set(plan.tasks.flatMap(t => t.candidates.flatMap(c => c.connections)));
+  const versions = {};
+  const ledgerPath = join(root, statePath, 'budget.json');
+  const ledger = await optionalJSON(ledgerPath) ?? { version: 1, entries: [] };
+  const reservationUsd = reservationFor(config, plan), unlimitedBudget = config.budget.maxUsd === 'unlimited';
+  const exhausted = () => { const used = budgetUsed(ledger); return !unlimitedBudget && used + reservationUsd > config.budget.maxUsd + 1e-10; };
+  if (exhausted()) throw new Error('Evaluation budget is exhausted or disabled. Choose a budget with eval plan --budget <USD|unlimited>; prior reservations still count.');
+  const runnerVersion = (await readJSON(new URL('../../../package.json', import.meta.url))).version;
+  const commands = config.connections.filter(c => selected.has(c.id)).map(c => ({ id: c.id, args: invocation(c, reservationUsd, unlimitedBudget) }));
+  const report = { version: 2, runnerVersion, budgetLimitUsd: config.budget.maxUsd, reservationUsd, id: randomUUID(), startedAt: new Date().toISOString(), revision: snap.revision, compatibility: null, planHash: digest(plan.tasks), agentVersions: versions, status: 'running', rows: [], summary: [] };
+  const reportPath = join(root, statePath, 'runs', `${report.startedAt.replace(/[:.]/g, '-')}-${report.id}.json`);
+  const previous = await optionalJSON(join(root, statePath, 'latest.json'));
+  const temp = await mkdtemp(join(tmpdir(), 'openmerit-eval-'));
+  const checkpoint = async () => { report.summary = summarize(report.rows); await saveJSON(reportPath, report); };
+  try {
+    await checkpoint();
+    for (const connection of config.connections.filter(c => selected.has(c.id))) {
+      try { versions[connection.id] = await checkAgent(connection, { cwd: temp }); }
+      catch (error) { error.connection = connection.id; throw error; }
+    }
+    // Graders are copied outside all agent workspaces, before any trial runs.
+    for (const task of snap.tasks) await materialize(task.graderFiles, join(temp, 'graders', task.id));
+    outer: for (const task of snap.tasks) {
+      const taskPlan = plan.tasks.find(t => t.id === task.id);
+      const cells = taskPlan.candidates.flatMap(candidate => candidate.connections.flatMap(connection => snap.variants.map(variant => ({ connection: config.connections.find(c => c.id === connection), variant, candidate }))));
+      for (let round = 0; round < config.repeats.max; round++) {
+        // Rotate and reverse the balanced order so one candidate is not always warm/last.
+        const ordered = [...cells.slice(round % cells.length), ...cells.slice(0, round % cells.length)];
+        if (round % 2) ordered.reverse();
+        for (const { connection, variant, candidate } of ordered) {
+          if (signal?.aborted) { report.status = 'cancelled'; report.reason = 'Interrupted'; break outer; }
+          if (exhausted()) { report.status = 'budget_stopped'; report.reason = 'Cumulative budget cannot admit another reserved attempt.'; break outer; }
+          const entry = { id: randomUUID(), run: report.id, reservationUsd, costUsd: null, settled: false };
+          ledger.entries.push(entry); await saveJSON(ledgerPath, ledger);
+          const workspace = join(temp, entry.id); await mkdir(workspace);
+          await materialize(task.fixtureFiles, workspace, task.fixture);
+          await materialize(variant.sourceFiles, workspace);
+          const prompt = `${task.prompt}\n\n${variant.instructions}\n\nThis is an isolated evaluation attempt. Do not run OpenMerit evaluation commands or inspect paths outside this workspace.`;
+          const result = await agentRunner(connection, prompt, { cwd: workspace, timeoutMs: config.timeoutSeconds * 1000, signal, reservationUsd, unlimitedBudget });
+          entry.costUsd = result.costUsd; entry.costBasis = result.costBasis ?? (result.costUsd === null ? 'unknown' : 'agent_estimate');
+          entry.partialCostUsd = result.partialCostUsd ?? null; entry.reservationUsd = Math.max(entry.reservationUsd, result.partialCostUsd ?? 0); entry.settled = true; await saveJSON(ledgerPath, ledger);
+          let quality = result.status === 'ok' || result.failureReason ? null : 0, status = result.status;
+          if (result.status === 'ok') {
+            const grader = await execute(process.execPath, [join(temp, 'graders', task.id, task.grader)], { cwd: temp, env: agentEnvironment(), input: JSON.stringify({ workspace, output: result.output }), timeoutMs: 30_000, signal });
+            try {
+              const value = JSON.parse(grader.stdout);
+              if (grader.code !== 0 || grader.reason || typeof value.quality !== 'number' || !Number.isFinite(value.quality) || value.quality < 0 || value.quality > 1) throw new Error('Invalid grade');
+              quality = value.quality;
+            } catch { status = 'grader_error'; }
+          }
+          const row = { task: task.id, connection: connection.id, agent: connection.agent, requestedModel: connection.model, observedModels: result.observedModels, scoutId: candidate.scoutId, scoutRank: candidate.rank, variant: variant.id, round: round + 1, quality, status, failureReason: result.failureReason ?? null, usage: result.usage, costUsd: result.costUsd, partialCostUsd: result.partialCostUsd ?? null, costBasis: entry.costBasis, seconds: result.seconds };
+          report.rows.push(row); await checkpoint(); await rm(workspace, { recursive: true, force: true });
+          onProgress(row);
+          if (result.failureReason) {
+            report.status = result.failureReason === 'budget_exhausted' ? 'budget_stopped' : result.failureReason === 'cancelled' ? 'cancelled' : ['sandbox_unavailable', 'permission_denied', 'spawn_failed'].includes(result.failureReason) ? 'environment_blocked' : 'connection_blocked';
+            report.reason = `${connection.id}: ${result.failureReason}. This attempt is unscored; resolve the execution block before retrying.`;
+            break outer;
+          }
+          if (status === 'grader_error') { report.status = 'invalid'; report.reason = `Grader failed for ${task.id}. Fix it before collecting more trials.`; break outer; }
+        }
+        if (round + 1 >= config.repeats.min && !isVolatile(report.rows.filter(r => r.task === task.id))) break;
+      }
+    }
+    if (report.status === 'running') report.status = 'complete';
+  } catch (error) {
+    if (error.failureReason) {
+      report.status = 'environment_blocked'; report.reason = `${error.connection}: ${error.failureReason}. ${error.message}`;
+      report.blockers = [{ connection: error.connection, failureReason: error.failureReason }];
+    } else {
+      report.status = 'error'; report.reason = 'Execution failed; unsettled reservations remain charged. Check configuration and agent authentication.';
+      throw error;
+    }
+  } finally {
+    report.finishedAt = new Date().toISOString(); report.budgetUsedUsd = budgetUsed(ledger);
+    report.compatibility = digest({ accountingVersion: report.version, snapshot: snap.compatibility, versions, runnerVersion, commands, observedModels: Object.fromEntries([...selected].sort().map(id => [id, [...new Set(report.rows.filter(r => r.connection === id).flatMap(r => r.observedModels ?? []))].sort()])) });
+    report.summary = summarize(report.rows); report.comparison = compareReports(report, previous);
+    await saveJSON(reportPath, report); await saveJSON(join(root, statePath, 'latest.json'), report);
+    await rm(temp, { recursive: true, force: true });
+  }
+  return report;
+}
+
+export async function watch(root, { signal, onReport, onProgress, intervalMs = 1500, ...options } = {}) {
+  let last = null, pending = null;
+  while (!signal?.aborted) {
+    const config = await loadConfig(root), current = (await snapshot(root, config)).revision;
+    if (current !== last && current === pending) {
+      const result = await evaluate(root, { signal, onProgress, ...options });
+      onReport?.(result); last = result.revision;
+      if (result.status !== 'complete') return result;
+    }
+    pending = current;
+    await new Promise(resolve => { const done = () => { clearTimeout(timer); signal?.removeEventListener('abort', done); resolve(); }; const timer = setTimeout(done, intervalMs); signal?.addEventListener('abort', done, { once: true }); });
+  }
+}

package/dist/eval/src/scout.mjs

@@ -0,0 +1,37 @@
+import { digest, saveJSON, statePath } from './config.mjs';
+import { join } from 'node:path';
+
+export const selectionHash = config => digest({ scope: config.scope, connections: config.connections, tasks: config.tasks.map(t => ({ id: t.id, description: t.description })), scoutURL: config.scoutURL });
+export async function makePlan(config, fetcher = fetch) {
+  if (!['existing', 'all'].includes(config.scope)) throw new Error('Choose a scope with eval plan --scope existing or --scope all. Existing means the connections declared for this project.');
+  const modelIds = [...new Set(config.connections.map(c => c.scoutId))];
+  if (config.scope === 'existing' && !modelIds.length) throw new Error('Declare project connections in .openmerit/eval.json first. No credentials are sent to Scout.');
+  const tasks = [];
+  for (const task of config.tasks) {
+    const response = await fetcher(config.scoutURL ?? 'https://openmerit.ai/scout/api/recommend', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ prompt: task.description, ...(config.scope === 'existing' ? { modelIds } : {}) }), signal: AbortSignal.timeout(15_000), redirect: 'error' });
+    if (!response.ok) throw new Error(`Scout returned HTTP ${response.status}; no evaluation was started.`);
+    const body = await response.json();
+    if (!Array.isArray(body.recommendations) || body.recommendations.length > 5) throw new Error('Invalid Scout response.');
+    if (config.scope === 'existing' && (!Array.isArray(body.selection?.modelIds) || digest([...body.selection.modelIds].sort()) !== digest([...modelIds].sort()))) throw new Error('Scout did not acknowledge filtering before ranking. Update the Scout endpoint; no unscoped fallback is allowed.');
+    const seen = new Set();
+    const candidates = body.recommendations.map(m => {
+      if (typeof m.id !== 'string' || seen.has(m.id) || config.scope === 'existing' && !modelIds.includes(m.id)) throw new Error('Invalid Scout model selection.');
+      seen.add(m.id);
+      return { scoutId: m.id, name: m.name, rank: m.rank, connections: config.connections.filter(c => c.scoutId === m.id).map(c => c.id) };
+    });
+    tasks.push({ id: task.id, candidates, ranking: body.ranking, warnings: body.warnings ?? [] });
+  }
+  return { version: 1, createdAt: new Date().toISOString(), scope: config.scope, selectionHash: selectionHash(config), tasks };
+}
+export async function savePlan(root, plan) { await saveJSON(join(root, statePath, 'plan.json'), plan); }
+export function validatePlan(config, plan) {
+  if (plan.selectionHash !== selectionHash(config)) throw new Error('Task descriptions, scope or connections changed. Run eval plan again.');
+  if (!Array.isArray(plan.tasks) || plan.tasks.length !== config.tasks.length) throw new Error('Incomplete evaluation plan. Run eval plan again.');
+  for (const task of config.tasks) {
+    const entry = plan.tasks.find(t => t.id === task.id);
+    if (!entry?.candidates?.length) throw new Error(`Scout has no supported candidates for ${task.id}. Broaden scope or revise the task description.`);
+    for (const model of entry.candidates) {
+      if (!model.connections.length || model.connections.some(id => !config.connections.some(c => c.id === id && c.scoutId === model.scoutId))) throw new Error(`Configure an authenticated agent connection for ${model.scoutId}, then plan again. All-model discovery does not grant endpoint access.`);
+    }
+  }
+}