openmerit 0.1.6 → 0.1.7-preview.1

package/dist/eval/src/runner.mjs

@@ -0,0 +1,106 @@
+import { mkdir, mkdtemp, rm, readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { randomUUID } from 'node:crypto';
+import { loadConfig, snapshot, materialize, readJSON, saveJSON, statePath, digest } from './config.mjs';
+import { validatePlan } from './scout.mjs';
+import { preflight, runAgent, invocation } from './adapters.mjs';
+import { execute, agentEnvironment } from './process.mjs';
+import { summarize, isVolatile, compareReports } from './report.mjs';
+
+async function optionalJSON(path) { try { return await readJSON(path); } catch (e) { if (e.code === 'ENOENT') return null; throw e; } }
+export async function acquireLock(root) {
+  const lock = join(root, statePath, 'lock'); await mkdir(join(root, statePath), { recursive: true });
+  try { await mkdir(lock); } catch (error) { if (error.code === 'EEXIST') throw new Error('An eval run/watch owns this project. Stop it first. After a crash, verify the PID in .openmerit/eval/lock/pid before removing that lock directory.'); throw error; }
+  await writeFile(join(lock, 'pid'), String(process.pid));
+  return () => rm(lock, { recursive: true, force: true });
+}
+export function budgetUsed(ledger) {
+  if (ledger?.version !== 1 || !Array.isArray(ledger.entries) || ledger.entries.some(e => !Number.isFinite(e.reservationUsd) || e.reservationUsd <= 0 || e.costUsd !== null && (!Number.isFinite(e.costUsd) || e.costUsd < 0))) throw new Error('Invalid budget ledger. Restore it before running; never discard unknown charges.');
+  return ledger.entries.reduce((sum, e) => sum + (e.costUsd ?? e.reservationUsd), 0);
+}
+
+export async function evaluate(root, { signal, onProgress = () => {}, agentRunner = runAgent, checkAgent = preflight } = {}) {
+  const config = await loadConfig(root), plan = await readJSON(join(root, statePath, 'plan.json'));
+  validatePlan(config, plan);
+  const snap = await snapshot(root, config);
+  const selected = new Set(plan.tasks.flatMap(t => t.candidates.flatMap(c => c.connections)));
+  const versions = {};
+  const ledgerPath = join(root, statePath, 'budget.json');
+  const ledger = await optionalJSON(ledgerPath) ?? { version: 1, entries: [] };
+  if (budgetUsed(ledger) + config.budget.reservationUsd > config.budget.maxUsd + 1e-10) throw new Error('Evaluation budget is exhausted or disabled. Set budget.maxUsd in .openmerit/eval.json; prior reservations still count.');
+  for (const connection of config.connections.filter(c => selected.has(c.id))) versions[connection.id] = await checkAgent(connection);
+  const runnerVersion = (await readJSON(new URL('../../../package.json', import.meta.url))).version;
+  const commands = config.connections.filter(c => selected.has(c.id)).map(c => ({ id: c.id, args: invocation(c, config.budget.reservationUsd) }));
+  const report = { version: 1, runnerVersion, id: randomUUID(), startedAt: new Date().toISOString(), revision: snap.revision, compatibility: digest({ snapshot: snap.compatibility, versions, runnerVersion, commands }), planHash: digest(plan.tasks), agentVersions: versions, status: 'running', rows: [], summary: [] };
+  const reportPath = join(root, statePath, 'runs', `${report.startedAt.replace(/[:.]/g, '-')}-${report.id}.json`);
+  const previous = await optionalJSON(join(root, statePath, 'latest.json'));
+  const temp = await mkdtemp(join(tmpdir(), 'openmerit-eval-'));
+  const checkpoint = async () => { report.summary = summarize(report.rows); await saveJSON(reportPath, report); };
+  try {
+    await checkpoint();
+    // Graders are copied outside all agent workspaces, before any trial runs.
+    for (const task of snap.tasks) await materialize(task.graderFiles, join(temp, 'graders', task.id));
+    outer: for (const task of snap.tasks) {
+      const taskPlan = plan.tasks.find(t => t.id === task.id);
+      const cells = taskPlan.candidates.flatMap(candidate => candidate.connections.flatMap(connection => snap.variants.map(variant => ({ connection: config.connections.find(c => c.id === connection), variant, candidate }))));
+      for (let round = 0; round < config.repeats.max; round++) {
+        // Rotate and reverse the balanced order so one candidate is not always warm/last.
+        const ordered = [...cells.slice(round % cells.length), ...cells.slice(0, round % cells.length)];
+        if (round % 2) ordered.reverse();
+        for (const { connection, variant, candidate } of ordered) {
+          if (signal?.aborted) { report.status = 'cancelled'; report.reason = 'Interrupted'; break outer; }
+          if (budgetUsed(ledger) + config.budget.reservationUsd > config.budget.maxUsd + 1e-10) { report.status = 'budget_stopped'; report.reason = 'Cumulative budget cannot admit another reserved attempt.'; break outer; }
+          const entry = { id: randomUUID(), run: report.id, reservationUsd: config.budget.reservationUsd, costUsd: null, settled: false };
+          ledger.entries.push(entry); await saveJSON(ledgerPath, ledger);
+          const workspace = join(temp, entry.id); await mkdir(workspace);
+          await materialize(task.fixtureFiles, workspace, task.fixture);
+          await materialize(variant.sourceFiles, workspace);
+          const prompt = `${task.prompt}\n\n${variant.instructions}\n\nThis is an isolated evaluation attempt. Do not run OpenMerit evaluation commands or inspect paths outside this workspace.`;
+          const result = await agentRunner(connection, prompt, { cwd: workspace, timeoutMs: config.timeoutSeconds * 1000, signal, reservationUsd: config.budget.reservationUsd });
+          entry.costUsd = result.costUsd; entry.reservationUsd = Math.max(entry.reservationUsd, result.partialCostUsd ?? 0); entry.settled = true; await saveJSON(ledgerPath, ledger);
+          let quality = result.status === 'ok' || result.failureReason ? null : 0, status = result.status;
+          if (result.status === 'ok') {
+            const grader = await execute(process.execPath, [join(temp, 'graders', task.id, task.grader)], { cwd: temp, env: agentEnvironment(), input: JSON.stringify({ workspace, output: result.output }), timeoutMs: 30_000, signal });
+            try {
+              const value = JSON.parse(grader.stdout);
+              if (grader.code !== 0 || grader.reason || typeof value.quality !== 'number' || !Number.isFinite(value.quality) || value.quality < 0 || value.quality > 1) throw new Error('Invalid grade');
+              quality = value.quality;
+            } catch { status = 'grader_error'; }
+          }
+          const row = { task: task.id, connection: connection.id, agent: connection.agent, requestedModel: connection.model, observedModels: result.observedModels, scoutId: candidate.scoutId, scoutRank: candidate.rank, variant: variant.id, round: round + 1, quality, status, failureReason: result.failureReason ?? null, usage: result.usage, costUsd: result.costUsd, seconds: result.seconds };
+          report.rows.push(row); await checkpoint(); await rm(workspace, { recursive: true, force: true });
+          onProgress(row);
+          if (result.failureReason) { report.status = 'connection_blocked'; report.reason = `${connection.id}: ${result.failureReason}. Fix the native agent connection before retrying.`; break outer; }
+          if (status === 'grader_error') { report.status = 'invalid'; report.reason = `Grader failed for ${task.id}. Fix it before collecting more trials.`; break outer; }
+        }
+        if (round + 1 >= config.repeats.min && !isVolatile(report.rows.filter(r => r.task === task.id))) break;
+      }
+    }
+    if (report.status === 'running') report.status = 'complete';
+  } catch (error) {
+    report.status = 'error'; report.reason = 'Execution failed; unsettled reservations remain charged. Check configuration and agent authentication.';
+    throw error;
+  } finally {
+    report.finishedAt = new Date().toISOString(); report.budgetUsedUsd = budgetUsed(ledger);
+    report.compatibility = digest({ base: report.compatibility, observedModels: Object.fromEntries([...selected].sort().map(id => [id, [...new Set(report.rows.filter(r => r.connection === id).flatMap(r => r.observedModels ?? []))].sort()])) });
+    report.summary = summarize(report.rows); report.comparison = compareReports(report, previous);
+    await saveJSON(reportPath, report); await saveJSON(join(root, statePath, 'latest.json'), report);
+    await rm(temp, { recursive: true, force: true });
+  }
+  return report;
+}
+
+export async function watch(root, { signal, onReport, onProgress, intervalMs = 1500, ...options } = {}) {
+  let last = null, pending = null;
+  while (!signal?.aborted) {
+    const config = await loadConfig(root), current = (await snapshot(root, config)).revision;
+    if (current !== last && current === pending) {
+      const result = await evaluate(root, { signal, onProgress, ...options });
+      onReport?.(result); last = result.revision;
+      if (result.status !== 'complete') return result;
+    }
+    pending = current;
+    await new Promise(resolve => { const done = () => { clearTimeout(timer); signal?.removeEventListener('abort', done); resolve(); }; const timer = setTimeout(done, intervalMs); signal?.addEventListener('abort', done, { once: true }); });
+  }
+}

package/dist/eval/src/scout.mjs

@@ -0,0 +1,37 @@
+import { digest, saveJSON, statePath } from './config.mjs';
+import { join } from 'node:path';
+
+export const selectionHash = config => digest({ scope: config.scope, connections: config.connections, tasks: config.tasks.map(t => ({ id: t.id, description: t.description })), scoutURL: config.scoutURL });
+export async function makePlan(config, fetcher = fetch) {
+  if (!['existing', 'all'].includes(config.scope)) throw new Error('Choose a scope with eval plan --scope existing or --scope all. Existing means the connections declared for this project.');
+  const modelIds = [...new Set(config.connections.map(c => c.scoutId))];
+  if (config.scope === 'existing' && !modelIds.length) throw new Error('Declare project connections in .openmerit/eval.json first. No credentials are sent to Scout.');
+  const tasks = [];
+  for (const task of config.tasks) {
+    const response = await fetcher(config.scoutURL ?? 'https://openmerit.ai/scout/api/recommend', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ prompt: task.description, ...(config.scope === 'existing' ? { modelIds } : {}) }), signal: AbortSignal.timeout(15_000), redirect: 'error' });
+    if (!response.ok) throw new Error(`Scout returned HTTP ${response.status}; no evaluation was started.`);
+    const body = await response.json();
+    if (!Array.isArray(body.recommendations) || body.recommendations.length > 5) throw new Error('Invalid Scout response.');
+    if (config.scope === 'existing' && (!Array.isArray(body.selection?.modelIds) || digest([...body.selection.modelIds].sort()) !== digest([...modelIds].sort()))) throw new Error('Scout did not acknowledge filtering before ranking. Update the Scout endpoint; no unscoped fallback is allowed.');
+    const seen = new Set();
+    const candidates = body.recommendations.map(m => {
+      if (typeof m.id !== 'string' || seen.has(m.id) || config.scope === 'existing' && !modelIds.includes(m.id)) throw new Error('Invalid Scout model selection.');
+      seen.add(m.id);
+      return { scoutId: m.id, name: m.name, rank: m.rank, connections: config.connections.filter(c => c.scoutId === m.id).map(c => c.id) };
+    });
+    tasks.push({ id: task.id, candidates, ranking: body.ranking, warnings: body.warnings ?? [] });
+  }
+  return { version: 1, createdAt: new Date().toISOString(), scope: config.scope, selectionHash: selectionHash(config), tasks };
+}
+export async function savePlan(root, plan) { await saveJSON(join(root, statePath, 'plan.json'), plan); }
+export function validatePlan(config, plan) {
+  if (plan.selectionHash !== selectionHash(config)) throw new Error('Task descriptions, scope or connections changed. Run eval plan again.');
+  if (!Array.isArray(plan.tasks) || plan.tasks.length !== config.tasks.length) throw new Error('Incomplete evaluation plan. Run eval plan again.');
+  for (const task of config.tasks) {
+    const entry = plan.tasks.find(t => t.id === task.id);
+    if (!entry?.candidates?.length) throw new Error(`Scout has no supported candidates for ${task.id}. Broaden scope or revise the task description.`);
+    for (const model of entry.candidates) {
+      if (!model.connections.length || model.connections.some(id => !config.connections.some(c => c.id === id && c.scoutId === model.scoutId))) throw new Error(`Configure an authenticated agent connection for ${model.scoutId}, then plan again. All-model discovery does not grant endpoint access.`);
+    }
+  }
+}