openmerit 0.1.0

package/dist/daemon.js

@@ -0,0 +1,305 @@
+/**
+ * Trial engine shared by the pi extension's session-bound jobs and the optional
+ * CLI watcher. It reads traces, runs pi candidates, and writes recommendations
+ * for the extension to apply in the active session.
+ */
+import { fetchCatalog, diffNewModels, loadSnapshot, saveSnapshot } from "./catalog.js";
+import { categorizeTask, loadDigest, relevantBenchmarks, benchmarkPrior, openRouterBenchmarkCandidates } from "./benchmarks.js";
+import { loadKey } from "./llm.js";
+import { providerAllowed } from "./policy.js";
+import { buildRecommendation } from "./recommend.js";
+import { appendJsonl, paths, readJson, readJsonl, writeJson, } from "./store.js";
+import { ingestNewTraces } from "./traces.js";
+import { budgetOk, ensureRubric, recordTrialSpend, runTrial } from "./trials.js";
+import { JUDGE_PREFS } from "./judge.js";
+import { availablePiModels, runPiTrial, scorePiRun, settledActiveTask } from "./pi-trials.js";
+import { taskInputKey } from "./task-input.js";
+import { pickNext, STRAT_PREFS } from "./strategist.js";
+function emitTrialProgress(progress) {
+    console.log(`[openmerit/progress] ${JSON.stringify(progress)}`);
+}
+function resolvePref(cat, prefs, label) {
+    for (const p of prefs) {
+        if (p && cat.has(p))
+            return p;
+    }
+    for (const p of prefs) {
+        if (!p)
+            continue;
+        for (const id of [...cat.keys()].sort()) {
+            if (id.includes(p))
+                return id;
+        }
+    }
+    throw new Error(`could not resolve ${label} model from preferences`);
+}
+/** Ingest new session-trace bytes into the observation log. */
+export function tracesTick() {
+    const fresh = ingestNewTraces();
+    for (const o of fresh)
+        appendJsonl(paths.observations(), o);
+    if (fresh.length > 0)
+        console.log(`[openmerit] traces: ${fresh.length} new observation(s)`);
+    return fresh;
+}
+/** Diff the provider catalog; queue newly released models for trials. */
+export async function catalogTick(key, policy) {
+    const cat = await fetchCatalog(key);
+    const snapshot = loadSnapshot();
+    const fresh = snapshot.ids.length === 0 ? [] : diffNewModels(cat, snapshot);
+    saveSnapshot(cat);
+    const allowed = fresh.filter((e) => e.price <= policy.max_usd_per_m && providerAllowed(policy, e.id));
+    if (allowed.length > 0) {
+        const queue = readJson(paths.candidates(), { newModels: [] });
+        const known = new Set(queue.newModels.map((m) => m.id));
+        for (const e of allowed) {
+            if (!known.has(e.id)) {
+                queue.newModels.push({ id: e.id, firstSeen: new Date().toISOString(), price: e.price });
+            }
+        }
+        writeJson(paths.candidates(), queue);
+        console.log(`[openmerit] catalog: ${allowed.length} new model(s) queued: ${allowed.map((e) => e.id).join(", ")}`);
+    }
+    return allowed.map((e) => e.id);
+}
+/**
+ * Run due shadow trials: for each recently observed task, trial queued new
+ * models (or a strategist pick when the queue is empty and the frontier is
+ * thin). Budget-capped per policy.
+ */
+export async function trialTick(key, policy) {
+    const budget = budgetOk(policy);
+    if (!budget.ok) {
+        console.log(`[openmerit] trials paused: ${budget.reason}`);
+        return;
+    }
+    const observations = readJsonl(paths.observations());
+    const queue = readJson(paths.candidates(), { newModels: [] });
+    if (observations.length === 0 || queue.newModels.length === 0)
+        return;
+    const cat = await fetchCatalog(key);
+    const judge = resolvePref(cat, [policy.judge_model, ...JUDGE_PREFS], "judge");
+    // Most recent observation per task.
+    const latestByTask = new Map();
+    for (const o of observations)
+        latestByTask.set(o.taskKey, o);
+    const trials = readJsonl(paths.trials());
+    const rubricCache = new Map();
+    const digest = loadDigest();
+    for (const [tKey, obs] of latestByTask) {
+        const due = queue.newModels.filter((m) => !trials.some((t) => t.taskKey === tKey && t.model === m.id));
+        if (due.length === 0)
+            continue;
+        const { benchmarks } = relevantBenchmarks(obs.taskLabel);
+        const category = categorizeTask(obs.taskLabel);
+        // Rank due candidates by benchmark prior; cheapest first when unknown.
+        due.sort((a, b) => {
+            const pa = benchmarkPrior(category, a.id, digest);
+            const pb = benchmarkPrior(category, b.id, digest);
+            return (pb ?? -1) - (pa ?? -1) || a.price - b.price;
+        });
+        const candidate = due[0];
+        const b = budgetOk(policy);
+        if (!b.ok)
+            break;
+        console.log(`[openmerit] shadow trial: ${candidate.id} on task ${tKey} (${category}; benchmarks: ${benchmarks.join(", ")})`);
+        const rubric = await ensureRubric(key, judge, obs.taskLabel, rubricCache);
+        const { point, costUsd, error } = await runTrial(key, judge, obs.taskLabel, rubric, candidate.id, cat.get(candidate.id));
+        recordTrialSpend(costUsd);
+        appendJsonl(paths.trials(), { ...point, taskKey: tKey });
+        if (error)
+            console.log(`[openmerit] trial failed: ${error.slice(0, 120)}`);
+        else
+            console.log(`[openmerit] trial: ${candidate.id} score=${point.score.toFixed(2)} $${point.price.toFixed(2)}/M`);
+    }
+}
+/** Compare the one completed active pi task, sequentially, then target its session. */
+export async function autoTaskTick(key, policy, settledTask) {
+    const task = settledTask === undefined ? settledActiveTask() : settledTask;
+    if (!task)
+        return null;
+    const marker = settledTask === undefined
+        ? `${task.sessionFile}:${task.settledAt}`
+        : `${task.sessionFile}:${task.sessionBytes ?? task.settledAt}`;
+    const processedPath = settledTask === undefined ? paths.watchProcessed() : paths.sessionJob(marker);
+    const prior = readJson(processedPath, null);
+    if (prior?.marker === marker && prior.status !== "failed")
+        return null;
+    if (!budgetOk(policy).ok)
+        return null;
+    // Claim the settled turn before provider calls so the next poll cannot duplicate it.
+    writeJson(processedPath, { marker, status: "running", updatedAt: new Date().toISOString() });
+    try {
+        const cat = await fetchCatalog(key);
+        const piModels = availablePiModels();
+        for (const id of [...cat.keys()]) {
+            const e = cat.get(id);
+            if (!piModels.has(id) || e.price > policy.max_usd_per_m || !providerAllowed(policy, id) ||
+                (task.images.length > 0 && !e.inputModalities?.includes("image")))
+                cat.delete(id);
+        }
+        if (!cat.has(task.model))
+            throw new Error(`active model ${task.model} is not trialable in pi/OpenRouter`);
+        const judge = resolvePref(cat, [policy.judge_model, ...JUDGE_PREFS], "judge");
+        const strategist = resolvePref(cat, [policy.strategist_model, ...STRAT_PREFS], "strategist");
+        const { category, benchmarks } = relevantBenchmarks(task.task);
+        let benchmarkCandidates = [];
+        try {
+            benchmarkCandidates = await openRouterBenchmarkCandidates(key, category, cat);
+        }
+        catch (e) {
+            console.log(`[openmerit] benchmark shortlist unavailable: ${e.message}`);
+        }
+        const rubric = await ensureRubric(key, judge, task.task, new Map());
+        const tKey = taskInputKey(task.task, task.images);
+        const points = [];
+        const tried = new Set();
+        const failedVendors = new Set();
+        const total = Math.max(1, Math.floor(policy.watch.models_per_task));
+        if (settledTask !== undefined)
+            emitTrialProgress({ phase: "start", taskKey: tKey,
+                model: task.model, index: 1, total });
+        const baseline = await scorePiRun(key, judge, task.task, rubric, task.model, cat.get(task.model), task.run, "trace", task.images);
+        points.push(baseline.point);
+        tried.add(task.model);
+        appendJsonl(paths.trials(), { ...baseline.point, taskKey: tKey, sessionFile: task.sessionFile });
+        console.log(`[openmerit] task ${tKey}: A=${task.model} score=${baseline.point.score.toFixed(2)} from pi trace`);
+        if (settledTask !== undefined)
+            emitTrialProgress({ phase: "complete", taskKey: tKey,
+                model: task.model, index: 1, total, score: baseline.point.score,
+                price: baseline.point.price, latencyMs: baseline.point.latencyMs,
+                costUsd: baseline.costUsd });
+        for (let i = 1; i < total; i++) {
+            if (!budgetOk(policy).ok)
+                break;
+            const pick = await pickNext(key, strategist, task.task, rubric, benchmarks, points, cat, tried, policy.max_usd_per_m, failedVendors, benchmarkCandidates);
+            if (!pick)
+                break;
+            if (settledTask !== undefined)
+                emitTrialProgress({ phase: "start", taskKey: tKey,
+                    model: pick.model, index: i + 1, total });
+            const { point, costUsd, error } = await runPiTrial(key, judge, task.task, rubric, pick.model, cat.get(pick.model), task.cwd, task.images);
+            tried.add(pick.model);
+            points.push(point);
+            recordTrialSpend(costUsd);
+            appendJsonl(paths.trials(), { ...point, taskKey: tKey, sessionFile: task.sessionFile });
+            if (error)
+                failedVendors.add(pick.model.split("/")[0]);
+            console.log(`[openmerit] task ${tKey}: ${pick.model} score=${point.score.toFixed(2)} cost=$${costUsd.toFixed(4)}${error ? ` error=${error.slice(0, 80)}` : ""}`);
+            if (settledTask !== undefined)
+                emitTrialProgress({ phase: "complete", taskKey: tKey,
+                    model: pick.model, index: i + 1, total, score: point.score, price: point.price,
+                    latencyMs: point.latencyMs, costUsd, error: error?.slice(0, 120) });
+        }
+        const rec = buildRecommendation(tKey, task.task.slice(0, 120), task.model, points, policy, task.sessionFile);
+        if (rec) {
+            appendJsonl(paths.recommendations(), rec);
+            console.log(`[openmerit] task ${tKey}: selected ${rec.recommended.model}; auto=${rec.policy.autoApply}`);
+        }
+        else
+            console.log(`[openmerit] task ${tKey}: active model remains best`);
+        writeJson(processedPath, { marker, status: rec ? "complete" : "no_swap",
+            updatedAt: new Date().toISOString() });
+        return rec;
+    }
+    catch (error) {
+        writeJson(processedPath, { marker, status: "failed",
+            updatedAt: new Date().toISOString() });
+        throw error;
+    }
+}
+/** Rebuild frontiers from trials and emit recommendations for the harness's current model. */
+export function recommendTick(policy) {
+    const harness = readJson(paths.harnessState(), {});
+    const currentModel = harness.currentModel ?? null;
+    const trials = readJsonl(paths.trials());
+    const observations = readJsonl(paths.observations());
+    const labelByTask = new Map();
+    for (const o of observations)
+        if (!labelByTask.has(o.taskKey))
+            labelByTask.set(o.taskKey, o.taskLabel);
+    const byTask = new Map();
+    for (const t of trials) {
+        if (!t.taskKey)
+            continue;
+        const arr = byTask.get(t.taskKey) ?? [];
+        arr.push(t);
+        byTask.set(t.taskKey, arr);
+    }
+    // Dedupe on latest status per id: don't re-emit what is pending or was dismissed.
+    const all = readJsonl(paths.recommendations());
+    const latestById = new Map();
+    for (const r of all)
+        latestById.set(r.id, r);
+    const existing = new Set([...latestById.values()]
+        .filter((r) => r.status === "pending" || r.status === "dismissed")
+        .map((r) => `${r.taskKey}:${r.recommended.model}`));
+    const emitted = [];
+    for (const [tKey, points] of byTask) {
+        const rec = buildRecommendation(tKey, labelByTask.get(tKey) ?? tKey, currentModel, points, policy);
+        if (!rec)
+            continue;
+        const dedupeKey = `${rec.taskKey}:${rec.recommended.model}`;
+        if (existing.has(dedupeKey))
+            continue;
+        appendJsonl(paths.recommendations(), rec);
+        emitted.push(rec);
+        console.log(`[openmerit] recommendation: ${rec.recommended.model} <- ${currentModel ?? "unknown"} ` +
+            `(gain ${rec.evidence.scoreGain}, auto=${rec.policy.autoApply})`);
+    }
+    return emitted;
+}
+/** One full pass. */
+export async function tickOnce(policy) {
+    tracesTick();
+    let key = null;
+    try {
+        key = loadKey();
+    }
+    catch (e) {
+        console.log(`[openmerit] ${e.message}; skipping provider ticks`);
+    }
+    if (key) {
+        await catalogTick(key, policy);
+        await autoTaskTick(key, policy);
+    }
+}
+/** Run forever, honoring policy watch intervals. */
+export async function runDaemon(policy) {
+    console.log("[openmerit] background track started");
+    const tracesMs = Math.max(5, policy.watch.traces_interval_sec) * 1000;
+    let lastCatalog = 0;
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        try {
+            tracesTick();
+        }
+        catch (e) {
+            console.log(`[openmerit] traces tick failed: ${e.message}`);
+        }
+        const now = Date.now();
+        const wantCatalog = now - lastCatalog >= policy.watch.catalog_interval_min * 60_000;
+        {
+            let key = null;
+            try {
+                key = loadKey();
+            }
+            catch {
+                /* no key; skip provider ticks */
+            }
+            if (key) {
+                try {
+                    if (wantCatalog) {
+                        await catalogTick(key, policy);
+                        lastCatalog = now;
+                    }
+                    await autoTaskTick(key, policy);
+                }
+                catch (e) {
+                    console.log(`[openmerit] provider tick failed: ${e.message}`);
+                }
+            }
+        }
+        await new Promise((r) => setTimeout(r, tracesMs));
+    }
+}

package/dist/frontier.js

@@ -0,0 +1,49 @@
+/** Pareto frontier over (score up, price down, latency down) + best-fit/fallback picks. */
+/** a dominates b if a is no worse on every objective and strictly better on one. */
+export function dominates(a, b) {
+    const aLat = a.latencyMs ?? Number.POSITIVE_INFINITY;
+    const bLat = b.latencyMs ?? Number.POSITIVE_INFINITY;
+    const noWorse = a.score >= b.score && a.price <= b.price && aLat <= bLat;
+    const better = a.score > b.score || a.price < b.price || aLat < bLat;
+    return noWorse && better;
+}
+/**
+ * Non-dominated set, cheapest first. When no point carries latency this
+ * reduces to the classic "strictly increasing score as price rises" chain
+ * from the original model-search prototype.
+ */
+export function paretoFrontier(points) {
+    const fr = points.filter((p) => !points.some((q) => q !== p && dominates(q, p)));
+    return fr.sort((a, b) => a.price - b.price || b.score - a.score);
+}
+/** Simple 2-objective chain (score vs price only), cheapest first. */
+export function scorePriceChain(points) {
+    const pts = [...points].sort((a, b) => a.price - b.price || b.score - a.score);
+    const chain = [];
+    let best = -1;
+    for (const p of pts) {
+        if (p.score > best) {
+            chain.push(p);
+            best = p.score;
+        }
+    }
+    return chain;
+}
+/** Highest score; ties broken by lower price then lower latency. */
+export function pickBest(points) {
+    return [...points].sort((a, b) => b.score - a.score || a.price - b.price || (a.latencyMs ?? 1e18) - (b.latencyMs ?? 1e18))[0];
+}
+/**
+ * Fallback for `best`: prefer the cheapest frontier point that still clears
+ * `minScore`; otherwise the next-best scoring point overall. Never returns
+ * `best` itself.
+ */
+export function pickFallback(points, best, minScore) {
+    const others = points.filter((p) => p.model !== best.model);
+    if (others.length === 0)
+        return undefined;
+    const fr = paretoFrontier(others).filter((p) => p.score >= minScore);
+    if (fr.length > 0)
+        return fr[0]; // cheapest adequate frontier point
+    return pickBest(others);
+}

package/dist/invoice-eval.js

@@ -0,0 +1,33 @@
+/** Reuse the audited invoice benchmark scorer for its pinned image/prompt cases. */
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { INVOICE_BENCHMARK_PROMPT, parseInvoiceJson, projectVisibleInvoice, scoreInvoice, } from "./invoice-score.js";
+function root() { return join(dirname(fileURLToPath(import.meta.url)), ".."); }
+export async function knownInvoiceScore(task, images, answer) {
+    if (images.length !== 1 || images[0].mimeType !== "image/jpeg")
+        return null;
+    const benchDir = join(root(), "benchmark", "invoice_ocr");
+    const piPrompt = readFileSync(join(root(), "examples", "invoice-prompt.txt"), "utf8").trim();
+    if (task.trim() !== INVOICE_BENCHMARK_PROMPT.trim() && task.trim() !== piPrompt)
+        return null;
+    const manifest = JSON.parse(readFileSync(join(benchDir, "data", "manifest.json"), "utf8"));
+    const inputHash = createHash("sha256").update(Buffer.from(images[0].data, "base64")).digest("hex");
+    const row = manifest.rows.find((r) => createHash("sha256")
+        .update(readFileSync(join(benchDir, "data", r.image))).digest("hex") === inputHash);
+    if (!row)
+        return null;
+    const expected = JSON.parse(readFileSync(join(benchDir, "data", row.ground_truth), "utf8"));
+    let actual = null;
+    try {
+        actual = parseInvoiceJson(answer);
+    }
+    catch { /* invalid JSON scores zero */ }
+    const result = scoreInvoice(projectVisibleInvoice(expected), projectVisibleInvoice(actual));
+    return {
+        score: result.exact_leaf_accuracy,
+        why: `pinned ${row.invoice_id}: audited exact-leaf ${result.exact_leaf_accuracy.toFixed(3)}, ` +
+            `exact document=${result.exact_document}`,
+    };
+}

package/dist/invoice-score.js

@@ -0,0 +1,124 @@
+/** Runtime-safe copy of the audited invoice benchmark's exact scoring rules. */
+export const INVOICE_BENCHMARK_PROMPT = `Extract the invoice into the required JSON schema using only the image.
+
+Rules:
+- Copy names, identifiers, and descriptions exactly as printed.
+- Return dates as YYYY-MM-DD, regardless of the printed date format.
+- Return currency as its three-letter ISO code.
+- Return monetary values and quantities as JSON numbers without currency symbols or grouping separators.
+- Preserve line-item order and include every printed line item exactly once.
+- Use null only when a scalar field is not present. Never infer a missing value.
+- document_total_net is the subtotal/net amount before tax; document_total_amount is the final amount including tax.
+- Return only the schema-conforming JSON object.`;
+export const EXCLUDED_INVOICE_QUALITY_FIELDS = new Set(["po_number"]);
+function isPlainObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+/** Match the Python equality semantics used by the original audited scorer. */
+function pyEqual(a, b) {
+    if (typeof a === "boolean" || typeof b === "boolean") {
+        if (typeof a === "boolean" && typeof b === "boolean")
+            return a === b;
+        if (typeof a === "number" && typeof b === "boolean")
+            return a === Number(b);
+        if (typeof a === "boolean" && typeof b === "number")
+            return Number(a) === b;
+        return false;
+    }
+    if (typeof a === "number" && typeof b === "number")
+        return a === b;
+    if (typeof a === "string" && typeof b === "string")
+        return a === b;
+    if (a === null || b === null)
+        return a === b;
+    if (Array.isArray(a) && Array.isArray(b)) {
+        return a.length === b.length && a.every((value, index) => pyEqual(value, b[index]));
+    }
+    if (isPlainObject(a) && isPlainObject(b)) {
+        const aKeys = Object.keys(a);
+        const bKeys = Object.keys(b);
+        return aKeys.length === bKeys.length && aKeys.every((key) => key in b && pyEqual(a[key], b[key]));
+    }
+    return false;
+}
+function flatten(value, path = "") {
+    if (isPlainObject(value)) {
+        let flattened = {};
+        for (const [key, child] of Object.entries(value)) {
+            const childPath = path ? `${path}.${key}` : key;
+            flattened = Object.assign(flattened, flatten(child, childPath));
+        }
+        return flattened;
+    }
+    if (Array.isArray(value)) {
+        if (value.length === 0)
+            return { [`${path}.__length__`]: 0 };
+        let flattened = { [`${path}.__length__`]: value.length };
+        for (let index = 0; index < value.length; index++) {
+            flattened = Object.assign(flattened, flatten(value[index], `${path}[${index}]`));
+        }
+        return flattened;
+    }
+    return { [path]: value };
+}
+function exactValue(expected, actual) {
+    if (typeof expected === "boolean" || typeof actual === "boolean") {
+        return typeof expected === typeof actual && expected === actual;
+    }
+    if (typeof expected === "number" && typeof actual === "number")
+        return expected === actual;
+    return typeof expected === typeof actual && expected === actual;
+}
+export function scoreInvoice(expected, actual) {
+    if (actual === null) {
+        const expectedFlat = flatten(expected);
+        return {
+            exact_document: false,
+            matched_leaves: 0,
+            total_union_leaves: Object.keys(expectedFlat).length,
+            exact_leaf_accuracy: 0,
+            scalar_accuracy: 0,
+            line_item_accuracy: 0,
+        };
+    }
+    const expectedFlat = flatten(expected);
+    const actualFlat = flatten(actual);
+    const paths = new Set([...Object.keys(expectedFlat), ...Object.keys(actualFlat)]);
+    let matches = 0;
+    for (const path of paths) {
+        if (path in expectedFlat && path in actualFlat && exactValue(expectedFlat[path], actualFlat[path])) {
+            matches += 1;
+        }
+    }
+    const scalarPaths = Object.keys(expectedFlat).filter((path) => !path.startsWith("line_items"));
+    const itemPaths = [...paths].filter((path) => path.startsWith("line_items"));
+    const scalarMatches = scalarPaths.filter((path) => path in actualFlat && exactValue(expectedFlat[path], actualFlat[path])).length;
+    const itemMatches = itemPaths.filter((path) => path in expectedFlat && path in actualFlat && exactValue(expectedFlat[path], actualFlat[path])).length;
+    return {
+        exact_document: pyEqual(expected, actual),
+        matched_leaves: matches,
+        total_union_leaves: paths.size,
+        exact_leaf_accuracy: paths.size ? matches / paths.size : 1,
+        scalar_accuracy: scalarPaths.length ? scalarMatches / scalarPaths.length : 1,
+        line_item_accuracy: itemPaths.length ? itemMatches / itemPaths.length : 1,
+    };
+}
+export function parseInvoiceJson(content) {
+    let stripped = content.trim();
+    if (stripped.startsWith("```")) {
+        const lines = stripped.split(/\r\n|\r|\n/);
+        stripped = lines.slice(1, -1).join("\n");
+    }
+    const parsed = JSON.parse(stripped);
+    if (!isPlainObject(parsed))
+        throw new TypeError("Model output is not a JSON object");
+    return parsed;
+}
+export function projectVisibleInvoice(document) {
+    if (document === null || document === undefined)
+        return document;
+    const projected = structuredClone(document);
+    for (const field of EXCLUDED_INVOICE_QUALITY_FIELDS)
+        delete projected[field];
+    return projected;
+}

package/dist/judge.js

@@ -0,0 +1,43 @@
+/** Judge: score a model's answer against a rubric. */
+import { chat, chatWithImages } from "./llm.js";
+export const JUDGE_PREFS = ["openai/gpt-4o-mini", "openai/gpt-4o"];
+const JUDGE_PROMPT = `You are grading a model's answer.
+TASK: {task}
+RUBRIC: {rubric}
+ANSWER: {answer}
+Return ONLY json: {{"score": <0..1>, "why": "<one line>"}}`;
+/** Extract the first {...} JSON object from model output. */
+export function parseObj(txt) {
+    const m = txt.match(/\{.*\}/s) ?? txt.match(/\{[^}]*\}/s);
+    if (!m)
+        throw new Error("no json object found");
+    return JSON.parse(m[0]);
+}
+export async function judge(key, judgeModel, task, rubric, answer) {
+    const { content } = await chat(key, judgeModel, JUDGE_PROMPT.replace("{task}", task)
+        .replace("{rubric}", rubric)
+        .replace("{answer}", answer.slice(0, 16000)), 1024, 0);
+    try {
+        const j = parseObj(content);
+        return { score: Number(j.score ?? 0), why: String(j.why ?? "") };
+    }
+    catch {
+        return { score: 0, why: "unparseable judge output" };
+    }
+}
+/** Grade OCR answers against the actual uploaded pixels, not answer text alone. */
+export async function judgeWithImages(key, judgeModel, task, rubric, answer, images) {
+    const prompt = `You are grading an answer to a visual task. Inspect the attached image carefully.\n` +
+        `TASK: ${task}\nRUBRIC: ${rubric}\nANSWER: ${answer.slice(0, 16000)}\n` +
+        `For invoice extraction, check every visible field and line item against the image. ` +
+        `Penalize missing, invented, or mistyped values and invalid JSON. ` +
+        `Return ONLY json: {"score": <0..1>, "why": "<one line>"}`;
+    const { content } = await chatWithImages(key, judgeModel, prompt, images, 1024);
+    try {
+        const j = parseObj(content);
+        return { score: Number(j.score ?? 0), why: String(j.why ?? "") };
+    }
+    catch {
+        return { score: 0, why: "unparseable visual judge output" };
+    }
+}

package/dist/llm.js

@@ -0,0 +1,66 @@
+/** Minimal OpenRouter chat client for catalog and trial requests. */
+import { existsSync, readFileSync } from "node:fs";
+import { paths } from "./store.js";
+const OR = "https://openrouter.ai/api/v1";
+/** Load OPENROUTER_API_KEY from env or ~/.openmerit/.env. */
+export function loadKey() {
+    const envFile = paths.envFile();
+    if (existsSync(envFile)) {
+        for (const line of readFileSync(envFile, "utf8").split("\n")) {
+            const t = line.trim();
+            if (!t || t.startsWith("#") || !t.includes("="))
+                continue;
+            const [k, ...v] = t.split("=");
+            if (process.env[k.trim()] === undefined)
+                process.env[k.trim()] = v.join("=").trim();
+        }
+    }
+    const key = process.env.OPENROUTER_API_KEY;
+    if (!key)
+        throw new Error("missing OPENROUTER_API_KEY (env or ~/.openmerit/.env)");
+    return key;
+}
+async function request(key, path, body, tries = 3) {
+    for (let attempt = 0; attempt < tries; attempt++) {
+        const res = await fetch(OR + path, {
+            method: body === undefined ? "GET" : "POST",
+            headers: {
+                Authorization: `Bearer ${key}`,
+                ...(body === undefined ? {} : { "Content-Type": "application/json", "X-Title": "openmerit" }),
+            },
+            body: body === undefined ? undefined : JSON.stringify(body),
+        });
+        if (res.ok)
+            return (await res.json());
+        const detail = (await res.text()).slice(0, 400);
+        if ([429, 500, 502, 503].includes(res.status) && attempt < tries - 1) {
+            await new Promise((r) => setTimeout(r, 2000 * (attempt + 1)));
+            continue;
+        }
+        throw new Error(`OpenRouter HTTP ${res.status}: ${detail}`);
+    }
+    throw new Error("unreachable");
+}
+export async function chat(key, model, prompt, maxTokens, temperature) {
+    const r = await request(key, "/chat/completions", {
+        model,
+        messages: [{ role: "user", content: prompt }],
+        max_tokens: maxTokens,
+        temperature,
+    });
+    return { content: r.choices[0]?.message?.content ?? "", usage: r.usage ?? {} };
+}
+/** Send a visual grading request through the same OpenRouter account. */
+export async function chatWithImages(key, model, prompt, images, maxTokens) {
+    const r = await request(key, "/chat/completions", {
+        model,
+        messages: [{ role: "user", content: [
+                    { type: "text", text: prompt },
+                    ...images.map((image) => ({ type: "image_url",
+                        image_url: { url: `data:${image.mimeType};base64,${image.data}` } })),
+                ] }],
+        max_tokens: maxTokens,
+        temperature: 0,
+    });
+    return { content: r.choices[0]?.message?.content ?? "", usage: r.usage ?? {} };
+}