openmerit 0.1.0

package/benchmark/invoice_ocr/data/manifest.json

@@ -0,0 +1,97 @@
+{
+  "dataset": "alamgirqazi/invoice-ocr-synthetic",
+  "dataset_revision": "44ddfd3c0c373b69f377910f83c4164a1e52d52c",
+  "license": "CC BY 4.0",
+  "rows": [
+    {
+      "invoice_id": "invoice_01",
+      "row_idx": 2,
+      "file_name": "synth_0002_8f01361b.png",
+      "quality": "clean",
+      "template": "freelancer_invoice",
+      "image": "invoice_01_row_2.jpg",
+      "ground_truth": "invoice_01_ground_truth.json"
+    },
+    {
+      "invoice_id": "invoice_02",
+      "row_idx": 5,
+      "file_name": "synth_0005_553deef3.png",
+      "quality": "clean",
+      "template": "formal_corporate",
+      "image": "invoice_02_row_5.jpg",
+      "ground_truth": "invoice_02_ground_truth.json"
+    },
+    {
+      "invoice_id": "invoice_03",
+      "row_idx": 6,
+      "file_name": "synth_0006_474081ca.png",
+      "quality": "clean",
+      "template": "modern_minimal",
+      "image": "invoice_03_row_6.jpg",
+      "ground_truth": "invoice_03_ground_truth.json"
+    },
+    {
+      "invoice_id": "invoice_04",
+      "row_idx": 7,
+      "file_name": "synth_0007_7292e354.png",
+      "quality": "clean",
+      "template": "utility_bill",
+      "image": "invoice_04_row_7.jpg",
+      "ground_truth": "invoice_04_ground_truth.json"
+    },
+    {
+      "invoice_id": "invoice_05",
+      "row_idx": 947,
+      "file_name": "synth_0009_8f6b430f.png",
+      "quality": "faded",
+      "template": "freelancer_invoice",
+      "image": "invoice_05_row_947.jpg",
+      "ground_truth": "invoice_05_ground_truth.json"
+    },
+    {
+      "invoice_id": "invoice_06",
+      "row_idx": 948,
+      "file_name": "synth_0010_73bd8d75.png",
+      "quality": "faded",
+      "template": "formal_corporate",
+      "image": "invoice_06_row_948.jpg",
+      "ground_truth": "invoice_06_ground_truth.json"
+    },
+    {
+      "invoice_id": "invoice_07",
+      "row_idx": 949,
+      "file_name": "synth_0011_8888d5c6.png",
+      "quality": "faded",
+      "template": "utility_bill",
+      "image": "invoice_07_row_949.jpg",
+      "ground_truth": "invoice_07_ground_truth.json"
+    },
+    {
+      "invoice_id": "invoice_08",
+      "row_idx": 1888,
+      "file_name": "synth_0012_72cf2460.png",
+      "quality": "bad_scan",
+      "template": "modern_minimal",
+      "image": "invoice_08_row_1888.jpg",
+      "ground_truth": "invoice_08_ground_truth.json"
+    },
+    {
+      "invoice_id": "invoice_09",
+      "row_idx": 1890,
+      "file_name": "synth_0014_d3972e7f.png",
+      "quality": "bad_scan",
+      "template": "freelancer_invoice",
+      "image": "invoice_09_row_1890.jpg",
+      "ground_truth": "invoice_09_ground_truth.json"
+    },
+    {
+      "invoice_id": "invoice_10",
+      "row_idx": 1892,
+      "file_name": "synth_0016_223f5434.png",
+      "quality": "bad_scan",
+      "template": "modern_minimal",
+      "image": "invoice_10_row_1892.jpg",
+      "ground_truth": "invoice_10_ground_truth.json"
+    }
+  ]
+}

package/dist/benchmarks.js

@@ -0,0 +1,98 @@
+/**
+ * Public benchmark signals.
+ *
+ * Two jobs:
+ *  1. Map a task to the public benchmarks that matter for it (so the
+ *     strategist and the user know why a candidate is "worth testing").
+ *  2. Provide a prior score for a model from a benchmark digest, so a newly
+ *     released model can be ranked against the existing frontier before we
+ *     spend money on shadow trials.
+ *
+ * The digest lives at ~/.openmerit/benchmarks/digest.json and is a simple
+ * map: { "vendor/model": { "benchmark_name": score_0_to_100, ... }, ... }.
+ * Refresh it from public leaderboards (LMArena, Artificial Analysis,
+ * SWE-bench, ...) however you like; a small seed ships below so the system
+ * works offline.
+ */
+import { paths, readJson, writeJson } from "./store.js";
+/** Task category -> public benchmarks that are predictive for it. */
+export const TASK_BENCHMARKS = {
+    code: ["SWE-bench Verified", "HumanEval", "LMArena Code", "Aider Polyglot"],
+    agentic: ["SWE-bench Verified", "tau2-bench", "Terminal-Bench", "BrowseComp"],
+    knowledge: ["MMLU", "GPQA Diamond", "SimpleQA"],
+    math: ["MATH-500", "AIME", "HMMT"],
+    instruction: ["IFEval", "FollowBench"],
+    multimodal: ["MMMU", "MathVista", "DocVQA"],
+    long_context: ["RULER", "MRCR", "LongBench"],
+    general: ["LMArena Overall", "MMLU", "IFEval"],
+};
+/** Guess the task category from its text (deliberately simple keyword scan). */
+export function categorizeTask(taskText) {
+    const t = taskText.toLowerCase();
+    const has = (...words) => words.some((w) => t.includes(w));
+    if (has("image", "invoice", "screenshot", "ocr", "vision", "pdf page", "photo"))
+        return "multimodal";
+    if (has("agent", "tool call", "terminal", "browse", "multi-step", "workflow"))
+        return "agentic";
+    if (has("function", "code", "python", "typescript", "bug", "refactor", "compile", "sql"))
+        return "code";
+    if (has("prove", "equation", "math", "integral", "probability", "aime"))
+        return "math";
+    if (has("summarize", "extract", "format", "json", "follow the format", "instruction"))
+        return "instruction";
+    if (has("who ", "what is", "explain", "history", "science"))
+        return "knowledge";
+    return "general";
+}
+/** Small offline seed; overwrite/extend via `openmerit benchmarks import <file>`. */
+export const SEED_DIGEST = {
+    updatedAt: "2026-09-01T00:00:00.000Z",
+    scores: {},
+};
+export function loadDigest() {
+    return readJson(paths.benchmarksDigest(), SEED_DIGEST);
+}
+export function saveDigest(d) {
+    writeJson(paths.benchmarksDigest(), d);
+}
+/**
+ * Prior 0..1 for a model on a task category, from the digest. Averages the
+ * category's benchmarks the model has scores for; returns null when the
+ * digest knows nothing about this model.
+ */
+export function benchmarkPrior(taskCategory, modelId, digest) {
+    const benches = TASK_BENCHMARKS[taskCategory] ?? TASK_BENCHMARKS.general;
+    const modelScores = digest.scores[modelId];
+    if (!modelScores)
+        return null;
+    const hits = benches
+        .map((b) => modelScores[b])
+        .filter((s) => typeof s === "number");
+    if (hits.length === 0)
+        return null;
+    return hits.reduce((a, b) => a + b, 0) / hits.length / 100;
+}
+/**
+ * Which benchmarks should we check when a NEW model appears for this task?
+ * Returns the list to display in "worth testing" reasoning.
+ */
+export function relevantBenchmarks(taskText) {
+    const category = categorizeTask(taskText);
+    return { category, benchmarks: TASK_BENCHMARKS[category] ?? TASK_BENCHMARKS.general };
+}
+/** Live OpenRouter benchmark scores shortlist candidates; task trials still decide quality. */
+export async function openRouterBenchmarkCandidates(key, category, catalog) {
+    const metric = category === "code" ? "coding_index"
+        : category === "agentic" ? "agentic_index" : "intelligence_index";
+    const response = await fetch("https://openrouter.ai/api/v1/benchmarks?source=artificial-analysis", {
+        headers: { Authorization: `Bearer ${key}` },
+    });
+    if (!response.ok)
+        throw new Error(`OpenRouter benchmarks HTTP ${response.status}`);
+    const payload = await response.json();
+    return (payload.data ?? [])
+        .map((row) => ({ id: String(row.model_permaslug ?? ""), score: Number(row[metric]) }))
+        .filter((row) => catalog.has(row.id) && Number.isFinite(row.score))
+        .sort((a, b) => b.score - a.score)
+        .slice(0, 20).map((row) => row.id);
+}

package/dist/catalog.js

@@ -0,0 +1,51 @@
+/** OpenRouter catalog fetch + snapshot diffing: the "new model release" watch. */
+import { readJson, paths, writeJson } from "./store.js";
+const OR = "https://openrouter.ai/api/v1";
+export async function fetchCatalog(key) {
+    const res = await fetch(`${OR}/models`, {
+        headers: { Authorization: `Bearer ${key}` },
+    });
+    if (!res.ok)
+        throw new Error(`OpenRouter /models HTTP ${res.status}`);
+    const raw = (await res.json());
+    const cat = new Map();
+    for (const m of raw.data ?? []) {
+        const mm = m;
+        const id = String(mm.id ?? "");
+        if (!id || id.startsWith("openrouter/"))
+            continue; // skip meta-routers
+        const pricing = (mm.pricing ?? {});
+        const architecture = (mm.architecture ?? {});
+        const modalities = architecture.input_modalities;
+        const pp = Number(pricing.prompt ?? 0);
+        const pc = Number(pricing.completion ?? 0);
+        if (!(pp >= 0 && pc >= 0))
+            continue; // skip bogus pricing
+        cat.set(id, {
+            id,
+            name: String(mm.name ?? id),
+            ctx: Number(mm.context_length ?? 0),
+            pp,
+            pc,
+            price: Math.round((((pp + pc) / 2) * 1e6 + Number.EPSILON) * 1e4) / 1e4,
+            inputModalities: Array.isArray(modalities) ? modalities.map(String) : [],
+            created: typeof mm.created === "number" ? mm.created : undefined,
+        });
+    }
+    return cat;
+}
+/** Diff the live catalog against the stored snapshot; returns newly appeared models. */
+export function diffNewModels(cat, snapshot) {
+    const seen = new Set(snapshot.ids);
+    return [...cat.values()].filter((e) => !seen.has(e.id));
+}
+export function loadSnapshot() {
+    return readJson(paths.catalogSnapshot(), { takenAt: "", ids: [] });
+}
+export function saveSnapshot(cat) {
+    writeJson(paths.catalogSnapshot(), {
+        takenAt: new Date().toISOString(),
+        ids: [...cat.keys()].sort(),
+        entries: [...cat.values()].sort((a, b) => a.id.localeCompare(b.id)),
+    });
+}

package/dist/cli.js

@@ -0,0 +1,261 @@
+#!/usr/bin/env node
+/** openmerit CLI: init / watch / trial / frontier / recommend / status. */
+import { copyFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { loadPolicy } from "./policy.js";
+import { loadKey } from "./llm.js";
+import { fetchCatalog } from "./catalog.js";
+import { paretoFrontier, pickBest, pickFallback } from "./frontier.js";
+import { buildRecommendation } from "./recommend.js";
+import { appendJsonl, paths, readJson, readJsonl, taskKey } from "./store.js";
+import { autoTaskTick, recommendTick, runDaemon, tickOnce } from "./daemon.js";
+import { budgetOk, recordTrialSpend } from "./trials.js";
+import { availablePiModels, recordedActiveTask, runPiTrial, scorePiRun, settledActiveTask } from "./pi-trials.js";
+import { pickNext, STRAT_PREFS } from "./strategist.js";
+import { JUDGE_PREFS } from "./judge.js";
+import { openRouterBenchmarkCandidates, relevantBenchmarks } from "./benchmarks.js";
+const ROOT = dirname(dirname(fileURLToPath(import.meta.url)));
+function resolvePref(cat, prefs, label) {
+    for (const p of prefs)
+        if (p && cat.has(p))
+            return p;
+    for (const p of prefs) {
+        if (!p)
+            continue;
+        for (const id of [...cat.keys()].sort())
+            if (id.includes(p))
+                return id;
+    }
+    throw new Error(`could not resolve ${label} model`);
+}
+function cmdInit() {
+    const policyPath = paths.policy();
+    mkdirSync(dirname(policyPath), { recursive: true });
+    if (!existsSync(policyPath)) {
+        copyFileSync(join(ROOT, "instructions", "openmerit.policy.json"), policyPath);
+        console.log(`policy  -> ${policyPath}`);
+    }
+    else {
+        console.log(`policy  -> ${policyPath} (kept existing)`);
+    }
+    console.log(`instr.  -> ${join(ROOT, "instructions", "OPENMERIT.md")} (add to your harness's AGENTS.md/context)`);
+    console.log(`ext.    -> ${join(ROOT, "extension", "openmerit.ts")}`);
+    console.log(`pi npm  -> pi install npm:openmerit`);
+    console.log(`pi local-> pi install "${ROOT}"`);
+    console.log("\nNext: set OPENROUTER_API_KEY (env or ~/.openmerit/.env), install one extension source into pi, then start pi; the extension launches comparisons automatically.");
+}
+async function cmdTrial(taskFile, rounds) {
+    const cfg = JSON.parse(readFileSync(taskFile, "utf8"));
+    const policy = loadPolicy(paths.policy());
+    const key = loadKey();
+    console.log("fetching OpenRouter catalog...");
+    const cat = await fetchCatalog(key);
+    const piModels = availablePiModels();
+    for (const id of [...cat.keys()])
+        if (!piModels.has(id))
+            cat.delete(id);
+    if (!cat.has(cfg.initial_model))
+        throw new Error(`initial_model ${cfg.initial_model} is unavailable in pi's OpenRouter registry`);
+    console.log(`catalog: ${cat.size} models selectable in pi`);
+    const judge = resolvePref(cat, [cfg.judge_model ?? policy.judge_model, ...JUDGE_PREFS], "judge");
+    const strat = resolvePref(cat, [cfg.strategist_model ?? policy.strategist_model, ...STRAT_PREFS], "strategist");
+    const { category, benchmarks } = relevantBenchmarks(cfg.task);
+    let benchmarkCandidates = [];
+    try {
+        benchmarkCandidates = await openRouterBenchmarkCandidates(key, category, cat);
+    }
+    catch (e) {
+        console.log(`OpenRouter benchmark shortlist unavailable: ${e.message}`);
+    }
+    console.log(`judge: ${judge} | strategist: ${strat} | benchmarks: ${benchmarks.join(", ")}`);
+    console.log(`OpenRouter benchmark candidates: ${benchmarkCandidates.slice(0, 5).join(", ") || "none"}`);
+    const tKey = taskKey(cfg.task);
+    const maxPrice = cfg.max_usd_per_m ?? policy.max_usd_per_m;
+    const tried = new Set();
+    const results = [];
+    const failedVendors = new Set();
+    for (let i = 0; i < rounds; i++) {
+        const budget = budgetOk(policy);
+        if (!budget.ok) {
+            console.log(`budget: ${budget.reason}; stopping`);
+            break;
+        }
+        let model;
+        let why;
+        if (i === 0) {
+            model = cfg.initial_model;
+            why = "initial model";
+        }
+        else {
+            const pick = await pickNext(key, strat, cfg.task, cfg.eval, benchmarks, results, cat, tried, maxPrice, failedVendors, benchmarkCandidates);
+            if (!pick)
+                break;
+            model = pick.model;
+            why = pick.why;
+        }
+        if (tried.has(model))
+            break;
+        console.log(`[round ${i + 1}/${rounds}] ${model} (${why}) ...`);
+        const observed = i === 0 ? recordedActiveTask(cfg.task, model) : null;
+        if (i === 0)
+            console.log(observed ? "    using model A's active pi session trace" : "    active trace unavailable; running model A in a fresh pi session");
+        const { point, costUsd, error, sessionId } = observed
+            ? await scorePiRun(key, judge, cfg.task, cfg.eval, model, cat.get(model), observed, "trace")
+            : await runPiTrial(key, judge, cfg.task, cfg.eval, model, cat.get(model));
+        tried.add(model);
+        results.push(point);
+        recordTrialSpend(costUsd);
+        appendJsonl(paths.trials(), { ...point, taskKey: tKey });
+        if (sessionId)
+            console.log(`    pi trace session: ${sessionId}`);
+        if (error) {
+            failedVendors.add(model.split("/")[0]);
+            console.log(`    FAILED: ${error.slice(0, 160)}`);
+        }
+        else {
+            console.log(`    score=${point.score.toFixed(2)}  $/M=${point.price}  est_cost=$${costUsd.toFixed(4)}  ${(point.why ?? "").slice(0, 110)}`);
+        }
+    }
+    const pts = results.filter((p) => p.score > 0);
+    if (pts.length === 0) {
+        console.log("no trials completed");
+        process.exit(1);
+    }
+    const chain = paretoFrontier(pts);
+    const best = pickBest(pts);
+    const fb = pickFallback(pts, best, policy.fallback.min_score);
+    console.log("\n=== pareto frontier (nondominated: score up, price down, latency down) ===");
+    for (const p of chain)
+        console.log(`  ${p.model.padEnd(44)} score=${p.score.toFixed(2)}  $${p.price.toFixed(2)}/M`);
+    console.log(`\nBEST FIT : ${best.model}  (score ${best.score.toFixed(2)}, $${best.price.toFixed(2)}/M)`);
+    console.log(fb ? `FALLBACK : ${fb.model}  (score ${fb.score.toFixed(2)}, $${fb.price.toFixed(2)}/M)` : "FALLBACK : n/a");
+    const rec = buildRecommendation(tKey, cfg.task.slice(0, 120), cfg.initial_model, pts, policy);
+    if (rec) {
+        appendJsonl(paths.recommendations(), rec);
+        console.log(`\nrecommendation -> ${paths.recommendations()} (auto=${rec.policy.autoApply})`);
+    }
+}
+function loadFrontiers() {
+    const trials = readJsonl(paths.trials());
+    const observations = readJsonl(paths.observations());
+    const labelByTask = new Map();
+    for (const o of observations)
+        if (!labelByTask.has(o.taskKey))
+            labelByTask.set(o.taskKey, o.taskLabel);
+    const byTask = new Map();
+    for (const t of trials) {
+        if (!t.taskKey)
+            continue;
+        byTask.set(t.taskKey, [...(byTask.get(t.taskKey) ?? []), t]);
+    }
+    const policy = loadPolicy(paths.policy());
+    return [...byTask.entries()].map(([tKey, points]) => {
+        const scored = points.filter((p) => p.score > 0);
+        const best = pickBest(scored);
+        return {
+            taskKey: tKey,
+            taskLabel: labelByTask.get(tKey) ?? tKey,
+            points,
+            frontier: paretoFrontier(scored).map((p) => p.model),
+            bestFit: best?.model,
+            fallback: best ? pickFallback(scored, best, policy.fallback.min_score)?.model : undefined,
+            updatedAt: new Date().toISOString(),
+        };
+    });
+}
+function cmdFrontier(filter) {
+    const frontiers = loadFrontiers().filter((f) => !filter || f.taskKey.startsWith(filter));
+    if (frontiers.length === 0) {
+        console.log("no frontiers yet (run `openmerit trial` or let `openmerit watch` collect trials)");
+        return;
+    }
+    for (const f of frontiers) {
+        console.log(`\ntask ${f.taskKey}  "${f.taskLabel}"`);
+        const pts = [...f.points].sort((a, b) => a.price - b.price);
+        for (const p of pts) {
+            const mark = f.frontier.includes(p.model) ? "*" : " ";
+            console.log(` ${mark} ${p.model.padEnd(44)} score=${p.score.toFixed(2)}  $${p.price.toFixed(2)}/M` +
+                (p.latencyMs ? `  ${Math.round(p.latencyMs)}ms` : ""));
+        }
+        console.log(`  best=${f.bestFit ?? "n/a"}  fallback=${f.fallback ?? "n/a"}  (* = pareto frontier)`);
+    }
+}
+function cmdStatus() {
+    const harness = readJson(paths.harnessState(), {});
+    const queue = readJson(paths.candidates(), { newModels: [] });
+    const latestById = new Map();
+    for (const r of readJsonl(paths.recommendations()))
+        latestById.set(r.id, r);
+    const recs = [...latestById.values()].filter((r) => r.status === "pending");
+    const frontiers = loadFrontiers();
+    console.log(`harness model : ${harness.currentModel ?? "unknown"} (as of ${harness.updatedAt ?? "n/a"})`);
+    console.log(`tasks tracked : ${frontiers.length}`);
+    console.log(`new models queued for trial: ${queue.newModels.length}${queue.newModels.length ? " — " + queue.newModels.slice(0, 5).map((m) => m.id).join(", ") + (queue.newModels.length > 5 ? "…" : "") : ""}`);
+    console.log(`pending recommendations: ${recs.length}`);
+    for (const r of recs.slice(-5)) {
+        console.log(`  - ${r.recommended.model} <- ${r.currentModel ?? "?"} (gain ${r.evidence.scoreGain}, auto=${r.policy.autoApply})`);
+    }
+}
+async function main() {
+    const [cmd, ...args] = process.argv.slice(2);
+    const policy = loadPolicy(paths.policy());
+    switch (cmd) {
+        case "init":
+            cmdInit();
+            break;
+        case "session-trial": {
+            const [sessionFile, settledAt, currentModel, settledTaskKey, cwd, bytesText] = args;
+            if (!sessionFile || !settledAt || !currentModel || !settledTaskKey || !cwd || !bytesText)
+                throw new Error("usage: openmerit session-trial <session-file> <settled-at> <model> <task-key> <cwd> <session-bytes>");
+            const sessionBytes = Number(bytesText);
+            if (!Number.isSafeInteger(sessionBytes) || sessionBytes <= 0)
+                throw new Error("invalid session byte limit");
+            const task = settledActiveTask({ sessionFile, settledAt, currentModel, settledTaskKey, cwd, sessionBytes });
+            if (!task)
+                throw new Error("the specified pi session has no completed, supported task matching this marker");
+            await autoTaskTick(loadKey(), policy, task);
+            break;
+        }
+        case "watch":
+            if (args.includes("--once"))
+                await tickOnce(policy);
+            else
+                await runDaemon(policy);
+            break;
+        case "trial": {
+            const file = args[0];
+            if (!file)
+                throw new Error("usage: openmerit trial <task.json> [--rounds N]");
+            const rIdx = args.indexOf("--rounds");
+            await cmdTrial(file, rIdx >= 0 ? Number(args[rIdx + 1]) : 5);
+            break;
+        }
+        case "frontier":
+            cmdFrontier(args[0]);
+            break;
+        case "recommend":
+            recommendTick(policy);
+            break;
+        case "status":
+            cmdStatus();
+            break;
+        default:
+            console.log(`openmerit — external model-merit harness
+
+usage: openmerit <command>
+
+  init                 set up ~/.openmerit (policy, state dirs)
+  watch [--once]       observe completed pi tasks, trial candidates sequentially, and recommend session swaps
+  trial <task.json>    iterative model search on one task spec [--rounds N]
+  frontier [taskKey]   print pareto frontier(s)
+  recommend            emit recommendations now
+  status               harness model, queued models, pending recommendations`);
+            if (cmd && cmd !== "help" && cmd !== "--help")
+                process.exitCode = 1;
+    }
+}
+main().catch((e) => {
+    console.error(`openmerit: ${e.message}`);
+    process.exit(1);
+});