openmerit 0.1.3 → 0.1.5

package/dist/benchmarks.js

@@ -1,98 +0,0 @@
-/**
- * Public benchmark signals.
- *
- * Two jobs:
- *  1. Map a task to the public benchmarks that matter for it (so the
- *     strategist and the user know why a candidate is "worth testing").
- *  2. Provide a prior score for a model from a benchmark digest, so a newly
- *     released model can be ranked against the existing frontier before we
- *     spend money on shadow trials.
- *
- * The digest lives at ~/.openmerit/benchmarks/digest.json and is a simple
- * map: { "vendor/model": { "benchmark_name": score_0_to_100, ... }, ... }.
- * Refresh it from public leaderboards (LMArena, Artificial Analysis,
- * SWE-bench, ...) however you like; a small seed ships below so the system
- * works offline.
- */
-import { paths, readJson, writeJson } from "./store.js";
-/** Task category -> public benchmarks that are predictive for it. */
-export const TASK_BENCHMARKS = {
-    code: ["SWE-bench Verified", "HumanEval", "LMArena Code", "Aider Polyglot"],
-    agentic: ["SWE-bench Verified", "tau2-bench", "Terminal-Bench", "BrowseComp"],
-    knowledge: ["MMLU", "GPQA Diamond", "SimpleQA"],
-    math: ["MATH-500", "AIME", "HMMT"],
-    instruction: ["IFEval", "FollowBench"],
-    multimodal: ["MMMU", "MathVista", "DocVQA"],
-    long_context: ["RULER", "MRCR", "LongBench"],
-    general: ["LMArena Overall", "MMLU", "IFEval"],
-};
-/** Guess the task category from its text (deliberately simple keyword scan). */
-export function categorizeTask(taskText) {
-    const t = taskText.toLowerCase();
-    const has = (...words) => words.some((w) => t.includes(w));
-    if (has("image", "invoice", "screenshot", "ocr", "vision", "pdf page", "photo"))
-        return "multimodal";
-    if (has("agent", "tool call", "terminal", "browse", "multi-step", "workflow"))
-        return "agentic";
-    if (has("function", "code", "python", "typescript", "bug", "refactor", "compile", "sql"))
-        return "code";
-    if (has("prove", "equation", "math", "integral", "probability", "aime"))
-        return "math";
-    if (has("summarize", "extract", "format", "json", "follow the format", "instruction"))
-        return "instruction";
-    if (has("who ", "what is", "explain", "history", "science"))
-        return "knowledge";
-    return "general";
-}
-/** Small offline seed; overwrite/extend via `openmerit benchmarks import <file>`. */
-export const SEED_DIGEST = {
-    updatedAt: "2026-09-01T00:00:00.000Z",
-    scores: {},
-};
-export function loadDigest() {
-    return readJson(paths.benchmarksDigest(), SEED_DIGEST);
-}
-export function saveDigest(d) {
-    writeJson(paths.benchmarksDigest(), d);
-}
-/**
- * Prior 0..1 for a model on a task category, from the digest. Averages the
- * category's benchmarks the model has scores for; returns null when the
- * digest knows nothing about this model.
- */
-export function benchmarkPrior(taskCategory, modelId, digest) {
-    const benches = TASK_BENCHMARKS[taskCategory] ?? TASK_BENCHMARKS.general;
-    const modelScores = digest.scores[modelId];
-    if (!modelScores)
-        return null;
-    const hits = benches
-        .map((b) => modelScores[b])
-        .filter((s) => typeof s === "number");
-    if (hits.length === 0)
-        return null;
-    return hits.reduce((a, b) => a + b, 0) / hits.length / 100;
-}
-/**
- * Which benchmarks should we check when a NEW model appears for this task?
- * Returns the list to display in "worth testing" reasoning.
- */
-export function relevantBenchmarks(taskText) {
-    const category = categorizeTask(taskText);
-    return { category, benchmarks: TASK_BENCHMARKS[category] ?? TASK_BENCHMARKS.general };
-}
-/** Live OpenRouter benchmark scores shortlist candidates; task trials still decide quality. */
-export async function openRouterBenchmarkCandidates(key, category, catalog) {
-    const metric = category === "code" ? "coding_index"
-        : category === "agentic" ? "agentic_index" : "intelligence_index";
-    const response = await fetch("https://openrouter.ai/api/v1/benchmarks?source=artificial-analysis", {
-        headers: { Authorization: `Bearer ${key}` },
-    });
-    if (!response.ok)
-        throw new Error(`OpenRouter benchmarks HTTP ${response.status}`);
-    const payload = await response.json();
-    return (payload.data ?? [])
-        .map((row) => ({ id: String(row.model_permaslug ?? ""), score: Number(row[metric]) }))
-        .filter((row) => catalog.has(row.id) && Number.isFinite(row.score))
-        .sort((a, b) => b.score - a.score)
-        .slice(0, 20).map((row) => row.id);
-}

package/dist/catalog.js

@@ -1,61 +0,0 @@
-/** OpenRouter catalog fetch + snapshot diffing: the "new model release" watch. */
-import { readJson, paths, writeJson } from "./store.js";
-const OR = "https://openrouter.ai/api/v1";
-export class OpenRouterCatalogProvider {
-    key;
-    id = "openrouter";
-    constructor(key) {
-        this.key = key;
-    }
-    fetchCatalog() { return fetchCatalog(this.key); }
-}
-export async function fetchCatalog(key, provider = "openrouter") {
-    if (provider !== "openrouter")
-        return new Map();
-    const res = await fetch(`${OR}/models`, {
-        headers: { Authorization: `Bearer ${key}` },
-    });
-    if (!res.ok)
-        throw new Error(`OpenRouter /models HTTP ${res.status}`);
-    const raw = (await res.json());
-    const cat = new Map();
-    for (const m of raw.data ?? []) {
-        const mm = m;
-        const id = String(mm.id ?? "");
-        if (!id || id.startsWith("openrouter/"))
-            continue; // skip meta-routers
-        const pricing = (mm.pricing ?? {});
-        const architecture = (mm.architecture ?? {});
-        const modalities = architecture.input_modalities;
-        const pp = Number(pricing.prompt ?? 0);
-        const pc = Number(pricing.completion ?? 0);
-        if (!(pp >= 0 && pc >= 0))
-            continue; // skip bogus pricing
-        cat.set(id, {
-            id,
-            name: String(mm.name ?? id),
-            ctx: Number(mm.context_length ?? 0),
-            pp,
-            pc,
-            price: Math.round((((pp + pc) / 2) * 1e6 + Number.EPSILON) * 1e4) / 1e4,
-            inputModalities: Array.isArray(modalities) ? modalities.map(String) : [],
-            created: typeof mm.created === "number" ? mm.created : undefined,
-        });
-    }
-    return cat;
-}
-/** Diff the live catalog against the stored snapshot; returns newly appeared models. */
-export function diffNewModels(cat, snapshot) {
-    const seen = new Set(snapshot.ids);
-    return [...cat.values()].filter((e) => !seen.has(e.id));
-}
-export function loadSnapshot() {
-    return readJson(paths.catalogSnapshot(), { takenAt: "", ids: [] });
-}
-export function saveSnapshot(cat) {
-    writeJson(paths.catalogSnapshot(), {
-        takenAt: new Date().toISOString(),
-        ids: [...cat.keys()].sort(),
-        entries: [...cat.values()].sort((a, b) => a.id.localeCompare(b.id)),
-    });
-}

package/dist/cli.js

@@ -1,188 +0,0 @@
-#!/usr/bin/env node
-/** openmerit CLI: init / watch / trial / frontier / recommend / status / doctor / verify. */
-import { copyFileSync, existsSync, mkdirSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
-import { loadPolicy } from "./policy.js";
-import { loadKey } from "./llm.js";
-import { paretoFrontier, pickBest, pickFallback } from "./frontier.js";
-import { paths, readJson, readJsonl } from "./store.js";
-import { autoTaskTick, recommendTick, runDaemon, tickOnce } from "./daemon.js";
-import { settledActiveTask } from "./pi-trials.js";
-import { modelRoute, routeLabel } from "./routes.js";
-import { runStandaloneTrial } from "./standalone.js";
-import { collectDoctorReport, renderDiagnosticReport, runOfflineVerify } from "./diagnostics.js";
-const ROOT = dirname(dirname(fileURLToPath(import.meta.url)));
-function cmdInit() {
-    const policyPath = paths.policy();
-    mkdirSync(dirname(policyPath), { recursive: true });
-    if (!existsSync(policyPath)) {
-        copyFileSync(join(ROOT, "instructions", "openmerit.policy.json"), policyPath);
-        console.log(`policy  -> ${policyPath}`);
-    }
-    else {
-        console.log(`policy  -> ${policyPath} (kept existing)`);
-    }
-    console.log(`instr.  -> ${join(ROOT, "instructions", "OPENMERIT.md")} (add to your harness's AGENTS.md/context)`);
-    console.log(`ext.    -> ${join(ROOT, "extension", "openmerit.ts")}`);
-    console.log(`pi npm  -> pi install npm:openmerit`);
-    console.log(`pi local-> pi install "${ROOT}"`);
-    console.log("\nNext: authenticate at least two models in pi, install one extension source, then start pi. OpenRouter is optional enrichment; candidate tools are disabled by default.");
-}
-function optionalOpenRouterKey() {
-    try {
-        return loadKey();
-    }
-    catch {
-        return null;
-    }
-}
-function printReport(report, json) {
-    console.log(json ? JSON.stringify(report, null, 2) : renderDiagnosticReport(report));
-    if (!report.ok)
-        process.exitCode = 1;
-}
-function cmdDoctor(json) {
-    printReport(collectDoctorReport(), json);
-}
-function cmdVerify(json) {
-    printReport(runOfflineVerify(), json);
-}
-function loadFrontiers() {
-    const trials = readJsonl(paths.trials());
-    const observations = readJsonl(paths.observations());
-    const labelByTask = new Map();
-    for (const o of observations)
-        if (!labelByTask.has(o.taskKey))
-            labelByTask.set(o.taskKey, o.taskLabel);
-    const byTask = new Map();
-    for (const t of trials) {
-        if (!t.taskKey)
-            continue;
-        byTask.set(t.taskKey, [...(byTask.get(t.taskKey) ?? []), t]);
-    }
-    const policy = loadPolicy(paths.policy());
-    return [...byTask.entries()].map(([tKey, points]) => {
-        const scored = points.filter((p) => p.score > 0);
-        const best = pickBest(scored);
-        return {
-            taskKey: tKey,
-            taskLabel: labelByTask.get(tKey) ?? tKey,
-            points,
-            frontier: paretoFrontier(scored).map((p) => p.model),
-            bestFit: best?.model,
-            fallback: best ? pickFallback(scored, best, policy.fallback.min_score)?.model : undefined,
-            updatedAt: new Date().toISOString(),
-        };
-    });
-}
-function cmdFrontier(filter) {
-    const frontiers = loadFrontiers().filter((f) => !filter || f.taskKey.startsWith(filter));
-    if (frontiers.length === 0) {
-        console.log("no frontiers yet (run `openmerit trial` or let `openmerit watch` collect trials)");
-        return;
-    }
-    for (const f of frontiers) {
-        console.log(`\ntask ${f.taskKey}  "${f.taskLabel}"`);
-        const pts = [...f.points].sort((a, b) => a.price - b.price);
-        for (const p of pts) {
-            const mark = f.frontier.includes(p.model) ? "*" : " ";
-            console.log(` ${mark} ${p.model.padEnd(44)} score=${p.score.toFixed(2)}  $${p.price.toFixed(2)}/M` +
-                (p.latencyMs ? `  ${Math.round(p.latencyMs)}ms` : ""));
-        }
-        console.log(`  best=${f.bestFit ?? "n/a"}  fallback=${f.fallback ?? "n/a"}  (* = pareto frontier)`);
-    }
-}
-function cmdStatus() {
-    const harness = readJson(paths.harnessState(), {});
-    const queue = readJson(paths.candidates(), { newModels: [] });
-    const latestById = new Map();
-    for (const r of readJsonl(paths.recommendations()))
-        latestById.set(r.id, r);
-    const recs = [...latestById.values()].filter((r) => r.status === "pending");
-    const frontiers = loadFrontiers();
-    console.log(`harness model : ${harness.currentRoute ? routeLabel(harness.currentRoute) : harness.currentModel ?? "unknown"} ` +
-        `(as of ${harness.updatedAt ?? "n/a"})`);
-    console.log(`tasks tracked : ${frontiers.length}`);
-    console.log(`new models queued for trial: ${queue.newModels.length}${queue.newModels.length ? " — " + queue.newModels.slice(0, 5).map((m) => m.id).join(", ") + (queue.newModels.length > 5 ? "…" : "") : ""}`);
-    console.log(`pending recommendations: ${recs.length}`);
-    for (const r of recs.slice(-5)) {
-        console.log(`  - ${r.recommended.model} <- ${r.currentModel ?? "?"} (gain ${r.evidence.scoreGain}, auto=${r.policy.autoApply})`);
-    }
-}
-async function main() {
-    const [cmd, ...args] = process.argv.slice(2);
-    switch (cmd) {
-        case "init":
-            cmdInit();
-            break;
-        case "session-trial": {
-            const policy = loadPolicy(paths.policy());
-            const [sessionFile, settledAt, currentModel, settledTaskKey, cwd, bytesText, provider, providerModelId] = args;
-            if (!sessionFile || !settledAt || !currentModel || !settledTaskKey || !cwd || !bytesText)
-                throw new Error("usage: openmerit session-trial <session-file> <settled-at> <model> <task-key> <cwd> <session-bytes>");
-            const sessionBytes = Number(bytesText);
-            if (!Number.isSafeInteger(sessionBytes) || sessionBytes <= 0)
-                throw new Error("invalid session byte limit");
-            const state = readJson(paths.harnessState(), {});
-            const currentRoute = provider && providerModelId
-                ? state.routes?.find((route) => route.provider === provider && route.modelId === providerModelId) ??
-                    modelRoute(provider, providerModelId)
-                : state.currentRoute;
-            const task = settledActiveTask({ sessionFile, settledAt, currentModel, currentRoute,
-                routes: state.routes, settledTaskKey, cwd, sessionBytes });
-            if (!task)
-                throw new Error("the specified pi session has no completed, supported task matching this marker");
-            await autoTaskTick(optionalOpenRouterKey(), policy, task);
-            break;
-        }
-        case "watch":
-            if (args.includes("--once"))
-                await tickOnce(loadPolicy(paths.policy()));
-            else
-                await runDaemon(loadPolicy(paths.policy()));
-            break;
-        case "trial": {
-            const file = args[0];
-            if (!file)
-                throw new Error("usage: openmerit trial <task.json> [--rounds N]");
-            const rIdx = args.indexOf("--rounds");
-            await runStandaloneTrial(file, rIdx >= 0 ? Number(args[rIdx + 1]) : 5);
-            break;
-        }
-        case "frontier":
-            cmdFrontier(args[0]);
-            break;
-        case "recommend":
-            recommendTick(loadPolicy(paths.policy()));
-            break;
-        case "status":
-            cmdStatus();
-            break;
-        case "doctor":
-            cmdDoctor(args.includes("--json"));
-            break;
-        case "verify":
-            cmdVerify(args.includes("--json"));
-            break;
-        default:
-            console.log(`openmerit — external model-merit harness
-
-usage: openmerit <command>
-
-  init                 set up ~/.openmerit (policy, state dirs)
-  watch [--once]       observe completed pi tasks, trial candidates sequentially, and recommend session swaps
-  trial <task.json>    iterative model search on one task spec [--rounds N]
-  frontier [taskKey]   print pareto frontier(s)
-  recommend            emit recommendations now
-  status               harness model, queued models, pending recommendations
-  doctor [--json]      diagnose pi, policy, routes, state, and installation
-  verify [--json]      run an offline, provider-free core self-test`);
-            if (cmd && cmd !== "help" && cmd !== "--help")
-                process.exitCode = 1;
-    }
-}
-main().catch((e) => {
-    console.error(`openmerit: ${e.message}`);
-    process.exit(1);
-});