openmerit 0.1.1 → 0.1.3

package/dist/llm.js

@@ -1,6 +1,10 @@
 /** Minimal provider clients for judge and strategist requests. */
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { spawn } from "node:child_process";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { paths } from "./store.js";
+import { legacyOpenRouterRoute } from "./routes.js";
 const OR = "https://openrouter.ai/api/v1";
 /** Load OPENROUTER_API_KEY from env or ~/.openmerit/.env. */
 export function loadKey() {
@@ -100,3 +104,100 @@ export async function chat(key, model, prompt, maxTokens, temperature) {
 export async function chatWithImages(key, model, prompt, images, maxTokens) {
     return providerFor(model, key).chatWithImages(model, prompt, images, maxTokens);
 }
+export function directChatClient(key) {
+    const id = (model) => typeof model === "string" ? model : model.meritId;
+    return {
+        chat: (model, prompt, maxTokens, temperature) => chat(key, id(model), prompt, maxTokens, temperature),
+        chatWithImages: (model, prompt, images, maxTokens) => chatWithImages(key, id(model), prompt, images, maxTokens),
+    };
+}
+function textContent(content) {
+    if (typeof content === "string")
+        return content;
+    if (!Array.isArray(content))
+        return "";
+    return content.filter((part) => !!part && typeof part === "object" && part.type === "text")
+        .map((part) => part.text ?? "").join("\n");
+}
+/** Use pi's model registry and credential store without exposing provider secrets to OpenMerit. */
+export class PiCliChatClient {
+    executable;
+    onSpend;
+    id = "pi-cli";
+    constructor(executable = process.env.OPENMERIT_PI_BIN?.trim() || "pi", onSpend) {
+        this.executable = executable;
+        this.onSpend = onSpend;
+    }
+    chat(model, prompt, _maxTokens, _temperature) {
+        return this.run(model, prompt, []);
+    }
+    chatWithImages(model, prompt, images, _maxTokens) {
+        return this.run(model, prompt, images);
+    }
+    async run(target, prompt, images) {
+        const route = typeof target === "string" ? legacyOpenRouterRoute(target) : target;
+        const dir = mkdtempSync(join(tmpdir(), "openmerit-pi-chat-"));
+        try {
+            const imageArgs = images.map((image, index) => {
+                const suffix = { "image/jpeg": "jpg", "image/png": "png", "image/webp": "webp", "image/gif": "gif" }[image.mimeType];
+                if (!suffix)
+                    throw new Error(`unsupported pi image: ${image.mimeType}`);
+                const file = join(dir, `image-${index}.${suffix}`);
+                writeFileSync(file, Buffer.from(image.data, "base64"));
+                return `@${file}`;
+            });
+            const child = spawn(this.executable, [
+                "--provider", route.provider, "--model", route.modelId, "--mode", "json", "--offline",
+                "--no-extensions", "--no-tools", "--print", ...imageArgs, prompt,
+            ], { cwd: dir, stdio: ["ignore", "pipe", "pipe"] });
+            let stdout = "";
+            let stderr = "";
+            child.stdout.on("data", (chunk) => { stdout += chunk.toString("utf8"); });
+            child.stderr.on("data", (chunk) => { stderr += chunk.toString("utf8"); });
+            const code = await new Promise((resolve, reject) => {
+                child.on("error", reject);
+                child.on("close", (value) => resolve(value ?? 1));
+            });
+            let answer = "";
+            let promptTokens = 0;
+            let completionTokens = 0;
+            let totalTokens = 0;
+            let costUsd = 0;
+            let failed = false;
+            for (const line of stdout.split("\n")) {
+                if (!line.trim())
+                    continue;
+                let event;
+                try {
+                    event = JSON.parse(line);
+                }
+                catch {
+                    continue;
+                }
+                if (event.type !== "message_end" || event.message?.role !== "assistant")
+                    continue;
+                answer = textContent(event.message.content) || answer;
+                promptTokens += event.message.usage?.input ?? 0;
+                completionTokens += event.message.usage?.output ?? 0;
+                totalTokens += event.message.usage?.totalTokens ?? 0;
+                costUsd += event.message.usage?.cost?.total ?? 0;
+                if (event.message.stopReason === "error")
+                    failed = true;
+            }
+            if (code !== 0 || failed || !answer.trim()) {
+                throw new Error(`pi model call ${route.provider}/${route.modelId} failed: ` +
+                    (stderr.trim().slice(0, 240) || `exit ${code}, empty answer`));
+            }
+            this.onSpend?.(costUsd);
+            return { content: answer, usage: {
+                    prompt_tokens: promptTokens || undefined,
+                    completion_tokens: completionTokens || undefined,
+                    total_tokens: totalTokens || promptTokens + completionTokens || undefined,
+                    costUsd,
+                } };
+        }
+        finally {
+            rmSync(dir, { recursive: true, force: true });
+        }
+    }
+}

package/dist/pi-trials.js

@@ -8,6 +8,8 @@ import { judge, judgeWithImages } from "./judge.js";
 import { knownInvoiceScore } from "./invoice-eval.js";
 import { paths, readJson } from "./store.js";
 import { sessionFiles, sessionImages, taskInputKey } from "./task-input.js";
+import { legacyOpenRouterRoute, meritModelId, modelRoute, routeKey } from "./routes.js";
+import { directChatClient } from "./llm.js";
 function answerText(content) {
     if (typeof content === "string")
         return content;
@@ -107,10 +109,13 @@ export function recordedActiveTask(task, model, images = [], sessionFile, sessio
             toolErrors++;
     }
     const userImages = user ? sessionImages(user.content) : null;
-    if (!user || !userImages || taskInputKey(answerText(user.content), userImages) !==
-        taskInputKey(task, images))
+    const files = sessionFiles(task);
+    if (!user || !userImages || taskInputKey(answerText(user.content), userImages, sessionFiles(answerText(user.content))) !==
+        taskInputKey(task, images, files))
         return null;
-    const matching = assistants.filter((m) => (m.provider === "openrouter" ? m.model : `${m.provider}/${m.model}`) === model);
+    const matching = assistants.filter((m) => typeof model === "string"
+        ? meritModelId(m.provider ?? "openrouter", m.model ?? "") === model
+        : m.provider === model.provider && m.model === model.modelId);
     if (matching.length === 0 || matching.some((m) => m.stopReason === "error"))
         return null;
     const answer = matching.map((m) => answerText(m.content)).filter(Boolean).at(-1) ?? "";
@@ -167,23 +172,52 @@ export function settledActiveTask(snapshot) {
     const files = sessionFiles(task);
     if (!task.trim() || taskInputKey(task, images, files) !== st.settledTaskKey)
         return null;
-    const run = recordedActiveTask(task, st.currentModel, images, st.sessionFile, st.sessionBytes);
+    const route = st.currentRoute ?? legacyOpenRouterRoute(st.currentModel);
+    const run = recordedActiveTask(task, route, images, st.sessionFile, st.sessionBytes);
     if (!run)
         return null;
+    const routes = st.routes?.length ? st.routes : [route];
     return { task, images, model: st.currentModel, run, sessionFile: st.sessionFile,
-        settledAt: st.settledAt, sessionBytes: st.sessionBytes, cwd: st.cwd ?? process.cwd(), usedTools, files };
+        settledAt: st.settledAt, sessionBytes: st.sessionBytes, cwd: st.cwd ?? process.cwd(), usedTools, files,
+        route, routes };
 }
-/** Limit candidates to models this installed pi can actually select. */
-export function availablePiModels() {
-    const result = spawnSync("pi", ["--offline", "--list-models", "openrouter"], { encoding: "utf8" });
+export function piExecutable() {
+    return process.env.OPENMERIT_PI_BIN?.trim() || "pi";
+}
+function compactNumber(value) {
+    const match = value.match(/^(\d+(?:\.\d+)?)([KMG])?$/i);
+    if (!match)
+        return undefined;
+    const scale = { K: 1_000, M: 1_000_000, G: 1_000_000_000 }[(match[2] ?? "").toUpperCase()] ?? 1;
+    return Math.round(Number(match[1]) * scale);
+}
+/** Read provider/model routes from pi when no live extension snapshot is available. */
+export function availablePiRoutes(snapshot) {
+    if (snapshot?.length)
+        return snapshot;
+    const result = spawnSync(piExecutable(), ["--offline", "--list-models"], { encoding: "utf8" });
     if (result.error || result.status !== 0)
-        throw new Error("could not list pi OpenRouter models");
-    return new Set(result.stdout.split("\n").slice(1)
-        .map((line) => line.trim().split(/\s+/)[1])
-        .filter((id) => !!id && !id.startsWith("~")));
+        throw new Error("could not list models from pi");
+    const routes = [];
+    for (const line of result.stdout.split("\n").slice(1)) {
+        const [provider, id, context, maxOut, , images] = line.trim().split(/\s+/);
+        if (!provider || !id || id.startsWith("~"))
+            continue;
+        routes.push(modelRoute(provider, id, {
+            input: images === "yes" ? ["text", "image"] : ["text"],
+            contextWindow: compactNumber(context),
+            maxTokens: compactNumber(maxOut),
+        }));
+    }
+    return routes;
+}
+/** Backward-compatible logical-id view used by the standalone OpenRouter trial command. */
+export function availablePiModels() {
+    return new Set(availablePiRoutes().map((route) => route.meritId));
 }
 /** Each invocation is a fresh pi run, so model B does not inherit model A's answer. */
 export async function executePiTask(model, task, cwd = process.cwd(), images = [], files = []) {
+    const route = typeof model === "string" ? legacyOpenRouterRoute(model) : model;
     const imageDir = images.length ? mkdtempSync(join(tmpdir(), "openmerit-pi-image-")) : null;
     const suffix = {
         "image/jpeg": "jpg", "image/png": "png", "image/webp": "webp", "image/gif": "gif",
@@ -209,10 +243,10 @@ export async function executePiTask(model, task, cwd = process.cwd(), images = [
         });
         const candidateTask = [...attachmentPaths.entries()].reduce((text, [source, target]) => text.split(source).join(target), task);
         const before = fileSnapshot(workspace);
-        const traceFile = paths.trialTrace(`${model}:${task}:${Date.now()}`);
+        const traceFile = paths.trialTrace(`${routeKey(route)}:${task}:${Date.now()}`);
         const traceLines = [];
-        const child = spawn("pi", [
-            "--provider", "openrouter", "--model", model, "--mode", "json",
+        const child = spawn(piExecutable(), [
+            "--provider", route.provider, "--model", route.modelId, "--mode", "json",
             "--offline", "--no-extensions", "--approve", ...piTrialToolArgs(),
             "--print", ...imageArgs, ...fileArgs, candidateTask,
         ], { cwd: workspace, stdio: ["ignore", "pipe", "pipe"] });
@@ -275,7 +309,7 @@ export async function executePiTask(model, task, cwd = process.cwd(), images = [
             if (!after.has(file))
                 changedFiles++;
         if (exitCode !== 0 || !answer.trim()) {
-            throw new Error(`pi trial ${model} failed: ${stderr.trim().slice(0, 180) || `exit ${exitCode}, empty answer`}`);
+            throw new Error(`pi trial ${route.meritId} via ${route.provider} failed: ${stderr.trim().slice(0, 180) || `exit ${exitCode}, empty answer`}`);
         }
         return { answer, costUsd, latencyMs: Date.now() - started, errors, sessionId,
             toolCalls, toolErrors, changedFiles, traceFile };
@@ -287,29 +321,30 @@ export async function executePiTask(model, task, cwd = process.cwd(), images = [
     }
 }
 /** Quality uses the same OpenMerit judge; cost and latency come from pi itself. */
-export async function runPiTrial(key, judgeModel, task, rubric, model, entry, cwd = process.cwd(), images = [], files = [], harness = defaultHarness) {
+export async function runPiTrial(key, judgeModel, task, rubric, model, entry, cwd = process.cwd(), images = [], files = [], harness = defaultHarness, client = directChatClient(key)) {
     try {
-        const run = await harness.runTask(model, { task, cwd, images, files });
-        return await scorePiRun(key, judgeModel, task, rubric, model, entry, run, "pi_trial", images);
+        const run = await harness.runTask(entry?.route ?? model, { task, cwd, images, files });
+        return await scorePiRun(key, judgeModel, task, rubric, model, entry, run, "pi_trial", images, client);
     }
     catch (e) {
         const error = String(e);
         return {
-            point: { model, score: 0, price: entry?.price ?? 0,
+            point: { schemaVersion: 1, model, route: entry?.route, score: 0, price: entry?.price ?? 0,
+                priceKnown: entry?.priceKnown ?? !!entry,
                 ts: new Date().toISOString(), source: "pi_trial", toolCalls: 0, toolErrors: 1,
                 changedFiles: 0, why: error.slice(0, 160) },
             costUsd: 0, error,
         };
     }
 }
-export async function scorePiRun(key, judgeModel, task, rubric, model, entry, run, source, images = []) {
+export async function scorePiRun(key, judgeModel, task, rubric, model, entry, run, source, images = [], client = directChatClient(key)) {
     let score = 0;
     let why = "judge unavailable";
     try {
         const pinned = images.length ? await knownInvoiceScore(task, images, run.answer) : null;
         const j = pinned ?? (images.length
-            ? await judgeWithImages(key, judgeModel, task, rubric, run.answer, images)
-            : await judge(key, judgeModel, task, rubric, run.answer));
+            ? await judgeWithImages(key, judgeModel, task, rubric, run.answer, images, client)
+            : await judge(key, judgeModel, task, rubric, run.answer, client));
         score = Math.max(0, Math.min(1, j.score));
         why = j.why;
     }
@@ -318,7 +353,8 @@ export async function scorePiRun(key, judgeModel, task, rubric, model, entry, ru
     }
     return {
         point: {
-            model, score, price: entry?.price ?? 0,
+            schemaVersion: 1, model, route: entry?.route, score, price: entry?.price ?? 0,
+            priceKnown: entry?.priceKnown ?? !!entry,
             latencyMs: run.latencyMs, ts: new Date().toISOString(), source,
             why: `${why}${run.errors ? `; pi errors: ${run.errors}` : ""}; ` +
                 `tools=${run.toolCalls}, toolErrors=${run.toolErrors}, changedFiles=${run.changedFiles}`,

package/dist/policy.js

@@ -60,10 +60,11 @@ export function loadPolicy(path) {
         max_usd_per_m: num(raw.max_usd_per_m, d.max_usd_per_m),
     };
 }
-/** Is a model id allowed by the provider allow/deny lists? ("*" matches all; "vendor/*" matches a vendor.) */
-export function providerAllowed(policy, modelId) {
+/** Match either a logical vendor/model id or its concrete route provider. */
+export function providerAllowed(policy, modelId, routeProvider) {
     const vendor = modelId.split("/")[0];
-    const match = (pat) => pat === "*" || pat === modelId || (pat.endsWith("/*") && pat.slice(0, -2) === vendor);
+    const match = (pat) => pat === "*" || pat === modelId || pat === routeProvider ||
+        (pat.endsWith("/*") && (pat.slice(0, -2) === vendor || pat.slice(0, -2) === routeProvider));
     if (policy.providers.deny.some(match))
         return false;
     return policy.providers.allow.some(match);
@@ -87,6 +88,10 @@ export function gate(policy, input) {
         reasons.push("recommendation is not bound to a pi session");
         return { autoApply: false, reasons };
     }
+    if (input.priceKnown === false) {
+        reasons.push("candidate or current-model price is unknown");
+        return { autoApply: false, reasons };
+    }
     if (policy.mode !== "auto" || !policy.auto_apply.enabled) {
         reasons.push(`policy mode is "${policy.mode}" (auto_apply ${policy.auto_apply.enabled ? "enabled" : "disabled"})`);
         return { autoApply: false, reasons };

package/dist/recommend.js

@@ -2,40 +2,49 @@
 import { createHash } from "node:crypto";
 import { paretoFrontier, pickBest, pickFallback } from "./frontier.js";
 import { gate, providerAllowed } from "./policy.js";
-export function buildRecommendation(taskKey, taskLabel, currentModel, points, policy, sessionFile = null) {
+import { pointKey, routeLabel } from "./routes.js";
+export function buildRecommendation(taskKey, taskLabel, currentModel, points, policy, sessionFile = null, currentRoute = currentModel ? points.find((p) => p.model === currentModel)?.route ?? null : null) {
     const scored = points.filter((p) => p.score > 0);
     if (scored.length === 0)
         return null;
     const best = pickBest(scored);
     if (!best)
         return null;
-    if (currentModel && best.model === currentModel)
-        return null; // already on the best model
-    const current = currentModel ? scored.find((p) => p.model === currentModel) : undefined;
+    if (currentModel && best.model === currentModel &&
+        (!currentRoute || pointKey(best) === `${currentRoute.provider}:${currentRoute.modelId}`))
+        return null;
+    const current = currentModel ? scored.find((p) => currentRoute
+        ? pointKey(p) === `${currentRoute.provider}:${currentRoute.modelId}`
+        : p.model === currentModel) : undefined;
     const scoreGain = current ? best.score - current.score : 0;
     // pickBest may choose a cheaper or faster model at equal quality.
     if (current && scoreGain < 0)
         return null;
-    const priceRatio = current && current.price > 0 ? best.price / current.price : 1;
-    const frontierModels = paretoFrontier(scored).map((p) => p.model);
-    const onFrontier = frontierModels.includes(best.model);
+    const priceKnown = best.priceKnown !== false && (!current || current.priceKnown !== false);
+    const priceRatio = priceKnown && current && current.price > 0 ? best.price / current.price : 1;
+    const frontierModels = paretoFrontier(scored).map(pointKey);
+    const onFrontier = frontierModels.includes(pointKey(best));
     const fb = pickFallback(scored, best, policy.fallback.min_score);
     const decision = gate(policy, {
         scoreGain,
         priceRatio,
+        priceKnown,
         onFrontier,
-        providerOk: providerAllowed(policy, best.model),
+        providerOk: providerAllowed(policy, best.model, best.route?.provider),
         baselineMeasured: !!current,
         sessionBound: !!sessionFile,
     });
-    const reason = `${best.model} scores ${best.score.toFixed(2)} vs ` +
+    const bestLabel = best.route ? routeLabel(best.route) : best.model;
+    const reason = `${bestLabel} scores ${best.score.toFixed(2)} vs ` +
         (current ? `${current.score.toFixed(2)} for ${current.model}` : "no baseline measured") +
-        ` (${scoreGain > 0 ? `quality gain ${scoreGain.toFixed(2)}` : "equal measured quality"}), at $${best.price.toFixed(2)}/M` +
-        (current ? ` (${priceRatio.toFixed(2)}x current price)` : "") +
+        ` (${scoreGain > 0 ? `quality gain ${scoreGain.toFixed(2)}` : "equal measured quality"}), ` +
+        (best.priceKnown === false ? "with unknown price" : `at $${best.price.toFixed(2)}/M`) +
+        (current && priceKnown ? ` (${priceRatio.toFixed(2)}x current price)` : "") +
         (best.why ? `. Judge: ${best.why}` : "");
     return {
+        schemaVersion: 1,
         id: createHash("sha1")
-            .update(`${taskKey}:${best.model}:${Date.now()}`)
+            .update(`${taskKey}:${pointKey(best)}:${Date.now()}`)
             .digest("hex")
             .slice(0, 10),
         ts: new Date().toISOString(),
@@ -43,17 +52,21 @@ export function buildRecommendation(taskKey, taskLabel, currentModel, points, po
         taskLabel,
         sessionFile,
         currentModel,
-        recommended: { model: best.model, reason },
+        currentRoute,
+        recommended: { model: best.model, route: best.route, reason },
         fallback: fb
             ? {
                 model: fb.model,
-                reason: `score ${fb.score.toFixed(2)} at $${fb.price.toFixed(2)}/M` +
+                route: fb.route,
+                reason: `score ${fb.score.toFixed(2)} ` +
+                    (fb.priceKnown === false ? "with unknown price" : `at $${fb.price.toFixed(2)}/M`) +
                     (fb.latencyMs ? `, ~${Math.round(fb.latencyMs)}ms` : ""),
             }
             : null,
         evidence: {
             scoreGain: Math.round(scoreGain * 1000) / 1000,
             priceRatio: Math.round(priceRatio * 100) / 100,
+            priceKnown,
             onFrontier,
             trials: scored.length,
         },

package/dist/routes.js

@@ -0,0 +1,59 @@
+/** Provider-neutral model-route normalization. */
+export function meritModelId(provider, modelId) {
+    if (provider === "openrouter")
+        return modelId;
+    return modelId.startsWith(`${provider}/`) ? modelId : `${provider}/${modelId}`;
+}
+export function modelRoute(provider, modelId, details = {}) {
+    return {
+        provider,
+        modelId,
+        meritId: meritModelId(provider, modelId),
+        input: details.input ?? ["text"],
+        ...details,
+    };
+}
+export function legacyOpenRouterRoute(meritId) {
+    return modelRoute("openrouter", meritId);
+}
+export function routeKey(route) {
+    return `${route.provider}:${route.modelId}`;
+}
+export function pointKey(point) {
+    return point.route ? routeKey(point.route) : `openrouter:${point.model}`;
+}
+export function routeLabel(route) {
+    return route.provider === "openrouter" ? route.meritId : `${route.meritId} via ${route.provider}`;
+}
+export function routePrice(route) {
+    if (!route.cost || !Number.isFinite(route.cost.input) || !Number.isFinite(route.cost.output))
+        return { price: 0, known: false };
+    return { price: Math.round(((route.cost.input + route.cost.output) / 2 + Number.EPSILON) * 1e4) / 1e4,
+        known: true };
+}
+export function catalogEntryFromRoute(route) {
+    const priced = routePrice(route);
+    return {
+        id: route.meritId,
+        name: routeLabel(route),
+        ctx: route.contextWindow ?? 0,
+        pp: route.cost ? route.cost.input / 1e6 : 0,
+        pc: route.cost ? route.cost.output / 1e6 : 0,
+        price: priced.price,
+        priceKnown: priced.known,
+        inputModalities: route.input,
+        route,
+    };
+}
+export function routeCatalog(routes) {
+    const out = new Map();
+    for (const route of routes)
+        out.set(routeKey(route), catalogEntryFromRoute(route));
+    return out;
+}
+/** Preserve the route while enriching an OpenRouter entry with its live catalog metadata. */
+export function enrichRouteEntry(entry, live) {
+    if (!live)
+        return entry;
+    return { ...live, route: entry.route, id: entry.id, name: entry.name, priceKnown: true };
+}