openmerit 0.1.1 → 0.1.2

package/dist/frontier.js

@@ -1,10 +1,16 @@
 /** Pareto frontier over (score up, price down, latency down) + best-fit/fallback picks. */
+import { pointKey } from "./routes.js";
+function effectivePrice(point) {
+    return point.priceKnown === false ? Number.POSITIVE_INFINITY : point.price;
+}
 /** a dominates b if a is no worse on every objective and strictly better on one. */
 export function dominates(a, b) {
     const aLat = a.latencyMs ?? Number.POSITIVE_INFINITY;
     const bLat = b.latencyMs ?? Number.POSITIVE_INFINITY;
-    const noWorse = a.score >= b.score && a.price <= b.price && aLat <= bLat;
-    const better = a.score > b.score || a.price < b.price || aLat < bLat;
+    const aPrice = effectivePrice(a);
+    const bPrice = effectivePrice(b);
+    const noWorse = a.score >= b.score && aPrice <= bPrice && aLat <= bLat;
+    const better = a.score > b.score || aPrice < bPrice || aLat < bLat;
     return noWorse && better;
 }
 /**
@@ -14,11 +20,11 @@ export function dominates(a, b) {
  */
 export function paretoFrontier(points) {
     const fr = points.filter((p) => !points.some((q) => q !== p && dominates(q, p)));
-    return fr.sort((a, b) => a.price - b.price || b.score - a.score);
+    return fr.sort((a, b) => effectivePrice(a) - effectivePrice(b) || b.score - a.score);
 }
 /** Simple 2-objective chain (score vs price only), cheapest first. */
 export function scorePriceChain(points) {
-    const pts = [...points].sort((a, b) => a.price - b.price || b.score - a.score);
+    const pts = [...points].sort((a, b) => effectivePrice(a) - effectivePrice(b) || b.score - a.score);
     const chain = [];
     let best = -1;
     for (const p of pts) {
@@ -31,7 +37,8 @@ export function scorePriceChain(points) {
 }
 /** Highest score; ties broken by lower price then lower latency. */
 export function pickBest(points) {
-    return [...points].sort((a, b) => b.score - a.score || a.price - b.price || (a.latencyMs ?? 1e18) - (b.latencyMs ?? 1e18))[0];
+    return [...points].sort((a, b) => b.score - a.score || effectivePrice(a) - effectivePrice(b) ||
+        (a.latencyMs ?? 1e18) - (b.latencyMs ?? 1e18))[0];
 }
 /**
  * Fallback for `best`: prefer the cheapest frontier point that still clears
@@ -39,7 +46,7 @@ export function pickBest(points) {
  * `best` itself.
  */
 export function pickFallback(points, best, minScore) {
-    const others = points.filter((p) => p.model !== best.model);
+    const others = points.filter((p) => pointKey(p) !== pointKey(best));
     if (others.length === 0)
         return undefined;
     const fr = paretoFrontier(others).filter((p) => p.score >= minScore);

package/dist/integrations.js

@@ -0,0 +1,19 @@
+/** Stable boundaries for trace inputs and telemetry outputs. */
+import { randomUUID } from "node:crypto";
+import { appendJsonl, paths } from "./store.js";
+import { ingestNewTraces } from "./traces.js";
+export class PiTraceObservationSource {
+    sessionsDir;
+    id = "pi-jsonl";
+    constructor(sessionsDir) {
+        this.sessionsDir = sessionsDir;
+    }
+    read() { return ingestNewTraces(this.sessionsDir); }
+}
+export class LocalJsonlEventSink {
+    id = "local-jsonl";
+    emit(event) { appendJsonl(paths.events(), event); }
+}
+export function meritEvent(type, payload, source = "openmerit") {
+    return { schemaVersion: 1, id: randomUUID(), ts: new Date().toISOString(), source, type, payload };
+}

package/dist/judge.js

@@ -1,5 +1,5 @@
 /** Judge: score a model's answer against a rubric. */
-import { chat, chatWithImages } from "./llm.js";
+import { directChatClient } from "./llm.js";
 export const JUDGE_PREFS = ["openai/gpt-4o-mini", "openai/gpt-4o"];
 const JUDGE_PROMPT = `You are grading a model's answer.
 TASK: {task}
@@ -13,8 +13,8 @@ export function parseObj(txt) {
         throw new Error("no json object found");
     return JSON.parse(m[0]);
 }
-export async function judge(key, judgeModel, task, rubric, answer) {
-    const { content } = await chat(key, judgeModel, JUDGE_PROMPT.replace("{task}", task)
+export async function judge(key, judgeModel, task, rubric, answer, client = directChatClient(key)) {
+    const { content } = await client.chat(judgeModel, JUDGE_PROMPT.replace("{task}", task)
         .replace("{rubric}", rubric)
         .replace("{answer}", answer.slice(0, 16000)), 1024, 0);
     try {
@@ -26,13 +26,13 @@ export async function judge(key, judgeModel, task, rubric, answer) {
     }
 }
 /** Grade OCR answers against the actual uploaded pixels, not answer text alone. */
-export async function judgeWithImages(key, judgeModel, task, rubric, answer, images) {
+export async function judgeWithImages(key, judgeModel, task, rubric, answer, images, client = directChatClient(key)) {
     const prompt = `You are grading an answer to a visual task. Inspect the attached image carefully.\n` +
         `TASK: ${task}\nRUBRIC: ${rubric}\nANSWER: ${answer.slice(0, 16000)}\n` +
         `For invoice extraction, check every visible field and line item against the image. ` +
         `Penalize missing, invented, or mistyped values and invalid JSON. ` +
         `Return ONLY json: {"score": <0..1>, "why": "<one line>"}`;
-    const { content } = await chatWithImages(key, judgeModel, prompt, images, 1024);
+    const { content } = await client.chatWithImages(judgeModel, prompt, images, 1024);
     try {
         const j = parseObj(content);
         return { score: Number(j.score ?? 0), why: String(j.why ?? "") };

package/dist/llm.js

@@ -1,6 +1,10 @@
 /** Minimal provider clients for judge and strategist requests. */
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { spawn } from "node:child_process";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { paths } from "./store.js";
+import { legacyOpenRouterRoute } from "./routes.js";
 const OR = "https://openrouter.ai/api/v1";
 /** Load OPENROUTER_API_KEY from env or ~/.openmerit/.env. */
 export function loadKey() {
@@ -100,3 +104,100 @@ export async function chat(key, model, prompt, maxTokens, temperature) {
 export async function chatWithImages(key, model, prompt, images, maxTokens) {
     return providerFor(model, key).chatWithImages(model, prompt, images, maxTokens);
 }
+export function directChatClient(key) {
+    const id = (model) => typeof model === "string" ? model : model.meritId;
+    return {
+        chat: (model, prompt, maxTokens, temperature) => chat(key, id(model), prompt, maxTokens, temperature),
+        chatWithImages: (model, prompt, images, maxTokens) => chatWithImages(key, id(model), prompt, images, maxTokens),
+    };
+}
+function textContent(content) {
+    if (typeof content === "string")
+        return content;
+    if (!Array.isArray(content))
+        return "";
+    return content.filter((part) => !!part && typeof part === "object" && part.type === "text")
+        .map((part) => part.text ?? "").join("\n");
+}
+/** Use pi's model registry and credential store without exposing provider secrets to OpenMerit. */
+export class PiCliChatClient {
+    executable;
+    onSpend;
+    id = "pi-cli";
+    constructor(executable = process.env.OPENMERIT_PI_BIN?.trim() || "pi", onSpend) {
+        this.executable = executable;
+        this.onSpend = onSpend;
+    }
+    chat(model, prompt, _maxTokens, _temperature) {
+        return this.run(model, prompt, []);
+    }
+    chatWithImages(model, prompt, images, _maxTokens) {
+        return this.run(model, prompt, images);
+    }
+    async run(target, prompt, images) {
+        const route = typeof target === "string" ? legacyOpenRouterRoute(target) : target;
+        const dir = mkdtempSync(join(tmpdir(), "openmerit-pi-chat-"));
+        try {
+            const imageArgs = images.map((image, index) => {
+                const suffix = { "image/jpeg": "jpg", "image/png": "png", "image/webp": "webp", "image/gif": "gif" }[image.mimeType];
+                if (!suffix)
+                    throw new Error(`unsupported pi image: ${image.mimeType}`);
+                const file = join(dir, `image-${index}.${suffix}`);
+                writeFileSync(file, Buffer.from(image.data, "base64"));
+                return `@${file}`;
+            });
+            const child = spawn(this.executable, [
+                "--provider", route.provider, "--model", route.modelId, "--mode", "json", "--offline",
+                "--no-extensions", "--no-tools", "--print", ...imageArgs, prompt,
+            ], { cwd: dir, stdio: ["ignore", "pipe", "pipe"] });
+            let stdout = "";
+            let stderr = "";
+            child.stdout.on("data", (chunk) => { stdout += chunk.toString("utf8"); });
+            child.stderr.on("data", (chunk) => { stderr += chunk.toString("utf8"); });
+            const code = await new Promise((resolve, reject) => {
+                child.on("error", reject);
+                child.on("close", (value) => resolve(value ?? 1));
+            });
+            let answer = "";
+            let promptTokens = 0;
+            let completionTokens = 0;
+            let totalTokens = 0;
+            let costUsd = 0;
+            let failed = false;
+            for (const line of stdout.split("\n")) {
+                if (!line.trim())
+                    continue;
+                let event;
+                try {
+                    event = JSON.parse(line);
+                }
+                catch {
+                    continue;
+                }
+                if (event.type !== "message_end" || event.message?.role !== "assistant")
+                    continue;
+                answer = textContent(event.message.content) || answer;
+                promptTokens += event.message.usage?.input ?? 0;
+                completionTokens += event.message.usage?.output ?? 0;
+                totalTokens += event.message.usage?.totalTokens ?? 0;
+                costUsd += event.message.usage?.cost?.total ?? 0;
+                if (event.message.stopReason === "error")
+                    failed = true;
+            }
+            if (code !== 0 || failed || !answer.trim()) {
+                throw new Error(`pi model call ${route.provider}/${route.modelId} failed: ` +
+                    (stderr.trim().slice(0, 240) || `exit ${code}, empty answer`));
+            }
+            this.onSpend?.(costUsd);
+            return { content: answer, usage: {
+                    prompt_tokens: promptTokens || undefined,
+                    completion_tokens: completionTokens || undefined,
+                    total_tokens: totalTokens || promptTokens + completionTokens || undefined,
+                    costUsd,
+                } };
+        }
+        finally {
+            rmSync(dir, { recursive: true, force: true });
+        }
+    }
+}

package/dist/pi-trials.js

@@ -8,6 +8,8 @@ import { judge, judgeWithImages } from "./judge.js";
 import { knownInvoiceScore } from "./invoice-eval.js";
 import { paths, readJson } from "./store.js";
 import { sessionFiles, sessionImages, taskInputKey } from "./task-input.js";
+import { legacyOpenRouterRoute, meritModelId, modelRoute, routeKey } from "./routes.js";
+import { directChatClient } from "./llm.js";
 function answerText(content) {
     if (typeof content === "string")
         return content;
@@ -107,10 +109,11 @@ export function recordedActiveTask(task, model, images = [], sessionFile, sessio
             toolErrors++;
     }
     const userImages = user ? sessionImages(user.content) : null;
-    if (!user || !userImages || taskInputKey(answerText(user.content), userImages) !==
-        taskInputKey(task, images))
+    const files = sessionFiles(task);
+    if (!user || !userImages || taskInputKey(answerText(user.content), userImages, sessionFiles(answerText(user.content))) !==
+        taskInputKey(task, images, files))
         return null;
-    const matching = assistants.filter((m) => (m.provider === "openrouter" ? m.model : `${m.provider}/${m.model}`) === model);
+    const matching = assistants.filter((m) => meritModelId(m.provider ?? "openrouter", m.model ?? "") === model);
     if (matching.length === 0 || matching.some((m) => m.stopReason === "error"))
         return null;
     const answer = matching.map((m) => answerText(m.content)).filter(Boolean).at(-1) ?? "";
@@ -170,20 +173,38 @@ export function settledActiveTask(snapshot) {
     const run = recordedActiveTask(task, st.currentModel, images, st.sessionFile, st.sessionBytes);
     if (!run)
         return null;
+    const route = st.currentRoute ?? legacyOpenRouterRoute(st.currentModel);
+    const routes = st.routes?.length ? st.routes : [route];
     return { task, images, model: st.currentModel, run, sessionFile: st.sessionFile,
-        settledAt: st.settledAt, sessionBytes: st.sessionBytes, cwd: st.cwd ?? process.cwd(), usedTools, files };
+        settledAt: st.settledAt, sessionBytes: st.sessionBytes, cwd: st.cwd ?? process.cwd(), usedTools, files,
+        route, routes };
 }
-/** Limit candidates to models this installed pi can actually select. */
-export function availablePiModels() {
-    const result = spawnSync("pi", ["--offline", "--list-models", "openrouter"], { encoding: "utf8" });
+export function piExecutable() {
+    return process.env.OPENMERIT_PI_BIN?.trim() || "pi";
+}
+/** Read provider/model routes from pi when no live extension snapshot is available. */
+export function availablePiRoutes(snapshot) {
+    if (snapshot?.length)
+        return snapshot;
+    const result = spawnSync(piExecutable(), ["--offline", "--list-models"], { encoding: "utf8" });
     if (result.error || result.status !== 0)
-        throw new Error("could not list pi OpenRouter models");
-    return new Set(result.stdout.split("\n").slice(1)
-        .map((line) => line.trim().split(/\s+/)[1])
-        .filter((id) => !!id && !id.startsWith("~")));
+        throw new Error("could not list models from pi");
+    const routes = [];
+    for (const line of result.stdout.split("\n").slice(1)) {
+        const [provider, id, , , , images] = line.trim().split(/\s+/);
+        if (!provider || !id || id.startsWith("~"))
+            continue;
+        routes.push(modelRoute(provider, id, { input: images === "yes" ? ["text", "image"] : ["text"] }));
+    }
+    return routes;
+}
+/** Backward-compatible logical-id view used by the standalone OpenRouter trial command. */
+export function availablePiModels() {
+    return new Set(availablePiRoutes().map((route) => route.meritId));
 }
 /** Each invocation is a fresh pi run, so model B does not inherit model A's answer. */
 export async function executePiTask(model, task, cwd = process.cwd(), images = [], files = []) {
+    const route = typeof model === "string" ? legacyOpenRouterRoute(model) : model;
     const imageDir = images.length ? mkdtempSync(join(tmpdir(), "openmerit-pi-image-")) : null;
     const suffix = {
         "image/jpeg": "jpg", "image/png": "png", "image/webp": "webp", "image/gif": "gif",
@@ -209,10 +230,10 @@ export async function executePiTask(model, task, cwd = process.cwd(), images = [
         });
         const candidateTask = [...attachmentPaths.entries()].reduce((text, [source, target]) => text.split(source).join(target), task);
         const before = fileSnapshot(workspace);
-        const traceFile = paths.trialTrace(`${model}:${task}:${Date.now()}`);
+        const traceFile = paths.trialTrace(`${routeKey(route)}:${task}:${Date.now()}`);
         const traceLines = [];
-        const child = spawn("pi", [
-            "--provider", "openrouter", "--model", model, "--mode", "json",
+        const child = spawn(piExecutable(), [
+            "--provider", route.provider, "--model", route.modelId, "--mode", "json",
             "--offline", "--no-extensions", "--approve", ...piTrialToolArgs(),
             "--print", ...imageArgs, ...fileArgs, candidateTask,
         ], { cwd: workspace, stdio: ["ignore", "pipe", "pipe"] });
@@ -275,7 +296,7 @@ export async function executePiTask(model, task, cwd = process.cwd(), images = [
             if (!after.has(file))
                 changedFiles++;
         if (exitCode !== 0 || !answer.trim()) {
-            throw new Error(`pi trial ${model} failed: ${stderr.trim().slice(0, 180) || `exit ${exitCode}, empty answer`}`);
+            throw new Error(`pi trial ${route.meritId} via ${route.provider} failed: ${stderr.trim().slice(0, 180) || `exit ${exitCode}, empty answer`}`);
         }
         return { answer, costUsd, latencyMs: Date.now() - started, errors, sessionId,
             toolCalls, toolErrors, changedFiles, traceFile };
@@ -287,29 +308,30 @@ export async function executePiTask(model, task, cwd = process.cwd(), images = [
     }
 }
 /** Quality uses the same OpenMerit judge; cost and latency come from pi itself. */
-export async function runPiTrial(key, judgeModel, task, rubric, model, entry, cwd = process.cwd(), images = [], files = [], harness = defaultHarness) {
+export async function runPiTrial(key, judgeModel, task, rubric, model, entry, cwd = process.cwd(), images = [], files = [], harness = defaultHarness, client = directChatClient(key)) {
     try {
-        const run = await harness.runTask(model, { task, cwd, images, files });
-        return await scorePiRun(key, judgeModel, task, rubric, model, entry, run, "pi_trial", images);
+        const run = await harness.runTask(entry?.route ?? model, { task, cwd, images, files });
+        return await scorePiRun(key, judgeModel, task, rubric, model, entry, run, "pi_trial", images, client);
     }
     catch (e) {
         const error = String(e);
         return {
-            point: { model, score: 0, price: entry?.price ?? 0,
+            point: { schemaVersion: 1, model, route: entry?.route, score: 0, price: entry?.price ?? 0,
+                priceKnown: entry?.priceKnown ?? !!entry,
                 ts: new Date().toISOString(), source: "pi_trial", toolCalls: 0, toolErrors: 1,
                 changedFiles: 0, why: error.slice(0, 160) },
             costUsd: 0, error,
         };
     }
 }
-export async function scorePiRun(key, judgeModel, task, rubric, model, entry, run, source, images = []) {
+export async function scorePiRun(key, judgeModel, task, rubric, model, entry, run, source, images = [], client = directChatClient(key)) {
     let score = 0;
     let why = "judge unavailable";
     try {
         const pinned = images.length ? await knownInvoiceScore(task, images, run.answer) : null;
         const j = pinned ?? (images.length
-            ? await judgeWithImages(key, judgeModel, task, rubric, run.answer, images)
-            : await judge(key, judgeModel, task, rubric, run.answer));
+            ? await judgeWithImages(key, judgeModel, task, rubric, run.answer, images, client)
+            : await judge(key, judgeModel, task, rubric, run.answer, client));
         score = Math.max(0, Math.min(1, j.score));
         why = j.why;
     }
@@ -318,7 +340,8 @@ export async function scorePiRun(key, judgeModel, task, rubric, model, entry, ru
     }
     return {
         point: {
-            model, score, price: entry?.price ?? 0,
+            schemaVersion: 1, model, route: entry?.route, score, price: entry?.price ?? 0,
+            priceKnown: entry?.priceKnown ?? !!entry,
             latencyMs: run.latencyMs, ts: new Date().toISOString(), source,
             why: `${why}${run.errors ? `; pi errors: ${run.errors}` : ""}; ` +
                 `tools=${run.toolCalls}, toolErrors=${run.toolErrors}, changedFiles=${run.changedFiles}`,

package/dist/policy.js

@@ -60,10 +60,11 @@ export function loadPolicy(path) {
         max_usd_per_m: num(raw.max_usd_per_m, d.max_usd_per_m),
     };
 }
-/** Is a model id allowed by the provider allow/deny lists? ("*" matches all; "vendor/*" matches a vendor.) */
-export function providerAllowed(policy, modelId) {
+/** Match either a logical vendor/model id or its concrete route provider. */
+export function providerAllowed(policy, modelId, routeProvider) {
     const vendor = modelId.split("/")[0];
-    const match = (pat) => pat === "*" || pat === modelId || (pat.endsWith("/*") && pat.slice(0, -2) === vendor);
+    const match = (pat) => pat === "*" || pat === modelId || pat === routeProvider ||
+        (pat.endsWith("/*") && (pat.slice(0, -2) === vendor || pat.slice(0, -2) === routeProvider));
     if (policy.providers.deny.some(match))
         return false;
     return policy.providers.allow.some(match);
@@ -87,6 +88,10 @@ export function gate(policy, input) {
         reasons.push("recommendation is not bound to a pi session");
         return { autoApply: false, reasons };
     }
+    if (input.priceKnown === false) {
+        reasons.push("candidate or current-model price is unknown");
+        return { autoApply: false, reasons };
+    }
     if (policy.mode !== "auto" || !policy.auto_apply.enabled) {
         reasons.push(`policy mode is "${policy.mode}" (auto_apply ${policy.auto_apply.enabled ? "enabled" : "disabled"})`);
         return { autoApply: false, reasons };

package/dist/recommend.js

@@ -2,40 +2,49 @@
 import { createHash } from "node:crypto";
 import { paretoFrontier, pickBest, pickFallback } from "./frontier.js";
 import { gate, providerAllowed } from "./policy.js";
-export function buildRecommendation(taskKey, taskLabel, currentModel, points, policy, sessionFile = null) {
+import { pointKey, routeLabel } from "./routes.js";
+export function buildRecommendation(taskKey, taskLabel, currentModel, points, policy, sessionFile = null, currentRoute = currentModel ? points.find((p) => p.model === currentModel)?.route ?? null : null) {
     const scored = points.filter((p) => p.score > 0);
     if (scored.length === 0)
         return null;
     const best = pickBest(scored);
     if (!best)
         return null;
-    if (currentModel && best.model === currentModel)
-        return null; // already on the best model
-    const current = currentModel ? scored.find((p) => p.model === currentModel) : undefined;
+    if (currentModel && best.model === currentModel &&
+        (!currentRoute || pointKey(best) === `${currentRoute.provider}:${currentRoute.modelId}`))
+        return null;
+    const current = currentModel ? scored.find((p) => currentRoute
+        ? pointKey(p) === `${currentRoute.provider}:${currentRoute.modelId}`
+        : p.model === currentModel) : undefined;
     const scoreGain = current ? best.score - current.score : 0;
     // pickBest may choose a cheaper or faster model at equal quality.
     if (current && scoreGain < 0)
         return null;
-    const priceRatio = current && current.price > 0 ? best.price / current.price : 1;
-    const frontierModels = paretoFrontier(scored).map((p) => p.model);
-    const onFrontier = frontierModels.includes(best.model);
+    const priceKnown = best.priceKnown !== false && (!current || current.priceKnown !== false);
+    const priceRatio = priceKnown && current && current.price > 0 ? best.price / current.price : 1;
+    const frontierModels = paretoFrontier(scored).map(pointKey);
+    const onFrontier = frontierModels.includes(pointKey(best));
     const fb = pickFallback(scored, best, policy.fallback.min_score);
     const decision = gate(policy, {
         scoreGain,
         priceRatio,
+        priceKnown,
         onFrontier,
-        providerOk: providerAllowed(policy, best.model),
+        providerOk: providerAllowed(policy, best.model, best.route?.provider),
         baselineMeasured: !!current,
         sessionBound: !!sessionFile,
     });
-    const reason = `${best.model} scores ${best.score.toFixed(2)} vs ` +
+    const bestLabel = best.route ? routeLabel(best.route) : best.model;
+    const reason = `${bestLabel} scores ${best.score.toFixed(2)} vs ` +
         (current ? `${current.score.toFixed(2)} for ${current.model}` : "no baseline measured") +
-        ` (${scoreGain > 0 ? `quality gain ${scoreGain.toFixed(2)}` : "equal measured quality"}), at $${best.price.toFixed(2)}/M` +
-        (current ? ` (${priceRatio.toFixed(2)}x current price)` : "") +
+        ` (${scoreGain > 0 ? `quality gain ${scoreGain.toFixed(2)}` : "equal measured quality"}), ` +
+        (best.priceKnown === false ? "with unknown price" : `at $${best.price.toFixed(2)}/M`) +
+        (current && priceKnown ? ` (${priceRatio.toFixed(2)}x current price)` : "") +
         (best.why ? `. Judge: ${best.why}` : "");
     return {
+        schemaVersion: 1,
         id: createHash("sha1")
-            .update(`${taskKey}:${best.model}:${Date.now()}`)
+            .update(`${taskKey}:${pointKey(best)}:${Date.now()}`)
             .digest("hex")
             .slice(0, 10),
         ts: new Date().toISOString(),
@@ -43,17 +52,21 @@ export function buildRecommendation(taskKey, taskLabel, currentModel, points, po
         taskLabel,
         sessionFile,
         currentModel,
-        recommended: { model: best.model, reason },
+        currentRoute,
+        recommended: { model: best.model, route: best.route, reason },
         fallback: fb
             ? {
                 model: fb.model,
-                reason: `score ${fb.score.toFixed(2)} at $${fb.price.toFixed(2)}/M` +
+                route: fb.route,
+                reason: `score ${fb.score.toFixed(2)} ` +
+                    (fb.priceKnown === false ? "with unknown price" : `at $${fb.price.toFixed(2)}/M`) +
                     (fb.latencyMs ? `, ~${Math.round(fb.latencyMs)}ms` : ""),
             }
             : null,
         evidence: {
             scoreGain: Math.round(scoreGain * 1000) / 1000,
             priceRatio: Math.round(priceRatio * 100) / 100,
+            priceKnown,
             onFrontier,
             trials: scored.length,
         },

package/dist/routes.js

@@ -0,0 +1,59 @@
+/** Provider-neutral model-route normalization. */
+export function meritModelId(provider, modelId) {
+    if (provider === "openrouter")
+        return modelId;
+    return modelId.startsWith(`${provider}/`) ? modelId : `${provider}/${modelId}`;
+}
+export function modelRoute(provider, modelId, details = {}) {
+    return {
+        provider,
+        modelId,
+        meritId: meritModelId(provider, modelId),
+        input: details.input ?? ["text"],
+        ...details,
+    };
+}
+export function legacyOpenRouterRoute(meritId) {
+    return modelRoute("openrouter", meritId);
+}
+export function routeKey(route) {
+    return `${route.provider}:${route.modelId}`;
+}
+export function pointKey(point) {
+    return point.route ? routeKey(point.route) : `openrouter:${point.model}`;
+}
+export function routeLabel(route) {
+    return route.provider === "openrouter" ? route.meritId : `${route.meritId} via ${route.provider}`;
+}
+export function routePrice(route) {
+    if (!route.cost || !Number.isFinite(route.cost.input) || !Number.isFinite(route.cost.output))
+        return { price: 0, known: false };
+    return { price: Math.round(((route.cost.input + route.cost.output) / 2 + Number.EPSILON) * 1e4) / 1e4,
+        known: true };
+}
+export function catalogEntryFromRoute(route) {
+    const priced = routePrice(route);
+    return {
+        id: route.meritId,
+        name: routeLabel(route),
+        ctx: route.contextWindow ?? 0,
+        pp: route.cost ? route.cost.input / 1e6 : 0,
+        pc: route.cost ? route.cost.output / 1e6 : 0,
+        price: priced.price,
+        priceKnown: priced.known,
+        inputModalities: route.input,
+        route,
+    };
+}
+export function routeCatalog(routes) {
+    const out = new Map();
+    for (const route of routes)
+        out.set(routeKey(route), catalogEntryFromRoute(route));
+    return out;
+}
+/** Preserve the route while enriching an OpenRouter entry with its live catalog metadata. */
+export function enrichRouteEntry(entry, live) {
+    if (!live)
+        return entry;
+    return { ...live, route: entry.route, id: entry.id, name: entry.name, priceKnown: true };
+}

package/dist/store.js

@@ -18,6 +18,7 @@ export const paths = {
     benchmarksDigest: () => join(stateDir(), "benchmarks", "digest.json"),
     traceCursor: () => join(stateDir(), "traces", "cursor.json"),
     observations: () => join(stateDir(), "traces", "observations.jsonl"),
+    events: () => join(stateDir(), "events.jsonl"),
     harnessState: () => join(stateDir(), "harness-state.json"),
     ledger: () => join(stateDir(), "ledger.json"),
     watchProcessed: () => join(stateDir(), "watch", "processed.json"),

package/dist/strategist.js

@@ -1,12 +1,13 @@
 /** Strategist: pick the next untried model to trial, benchmark-aware. */
-import { chat } from "./llm.js";
+import { directChatClient } from "./llm.js";
 import { parseObj } from "./judge.js";
+import { routeKey, routeLabel } from "./routes.js";
 export const STRAT_PREFS = [
     "anthropic/claude-3.7-sonnet",
     "anthropic/claude-3.5-sonnet",
     "openai/gpt-4o",
 ];
-const STRAT_PROMPT = `We are finding the best OpenRouter model for a task via iterative trials.
+const STRAT_PROMPT = `We are finding the best configured model route for a task via iterative trials.
 TASK: {task}
 RUBRIC: {rubric}
 RELEVANT PUBLIC BENCHMARKS FOR THIS TASK: {benchmarks}
@@ -14,15 +15,16 @@ RESULTS SO FAR (model=score): {results}
 Pick ONE untried model from the catalog below that is likely to do well on this
 task, based on the benchmarks relevant to it. Balance quality against price so
 a pareto frontier emerges.
-Return ONLY json: {{"next_model": "<catalog id>", "why": "<one line>"}}
-CATALOG (id | blended $/1M tokens | ctx):
+Return ONLY json: {{"next_model": "<route id>", "why": "<one line>"}}
+ROUTES (route id | logical model | blended $/1M tokens | ctx):
 {catalog}`;
 /** Pick the next model to trial, or null when the catalog is exhausted. */
-export async function pickNext(key, stratModel, task, rubric, benchmarks, results, cat, tried, maxPrice, failedVendors, extraCandidates) {
-    const ok = (c) => c.price <= maxPrice &&
-        !tried.has(c.id) &&
+export async function pickNext(key, stratModel, task, rubric, benchmarks, results, cat, tried, maxPrice, failedVendors, extraCandidates, client = directChatClient(key)) {
+    const entryKey = (c) => c.route ? routeKey(c.route) : c.id;
+    const ok = (c) => (c.priceKnown === false || c.price <= maxPrice) &&
+        !tried.has(entryKey(c)) &&
         c.ctx >= 4096 &&
-        !failedVendors.has(c.id.split("/")[0]);
+        !failedVendors.has(c.route?.provider ?? c.id.split("/")[0]);
     // Benchmark-shortlisted models get priority in the listing.
     const pool = [...cat.values()].filter(ok);
     if (pool.length === 0)
@@ -31,15 +33,16 @@ export async function pickNext(key, stratModel, task, rubric, benchmarks, result
     pool.sort((a, b) => {
         const pa = priority.has(a.id) ? 0 : 1;
         const pb = priority.has(b.id) ? 0 : 1;
-        return pa - pb || a.price - b.price;
+        return pa - pb || (a.priceKnown === false ? 1 : 0) - (b.priceKnown === false ? 1 : 0) || a.price - b.price;
     });
     const lines = pool
-        .map((c) => `${c.id} | ${c.price.toFixed(2)} | ${c.ctx}${priority.has(c.id) ? " | BENCHMARK" : ""}`)
+        .map((c) => `${entryKey(c)} | ${c.route ? routeLabel(c.route) : c.id} | ` +
+        `${c.priceKnown === false ? "unknown" : c.price.toFixed(2)} | ${c.ctx}${priority.has(c.id) ? " | BENCHMARK" : ""}`)
         .join("\n");
     const res = results
         .map((r) => `${r.model}=${r.score.toFixed(2)}`)
         .join("  ") || "none yet";
-    const { content } = await chat(key, stratModel, STRAT_PROMPT.replace("{task}", task)
+    const { content } = await client.chat(stratModel, STRAT_PROMPT.replace("{task}", task)
         .replace("{rubric}", rubric)
         .replace("{benchmarks}", benchmarks.join(", "))
         .replace("{results}", res)
@@ -55,10 +58,11 @@ export async function pickNext(key, stratModel, task, rubric, benchmarks, result
         /* fall through to cheap pick */
     }
     if (pick) {
-        const entry = cat.get(pick);
+        const entry = cat.get(pick) ?? [...cat.values()].find((candidate) => candidate.id === pick);
         if (entry && ok(entry))
-            return { model: pick, why };
+            return { model: entry.id, route: entry.route, why };
     }
     const cheapest = pool[0];
-    return { model: cheapest.id, why: `fallback pick: cheapest untried (strategist pick ${pick ?? "none"} unusable)` };
+    return { model: cheapest.id, route: cheapest.route,
+        why: `fallback pick: cheapest untried (strategist pick ${pick ?? "none"} unusable)` };
 }

package/dist/traces.js

@@ -7,6 +7,7 @@ import { readdirSync, readFileSync, existsSync, statSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
 import { paths, readJson, taskKey, writeJson } from "./store.js";
+import { meritModelId, modelRoute } from "./routes.js";
 export function defaultSessionsDir() {
     return join(homedir(), ".pi", "agent", "sessions");
 }
@@ -53,6 +54,7 @@ export function observationsFromEntries(entries, sessionFile) {
                 continue;
             }
             current = {
+                schemaVersion: 1,
                 taskKey: taskKey(text),
                 taskLabel: text.replace(/\s+/g, " ").trim().slice(0, 120),
                 model: null,
@@ -68,8 +70,10 @@ export function observationsFromEntries(entries, sessionFile) {
         if (!current)
             continue;
         if (m.role === "assistant") {
-            if (!current.model && m.provider && m.model)
-                current.model = `${m.provider}/${m.model}`;
+            if (!current.model && m.provider && m.model) {
+                current.model = meritModelId(m.provider, m.model);
+                current.route = modelRoute(m.provider, m.model);
+            }
             current.costUsd += m.usage?.cost?.total ?? 0;
             current.tokens += m.usage?.totalTokens ?? 0;
             if (m.stopReason === "error")