openmerit 0.1.0 → 0.1.2

package/dist/pi-trials.js

@@ -1,12 +1,15 @@
 /** Run a task through pi for each model, using pi's JSON event stream as the measurement source. */
 import { spawn, spawnSync } from "node:child_process";
-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
-import { join } from "node:path";
+import { dirname, join, relative } from "node:path";
+import { createHash } from "node:crypto";
 import { judge, judgeWithImages } from "./judge.js";
 import { knownInvoiceScore } from "./invoice-eval.js";
 import { paths, readJson } from "./store.js";
-import { sessionImages, taskInputKey } from "./task-input.js";
+import { sessionFiles, sessionImages, taskInputKey } from "./task-input.js";
+import { legacyOpenRouterRoute, meritModelId, modelRoute, routeKey } from "./routes.js";
+import { directChatClient } from "./llm.js";
 function answerText(content) {
     if (typeof content === "string")
         return content;
@@ -15,6 +18,59 @@ function answerText(content) {
     return content.filter((x) => !!x && typeof x === "object" && x.type === "text")
         .map((x) => x.text ?? "").join("\n");
 }
+/** Pi-backed harness adapter. Future harnesses can implement the same seam. */
+export class PiHarnessAdapter {
+    runTask(model, input) {
+        return executePiTask(model, input.task, input.cwd, input.images, input.files);
+    }
+}
+const defaultHarness = new PiHarnessAdapter();
+const KNOWN_TRIAL_TOOLS = new Set([
+    "read", "grep", "find", "ls", "bash", "powershell", "edit", "write",
+]);
+/**
+ * Candidate runs have no tools by default. Users may explicitly opt into a
+ * list, but a copied cwd is not an OS sandbox for absolute paths or network.
+ */
+export function piTrialToolArgs(setting = process.env.OPENMERIT_PI_TRIAL_TOOLS) {
+    if (!setting?.trim())
+        return ["--no-tools"];
+    const requested = [...new Set(setting.split(",").map((tool) => tool.trim()).filter(Boolean))];
+    if (requested.length === 1 && requested[0] === "none")
+        return ["--no-tools"];
+    const unknown = requested.filter((tool) => !KNOWN_TRIAL_TOOLS.has(tool));
+    if (requested.length === 0 || unknown.length > 0) {
+        throw new Error(`invalid OPENMERIT_PI_TRIAL_TOOLS${unknown.length ? `: ${unknown.join(", ")}` : ""}`);
+    }
+    return ["--tools", requested.join(",")];
+}
+function fileSnapshot(root) {
+    const out = new Map();
+    function walk(dir) {
+        for (const name of readdirSync(dir)) {
+            if (name === ".git" || name === "node_modules" || name === ".openmerit")
+                continue;
+            const file = join(dir, name);
+            const rel = relative(root, file);
+            const st = statSync(file);
+            if (st.isDirectory())
+                walk(file);
+            else if (st.isFile() && st.size < 20_000_000) {
+                out.set(rel, createHash("sha1").update(readFileSync(file)).digest("hex"));
+            }
+        }
+    }
+    walk(root);
+    return out;
+}
+function isolatedWorkspace(cwd) {
+    const root = mkdtempSync(join(tmpdir(), "openmerit-pi-workspace-"));
+    cpSync(cwd, root, { recursive: true, filter: (src) => {
+            const rel = relative(cwd, src);
+            return !rel.split("/").some((part) => part === ".git" || part === "node_modules" || part === ".openmerit");
+        } });
+    return root;
+}
 /** Use model A's completed result from the active pi session when it matches this task. */
 export function recordedActiveTask(task, model, images = [], sessionFile, sessionBytes) {
     const state = sessionFile ? null : readJson(paths.harnessState(), {});
@@ -23,6 +79,8 @@ export function recordedActiveTask(task, model, images = [], sessionFile, sessio
         return null;
     let user = null;
     const assistants = [];
+    let toolCalls = 0;
+    let toolErrors = 0;
     for (const line of readFileSync(file).subarray(0, sessionBytes).toString("utf8").split("\n")) {
         if (!line.trim())
             continue;
@@ -38,15 +96,24 @@ export function recordedActiveTask(task, model, images = [], sessionFile, sessio
         if (entry.message.role === "user") {
             user = entry.message;
             assistants.length = 0;
+            toolCalls = 0;
+            toolErrors = 0;
         }
-        else if (user && entry.message.role === "assistant")
+        else if (user && entry.message.role === "assistant") {
             assistants.push(entry.message);
+            if (Array.isArray(entry.message.content))
+                toolCalls += entry.message.content.filter((part) => part?.type === "toolCall").length;
+        }
+        else if (user && entry.message.role === "toolResult" &&
+            entry.message.isError)
+            toolErrors++;
     }
     const userImages = user ? sessionImages(user.content) : null;
-    if (!user || !userImages || taskInputKey(answerText(user.content), userImages) !==
-        taskInputKey(task, images))
+    const files = sessionFiles(task);
+    if (!user || !userImages || taskInputKey(answerText(user.content), userImages, sessionFiles(answerText(user.content))) !==
+        taskInputKey(task, images, files))
         return null;
-    const matching = assistants.filter((m) => (m.provider === "openrouter" ? m.model : `${m.provider}/${m.model}`) === model);
+    const matching = assistants.filter((m) => meritModelId(m.provider ?? "openrouter", m.model ?? "") === model);
     if (matching.length === 0 || matching.some((m) => m.stopReason === "error"))
         return null;
     const answer = matching.map((m) => answerText(m.content)).filter(Boolean).at(-1) ?? "";
@@ -59,6 +126,9 @@ export function recordedActiveTask(task, model, images = [], sessionFile, sessio
         costUsd: matching.reduce((n, m) => n + (m.usage?.cost?.total ?? 0), 0),
         latencyMs: Math.max(0, last - first),
         errors: 0,
+        toolCalls,
+        toolErrors,
+        changedFiles: 0,
     };
 }
 /** Read the exact latest completed text task from the pi extension's session marker. */
@@ -91,31 +161,50 @@ export function settledActiveTask(snapshot) {
             Array.isArray(entry.message.content) && entry.message.content.some((b) => b?.type === "toolCall"))
             usedTools = true;
     }
-    if (!userContent || usedTools)
+    if (!userContent)
         return null;
     const images = sessionImages(userContent);
     if (!images)
         return null;
     const task = answerText(userContent);
-    if (!task.trim() || taskInputKey(task, images) !== st.settledTaskKey)
+    const files = sessionFiles(task);
+    if (!task.trim() || taskInputKey(task, images, files) !== st.settledTaskKey)
         return null;
     const run = recordedActiveTask(task, st.currentModel, images, st.sessionFile, st.sessionBytes);
     if (!run)
         return null;
+    const route = st.currentRoute ?? legacyOpenRouterRoute(st.currentModel);
+    const routes = st.routes?.length ? st.routes : [route];
     return { task, images, model: st.currentModel, run, sessionFile: st.sessionFile,
-        settledAt: st.settledAt, sessionBytes: st.sessionBytes, cwd: st.cwd ?? process.cwd() };
+        settledAt: st.settledAt, sessionBytes: st.sessionBytes, cwd: st.cwd ?? process.cwd(), usedTools, files,
+        route, routes };
 }
-/** Limit candidates to models this installed pi can actually select. */
-export function availablePiModels() {
-    const result = spawnSync("pi", ["--offline", "--list-models", "openrouter"], { encoding: "utf8" });
+export function piExecutable() {
+    return process.env.OPENMERIT_PI_BIN?.trim() || "pi";
+}
+/** Read provider/model routes from pi when no live extension snapshot is available. */
+export function availablePiRoutes(snapshot) {
+    if (snapshot?.length)
+        return snapshot;
+    const result = spawnSync(piExecutable(), ["--offline", "--list-models"], { encoding: "utf8" });
     if (result.error || result.status !== 0)
-        throw new Error("could not list pi OpenRouter models");
-    return new Set(result.stdout.split("\n").slice(1)
-        .map((line) => line.trim().split(/\s+/)[1])
-        .filter((id) => !!id && !id.startsWith("~")));
+        throw new Error("could not list models from pi");
+    const routes = [];
+    for (const line of result.stdout.split("\n").slice(1)) {
+        const [provider, id, , , , images] = line.trim().split(/\s+/);
+        if (!provider || !id || id.startsWith("~"))
+            continue;
+        routes.push(modelRoute(provider, id, { input: images === "yes" ? ["text", "image"] : ["text"] }));
+    }
+    return routes;
+}
+/** Backward-compatible logical-id view used by the standalone OpenRouter trial command. */
+export function availablePiModels() {
+    return new Set(availablePiRoutes().map((route) => route.meritId));
 }
 /** Each invocation is a fresh pi run, so model B does not inherit model A's answer. */
-export async function executePiTask(model, task, cwd = process.cwd(), images = []) {
+export async function executePiTask(model, task, cwd = process.cwd(), images = [], files = []) {
+    const route = typeof model === "string" ? legacyOpenRouterRoute(model) : model;
     const imageDir = images.length ? mkdtempSync(join(tmpdir(), "openmerit-pi-image-")) : null;
     const suffix = {
         "image/jpeg": "jpg", "image/png": "png", "image/webp": "webp", "image/gif": "gif",
@@ -127,85 +216,122 @@ export async function executePiTask(model, task, cwd = process.cwd(), images = [
         writeFileSync(file, Buffer.from(image.data, "base64"));
         return `@${file}`;
     });
-    const child = spawn("pi", [
-        "--provider", "openrouter", "--model", model, "--mode", "json",
-        "--offline", "--no-extensions", "--no-tools", "--print", ...imageArgs, task,
-    ], { cwd, stdio: ["ignore", "pipe", "pipe"] });
-    let buffer = "";
-    let stderr = "";
-    let answer = "";
-    let costUsd = 0;
-    let errors = 0;
-    let sessionId;
-    const started = Date.now();
-    function consume(line) {
-        if (!line.trim())
-            return;
-        let event;
-        try {
-            event = JSON.parse(line);
-        }
-        catch {
-            return;
-        }
-        if (event.type === "session" && event.id)
-            sessionId = event.id;
-        if (event.type !== "message_end" || event.message?.role !== "assistant")
-            return;
-        answer = answerText(event.message.content) || answer;
-        costUsd += event.message.usage?.cost?.total ?? 0;
-        if (event.message.stopReason === "error")
-            errors++;
-    }
-    child.stdout.on("data", (chunk) => {
-        buffer += chunk.toString("utf8");
-        let n;
-        while ((n = buffer.indexOf("\n")) >= 0) {
-            consume(buffer.slice(0, n));
-            buffer = buffer.slice(n + 1);
-        }
-    });
-    child.stderr.on("data", (chunk) => { stderr += chunk.toString("utf8"); });
-    let exitCode;
+    const workspace = isolatedWorkspace(cwd);
     try {
-        exitCode = await new Promise((resolve, reject) => {
+        const attachmentDir = join(workspace, ".openmerit-attachments");
+        mkdirSync(attachmentDir, { recursive: true });
+        const attachmentPaths = new Map();
+        const fileArgs = files.map((file, i) => {
+            const safeName = `${String(i).padStart(3, "0")}-${file.name.replace(/[^A-Za-z0-9._-]/g, "_")}`;
+            const target = join(attachmentDir, safeName);
+            cpSync(file.path, target);
+            attachmentPaths.set(file.path, target);
+            return `@${target}`;
+        });
+        const candidateTask = [...attachmentPaths.entries()].reduce((text, [source, target]) => text.split(source).join(target), task);
+        const before = fileSnapshot(workspace);
+        const traceFile = paths.trialTrace(`${routeKey(route)}:${task}:${Date.now()}`);
+        const traceLines = [];
+        const child = spawn(piExecutable(), [
+            "--provider", route.provider, "--model", route.modelId, "--mode", "json",
+            "--offline", "--no-extensions", "--approve", ...piTrialToolArgs(),
+            "--print", ...imageArgs, ...fileArgs, candidateTask,
+        ], { cwd: workspace, stdio: ["ignore", "pipe", "pipe"] });
+        let buffer = "";
+        let stderr = "";
+        let answer = "";
+        let costUsd = 0;
+        let errors = 0;
+        let sessionId;
+        let toolCalls = 0;
+        let toolErrors = 0;
+        const started = Date.now();
+        function consume(line) {
+            if (!line.trim())
+                return;
+            traceLines.push(line);
+            let event;
+            try {
+                event = JSON.parse(line);
+            }
+            catch {
+                return;
+            }
+            if (event.type === "session" && event.id)
+                sessionId = event.id;
+            const content = event.message?.content;
+            if (event.message?.role === "assistant" && Array.isArray(content))
+                toolCalls += content.filter((part) => part?.type === "toolCall").length;
+            if (event.message?.role === "toolResult" && event.message.isError)
+                toolErrors++;
+            if (event.type !== "message_end" || event.message?.role !== "assistant")
+                return;
+            answer = answerText(event.message.content) || answer;
+            costUsd += event.message.usage?.cost?.total ?? 0;
+            if (event.message.stopReason === "error")
+                errors++;
+        }
+        child.stdout.on("data", (chunk) => {
+            buffer += chunk.toString("utf8");
+            let n;
+            while ((n = buffer.indexOf("\n")) >= 0) {
+                consume(buffer.slice(0, n));
+                buffer = buffer.slice(n + 1);
+            }
+        });
+        child.stderr.on("data", (chunk) => { stderr += chunk.toString("utf8"); });
+        const exitCode = await new Promise((resolve, reject) => {
             child.on("error", reject);
             child.on("close", (code) => resolve(code ?? 1));
         });
+        consume(buffer);
+        mkdirSync(dirname(traceFile), { recursive: true });
+        writeFileSync(traceFile, traceLines.join("\n") + (traceLines.length ? "\n" : ""));
+        const after = fileSnapshot(workspace);
+        let changedFiles = 0;
+        for (const [file, hash] of after)
+            if (before.get(file) !== hash)
+                changedFiles++;
+        for (const file of before.keys())
+            if (!after.has(file))
+                changedFiles++;
+        if (exitCode !== 0 || !answer.trim()) {
+            throw new Error(`pi trial ${route.meritId} via ${route.provider} failed: ${stderr.trim().slice(0, 180) || `exit ${exitCode}, empty answer`}`);
+        }
+        return { answer, costUsd, latencyMs: Date.now() - started, errors, sessionId,
+            toolCalls, toolErrors, changedFiles, traceFile };
     }
     finally {
         if (imageDir)
             rmSync(imageDir, { recursive: true, force: true });
+        rmSync(workspace, { recursive: true, force: true });
     }
-    consume(buffer);
-    if (exitCode !== 0 || !answer.trim()) {
-        throw new Error(`pi trial ${model} failed: ${stderr.trim().slice(0, 180) || `exit ${exitCode}, empty answer`}`);
-    }
-    return { answer, costUsd, latencyMs: Date.now() - started, errors, sessionId };
 }
 /** Quality uses the same OpenMerit judge; cost and latency come from pi itself. */
-export async function runPiTrial(key, judgeModel, task, rubric, model, entry, cwd = process.cwd(), images = []) {
+export async function runPiTrial(key, judgeModel, task, rubric, model, entry, cwd = process.cwd(), images = [], files = [], harness = defaultHarness, client = directChatClient(key)) {
     try {
-        const run = await executePiTask(model, task, cwd, images);
-        return await scorePiRun(key, judgeModel, task, rubric, model, entry, run, "pi_trial", images);
+        const run = await harness.runTask(entry?.route ?? model, { task, cwd, images, files });
+        return await scorePiRun(key, judgeModel, task, rubric, model, entry, run, "pi_trial", images, client);
     }
     catch (e) {
         const error = String(e);
         return {
-            point: { model, score: 0, price: entry?.price ?? 0,
-                ts: new Date().toISOString(), source: "pi_trial", why: error.slice(0, 160) },
+            point: { schemaVersion: 1, model, route: entry?.route, score: 0, price: entry?.price ?? 0,
+                priceKnown: entry?.priceKnown ?? !!entry,
+                ts: new Date().toISOString(), source: "pi_trial", toolCalls: 0, toolErrors: 1,
+                changedFiles: 0, why: error.slice(0, 160) },
             costUsd: 0, error,
         };
     }
 }
-export async function scorePiRun(key, judgeModel, task, rubric, model, entry, run, source, images = []) {
+export async function scorePiRun(key, judgeModel, task, rubric, model, entry, run, source, images = [], client = directChatClient(key)) {
     let score = 0;
     let why = "judge unavailable";
     try {
         const pinned = images.length ? await knownInvoiceScore(task, images, run.answer) : null;
         const j = pinned ?? (images.length
-            ? await judgeWithImages(key, judgeModel, task, rubric, run.answer, images)
-            : await judge(key, judgeModel, task, rubric, run.answer));
+            ? await judgeWithImages(key, judgeModel, task, rubric, run.answer, images, client)
+            : await judge(key, judgeModel, task, rubric, run.answer, client));
         score = Math.max(0, Math.min(1, j.score));
         why = j.why;
     }
@@ -214,9 +340,12 @@ export async function scorePiRun(key, judgeModel, task, rubric, model, entry, ru
     }
     return {
         point: {
-            model, score, price: entry?.price ?? 0,
+            schemaVersion: 1, model, route: entry?.route, score, price: entry?.price ?? 0,
+            priceKnown: entry?.priceKnown ?? !!entry,
             latencyMs: run.latencyMs, ts: new Date().toISOString(), source,
-            why: `${why}${run.errors ? `; pi errors: ${run.errors}` : ""}`,
+            why: `${why}${run.errors ? `; pi errors: ${run.errors}` : ""}; ` +
+                `tools=${run.toolCalls}, toolErrors=${run.toolErrors}, changedFiles=${run.changedFiles}`,
+            toolCalls: run.toolCalls, toolErrors: run.toolErrors, changedFiles: run.changedFiles,
         },
         costUsd: run.costUsd,
         sessionId: run.sessionId,

package/dist/policy.js

@@ -60,10 +60,11 @@ export function loadPolicy(path) {
         max_usd_per_m: num(raw.max_usd_per_m, d.max_usd_per_m),
     };
 }
-/** Is a model id allowed by the provider allow/deny lists? ("*" matches all; "vendor/*" matches a vendor.) */
-export function providerAllowed(policy, modelId) {
+/** Match either a logical vendor/model id or its concrete route provider. */
+export function providerAllowed(policy, modelId, routeProvider) {
     const vendor = modelId.split("/")[0];
-    const match = (pat) => pat === "*" || pat === modelId || (pat.endsWith("/*") && pat.slice(0, -2) === vendor);
+    const match = (pat) => pat === "*" || pat === modelId || pat === routeProvider ||
+        (pat.endsWith("/*") && (pat.slice(0, -2) === vendor || pat.slice(0, -2) === routeProvider));
     if (policy.providers.deny.some(match))
         return false;
     return policy.providers.allow.some(match);
@@ -87,6 +88,10 @@ export function gate(policy, input) {
         reasons.push("recommendation is not bound to a pi session");
         return { autoApply: false, reasons };
     }
+    if (input.priceKnown === false) {
+        reasons.push("candidate or current-model price is unknown");
+        return { autoApply: false, reasons };
+    }
     if (policy.mode !== "auto" || !policy.auto_apply.enabled) {
         reasons.push(`policy mode is "${policy.mode}" (auto_apply ${policy.auto_apply.enabled ? "enabled" : "disabled"})`);
         return { autoApply: false, reasons };

package/dist/providers.js

@@ -0,0 +1 @@
+export {};

package/dist/recommend.js

@@ -2,40 +2,49 @@
 import { createHash } from "node:crypto";
 import { paretoFrontier, pickBest, pickFallback } from "./frontier.js";
 import { gate, providerAllowed } from "./policy.js";
-export function buildRecommendation(taskKey, taskLabel, currentModel, points, policy, sessionFile = null) {
+import { pointKey, routeLabel } from "./routes.js";
+export function buildRecommendation(taskKey, taskLabel, currentModel, points, policy, sessionFile = null, currentRoute = currentModel ? points.find((p) => p.model === currentModel)?.route ?? null : null) {
     const scored = points.filter((p) => p.score > 0);
     if (scored.length === 0)
         return null;
     const best = pickBest(scored);
     if (!best)
         return null;
-    if (currentModel && best.model === currentModel)
-        return null; // already on the best model
-    const current = currentModel ? scored.find((p) => p.model === currentModel) : undefined;
+    if (currentModel && best.model === currentModel &&
+        (!currentRoute || pointKey(best) === `${currentRoute.provider}:${currentRoute.modelId}`))
+        return null;
+    const current = currentModel ? scored.find((p) => currentRoute
+        ? pointKey(p) === `${currentRoute.provider}:${currentRoute.modelId}`
+        : p.model === currentModel) : undefined;
     const scoreGain = current ? best.score - current.score : 0;
     // pickBest may choose a cheaper or faster model at equal quality.
     if (current && scoreGain < 0)
         return null;
-    const priceRatio = current && current.price > 0 ? best.price / current.price : 1;
-    const frontierModels = paretoFrontier(scored).map((p) => p.model);
-    const onFrontier = frontierModels.includes(best.model);
+    const priceKnown = best.priceKnown !== false && (!current || current.priceKnown !== false);
+    const priceRatio = priceKnown && current && current.price > 0 ? best.price / current.price : 1;
+    const frontierModels = paretoFrontier(scored).map(pointKey);
+    const onFrontier = frontierModels.includes(pointKey(best));
     const fb = pickFallback(scored, best, policy.fallback.min_score);
     const decision = gate(policy, {
         scoreGain,
         priceRatio,
+        priceKnown,
         onFrontier,
-        providerOk: providerAllowed(policy, best.model),
+        providerOk: providerAllowed(policy, best.model, best.route?.provider),
         baselineMeasured: !!current,
         sessionBound: !!sessionFile,
     });
-    const reason = `${best.model} scores ${best.score.toFixed(2)} vs ` +
+    const bestLabel = best.route ? routeLabel(best.route) : best.model;
+    const reason = `${bestLabel} scores ${best.score.toFixed(2)} vs ` +
         (current ? `${current.score.toFixed(2)} for ${current.model}` : "no baseline measured") +
-        ` (${scoreGain > 0 ? `quality gain ${scoreGain.toFixed(2)}` : "equal measured quality"}), at $${best.price.toFixed(2)}/M` +
-        (current ? ` (${priceRatio.toFixed(2)}x current price)` : "") +
+        ` (${scoreGain > 0 ? `quality gain ${scoreGain.toFixed(2)}` : "equal measured quality"}), ` +
+        (best.priceKnown === false ? "with unknown price" : `at $${best.price.toFixed(2)}/M`) +
+        (current && priceKnown ? ` (${priceRatio.toFixed(2)}x current price)` : "") +
         (best.why ? `. Judge: ${best.why}` : "");
     return {
+        schemaVersion: 1,
         id: createHash("sha1")
-            .update(`${taskKey}:${best.model}:${Date.now()}`)
+            .update(`${taskKey}:${pointKey(best)}:${Date.now()}`)
             .digest("hex")
             .slice(0, 10),
         ts: new Date().toISOString(),
@@ -43,17 +52,21 @@ export function buildRecommendation(taskKey, taskLabel, currentModel, points, po
         taskLabel,
         sessionFile,
         currentModel,
-        recommended: { model: best.model, reason },
+        currentRoute,
+        recommended: { model: best.model, route: best.route, reason },
         fallback: fb
             ? {
                 model: fb.model,
-                reason: `score ${fb.score.toFixed(2)} at $${fb.price.toFixed(2)}/M` +
+                route: fb.route,
+                reason: `score ${fb.score.toFixed(2)} ` +
+                    (fb.priceKnown === false ? "with unknown price" : `at $${fb.price.toFixed(2)}/M`) +
                     (fb.latencyMs ? `, ~${Math.round(fb.latencyMs)}ms` : ""),
             }
             : null,
         evidence: {
             scoreGain: Math.round(scoreGain * 1000) / 1000,
             priceRatio: Math.round(priceRatio * 100) / 100,
+            priceKnown,
             onFrontier,
             trials: scored.length,
         },

package/dist/routes.js

@@ -0,0 +1,59 @@
+/** Provider-neutral model-route normalization. */
+export function meritModelId(provider, modelId) {
+    if (provider === "openrouter")
+        return modelId;
+    return modelId.startsWith(`${provider}/`) ? modelId : `${provider}/${modelId}`;
+}
+export function modelRoute(provider, modelId, details = {}) {
+    return {
+        provider,
+        modelId,
+        meritId: meritModelId(provider, modelId),
+        input: details.input ?? ["text"],
+        ...details,
+    };
+}
+export function legacyOpenRouterRoute(meritId) {
+    return modelRoute("openrouter", meritId);
+}
+export function routeKey(route) {
+    return `${route.provider}:${route.modelId}`;
+}
+export function pointKey(point) {
+    return point.route ? routeKey(point.route) : `openrouter:${point.model}`;
+}
+export function routeLabel(route) {
+    return route.provider === "openrouter" ? route.meritId : `${route.meritId} via ${route.provider}`;
+}
+export function routePrice(route) {
+    if (!route.cost || !Number.isFinite(route.cost.input) || !Number.isFinite(route.cost.output))
+        return { price: 0, known: false };
+    return { price: Math.round(((route.cost.input + route.cost.output) / 2 + Number.EPSILON) * 1e4) / 1e4,
+        known: true };
+}
+export function catalogEntryFromRoute(route) {
+    const priced = routePrice(route);
+    return {
+        id: route.meritId,
+        name: routeLabel(route),
+        ctx: route.contextWindow ?? 0,
+        pp: route.cost ? route.cost.input / 1e6 : 0,
+        pc: route.cost ? route.cost.output / 1e6 : 0,
+        price: priced.price,
+        priceKnown: priced.known,
+        inputModalities: route.input,
+        route,
+    };
+}
+export function routeCatalog(routes) {
+    const out = new Map();
+    for (const route of routes)
+        out.set(routeKey(route), catalogEntryFromRoute(route));
+    return out;
+}
+/** Preserve the route while enriching an OpenRouter entry with its live catalog metadata. */
+export function enrichRouteEntry(entry, live) {
+    if (!live)
+        return entry;
+    return { ...live, route: entry.route, id: entry.id, name: entry.name, priceKnown: true };
+}

package/dist/store.js

@@ -18,10 +18,12 @@ export const paths = {
     benchmarksDigest: () => join(stateDir(), "benchmarks", "digest.json"),
     traceCursor: () => join(stateDir(), "traces", "cursor.json"),
     observations: () => join(stateDir(), "traces", "observations.jsonl"),
+    events: () => join(stateDir(), "events.jsonl"),
     harnessState: () => join(stateDir(), "harness-state.json"),
     ledger: () => join(stateDir(), "ledger.json"),
     watchProcessed: () => join(stateDir(), "watch", "processed.json"),
     sessionJob: (marker) => join(stateDir(), "watch", "jobs", sha1(marker) + ".json"),
+    trialTrace: (id) => join(stateDir(), "traces", "trials", `${sha1(id)}.jsonl`),
     envFile: () => join(stateDir(), ".env"),
 };
 export function sha1(text) {

package/dist/strategist.js

@@ -1,12 +1,13 @@
 /** Strategist: pick the next untried model to trial, benchmark-aware. */
-import { chat } from "./llm.js";
+import { directChatClient } from "./llm.js";
 import { parseObj } from "./judge.js";
+import { routeKey, routeLabel } from "./routes.js";
 export const STRAT_PREFS = [
     "anthropic/claude-3.7-sonnet",
     "anthropic/claude-3.5-sonnet",
     "openai/gpt-4o",
 ];
-const STRAT_PROMPT = `We are finding the best OpenRouter model for a task via iterative trials.
+const STRAT_PROMPT = `We are finding the best configured model route for a task via iterative trials.
 TASK: {task}
 RUBRIC: {rubric}
 RELEVANT PUBLIC BENCHMARKS FOR THIS TASK: {benchmarks}
@@ -14,15 +15,16 @@ RESULTS SO FAR (model=score): {results}
 Pick ONE untried model from the catalog below that is likely to do well on this
 task, based on the benchmarks relevant to it. Balance quality against price so
 a pareto frontier emerges.
-Return ONLY json: {{"next_model": "<catalog id>", "why": "<one line>"}}
-CATALOG (id | blended $/1M tokens | ctx):
+Return ONLY json: {{"next_model": "<route id>", "why": "<one line>"}}
+ROUTES (route id | logical model | blended $/1M tokens | ctx):
 {catalog}`;
 /** Pick the next model to trial, or null when the catalog is exhausted. */
-export async function pickNext(key, stratModel, task, rubric, benchmarks, results, cat, tried, maxPrice, failedVendors, extraCandidates) {
-    const ok = (c) => c.price <= maxPrice &&
-        !tried.has(c.id) &&
+export async function pickNext(key, stratModel, task, rubric, benchmarks, results, cat, tried, maxPrice, failedVendors, extraCandidates, client = directChatClient(key)) {
+    const entryKey = (c) => c.route ? routeKey(c.route) : c.id;
+    const ok = (c) => (c.priceKnown === false || c.price <= maxPrice) &&
+        !tried.has(entryKey(c)) &&
         c.ctx >= 4096 &&
-        !failedVendors.has(c.id.split("/")[0]);
+        !failedVendors.has(c.route?.provider ?? c.id.split("/")[0]);
     // Benchmark-shortlisted models get priority in the listing.
     const pool = [...cat.values()].filter(ok);
     if (pool.length === 0)
@@ -31,15 +33,16 @@ export async function pickNext(key, stratModel, task, rubric, benchmarks, result
     pool.sort((a, b) => {
         const pa = priority.has(a.id) ? 0 : 1;
         const pb = priority.has(b.id) ? 0 : 1;
-        return pa - pb || a.price - b.price;
+        return pa - pb || (a.priceKnown === false ? 1 : 0) - (b.priceKnown === false ? 1 : 0) || a.price - b.price;
     });
     const lines = pool
-        .map((c) => `${c.id} | ${c.price.toFixed(2)} | ${c.ctx}${priority.has(c.id) ? " | BENCHMARK" : ""}`)
+        .map((c) => `${entryKey(c)} | ${c.route ? routeLabel(c.route) : c.id} | ` +
+        `${c.priceKnown === false ? "unknown" : c.price.toFixed(2)} | ${c.ctx}${priority.has(c.id) ? " | BENCHMARK" : ""}`)
         .join("\n");
     const res = results
         .map((r) => `${r.model}=${r.score.toFixed(2)}`)
         .join("  ") || "none yet";
-    const { content } = await chat(key, stratModel, STRAT_PROMPT.replace("{task}", task)
+    const { content } = await client.chat(stratModel, STRAT_PROMPT.replace("{task}", task)
         .replace("{rubric}", rubric)
         .replace("{benchmarks}", benchmarks.join(", "))
         .replace("{results}", res)
@@ -55,10 +58,11 @@ export async function pickNext(key, stratModel, task, rubric, benchmarks, result
         /* fall through to cheap pick */
     }
     if (pick) {
-        const entry = cat.get(pick);
+        const entry = cat.get(pick) ?? [...cat.values()].find((candidate) => candidate.id === pick);
         if (entry && ok(entry))
-            return { model: pick, why };
+            return { model: entry.id, route: entry.route, why };
     }
     const cheapest = pool[0];
-    return { model: cheapest.id, why: `fallback pick: cheapest untried (strategist pick ${pick ?? "none"} unusable)` };
+    return { model: cheapest.id, route: cheapest.route,
+        why: `fallback pick: cheapest untried (strategist pick ${pick ?? "none"} unusable)` };
 }