openmerit 0.1.0 → 0.1.2

package/dist/task-input.js

@@ -1,6 +1,29 @@
 /** A pi text task may carry images embedded in its saved session message. */
 import { createHash } from "node:crypto";
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { basename } from "node:path";
 import { taskKey } from "./store.js";
+/** Recover files Pi expanded into a task message (for example @invoice.pdf). */
+export function sessionFiles(text) {
+    const files = [];
+    const seen = new Set();
+    const re = /<file\s+name=["']([^"']+)["'][^>]*>/g;
+    for (const match of text.matchAll(re)) {
+        const file = match[1];
+        if (!file || seen.has(file) || !existsSync(file))
+            continue;
+        try {
+            const stat = statSync(file);
+            if (!stat.isFile() || stat.size > 100_000_000)
+                continue;
+            files.push({ path: file, name: basename(file), size: stat.size,
+                sha256: createHash("sha256").update(readFileSync(file)).digest("hex") });
+            seen.add(file);
+        }
+        catch { /* inaccessible attachment */ }
+    }
+    return files;
+}
 const SUPPORTED = new Set(["image/jpeg", "image/png", "image/webp", "image/gif"]);
 export function sessionImages(content) {
     if (!Array.isArray(content))
@@ -20,11 +43,12 @@ export function sessionImages(content) {
     return images;
 }
 /** Text-only keys remain compatible; image bytes distinguish same-prompt documents. */
-export function taskInputKey(text, images) {
-    if (images.length === 0)
+export function taskInputKey(text, images, files = []) {
+    if (images.length === 0 && files.length === 0)
         return taskKey(text);
     const normalized = text.toLowerCase().replace(/\s+/g, " ").trim();
     const hashes = images.map((image) => `${image.mimeType}:` +
         createHash("sha256").update(Buffer.from(image.data, "base64")).digest("hex"));
-    return taskKey(`${normalized}\nimages:${hashes.join(",")}`);
+    const fileHashes = files.map((file) => `${file.name}:${file.sha256}`).join(",");
+    return taskKey(`${normalized}\nimages:${hashes.join(",")}\nfiles:${fileHashes}`);
 }

package/dist/traces.js

@@ -7,6 +7,7 @@ import { readdirSync, readFileSync, existsSync, statSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
 import { paths, readJson, taskKey, writeJson } from "./store.js";
+import { meritModelId, modelRoute } from "./routes.js";
 export function defaultSessionsDir() {
     return join(homedir(), ".pi", "agent", "sessions");
 }
@@ -53,6 +54,7 @@ export function observationsFromEntries(entries, sessionFile) {
                 continue;
             }
             current = {
+                schemaVersion: 1,
                 taskKey: taskKey(text),
                 taskLabel: text.replace(/\s+/g, " ").trim().slice(0, 120),
                 model: null,
@@ -68,8 +70,10 @@ export function observationsFromEntries(entries, sessionFile) {
         if (!current)
             continue;
         if (m.role === "assistant") {
-            if (!current.model && m.provider && m.model)
-                current.model = `${m.provider}/${m.model}`;
+            if (!current.model && m.provider && m.model) {
+                current.model = meritModelId(m.provider, m.model);
+                current.route = modelRoute(m.provider, m.model);
+            }
             current.costUsd += m.usage?.cost?.total ?? 0;
             current.tokens += m.usage?.totalTokens ?? 0;
             if (m.stopReason === "error")

package/dist/trials.js

@@ -1,5 +1,5 @@
 /** Shadow trials: run candidate models against observed/declared tasks on the background track. */
-import { chat } from "./llm.js";
+import { directChatClient } from "./llm.js";
 import { judge, parseObj } from "./judge.js";
 import { paths, readJson, writeJson } from "./store.js";
 function today() {
@@ -15,6 +15,21 @@ export function budgetOk(policy) {
         return { ok: false, reason: `daily budget exhausted ($${policy.budgets.max_usd_per_day})` };
     return { ok: true };
 }
+/** Conservative admission estimate for one candidate answer (rough input tokens + 4K output). */
+export function trialBudgetOk(policy, entry, inputChars) {
+    if (entry.priceKnown === false)
+        return { ok: false, reason: "route price is unknown" };
+    const inputTokens = Math.ceil(inputChars / 4);
+    const outputTokens = Math.min(entry.route?.maxTokens ?? 4096, 4096);
+    const estimatedUsd = entry.pp * inputTokens + entry.pc * outputTokens;
+    if (estimatedUsd > policy.budgets.max_usd_per_trial)
+        return {
+            ok: false,
+            estimatedUsd,
+            reason: `estimated trial cost $${estimatedUsd.toFixed(4)} exceeds $${policy.budgets.max_usd_per_trial.toFixed(4)} limit`,
+        };
+    return { ok: true, estimatedUsd };
+}
 export function recordTrialSpend(usd) {
     let ledger = readJson(paths.ledger(), { date: today(), trials: 0, usd: 0 });
     if (ledger.date !== today())
@@ -23,16 +38,26 @@ export function recordTrialSpend(usd) {
     ledger.usd = Math.round((ledger.usd + usd) * 1e6) / 1e6;
     writeJson(paths.ledger(), ledger);
 }
+/** Record non-candidate merit-loop spend without consuming a trial slot. */
+export function recordMeritSpend(usd) {
+    if (!(usd > 0))
+        return;
+    let ledger = readJson(paths.ledger(), { date: today(), trials: 0, usd: 0 });
+    if (ledger.date !== today())
+        ledger = { date: today(), trials: 0, usd: 0 };
+    ledger.usd = Math.round((ledger.usd + usd) * 1e6) / 1e6;
+    writeJson(paths.ledger(), ledger);
+}
 const RUBRIC_PROMPT = `Write a compact grading rubric for answers to the task below.
 It must list the concrete criteria for a score of 1.0 and how to deduct.
 TASK: {task}
 Return ONLY json: {{"rubric": "<text>"}}`;
 /** Derive (and cache) a grading rubric for a task that came from traces. */
-export async function ensureRubric(key, judgeModel, task, cache) {
+export async function ensureRubric(key, judgeModel, task, cache, client = directChatClient(key)) {
     const hit = cache.get(task);
     if (hit)
         return hit;
-    const { content } = await chat(key, judgeModel, RUBRIC_PROMPT.replace("{task}", task.slice(0, 8000)), 1024, 0);
+    const { content } = await client.chat(judgeModel, RUBRIC_PROMPT.replace("{task}", task.slice(0, 8000)), 1024, 0);
     let rubric = "Score 1.0 for a fully correct, complete, usable answer; deduct for errors, omissions, or unusable output.";
     try {
         const obj = parseObj(content);
@@ -46,19 +71,20 @@ export async function ensureRubric(key, judgeModel, task, cache) {
     return rubric;
 }
 /** Run one shadow trial: generate with `model`, judge the output, return a frontier point + cost. */
-export async function runTrial(key, judgeModel, task, rubric, model, entry) {
+export async function runTrial(key, judgeModel, task, rubric, model, entry, client = directChatClient(key)) {
     const started = Date.now();
     let gen;
     let usage = {};
     try {
-        const r = await chat(key, model, task, 4096, 0.2);
+        const r = await client.chat(entry?.route ?? model, task, 4096, 0.2);
         gen = r.content;
         usage = r.usage;
     }
     catch (e) {
         return {
             point: {
-                model, score: 0, price: entry?.price ?? 0,
+                schemaVersion: 1, model, route: entry?.route, score: 0, price: entry?.price ?? 0,
+                priceKnown: entry?.priceKnown ?? !!entry,
                 ts: new Date().toISOString(), source: "shadow_trial", why: String(e).slice(0, 160),
             },
             costUsd: 0,
@@ -70,7 +96,8 @@ export async function runTrial(key, judgeModel, task, rubric, model, entry) {
     if (!gen.trim()) {
         return {
             point: {
-                model, score: 0, price: entry?.price ?? 0, latencyMs,
+                schemaVersion: 1, model, route: entry?.route, score: 0, price: entry?.price ?? 0,
+                priceKnown: entry?.priceKnown ?? !!entry, latencyMs,
                 ts: new Date().toISOString(), source: "shadow_trial", why: "empty response",
             },
             costUsd,
@@ -79,7 +106,7 @@ export async function runTrial(key, judgeModel, task, rubric, model, entry) {
     let score = 0;
     let why = "judge error";
     try {
-        const j = await judge(key, judgeModel, task, rubric, gen);
+        const j = await judge(key, judgeModel, task, rubric, gen, client);
         score = j.score;
         why = j.why;
     }
@@ -88,9 +115,12 @@ export async function runTrial(key, judgeModel, task, rubric, model, entry) {
     }
     return {
         point: {
+            schemaVersion: 1,
             model,
+            route: entry?.route,
             score,
             price: entry?.price ?? 0,
+            priceKnown: entry?.priceKnown ?? !!entry,
             latencyMs,
             ts: new Date().toISOString(),
             source: "shadow_trial",

package/extension/openmerit.ts

@@ -19,7 +19,7 @@
 import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
 import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
-import { join } from "node:path";
+import { basename, join } from "node:path";
 import { createHash } from "node:crypto";
 import { spawn, type ChildProcessByStdio } from "node:child_process";
 import type { Readable } from "node:stream";
@@ -46,6 +46,9 @@ interface TrialProgress {
   latencyMs?: number;
   costUsd?: number;
   error?: string;
+  toolCalls?: number;
+  toolErrors?: number;
+  changedFiles?: number;
 }
 
 function parseTrialProgress(line: string): TrialProgress | null {
@@ -63,7 +66,9 @@ function parseTrialProgress(line: string): TrialProgress | null {
 function completedTrialText(trial: TrialProgress): string {
   return `${trial.model}: quality ${(trial.score ?? 0).toFixed(2)}, ` +
     `$${(trial.price ?? 0).toFixed(2)}/M, ${Math.round(trial.latencyMs ?? 0)}ms, ` +
-    `run $${(trial.costUsd ?? 0).toFixed(4)}` + (trial.error ? ` (failed: ${trial.error})` : "");
+    `run $${(trial.costUsd ?? 0).toFixed(4)}, tools ${trial.toolCalls ?? 0}` +
+    (trial.toolErrors || trial.changedFiles ? `, tool errors ${trial.toolErrors ?? 0}, changed files ${trial.changedFiles ?? 0}` : "") +
+    (trial.error ? ` (failed: ${trial.error})` : "");
 }
 
 interface ExtensionPolicy {
@@ -110,19 +115,31 @@ function loadPolicy(): ExtensionPolicy {
 }
 
 interface Recommendation {
+  schemaVersion?: 1;
   id: string;
   ts: string;
   taskKey: string;
   taskLabel: string;
   sessionFile?: string | null;
   currentModel: string | null;
-  recommended: { model: string; reason: string };
-  fallback: { model: string; reason: string } | null;
-  evidence: { scoreGain: number; priceRatio: number; onFrontier: boolean; trials: number };
+  currentRoute?: ModelRoute | null;
+  recommended: { model: string; route?: ModelRoute; reason: string };
+  fallback: { model: string; route?: ModelRoute; reason: string } | null;
+  evidence: { scoreGain: number; priceRatio: number; priceKnown?: boolean; onFrontier: boolean; trials: number };
   policy: { autoApply: boolean; reasons: string[] };
   status: "pending" | "applied" | "dismissed" | "expired";
 }
 
+interface ModelRoute {
+  provider: string;
+  modelId: string;
+  meritId: string;
+  input: ("text" | "image")[];
+  cost?: { input: number; output: number; cacheRead?: number; cacheWrite?: number };
+  contextWindow?: number;
+  maxTokens?: number;
+}
+
 interface RecordedTrial {
   taskKey?: string;
   sessionFile?: string;
@@ -171,10 +188,18 @@ function userTaskKey(content: unknown): string | null {
   const images = parts.filter((b) => b?.type === "image");
   if (parts.length !== parts.filter((b) => b?.type === "text" || b?.type === "image").length ||
       images.some((b) => typeof b.data !== "string" || typeof b.mimeType !== "string")) return null;
-  if (images.length === 0) return taskKey(text);
+  const files: string[] = [];
+  const fileRe = /<file\s+name=["']([^"']+)["'][^>]*>/g;
+  for (const match of text.matchAll(fileRe)) {
+    const file = match[1];
+    if (!file || !existsSync(file) || files.includes(file)) continue;
+    try { files.push(`${basename(file)}:` + createHash("sha256").update(readFileSync(file)).digest("hex")); }
+    catch { /* inaccessible attachment */ }
+  }
+  if (images.length === 0 && files.length === 0) return taskKey(text);
   const hashes = images.map((b) => `${b.mimeType}:` +
     createHash("sha256").update(Buffer.from(b.data, "base64")).digest("hex"));
-  return taskKey(`${text.toLowerCase().replace(/\s+/g, " ").trim()}\nimages:${hashes.join(",")}`);
+  return taskKey(`${text.toLowerCase().replace(/\s+/g, " ").trim()}\nimages:${hashes.join(",")}\nfiles:${files.join(",")}`);
 }
 
 function latestSessionTaskKey(ctx: ExtensionContext): string | null {
@@ -235,7 +260,11 @@ function suggestionText(rec: Recommendation, trials: RecordedTrial[]): string {
 }
 
 function canonicalModel(provider: string, id: string): string {
-  return provider === "openrouter" ? id : `${provider}/${id}`;
+  return provider === "openrouter" || id.startsWith(`${provider}/`) ? id : `${provider}/${id}`;
+}
+
+function routeDisplay(route: ModelRoute | undefined, fallback: string): string {
+  return route && route.provider !== "openrouter" ? `${route.meritId} via ${route.provider}` : fallback;
 }
 
 function setStatus(rec: Recommendation, status: Recommendation["status"]): void {
@@ -243,11 +272,44 @@ function setStatus(rec: Recommendation, status: Recommendation["status"]): void
 }
 
 let fallbackModel: string | null = null;
+let fallbackRoute: ModelRoute | null = null;
+
+function routeFor(model: { provider: string; id: string; input?: ("text" | "image")[];
+  cost?: { input?: number; output?: number; cacheRead?: number; cacheWrite?: number };
+  contextWindow?: number; maxTokens?: number }): ModelRoute {
+  return {
+    provider: model.provider,
+    modelId: model.id,
+    meritId: canonicalModel(model.provider, model.id),
+    input: model.input ?? ["text"],
+    cost: model.cost && typeof model.cost.input === "number" && typeof model.cost.output === "number"
+      ? { input: model.cost.input, output: model.cost.output,
+          cacheRead: model.cost.cacheRead, cacheWrite: model.cost.cacheWrite }
+      : undefined,
+    contextWindow: model.contextWindow,
+    maxTokens: model.maxTokens,
+  };
+}
+
+function eligibleRoutes(ctx: ExtensionContext): ModelRoute[] {
+  const registry = ctx.modelRegistry as unknown as { getAvailable?: () => Array<Parameters<typeof routeFor>[0]> };
+  const scoped = (ctx.scopedModels ?? []) as readonly { model: Parameters<typeof routeFor>[0] }[];
+  const models = scoped.length ? scoped.map((item) => item.model) : registry.getAvailable?.() ?? [];
+  if (ctx.model && !models.some((model) => model.provider === ctx.model!.provider && model.id === ctx.model!.id))
+    models.unshift(ctx.model);
+  const byRoute = new Map<string, ModelRoute>();
+  for (const model of models) {
+    const route = routeFor(model);
+    byRoute.set(`${route.provider}:${route.modelId}`, route);
+  }
+  return [...byRoute.values()];
+}
 
 function reportHarnessState(ctx: ExtensionContext, settled = false): void {
   try {
     mkdirSync(HOME, { recursive: true });
     const model = ctx.model ? canonicalModel(ctx.model.provider, ctx.model.id) : null;
+    const currentRoute = ctx.model ? routeFor(ctx.model) : null;
     if (settled) settledTaskKey = latestSessionTaskKey(ctx);
     const prior = existsSync(STATE_FILE)
       ? JSON.parse(readFileSync(STATE_FILE, "utf8")) as { sessionFile?: string | null; settledAt?: string }
@@ -258,8 +320,12 @@ function reportHarnessState(ctx: ExtensionContext, settled = false): void {
     writeFileSync(
       STATE_FILE,
       JSON.stringify({
+        schemaVersion: 1,
         currentModel: model,
+        currentRoute,
+        routes: eligibleRoutes(ctx),
         fallbackModel,
+        fallbackRoute,
         sessionFile,
         settledTaskKey,
         settledAt,
@@ -275,8 +341,11 @@ function reportHarnessState(ctx: ExtensionContext, settled = false): void {
 /** Restore the persisted fallback so a fresh session keeps it even with no pending recs. */
 function restoreFallback(): void {
   try {
-    const st = JSON.parse(readFileSync(STATE_FILE, "utf8")) as { fallbackModel?: string | null };
+    const st = JSON.parse(readFileSync(STATE_FILE, "utf8")) as {
+      fallbackModel?: string | null; fallbackRoute?: ModelRoute | null;
+    };
     if (st.fallbackModel) fallbackModel = st.fallbackModel;
+    if (st.fallbackRoute) fallbackRoute = st.fallbackRoute;
   } catch {
     /* no state yet */
   }
@@ -331,7 +400,15 @@ function catalogEntry(openrouterId: string): CatalogEntryLite | undefined {
  * OpenRouter catalog, so inject anything missing via registerProvider
  * (takes effect immediately, no /reload needed).
  */
-function ensureModel(pi: ExtensionAPI, ctx: ExtensionContext, openrouterId: string) {
+function ensureModel(pi: ExtensionAPI, ctx: ExtensionContext, openrouterId: string, route?: ModelRoute) {
+  const registry = ctx.modelRegistry as unknown as {
+    find?: (provider: string, id: string) => unknown | undefined;
+  };
+  if (route) {
+    const exact = registry.find?.(route.provider, route.modelId);
+    if (exact) return exact;
+    if (route.provider !== "openrouter") return undefined;
+  }
   const existing = resolveModel(ctx, openrouterId);
   if (existing) return existing;
 
@@ -359,16 +436,14 @@ function ensureModel(pi: ExtensionAPI, ctx: ExtensionContext, openrouterId: stri
     })),
   });
 
-  const reg = ctx.modelRegistry as unknown as {
-    find?: (provider: string, id: string) => unknown | undefined;
-  };
-  return reg.find?.(INJECT_PROVIDER, openrouterId);
+  return registry.find?.(INJECT_PROVIDER, openrouterId);
 }
 
 export default function openmerit(pi: ExtensionAPI) {
   let pollTimer: ReturnType<typeof setInterval> | null = null;
   let trialJob: ChildProcessByStdio<null, Readable, Readable> | null = null;
-  const queuedJobs: { sessionFile: string; settledAt: string; model: string; key: string; cwd: string; bytes: number }[] = [];
+  const queuedJobs: { sessionFile: string; settledAt: string; model: string; route: ModelRoute;
+    key: string; cwd: string; bytes: number }[] = [];
   const startedJobs = new Set<string>();
   let activeSession: string | null = null;
   let progress: { sessionFile: string; taskKey: string; current: TrialProgress | null;
@@ -395,7 +470,7 @@ export default function openmerit(pi: ExtensionAPI) {
       return;
     }
     const child = spawn(process.execPath, [ENGINE_FILE, "session-trial", job.sessionFile,
-      job.settledAt, job.model, job.key, job.cwd, String(job.bytes)], {
+      job.settledAt, job.model, job.key, job.cwd, String(job.bytes), job.route.provider, job.route.modelId], {
       cwd: job.cwd, env: process.env, stdio: ["ignore", "pipe", "pipe"],
       detached: process.platform !== "win32",
     });
@@ -453,7 +528,8 @@ export default function openmerit(pi: ExtensionAPI) {
   function queueSettledTask(ctx: ExtensionContext): void {
     const sessionFile = ctx.sessionManager.getSessionFile();
     const model = ctx.model ? canonicalModel(ctx.model.provider, ctx.model.id) : null;
-    if (!sessionFile || !model || !settledTaskKey || !existsSync(sessionFile)) return;
+    const route = ctx.model ? routeFor(ctx.model) : null;
+    if (!sessionFile || !model || !route || !settledTaskKey || !existsSync(sessionFile)) return;
     // The alpha compares completed response tasks. The engine validates images,
     // tool use and the baseline answer before any provider calls.
     let completed = false;
@@ -477,7 +553,7 @@ export default function openmerit(pi: ExtensionAPI) {
       if (ctx.hasUI) ctx.ui.notify(`openmerit: comparison skipped: ${budget.reason}. ${budget.summary}`, "warning");
       return;
     }
-    queuedJobs.push({ sessionFile, settledAt, model, key: settledTaskKey, cwd: ctx.cwd, bytes });
+    queuedJobs.push({ sessionFile, settledAt, model, route, key: settledTaskKey, cwd: ctx.cwd, bytes });
     startNextJob(ctx);
   }
 
@@ -489,11 +565,18 @@ export default function openmerit(pi: ExtensionAPI) {
 
   async function applyRecommendation(ctx: ExtensionContext, rec: Recommendation): Promise<boolean> {
     const current = ctx.model ? canonicalModel(ctx.model.provider, ctx.model.id) : null;
-    if (rec.currentModel && current !== rec.currentModel) {
+    const currentRoute = ctx.model ? routeFor(ctx.model) : null;
+    if (rec.currentRoute && currentRoute &&
+        (rec.currentRoute.provider !== currentRoute.provider || rec.currentRoute.modelId !== currentRoute.modelId)) {
+      ctx.ui.notify(`openmerit: recommendation targets ${rec.currentRoute.provider}/${rec.currentRoute.modelId}; ` +
+        `current route is ${currentRoute.provider}/${currentRoute.modelId}`, "warning");
+      return false;
+    }
+    if (!rec.currentRoute && rec.currentModel && current !== rec.currentModel) {
       ctx.ui.notify(`openmerit: recommendation was measured against ${rec.currentModel}; current model is ${current ?? "unknown"}`, "warning");
       return false;
     }
-    const model = ensureModel(pi, ctx, rec.recommended.model);
+    const model = ensureModel(pi, ctx, rec.recommended.model, rec.recommended.route);
     if (!model) {
       ctx.ui.notify(
         `openmerit: ${rec.recommended.model} not found in pi's model registry (add the provider/model first)`,
@@ -504,8 +587,12 @@ export default function openmerit(pi: ExtensionAPI) {
     const ok = await pi.setModel(model as Parameters<typeof pi.setModel>[0]);
     if (ok) {
       setStatus(rec, "applied");
-      if (rec.fallback) fallbackModel = rec.fallback.model;
-      ctx.ui.notify(`openmerit: switched to ${rec.recommended.model}\n${rec.recommended.reason}`, "info");
+      if (rec.fallback) {
+        fallbackModel = rec.fallback.model;
+        fallbackRoute = rec.fallback.route ?? null;
+      }
+      ctx.ui.notify(`openmerit: switched to ${routeDisplay(rec.recommended.route, rec.recommended.model)}\n` +
+        rec.recommended.reason, "info");
       reportHarnessState(ctx);
     } else {
       ctx.ui.notify(`openmerit: no auth for ${rec.recommended.model}`, "error");
@@ -537,7 +624,7 @@ export default function openmerit(pi: ExtensionAPI) {
       // startup. Notify (fire-and-forget) and let the user act via /openmerit.
       if (ctx.hasUI) {
         ctx.ui.notify(
-          `openmerit recommends: ${rec.recommended.model}\n${rec.recommended.reason}\n` +
+          `openmerit recommends: ${routeDisplay(rec.recommended.route, rec.recommended.model)}\n${rec.recommended.reason}\n` +
             `gate: ${rec.policy.reasons.join("; ")}\n(/openmerit apply to switch, /openmerit dismiss to ignore)`,
           "info",
         );
@@ -608,7 +695,7 @@ export default function openmerit(pi: ExtensionAPI) {
     const policy = loadPolicy();
     if (policy.fallback?.apply_on_error === false) return;
     const current = ctx.model ? canonicalModel(ctx.model.provider, ctx.model.id) : "current model";
-    const model = ensureModel(pi, ctx, fallbackModel);
+    const model = ensureModel(pi, ctx, fallbackModel, fallbackRoute ?? undefined);
     if (!model) return;
 
     const auto = policy.mode === "auto";
@@ -660,7 +747,7 @@ export default function openmerit(pi: ExtensionAPI) {
           ctx.ui.notify("openmerit: no fallback set yet", "info");
           return;
         }
-        const model = ensureModel(pi, ctx, fallbackModel);
+        const model = ensureModel(pi, ctx, fallbackModel, fallbackRoute ?? undefined);
         if (model && (await pi.setModel(model as Parameters<typeof pi.setModel>[0]))) {
           ctx.ui.notify(`openmerit: switched to fallback ${fallbackModel}`, "info");
           reportHarnessState(ctx);
@@ -670,7 +757,8 @@ export default function openmerit(pi: ExtensionAPI) {
         return;
       }
 
-      const current = ctx.model ? canonicalModel(ctx.model.provider, ctx.model.id) : "unknown";
+      const currentRoute = ctx.model ? routeFor(ctx.model) : undefined;
+      const current = currentRoute ? routeDisplay(currentRoute, currentRoute.meritId) : "unknown";
       const budget = trialBudget();
       const liveProgress = progress?.sessionFile === ctx.sessionManager.getSessionFile() ? progress : null;
       const comparison = trialJob ? liveProgress?.current
@@ -693,7 +781,8 @@ export default function openmerit(pi: ExtensionAPI) {
       for (const r of pending.slice(-5)) {
         lines.push(
           "",
-          `-> ${r.recommended.model} (gain ${r.evidence.scoreGain}, ${r.evidence.priceRatio}x price, frontier=${r.evidence.onFrontier})`,
+          `-> ${routeDisplay(r.recommended.route, r.recommended.model)} (gain ${r.evidence.scoreGain}, ` +
+            `${r.evidence.priceKnown === false ? "price unknown" : `${r.evidence.priceRatio}x price`}, frontier=${r.evidence.onFrontier})`,
           `   ${r.recommended.reason}`,
           `   gate: ${r.policy.reasons.join("; ") || "n/a"}`,
         );

package/instructions/OPENMERIT.md

@@ -8,8 +8,9 @@ the best model for the task at hand, at the best price, with a vetted fallback.
 
 1. **Observes** this harness's session traces (models used, tokens, cost,
    latency, errors) without intercepting or slowing down your work.
-2. **Evaluates** candidate models for each completed text or image task sequentially
-   through pi and scores their outputs with a judge model.
+2. **Evaluates** candidate models for each completed task sequentially through
+   pi and scores their outputs with a judge model. Candidate tools are disabled
+   by default.
 3. **Maintains a pareto frontier** per task (quality vs. cost vs. latency) and
    an aggregate frontier across all of your tasks.
 4. **Watches for new model releases** (provider catalogs + public benchmarks)
@@ -43,9 +44,11 @@ the best model for the task at hand, at the best price, with a vetted fallback.
 
 ## Guarantees
 
-- Per-task comparisons send the task text, uploaded images (when present), and
-  candidate answers through pi/OpenRouter for model runs and judging. Use
-  non-sensitive examples while evaluating this alpha.
+- Per-task comparisons send the task text, uploaded files or images (when
+  present), and candidate answers through the model routes configured in Pi
+  for model runs and judging. Candidate Pi runs default to no tools; any tool access is an
+  explicit user opt-in. Use non-sensitive examples while
+  evaluating this alpha.
 - The shipped policy is supervised. Swaps happen automatically only after the
   user opts in and the configured quality/cost guardrails pass; otherwise they
   remain recommendations for a human to approve.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openmerit",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Find better models for each pi task by comparing quality, cost, and latency.",
   "type": "module",
   "keywords": [
@@ -17,10 +17,13 @@
     "extension/openmerit.ts",
     "instructions/",
     "examples/",
+    "rules.md",
     "benchmark/invoice_ocr/data/"
   ],
   "pi": {
-    "extensions": ["./extension/openmerit.ts"]
+    "extensions": [
+      "./extension/openmerit.ts"
+    ]
   },
   "bin": {
     "openmerit": "dist/cli.js"

package/rules.md

@@ -0,0 +1,39 @@
+# OpenMerit product and architecture rules
+
+- **Optimize for useful model choices.** OpenMerit exists to find better task-specific tradeoffs between quality, cost, and latency—not to become a general-purpose agent or telemetry platform.
+
+- **Act as a control plane, not a gateway.** Normal traffic stays between the harness and provider; OpenMerit observes evidence, runs explicit trials, and returns recommendations without intercepting ordinary work.
+
+- **Make Pi plug-and-play first.** Pi and Pi-based harnesses are the first supported user experience because that is where early users already work; broader harness support can follow without weakening this path.
+
+- **Keep the core harness-neutral.** Pi is the first `HarnessAdapter`, not a permanent assumption in scoring, recommendations, policy, or stored data, so other harnesses can be added without rewriting the merit loop.
+
+- **Keep model providers interchangeable.** OpenRouter is a first-class provider and discovery source, not the foundation of the domain model; provider-specific behavior belongs behind a `ModelProviderAdapter`.
+
+- **Require end-to-end provider neutrality.** Discovery, authentication, execution, scoring, and swapping must preserve the selected route; a feature is not provider-neutral if only its API client is abstracted.
+
+- **Separate model identity from execution route.** Keep logical model IDs in stable `vendor/model` form, while recording the actual provider or route separately, so the same model can be compared through OpenRouter, a native provider, or a local provider.
+
+- **Respect the harness's eligible model pool.** Prefer models already available and configured in the active harness; external catalogs may enrich or expand discovery but must not silently override harness scope or credentials.
+
+- **Treat public benchmarks as priors, not proof.** Benchmarks help shortlist candidates, but merit comes from trials on the user's actual task.
+
+- **Keep four integration boundaries distinct.** Model execution (`ModelProviderAdapter`), agent execution (`HarnessAdapter`), incoming traces (`ObservationSource`), and outgoing telemetry (`EventSink`) solve different problems and must not be coupled.
+
+- **Use provider-neutral core records.** Normalize integrations into stable concepts such as `TaskObservation`, `CandidateRun`, `TrialScore`, `Frontier`, `Recommendation`, and `MeritEvent`, so integrations do not leak their schemas into the decision engine.
+
+- **Optimize per task before aggregating per agent.** Agent-level conclusions must be built from measured task evidence rather than assumed from global model rankings.
+
+- **Version persisted events and evolve them additively.** Existing 0.1.x state must remain readable, and append-only recommendation history must stay intact; migrations should normalize old records rather than invalidate them.
+
+- **Keep policy as the sole auto-swap authority.** Trials and strategists may recommend changes, but only the policy gate may approve automatic application, and every recommendation must retain its gate reasons.
+
+- **Default to safe, explicit trials.** Candidate tool access stays off unless deliberately allowed, trials remain isolated, and temporary workspaces are cleaned up because trying a model must not expose or damage a user's project by surprise.
+
+- **Keep the shadow track isolated, not necessarily concurrent.** Candidate trials may run sequentially to respect cost, rate, and safety limits while remaining separate from the user's live task.
+
+- **Treat local JSONL as the default integration, not a lock-in.** Local traces and events should work without an external service; systems such as Langfuse can later plug in as observation sources or event sinks.
+
+- **Keep integrations optional and the core lightweight.** New providers, harnesses, and observability services should not impose credentials, network calls, or heavy dependencies on users who do not enable them.
+
+- **Keep 0.1.x focused.** The near-term bar is reliable, provider-neutral use for Pi tinkerers and solo hackers; additional harnesses and hosted observability integrations belong in later releases unless required to prove the boundaries work.