openmerit 0.1.1 → 0.1.2

package/dist/trials.js

@@ -1,5 +1,5 @@
 /** Shadow trials: run candidate models against observed/declared tasks on the background track. */
-import { chat } from "./llm.js";
+import { directChatClient } from "./llm.js";
 import { judge, parseObj } from "./judge.js";
 import { paths, readJson, writeJson } from "./store.js";
 function today() {
@@ -15,6 +15,21 @@ export function budgetOk(policy) {
         return { ok: false, reason: `daily budget exhausted ($${policy.budgets.max_usd_per_day})` };
     return { ok: true };
 }
+/** Conservative admission estimate for one candidate answer (rough input tokens + 4K output). */
+export function trialBudgetOk(policy, entry, inputChars) {
+    if (entry.priceKnown === false)
+        return { ok: false, reason: "route price is unknown" };
+    const inputTokens = Math.ceil(inputChars / 4);
+    const outputTokens = Math.min(entry.route?.maxTokens ?? 4096, 4096);
+    const estimatedUsd = entry.pp * inputTokens + entry.pc * outputTokens;
+    if (estimatedUsd > policy.budgets.max_usd_per_trial)
+        return {
+            ok: false,
+            estimatedUsd,
+            reason: `estimated trial cost $${estimatedUsd.toFixed(4)} exceeds $${policy.budgets.max_usd_per_trial.toFixed(4)} limit`,
+        };
+    return { ok: true, estimatedUsd };
+}
 export function recordTrialSpend(usd) {
     let ledger = readJson(paths.ledger(), { date: today(), trials: 0, usd: 0 });
     if (ledger.date !== today())
@@ -23,16 +38,26 @@ export function recordTrialSpend(usd) {
     ledger.usd = Math.round((ledger.usd + usd) * 1e6) / 1e6;
     writeJson(paths.ledger(), ledger);
 }
+/** Record non-candidate merit-loop spend without consuming a trial slot. */
+export function recordMeritSpend(usd) {
+    if (!(usd > 0))
+        return;
+    let ledger = readJson(paths.ledger(), { date: today(), trials: 0, usd: 0 });
+    if (ledger.date !== today())
+        ledger = { date: today(), trials: 0, usd: 0 };
+    ledger.usd = Math.round((ledger.usd + usd) * 1e6) / 1e6;
+    writeJson(paths.ledger(), ledger);
+}
 const RUBRIC_PROMPT = `Write a compact grading rubric for answers to the task below.
 It must list the concrete criteria for a score of 1.0 and how to deduct.
 TASK: {task}
 Return ONLY json: {{"rubric": "<text>"}}`;
 /** Derive (and cache) a grading rubric for a task that came from traces. */
-export async function ensureRubric(key, judgeModel, task, cache) {
+export async function ensureRubric(key, judgeModel, task, cache, client = directChatClient(key)) {
     const hit = cache.get(task);
     if (hit)
         return hit;
-    const { content } = await chat(key, judgeModel, RUBRIC_PROMPT.replace("{task}", task.slice(0, 8000)), 1024, 0);
+    const { content } = await client.chat(judgeModel, RUBRIC_PROMPT.replace("{task}", task.slice(0, 8000)), 1024, 0);
     let rubric = "Score 1.0 for a fully correct, complete, usable answer; deduct for errors, omissions, or unusable output.";
     try {
         const obj = parseObj(content);
@@ -46,19 +71,20 @@ export async function ensureRubric(key, judgeModel, task, cache) {
     return rubric;
 }
 /** Run one shadow trial: generate with `model`, judge the output, return a frontier point + cost. */
-export async function runTrial(key, judgeModel, task, rubric, model, entry) {
+export async function runTrial(key, judgeModel, task, rubric, model, entry, client = directChatClient(key)) {
     const started = Date.now();
     let gen;
     let usage = {};
     try {
-        const r = await chat(key, model, task, 4096, 0.2);
+        const r = await client.chat(entry?.route ?? model, task, 4096, 0.2);
         gen = r.content;
         usage = r.usage;
     }
     catch (e) {
         return {
             point: {
-                model, score: 0, price: entry?.price ?? 0,
+                schemaVersion: 1, model, route: entry?.route, score: 0, price: entry?.price ?? 0,
+                priceKnown: entry?.priceKnown ?? !!entry,
                 ts: new Date().toISOString(), source: "shadow_trial", why: String(e).slice(0, 160),
             },
             costUsd: 0,
@@ -70,7 +96,8 @@ export async function runTrial(key, judgeModel, task, rubric, model, entry) {
     if (!gen.trim()) {
         return {
             point: {
-                model, score: 0, price: entry?.price ?? 0, latencyMs,
+                schemaVersion: 1, model, route: entry?.route, score: 0, price: entry?.price ?? 0,
+                priceKnown: entry?.priceKnown ?? !!entry, latencyMs,
                 ts: new Date().toISOString(), source: "shadow_trial", why: "empty response",
             },
             costUsd,
@@ -79,7 +106,7 @@ export async function runTrial(key, judgeModel, task, rubric, model, entry) {
     let score = 0;
     let why = "judge error";
     try {
-        const j = await judge(key, judgeModel, task, rubric, gen);
+        const j = await judge(key, judgeModel, task, rubric, gen, client);
         score = j.score;
         why = j.why;
     }
@@ -88,9 +115,12 @@ export async function runTrial(key, judgeModel, task, rubric, model, entry) {
     }
     return {
         point: {
+            schemaVersion: 1,
             model,
+            route: entry?.route,
             score,
             price: entry?.price ?? 0,
+            priceKnown: entry?.priceKnown ?? !!entry,
             latencyMs,
             ts: new Date().toISOString(),
             source: "shadow_trial",

package/extension/openmerit.ts

@@ -115,19 +115,31 @@ function loadPolicy(): ExtensionPolicy {
 }
 
 interface Recommendation {
+  schemaVersion?: 1;
   id: string;
   ts: string;
   taskKey: string;
   taskLabel: string;
   sessionFile?: string | null;
   currentModel: string | null;
-  recommended: { model: string; reason: string };
-  fallback: { model: string; reason: string } | null;
-  evidence: { scoreGain: number; priceRatio: number; onFrontier: boolean; trials: number };
+  currentRoute?: ModelRoute | null;
+  recommended: { model: string; route?: ModelRoute; reason: string };
+  fallback: { model: string; route?: ModelRoute; reason: string } | null;
+  evidence: { scoreGain: number; priceRatio: number; priceKnown?: boolean; onFrontier: boolean; trials: number };
   policy: { autoApply: boolean; reasons: string[] };
   status: "pending" | "applied" | "dismissed" | "expired";
 }
 
+interface ModelRoute {
+  provider: string;
+  modelId: string;
+  meritId: string;
+  input: ("text" | "image")[];
+  cost?: { input: number; output: number; cacheRead?: number; cacheWrite?: number };
+  contextWindow?: number;
+  maxTokens?: number;
+}
+
 interface RecordedTrial {
   taskKey?: string;
   sessionFile?: string;
@@ -248,7 +260,11 @@ function suggestionText(rec: Recommendation, trials: RecordedTrial[]): string {
 }
 
 function canonicalModel(provider: string, id: string): string {
-  return provider === "openrouter" ? id : `${provider}/${id}`;
+  return provider === "openrouter" || id.startsWith(`${provider}/`) ? id : `${provider}/${id}`;
+}
+
+function routeDisplay(route: ModelRoute | undefined, fallback: string): string {
+  return route && route.provider !== "openrouter" ? `${route.meritId} via ${route.provider}` : fallback;
 }
 
 function setStatus(rec: Recommendation, status: Recommendation["status"]): void {
@@ -256,11 +272,44 @@ function setStatus(rec: Recommendation, status: Recommendation["status"]): void
 }
 
 let fallbackModel: string | null = null;
+let fallbackRoute: ModelRoute | null = null;
+
+function routeFor(model: { provider: string; id: string; input?: ("text" | "image")[];
+  cost?: { input?: number; output?: number; cacheRead?: number; cacheWrite?: number };
+  contextWindow?: number; maxTokens?: number }): ModelRoute {
+  return {
+    provider: model.provider,
+    modelId: model.id,
+    meritId: canonicalModel(model.provider, model.id),
+    input: model.input ?? ["text"],
+    cost: model.cost && typeof model.cost.input === "number" && typeof model.cost.output === "number"
+      ? { input: model.cost.input, output: model.cost.output,
+          cacheRead: model.cost.cacheRead, cacheWrite: model.cost.cacheWrite }
+      : undefined,
+    contextWindow: model.contextWindow,
+    maxTokens: model.maxTokens,
+  };
+}
+
+function eligibleRoutes(ctx: ExtensionContext): ModelRoute[] {
+  const registry = ctx.modelRegistry as unknown as { getAvailable?: () => Array<Parameters<typeof routeFor>[0]> };
+  const scoped = (ctx.scopedModels ?? []) as readonly { model: Parameters<typeof routeFor>[0] }[];
+  const models = scoped.length ? scoped.map((item) => item.model) : registry.getAvailable?.() ?? [];
+  if (ctx.model && !models.some((model) => model.provider === ctx.model!.provider && model.id === ctx.model!.id))
+    models.unshift(ctx.model);
+  const byRoute = new Map<string, ModelRoute>();
+  for (const model of models) {
+    const route = routeFor(model);
+    byRoute.set(`${route.provider}:${route.modelId}`, route);
+  }
+  return [...byRoute.values()];
+}
 
 function reportHarnessState(ctx: ExtensionContext, settled = false): void {
   try {
     mkdirSync(HOME, { recursive: true });
     const model = ctx.model ? canonicalModel(ctx.model.provider, ctx.model.id) : null;
+    const currentRoute = ctx.model ? routeFor(ctx.model) : null;
     if (settled) settledTaskKey = latestSessionTaskKey(ctx);
     const prior = existsSync(STATE_FILE)
       ? JSON.parse(readFileSync(STATE_FILE, "utf8")) as { sessionFile?: string | null; settledAt?: string }
@@ -271,8 +320,12 @@ function reportHarnessState(ctx: ExtensionContext, settled = false): void {
     writeFileSync(
       STATE_FILE,
       JSON.stringify({
+        schemaVersion: 1,
         currentModel: model,
+        currentRoute,
+        routes: eligibleRoutes(ctx),
         fallbackModel,
+        fallbackRoute,
         sessionFile,
         settledTaskKey,
         settledAt,
@@ -288,8 +341,11 @@ function reportHarnessState(ctx: ExtensionContext, settled = false): void {
 /** Restore the persisted fallback so a fresh session keeps it even with no pending recs. */
 function restoreFallback(): void {
   try {
-    const st = JSON.parse(readFileSync(STATE_FILE, "utf8")) as { fallbackModel?: string | null };
+    const st = JSON.parse(readFileSync(STATE_FILE, "utf8")) as {
+      fallbackModel?: string | null; fallbackRoute?: ModelRoute | null;
+    };
     if (st.fallbackModel) fallbackModel = st.fallbackModel;
+    if (st.fallbackRoute) fallbackRoute = st.fallbackRoute;
   } catch {
     /* no state yet */
   }
@@ -344,7 +400,15 @@ function catalogEntry(openrouterId: string): CatalogEntryLite | undefined {
  * OpenRouter catalog, so inject anything missing via registerProvider
  * (takes effect immediately, no /reload needed).
  */
-function ensureModel(pi: ExtensionAPI, ctx: ExtensionContext, openrouterId: string) {
+function ensureModel(pi: ExtensionAPI, ctx: ExtensionContext, openrouterId: string, route?: ModelRoute) {
+  const registry = ctx.modelRegistry as unknown as {
+    find?: (provider: string, id: string) => unknown | undefined;
+  };
+  if (route) {
+    const exact = registry.find?.(route.provider, route.modelId);
+    if (exact) return exact;
+    if (route.provider !== "openrouter") return undefined;
+  }
   const existing = resolveModel(ctx, openrouterId);
   if (existing) return existing;
 
@@ -372,16 +436,14 @@ function ensureModel(pi: ExtensionAPI, ctx: ExtensionContext, openrouterId: stri
     })),
   });
 
-  const reg = ctx.modelRegistry as unknown as {
-    find?: (provider: string, id: string) => unknown | undefined;
-  };
-  return reg.find?.(INJECT_PROVIDER, openrouterId);
+  return registry.find?.(INJECT_PROVIDER, openrouterId);
 }
 
 export default function openmerit(pi: ExtensionAPI) {
   let pollTimer: ReturnType<typeof setInterval> | null = null;
   let trialJob: ChildProcessByStdio<null, Readable, Readable> | null = null;
-  const queuedJobs: { sessionFile: string; settledAt: string; model: string; key: string; cwd: string; bytes: number }[] = [];
+  const queuedJobs: { sessionFile: string; settledAt: string; model: string; route: ModelRoute;
+    key: string; cwd: string; bytes: number }[] = [];
   const startedJobs = new Set<string>();
   let activeSession: string | null = null;
   let progress: { sessionFile: string; taskKey: string; current: TrialProgress | null;
@@ -408,7 +470,7 @@ export default function openmerit(pi: ExtensionAPI) {
       return;
     }
     const child = spawn(process.execPath, [ENGINE_FILE, "session-trial", job.sessionFile,
-      job.settledAt, job.model, job.key, job.cwd, String(job.bytes)], {
+      job.settledAt, job.model, job.key, job.cwd, String(job.bytes), job.route.provider, job.route.modelId], {
       cwd: job.cwd, env: process.env, stdio: ["ignore", "pipe", "pipe"],
       detached: process.platform !== "win32",
     });
@@ -466,7 +528,8 @@ export default function openmerit(pi: ExtensionAPI) {
   function queueSettledTask(ctx: ExtensionContext): void {
     const sessionFile = ctx.sessionManager.getSessionFile();
     const model = ctx.model ? canonicalModel(ctx.model.provider, ctx.model.id) : null;
-    if (!sessionFile || !model || !settledTaskKey || !existsSync(sessionFile)) return;
+    const route = ctx.model ? routeFor(ctx.model) : null;
+    if (!sessionFile || !model || !route || !settledTaskKey || !existsSync(sessionFile)) return;
     // The alpha compares completed response tasks. The engine validates images,
     // tool use and the baseline answer before any provider calls.
     let completed = false;
@@ -490,7 +553,7 @@ export default function openmerit(pi: ExtensionAPI) {
       if (ctx.hasUI) ctx.ui.notify(`openmerit: comparison skipped: ${budget.reason}. ${budget.summary}`, "warning");
       return;
     }
-    queuedJobs.push({ sessionFile, settledAt, model, key: settledTaskKey, cwd: ctx.cwd, bytes });
+    queuedJobs.push({ sessionFile, settledAt, model, route, key: settledTaskKey, cwd: ctx.cwd, bytes });
     startNextJob(ctx);
   }
 
@@ -502,11 +565,18 @@ export default function openmerit(pi: ExtensionAPI) {
 
   async function applyRecommendation(ctx: ExtensionContext, rec: Recommendation): Promise<boolean> {
     const current = ctx.model ? canonicalModel(ctx.model.provider, ctx.model.id) : null;
-    if (rec.currentModel && current !== rec.currentModel) {
+    const currentRoute = ctx.model ? routeFor(ctx.model) : null;
+    if (rec.currentRoute && currentRoute &&
+        (rec.currentRoute.provider !== currentRoute.provider || rec.currentRoute.modelId !== currentRoute.modelId)) {
+      ctx.ui.notify(`openmerit: recommendation targets ${rec.currentRoute.provider}/${rec.currentRoute.modelId}; ` +
+        `current route is ${currentRoute.provider}/${currentRoute.modelId}`, "warning");
+      return false;
+    }
+    if (!rec.currentRoute && rec.currentModel && current !== rec.currentModel) {
       ctx.ui.notify(`openmerit: recommendation was measured against ${rec.currentModel}; current model is ${current ?? "unknown"}`, "warning");
       return false;
     }
-    const model = ensureModel(pi, ctx, rec.recommended.model);
+    const model = ensureModel(pi, ctx, rec.recommended.model, rec.recommended.route);
     if (!model) {
       ctx.ui.notify(
         `openmerit: ${rec.recommended.model} not found in pi's model registry (add the provider/model first)`,
@@ -517,8 +587,12 @@ export default function openmerit(pi: ExtensionAPI) {
     const ok = await pi.setModel(model as Parameters<typeof pi.setModel>[0]);
     if (ok) {
       setStatus(rec, "applied");
-      if (rec.fallback) fallbackModel = rec.fallback.model;
-      ctx.ui.notify(`openmerit: switched to ${rec.recommended.model}\n${rec.recommended.reason}`, "info");
+      if (rec.fallback) {
+        fallbackModel = rec.fallback.model;
+        fallbackRoute = rec.fallback.route ?? null;
+      }
+      ctx.ui.notify(`openmerit: switched to ${routeDisplay(rec.recommended.route, rec.recommended.model)}\n` +
+        rec.recommended.reason, "info");
       reportHarnessState(ctx);
     } else {
       ctx.ui.notify(`openmerit: no auth for ${rec.recommended.model}`, "error");
@@ -550,7 +624,7 @@ export default function openmerit(pi: ExtensionAPI) {
       // startup. Notify (fire-and-forget) and let the user act via /openmerit.
       if (ctx.hasUI) {
         ctx.ui.notify(
-          `openmerit recommends: ${rec.recommended.model}\n${rec.recommended.reason}\n` +
+          `openmerit recommends: ${routeDisplay(rec.recommended.route, rec.recommended.model)}\n${rec.recommended.reason}\n` +
             `gate: ${rec.policy.reasons.join("; ")}\n(/openmerit apply to switch, /openmerit dismiss to ignore)`,
           "info",
         );
@@ -621,7 +695,7 @@ export default function openmerit(pi: ExtensionAPI) {
     const policy = loadPolicy();
     if (policy.fallback?.apply_on_error === false) return;
     const current = ctx.model ? canonicalModel(ctx.model.provider, ctx.model.id) : "current model";
-    const model = ensureModel(pi, ctx, fallbackModel);
+    const model = ensureModel(pi, ctx, fallbackModel, fallbackRoute ?? undefined);
     if (!model) return;
 
     const auto = policy.mode === "auto";
@@ -673,7 +747,7 @@ export default function openmerit(pi: ExtensionAPI) {
           ctx.ui.notify("openmerit: no fallback set yet", "info");
           return;
         }
-        const model = ensureModel(pi, ctx, fallbackModel);
+        const model = ensureModel(pi, ctx, fallbackModel, fallbackRoute ?? undefined);
         if (model && (await pi.setModel(model as Parameters<typeof pi.setModel>[0]))) {
           ctx.ui.notify(`openmerit: switched to fallback ${fallbackModel}`, "info");
           reportHarnessState(ctx);
@@ -683,7 +757,8 @@ export default function openmerit(pi: ExtensionAPI) {
         return;
       }
 
-      const current = ctx.model ? canonicalModel(ctx.model.provider, ctx.model.id) : "unknown";
+      const currentRoute = ctx.model ? routeFor(ctx.model) : undefined;
+      const current = currentRoute ? routeDisplay(currentRoute, currentRoute.meritId) : "unknown";
       const budget = trialBudget();
       const liveProgress = progress?.sessionFile === ctx.sessionManager.getSessionFile() ? progress : null;
       const comparison = trialJob ? liveProgress?.current
@@ -706,7 +781,8 @@ export default function openmerit(pi: ExtensionAPI) {
       for (const r of pending.slice(-5)) {
         lines.push(
           "",
-          `-> ${r.recommended.model} (gain ${r.evidence.scoreGain}, ${r.evidence.priceRatio}x price, frontier=${r.evidence.onFrontier})`,
+          `-> ${routeDisplay(r.recommended.route, r.recommended.model)} (gain ${r.evidence.scoreGain}, ` +
+            `${r.evidence.priceKnown === false ? "price unknown" : `${r.evidence.priceRatio}x price`}, frontier=${r.evidence.onFrontier})`,
           `   ${r.recommended.reason}`,
           `   gate: ${r.policy.reasons.join("; ") || "n/a"}`,
         );

package/instructions/OPENMERIT.md

@@ -45,8 +45,8 @@ the best model for the task at hand, at the best price, with a vetted fallback.
 ## Guarantees
 
 - Per-task comparisons send the task text, uploaded files or images (when
-  present), and candidate answers through pi/OpenRouter for model runs and
-  judging. Candidate Pi runs default to no tools; any tool access is an
+  present), and candidate answers through the model routes configured in Pi
+  for model runs and judging. Candidate Pi runs default to no tools; any tool access is an
   explicit user opt-in. Use non-sensitive examples while
   evaluating this alpha.
 - The shipped policy is supervised. Swaps happen automatically only after the

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openmerit",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Find better models for each pi task by comparing quality, cost, and latency.",
   "type": "module",
   "keywords": [
@@ -17,10 +17,13 @@
     "extension/openmerit.ts",
     "instructions/",
     "examples/",
+    "rules.md",
     "benchmark/invoice_ocr/data/"
   ],
   "pi": {
-    "extensions": ["./extension/openmerit.ts"]
+    "extensions": [
+      "./extension/openmerit.ts"
+    ]
   },
   "bin": {
     "openmerit": "dist/cli.js"

package/rules.md

@@ -0,0 +1,39 @@
+# OpenMerit product and architecture rules
+
+- **Optimize for useful model choices.** OpenMerit exists to find better task-specific tradeoffs between quality, cost, and latency—not to become a general-purpose agent or telemetry platform.
+
+- **Act as a control plane, not a gateway.** Normal traffic stays between the harness and provider; OpenMerit observes evidence, runs explicit trials, and returns recommendations without intercepting ordinary work.
+
+- **Make Pi plug-and-play first.** Pi and Pi-based harnesses are the first supported user experience because that is where early users already work; broader harness support can follow without weakening this path.
+
+- **Keep the core harness-neutral.** Pi is the first `HarnessAdapter`, not a permanent assumption in scoring, recommendations, policy, or stored data, so other harnesses can be added without rewriting the merit loop.
+
+- **Keep model providers interchangeable.** OpenRouter is a first-class provider and discovery source, not the foundation of the domain model; provider-specific behavior belongs behind a `ModelProviderAdapter`.
+
+- **Require end-to-end provider neutrality.** Discovery, authentication, execution, scoring, and swapping must preserve the selected route; a feature is not provider-neutral if only its API client is abstracted.
+
+- **Separate model identity from execution route.** Keep logical model IDs in stable `vendor/model` form, while recording the actual provider or route separately, so the same model can be compared through OpenRouter, a native provider, or a local provider.
+
+- **Respect the harness's eligible model pool.** Prefer models already available and configured in the active harness; external catalogs may enrich or expand discovery but must not silently override harness scope or credentials.
+
+- **Treat public benchmarks as priors, not proof.** Benchmarks help shortlist candidates, but merit comes from trials on the user's actual task.
+
+- **Keep four integration boundaries distinct.** Model execution (`ModelProviderAdapter`), agent execution (`HarnessAdapter`), incoming traces (`ObservationSource`), and outgoing telemetry (`EventSink`) solve different problems and must not be coupled.
+
+- **Use provider-neutral core records.** Normalize integrations into stable concepts such as `TaskObservation`, `CandidateRun`, `TrialScore`, `Frontier`, `Recommendation`, and `MeritEvent`, so integrations do not leak their schemas into the decision engine.
+
+- **Optimize per task before aggregating per agent.** Agent-level conclusions must be built from measured task evidence rather than assumed from global model rankings.
+
+- **Version persisted events and evolve them additively.** Existing 0.1.x state must remain readable, and append-only recommendation history must stay intact; migrations should normalize old records rather than invalidate them.
+
+- **Keep policy as the sole auto-swap authority.** Trials and strategists may recommend changes, but only the policy gate may approve automatic application, and every recommendation must retain its gate reasons.
+
+- **Default to safe, explicit trials.** Candidate tool access stays off unless deliberately allowed, trials remain isolated, and temporary workspaces are cleaned up because trying a model must not expose or damage a user's project by surprise.
+
+- **Keep the shadow track isolated, not necessarily concurrent.** Candidate trials may run sequentially to respect cost, rate, and safety limits while remaining separate from the user's live task.
+
+- **Treat local JSONL as the default integration, not a lock-in.** Local traces and events should work without an external service; systems such as Langfuse can later plug in as observation sources or event sinks.
+
+- **Keep integrations optional and the core lightweight.** New providers, harnesses, and observability services should not impose credentials, network calls, or heavy dependencies on users who do not enable them.
+
+- **Keep 0.1.x focused.** The near-term bar is reliable, provider-neutral use for Pi tinkerers and solo hackers; additional harnesses and hosted observability integrations belong in later releases unless required to prove the boundaries work.