openmerit 0.1.1 → 0.1.3

package/extension/openmerit.ts

@@ -17,7 +17,9 @@
  */
 
 import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
-import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
+import {
+  appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync,
+} from "node:fs";
 import { homedir } from "node:os";
 import { basename, join } from "node:path";
 import { createHash } from "node:crypto";
@@ -84,7 +86,12 @@ function trialBudget(): { summary: string; reason: string | null } {
   const today = new Date().toISOString().slice(0, 10);
   let ledger: { date?: string; trials?: number; usd?: number } = {};
   try { ledger = JSON.parse(readFileSync(LEDGER_FILE, "utf8")) as typeof ledger; }
-  catch { /* no trial spend yet */ }
+  catch {
+    if (existsSync(LEDGER_FILE)) return {
+      summary: "ledger unreadable",
+      reason: "trial ledger is malformed; run `openmerit doctor` before spending",
+    };
+  }
   const trials = ledger.date === today ? ledger.trials ?? 0 : 0;
   const usd = ledger.date === today ? ledger.usd ?? 0 : 0;
   const reason = trials >= limit ? `daily trial cap reached (${limit})`
@@ -107,27 +114,41 @@ function latestSessionJob(sessionFile: string | undefined): string | null {
 }
 
 function loadPolicy(): ExtensionPolicy {
+  if (!existsSync(POLICY_FILE)) return {};
   try {
     return JSON.parse(readFileSync(POLICY_FILE, "utf8")) as ExtensionPolicy;
   } catch {
-    return {};
+    return { mode: "recommend", fallback: { apply_on_error: false },
+      budgets: { max_trials_per_day: 0, max_usd_per_day: 0 } };
   }
 }
 
 interface Recommendation {
+  schemaVersion?: 1;
   id: string;
   ts: string;
   taskKey: string;
   taskLabel: string;
   sessionFile?: string | null;
   currentModel: string | null;
-  recommended: { model: string; reason: string };
-  fallback: { model: string; reason: string } | null;
-  evidence: { scoreGain: number; priceRatio: number; onFrontier: boolean; trials: number };
+  currentRoute?: ModelRoute | null;
+  recommended: { model: string; route?: ModelRoute; reason: string };
+  fallback: { model: string; route?: ModelRoute; reason: string } | null;
+  evidence: { scoreGain: number; priceRatio: number; priceKnown?: boolean; onFrontier: boolean; trials: number };
   policy: { autoApply: boolean; reasons: string[] };
   status: "pending" | "applied" | "dismissed" | "expired";
 }
 
+interface ModelRoute {
+  provider: string;
+  modelId: string;
+  meritId: string;
+  input: ("text" | "image")[];
+  cost?: { input: number; output: number; cacheRead?: number; cacheWrite?: number };
+  contextWindow?: number;
+  maxTokens?: number;
+}
+
 interface RecordedTrial {
   taskKey?: string;
   sessionFile?: string;
@@ -161,7 +182,17 @@ function appendJsonl(file: string, obj: unknown): void {
 function latestRecommendations(): Recommendation[] {
   const byId = new Map<string, Recommendation>();
   for (const r of readJsonl<Recommendation>(RECS_FILE)) byId.set(r.id, r);
-  return [...byId.values()];
+  let malformed = false;
+  if (existsSync(RECS_FILE)) {
+    for (const line of readFileSync(RECS_FILE, "utf8").split("\n")) {
+      if (!line.trim()) continue;
+      try { JSON.parse(line); } catch { malformed = true; break; }
+    }
+  }
+  return [...byId.values()].map((rec) => malformed && rec.status === "pending" && rec.policy.autoApply
+    ? { ...rec, policy: { autoApply: false,
+        reasons: [...rec.policy.reasons, "recommendation history contains a malformed line; manual review required"] } }
+    : rec);
 }
 
 function taskKey(text: string): string {
@@ -248,7 +279,11 @@ function suggestionText(rec: Recommendation, trials: RecordedTrial[]): string {
 }
 
 function canonicalModel(provider: string, id: string): string {
-  return provider === "openrouter" ? id : `${provider}/${id}`;
+  return provider === "openrouter" || id.startsWith(`${provider}/`) ? id : `${provider}/${id}`;
+}
+
+function routeDisplay(route: ModelRoute | undefined, fallback: string): string {
+  return route && route.provider !== "openrouter" ? `${route.meritId} via ${route.provider}` : fallback;
 }
 
 function setStatus(rec: Recommendation, status: Recommendation["status"]): void {
@@ -256,11 +291,54 @@ function setStatus(rec: Recommendation, status: Recommendation["status"]): void
 }
 
 let fallbackModel: string | null = null;
+let fallbackRoute: ModelRoute | null = null;
+
+function routeFor(model: { provider: string; id: string; input?: ("text" | "image")[];
+  cost?: { input?: number; output?: number; cacheRead?: number; cacheWrite?: number };
+  contextWindow?: number; maxTokens?: number }): ModelRoute {
+  return {
+    provider: model.provider,
+    modelId: model.id,
+    meritId: canonicalModel(model.provider, model.id),
+    input: model.input ?? ["text"],
+    cost: model.cost && typeof model.cost.input === "number" && typeof model.cost.output === "number"
+      ? { input: model.cost.input, output: model.cost.output,
+          cacheRead: model.cost.cacheRead, cacheWrite: model.cost.cacheWrite }
+      : undefined,
+    contextWindow: model.contextWindow,
+    maxTokens: model.maxTokens,
+  };
+}
+
+function eligibleRoutes(ctx: ExtensionContext): ModelRoute[] {
+  const registry = ctx.modelRegistry as unknown as { getAvailable?: () => Array<Parameters<typeof routeFor>[0]> };
+  const scoped = (ctx.scopedModels ?? []) as readonly { model: Parameters<typeof routeFor>[0] }[];
+  const models = scoped.length ? scoped.map((item) => item.model) : registry.getAvailable?.() ?? [];
+  if (ctx.model && !models.some((model) => model.provider === ctx.model!.provider && model.id === ctx.model!.id))
+    models.unshift(ctx.model);
+  const byRoute = new Map<string, ModelRoute>();
+  for (const model of models) {
+    const route = routeFor(model);
+    byRoute.set(`${route.provider}:${route.modelId}`, route);
+  }
+  return [...byRoute.values()];
+}
+
+function writeState(value: unknown): void {
+  const temp = `${STATE_FILE}.tmp-${process.pid}-${Date.now()}`;
+  try {
+    writeFileSync(temp, JSON.stringify(value) + "\n");
+    renameSync(temp, STATE_FILE);
+  } finally {
+    if (existsSync(temp)) rmSync(temp, { force: true });
+  }
+}
 
 function reportHarnessState(ctx: ExtensionContext, settled = false): void {
   try {
     mkdirSync(HOME, { recursive: true });
     const model = ctx.model ? canonicalModel(ctx.model.provider, ctx.model.id) : null;
+    const currentRoute = ctx.model ? routeFor(ctx.model) : null;
     if (settled) settledTaskKey = latestSessionTaskKey(ctx);
     const prior = existsSync(STATE_FILE)
       ? JSON.parse(readFileSync(STATE_FILE, "utf8")) as { sessionFile?: string | null; settledAt?: string }
@@ -268,18 +346,19 @@ function reportHarnessState(ctx: ExtensionContext, settled = false): void {
     const sessionFile = ctx.sessionManager.getSessionFile() ?? null;
     const settledAt = settled ? new Date().toISOString()
       : prior.sessionFile === sessionFile ? prior.settledAt ?? null : null;
-    writeFileSync(
-      STATE_FILE,
-      JSON.stringify({
+    writeState({
+        schemaVersion: 1,
         currentModel: model,
+        currentRoute,
+        routes: eligibleRoutes(ctx),
         fallbackModel,
+        fallbackRoute,
         sessionFile,
         settledTaskKey,
         settledAt,
         cwd: ctx.cwd,
         updatedAt: new Date().toISOString(),
-      }) + "\n",
-    );
+      });
   } catch {
     /* never break the host session over reporting */
   }
@@ -288,8 +367,11 @@ function reportHarnessState(ctx: ExtensionContext, settled = false): void {
 /** Restore the persisted fallback so a fresh session keeps it even with no pending recs. */
 function restoreFallback(): void {
   try {
-    const st = JSON.parse(readFileSync(STATE_FILE, "utf8")) as { fallbackModel?: string | null };
+    const st = JSON.parse(readFileSync(STATE_FILE, "utf8")) as {
+      fallbackModel?: string | null; fallbackRoute?: ModelRoute | null;
+    };
     if (st.fallbackModel) fallbackModel = st.fallbackModel;
+    if (st.fallbackRoute) fallbackRoute = st.fallbackRoute;
   } catch {
     /* no state yet */
   }
@@ -344,7 +426,15 @@ function catalogEntry(openrouterId: string): CatalogEntryLite | undefined {
  * OpenRouter catalog, so inject anything missing via registerProvider
  * (takes effect immediately, no /reload needed).
  */
-function ensureModel(pi: ExtensionAPI, ctx: ExtensionContext, openrouterId: string) {
+function ensureModel(pi: ExtensionAPI, ctx: ExtensionContext, openrouterId: string, route?: ModelRoute) {
+  const registry = ctx.modelRegistry as unknown as {
+    find?: (provider: string, id: string) => unknown | undefined;
+  };
+  if (route) {
+    const exact = registry.find?.(route.provider, route.modelId);
+    if (exact) return exact;
+    if (route.provider !== "openrouter") return undefined;
+  }
   const existing = resolveModel(ctx, openrouterId);
   if (existing) return existing;
 
@@ -372,16 +462,14 @@ function ensureModel(pi: ExtensionAPI, ctx: ExtensionContext, openrouterId: stri
     })),
   });
 
-  const reg = ctx.modelRegistry as unknown as {
-    find?: (provider: string, id: string) => unknown | undefined;
-  };
-  return reg.find?.(INJECT_PROVIDER, openrouterId);
+  return registry.find?.(INJECT_PROVIDER, openrouterId);
 }
 
 export default function openmerit(pi: ExtensionAPI) {
   let pollTimer: ReturnType<typeof setInterval> | null = null;
   let trialJob: ChildProcessByStdio<null, Readable, Readable> | null = null;
-  const queuedJobs: { sessionFile: string; settledAt: string; model: string; key: string; cwd: string; bytes: number }[] = [];
+  const queuedJobs: { sessionFile: string; settledAt: string; model: string; route: ModelRoute;
+    key: string; cwd: string; bytes: number }[] = [];
   const startedJobs = new Set<string>();
   let activeSession: string | null = null;
   let progress: { sessionFile: string; taskKey: string; current: TrialProgress | null;
@@ -408,7 +496,7 @@ export default function openmerit(pi: ExtensionAPI) {
       return;
     }
     const child = spawn(process.execPath, [ENGINE_FILE, "session-trial", job.sessionFile,
-      job.settledAt, job.model, job.key, job.cwd, String(job.bytes)], {
+      job.settledAt, job.model, job.key, job.cwd, String(job.bytes), job.route.provider, job.route.modelId], {
       cwd: job.cwd, env: process.env, stdio: ["ignore", "pipe", "pipe"],
       detached: process.platform !== "win32",
     });
@@ -466,7 +554,8 @@ export default function openmerit(pi: ExtensionAPI) {
   function queueSettledTask(ctx: ExtensionContext): void {
     const sessionFile = ctx.sessionManager.getSessionFile();
     const model = ctx.model ? canonicalModel(ctx.model.provider, ctx.model.id) : null;
-    if (!sessionFile || !model || !settledTaskKey || !existsSync(sessionFile)) return;
+    const route = ctx.model ? routeFor(ctx.model) : null;
+    if (!sessionFile || !model || !route || !settledTaskKey || !existsSync(sessionFile)) return;
     // The alpha compares completed response tasks. The engine validates images,
     // tool use and the baseline answer before any provider calls.
     let completed = false;
@@ -490,7 +579,7 @@ export default function openmerit(pi: ExtensionAPI) {
       if (ctx.hasUI) ctx.ui.notify(`openmerit: comparison skipped: ${budget.reason}. ${budget.summary}`, "warning");
       return;
     }
-    queuedJobs.push({ sessionFile, settledAt, model, key: settledTaskKey, cwd: ctx.cwd, bytes });
+    queuedJobs.push({ sessionFile, settledAt, model, route, key: settledTaskKey, cwd: ctx.cwd, bytes });
     startNextJob(ctx);
   }
 
@@ -502,11 +591,18 @@ export default function openmerit(pi: ExtensionAPI) {
 
   async function applyRecommendation(ctx: ExtensionContext, rec: Recommendation): Promise<boolean> {
     const current = ctx.model ? canonicalModel(ctx.model.provider, ctx.model.id) : null;
-    if (rec.currentModel && current !== rec.currentModel) {
+    const currentRoute = ctx.model ? routeFor(ctx.model) : null;
+    if (rec.currentRoute && currentRoute &&
+        (rec.currentRoute.provider !== currentRoute.provider || rec.currentRoute.modelId !== currentRoute.modelId)) {
+      ctx.ui.notify(`openmerit: recommendation targets ${rec.currentRoute.provider}/${rec.currentRoute.modelId}; ` +
+        `current route is ${currentRoute.provider}/${currentRoute.modelId}`, "warning");
+      return false;
+    }
+    if (!rec.currentRoute && rec.currentModel && current !== rec.currentModel) {
       ctx.ui.notify(`openmerit: recommendation was measured against ${rec.currentModel}; current model is ${current ?? "unknown"}`, "warning");
       return false;
     }
-    const model = ensureModel(pi, ctx, rec.recommended.model);
+    const model = ensureModel(pi, ctx, rec.recommended.model, rec.recommended.route);
     if (!model) {
       ctx.ui.notify(
         `openmerit: ${rec.recommended.model} not found in pi's model registry (add the provider/model first)`,
@@ -517,8 +613,12 @@ export default function openmerit(pi: ExtensionAPI) {
     const ok = await pi.setModel(model as Parameters<typeof pi.setModel>[0]);
     if (ok) {
       setStatus(rec, "applied");
-      if (rec.fallback) fallbackModel = rec.fallback.model;
-      ctx.ui.notify(`openmerit: switched to ${rec.recommended.model}\n${rec.recommended.reason}`, "info");
+      if (rec.fallback) {
+        fallbackModel = rec.fallback.model;
+        fallbackRoute = rec.fallback.route ?? null;
+      }
+      ctx.ui.notify(`openmerit: switched to ${routeDisplay(rec.recommended.route, rec.recommended.model)}\n` +
+        rec.recommended.reason, "info");
       reportHarnessState(ctx);
     } else {
       ctx.ui.notify(`openmerit: no auth for ${rec.recommended.model}`, "error");
@@ -550,7 +650,7 @@ export default function openmerit(pi: ExtensionAPI) {
       // startup. Notify (fire-and-forget) and let the user act via /openmerit.
       if (ctx.hasUI) {
         ctx.ui.notify(
-          `openmerit recommends: ${rec.recommended.model}\n${rec.recommended.reason}\n` +
+          `openmerit recommends: ${routeDisplay(rec.recommended.route, rec.recommended.model)}\n${rec.recommended.reason}\n` +
             `gate: ${rec.policy.reasons.join("; ")}\n(/openmerit apply to switch, /openmerit dismiss to ignore)`,
           "info",
         );
@@ -621,7 +721,7 @@ export default function openmerit(pi: ExtensionAPI) {
     const policy = loadPolicy();
     if (policy.fallback?.apply_on_error === false) return;
     const current = ctx.model ? canonicalModel(ctx.model.provider, ctx.model.id) : "current model";
-    const model = ensureModel(pi, ctx, fallbackModel);
+    const model = ensureModel(pi, ctx, fallbackModel, fallbackRoute ?? undefined);
     if (!model) return;
 
     const auto = policy.mode === "auto";
@@ -673,7 +773,7 @@ export default function openmerit(pi: ExtensionAPI) {
           ctx.ui.notify("openmerit: no fallback set yet", "info");
           return;
         }
-        const model = ensureModel(pi, ctx, fallbackModel);
+        const model = ensureModel(pi, ctx, fallbackModel, fallbackRoute ?? undefined);
         if (model && (await pi.setModel(model as Parameters<typeof pi.setModel>[0]))) {
           ctx.ui.notify(`openmerit: switched to fallback ${fallbackModel}`, "info");
           reportHarnessState(ctx);
@@ -683,7 +783,8 @@ export default function openmerit(pi: ExtensionAPI) {
         return;
       }
 
-      const current = ctx.model ? canonicalModel(ctx.model.provider, ctx.model.id) : "unknown";
+      const currentRoute = ctx.model ? routeFor(ctx.model) : undefined;
+      const current = currentRoute ? routeDisplay(currentRoute, currentRoute.meritId) : "unknown";
       const budget = trialBudget();
       const liveProgress = progress?.sessionFile === ctx.sessionManager.getSessionFile() ? progress : null;
       const comparison = trialJob ? liveProgress?.current
@@ -706,7 +807,8 @@ export default function openmerit(pi: ExtensionAPI) {
       for (const r of pending.slice(-5)) {
         lines.push(
           "",
-          `-> ${r.recommended.model} (gain ${r.evidence.scoreGain}, ${r.evidence.priceRatio}x price, frontier=${r.evidence.onFrontier})`,
+          `-> ${routeDisplay(r.recommended.route, r.recommended.model)} (gain ${r.evidence.scoreGain}, ` +
+            `${r.evidence.priceKnown === false ? "price unknown" : `${r.evidence.priceRatio}x price`}, frontier=${r.evidence.onFrontier})`,
           `   ${r.recommended.reason}`,
           `   gate: ${r.policy.reasons.join("; ") || "n/a"}`,
         );

package/instructions/OPENMERIT.md

@@ -45,8 +45,8 @@ the best model for the task at hand, at the best price, with a vetted fallback.
 ## Guarantees
 
 - Per-task comparisons send the task text, uploaded files or images (when
-  present), and candidate answers through pi/OpenRouter for model runs and
-  judging. Candidate Pi runs default to no tools; any tool access is an
+  present), and candidate answers through the model routes configured in Pi
+  for model runs and judging. Candidate Pi runs default to no tools; any tool access is an
   explicit user opt-in. Use non-sensitive examples while
   evaluating this alpha.
 - The shipped policy is supervised. Swaps happen automatically only after the

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openmerit",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "Find better models for each pi task by comparing quality, cost, and latency.",
   "type": "module",
   "keywords": [
@@ -17,10 +17,13 @@
     "extension/openmerit.ts",
     "instructions/",
     "examples/",
+    "rules.md",
     "benchmark/invoice_ocr/data/"
   ],
   "pi": {
-    "extensions": ["./extension/openmerit.ts"]
+    "extensions": [
+      "./extension/openmerit.ts"
+    ]
   },
   "bin": {
     "openmerit": "dist/cli.js"

package/rules.md

@@ -0,0 +1,39 @@
+# OpenMerit product and architecture rules
+
+- **Optimize for useful model choices.** OpenMerit exists to find better task-specific tradeoffs between quality, cost, and latency—not to become a general-purpose agent or telemetry platform.
+
+- **Act as a control plane, not a gateway.** Normal traffic stays between the harness and provider; OpenMerit observes evidence, runs explicit trials, and returns recommendations without intercepting ordinary work.
+
+- **Make Pi plug-and-play first.** Pi and Pi-based harnesses are the first supported user experience because that is where early users already work; broader harness support can follow without weakening this path.
+
+- **Keep the core harness-neutral.** Pi is the first `HarnessAdapter`, not a permanent assumption in scoring, recommendations, policy, or stored data, so other harnesses can be added without rewriting the merit loop.
+
+- **Keep model providers interchangeable.** OpenRouter is a first-class provider and discovery source, not the foundation of the domain model; provider-specific behavior belongs behind a `ModelProviderAdapter`.
+
+- **Require end-to-end provider neutrality.** Discovery, authentication, execution, scoring, and swapping must preserve the selected route; a feature is not provider-neutral if only its API client is abstracted.
+
+- **Separate model identity from execution route.** Keep logical model IDs in stable `vendor/model` form, while recording the actual provider or route separately, so the same model can be compared through OpenRouter, a native provider, or a local provider.
+
+- **Respect the harness's eligible model pool.** Prefer models already available and configured in the active harness; external catalogs may enrich or expand discovery but must not silently override harness scope or credentials.
+
+- **Treat public benchmarks as priors, not proof.** Benchmarks help shortlist candidates, but merit comes from trials on the user's actual task.
+
+- **Keep four integration boundaries distinct.** Model execution (`ModelProviderAdapter`), agent execution (`HarnessAdapter`), incoming traces (`ObservationSource`), and outgoing telemetry (`EventSink`) solve different problems and must not be coupled.
+
+- **Use provider-neutral core records.** Normalize integrations into stable concepts such as `TaskObservation`, `CandidateRun`, `TrialScore`, `Frontier`, `Recommendation`, and `MeritEvent`, so integrations do not leak their schemas into the decision engine.
+
+- **Optimize per task before aggregating per agent.** Agent-level conclusions must be built from measured task evidence rather than assumed from global model rankings.
+
+- **Version persisted events and evolve them additively.** Existing 0.1.x state must remain readable, and append-only recommendation history must stay intact; migrations should normalize old records rather than invalidate them.
+
+- **Keep policy as the sole auto-swap authority.** Trials and strategists may recommend changes, but only the policy gate may approve automatic application, and every recommendation must retain its gate reasons.
+
+- **Default to safe, explicit trials.** Candidate tool access stays off unless deliberately allowed, trials remain isolated, and temporary workspaces are cleaned up because trying a model must not expose or damage a user's project by surprise.
+
+- **Keep the shadow track isolated, not necessarily concurrent.** Candidate trials may run sequentially to respect cost, rate, and safety limits while remaining separate from the user's live task.
+
+- **Treat local JSONL as the default integration, not a lock-in.** Local traces and events should work without an external service; systems such as Langfuse can later plug in as observation sources or event sinks.
+
+- **Keep integrations optional and the core lightweight.** New providers, harnesses, and observability services should not impose credentials, network calls, or heavy dependencies on users who do not enable them.
+
+- **Keep 0.1.x focused.** The near-term bar is reliable, provider-neutral use for Pi tinkerers and solo hackers; additional harnesses and hosted observability integrations belong in later releases unless required to prove the boundaries work.