openmerit 0.1.1 → 0.1.3

package/dist/daemon.js

@@ -9,15 +9,68 @@ import { loadKey } from "./llm.js";
 import { providerAllowed } from "./policy.js";
 import { buildRecommendation } from "./recommend.js";
 import { appendJsonl, paths, readJson, readJsonl, writeJson, } from "./store.js";
-import { ingestNewTraces } from "./traces.js";
-import { budgetOk, ensureRubric, recordTrialSpend, runTrial } from "./trials.js";
+import { budgetOk, ensureRubric, recordMeritSpend, recordTrialSpend, runTrial, trialBudgetOk } from "./trials.js";
 import { JUDGE_PREFS } from "./judge.js";
-import { availablePiModels, runPiTrial, scorePiRun, settledActiveTask } from "./pi-trials.js";
+import { runPiTrial, scorePiRun, settledActiveTask } from "./pi-trials.js";
 import { taskInputKey } from "./task-input.js";
 import { pickNext, STRAT_PREFS } from "./strategist.js";
+import { enrichRouteEntry, routeCatalog, routeKey, routeLabel } from "./routes.js";
+import { PiCliChatClient } from "./llm.js";
+import { LocalJsonlEventSink, PiTraceObservationSource, meritEvent } from "./integrations.js";
+function resolveRoutePref(cat, prefs, label, requireImages = false) {
+    const available = [...cat.values()].filter((entry) => !!entry.route && entry.priceKnown !== false &&
+        (!requireImages || entry.inputModalities?.includes("image")));
+    for (const pref of prefs) {
+        if (!pref)
+            continue;
+        const routed = cat.get(pref);
+        if (routed && available.includes(routed))
+            return routed;
+        const exact = available.find((entry) => entry.id === pref);
+        if (exact)
+            return exact;
+    }
+    for (const pref of prefs) {
+        if (!pref)
+            continue;
+        const fuzzy = available.find((entry) => entry.id.includes(pref));
+        if (fuzzy)
+            return fuzzy;
+    }
+    const fallback = available.sort((a, b) => (a.priceKnown === false ? 1 : 0) - (b.priceKnown === false ? 1 : 0) || a.price - b.price)[0];
+    if (fallback)
+        return fallback;
+    throw new Error(`could not resolve ${label} route from Pi's eligible models`);
+}
+function processIsAlive(pid) {
+    if (!Number.isInteger(pid) || pid <= 0)
+        return false;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === "EPERM";
+    }
+}
+/** Completed jobs stay final; live owners retain their claim; abandoned claims can retry. */
+export function taskJobIsClaimed(prior, marker, now = Date.now()) {
+    if (!prior || prior.marker !== marker || prior.status === "failed")
+        return false;
+    if (prior.status !== "running")
+        return true;
+    if (prior.ownerPid !== undefined)
+        return processIsAlive(prior.ownerPid);
+    const updated = Date.parse(prior.updatedAt);
+    return Number.isFinite(updated) && now - updated < 15 * 60_000;
+}
 function emitTrialProgress(progress) {
     console.log(`[openmerit/progress] ${JSON.stringify(progress)}`);
 }
+function persistTrial(point, sink) {
+    appendJsonl(paths.trials(), point);
+    sink.emit(meritEvent("trial.completed", point));
+}
 function resolvePref(cat, prefs, label) {
     for (const p of prefs) {
         if (p && cat.has(p))
@@ -34,10 +87,12 @@ function resolvePref(cat, prefs, label) {
     throw new Error(`could not resolve ${label} model from preferences`);
 }
 /** Ingest new session-trace bytes into the observation log. */
-export function tracesTick() {
-    const fresh = ingestNewTraces();
-    for (const o of fresh)
+export function tracesTick(source = new PiTraceObservationSource(), sink = new LocalJsonlEventSink()) {
+    const fresh = source.read();
+    for (const o of fresh) {
         appendJsonl(paths.observations(), o);
+        sink.emit(meritEvent("observation.recorded", o, source.id));
+    }
     if (fresh.length > 0)
         console.log(`[openmerit] traces: ${fresh.length} new observation(s)`);
     return fresh;
@@ -68,6 +123,7 @@ export async function catalogTick(key, policy) {
  * thin). Budget-capped per policy.
  */
 export async function trialTick(key, policy) {
+    const sink = new LocalJsonlEventSink();
     const budget = budgetOk(policy);
     if (!budget.ok) {
         console.log(`[openmerit] trials paused: ${budget.reason}`);
@@ -106,7 +162,7 @@ export async function trialTick(key, policy) {
         const rubric = await ensureRubric(key, judge, obs.taskLabel, rubricCache);
         const { point, costUsd, error } = await runTrial(key, judge, obs.taskLabel, rubric, candidate.id, cat.get(candidate.id));
         recordTrialSpend(costUsd);
-        appendJsonl(paths.trials(), { ...point, taskKey: tKey });
+        persistTrial({ ...point, taskKey: tKey }, sink);
         if (error)
             console.log(`[openmerit] trial failed: ${error.slice(0, 120)}`);
         else
@@ -115,6 +171,7 @@ export async function trialTick(key, policy) {
 }
 /** Compare the one completed active pi task, sequentially, then target its session. */
 export async function autoTaskTick(key, policy, settledTask) {
+    const sink = new LocalJsonlEventSink();
     const task = settledTask === undefined ? settledActiveTask() : settledTask;
     if (!task)
         return null;
@@ -123,99 +180,122 @@ export async function autoTaskTick(key, policy, settledTask) {
         : `${task.sessionFile}:${task.sessionBytes ?? task.settledAt}`;
     const processedPath = settledTask === undefined ? paths.watchProcessed() : paths.sessionJob(marker);
     const prior = readJson(processedPath, null);
-    if (prior?.marker === marker && prior.status !== "failed")
+    if (taskJobIsClaimed(prior, marker))
         return null;
     if (!budgetOk(policy).ok)
         return null;
     // Claim the settled turn before provider calls so the next poll cannot duplicate it.
-    writeJson(processedPath, { marker, status: "running", updatedAt: new Date().toISOString() });
+    writeJson(processedPath, { marker, status: "running", ownerPid: process.pid,
+        updatedAt: new Date().toISOString() });
     try {
-        const cat = await fetchCatalog(key);
-        const piModels = availablePiModels();
-        for (const id of [...cat.keys()]) {
-            const e = cat.get(id);
-            if (!piModels.has(id) || e.price > policy.max_usd_per_m || !providerAllowed(policy, id) ||
-                (task.images.length > 0 && !e.inputModalities?.includes("image")))
+        let cat = routeCatalog(task.routes);
+        if (!cat.has(routeKey(task.route)))
+            cat.set(routeKey(task.route), routeCatalog([task.route]).values().next().value);
+        if (key) {
+            try {
+                const live = await fetchCatalog(key);
+                saveSnapshot(live);
+                cat = new Map([...cat].map(([id, entry]) => [id,
+                    entry.route?.provider === "openrouter" ? enrichRouteEntry(entry, live.get(entry.id)) : entry]));
+            }
+            catch (error) {
+                console.log(`[openmerit] OpenRouter enrichment unavailable: ${error.message}`);
+            }
+        }
+        for (const [id, entry] of [...cat]) {
+            if ((entry.priceKnown !== false && entry.price > policy.max_usd_per_m) ||
+                !providerAllowed(policy, entry.id, entry.route?.provider) ||
+                (task.images.length > 0 && !entry.inputModalities?.includes("image")) ||
+                (id !== routeKey(task.route) && !trialBudgetOk(policy, entry, task.task.length).ok))
                 cat.delete(id);
         }
-        if (!cat.has(task.model))
-            throw new Error(`active model ${task.model} is not trialable in pi/OpenRouter`);
-        const judge = resolvePref(cat, [policy.judge_model, ...JUDGE_PREFS], "judge");
-        const strategist = resolvePref(cat, [policy.strategist_model, ...STRAT_PREFS], "strategist");
+        const activeEntry = cat.get(routeKey(task.route));
+        if (!activeEntry)
+            throw new Error(`active route ${task.route.provider}/${task.route.modelId} is excluded by policy or task capabilities`);
+        if (cat.size < 2)
+            throw new Error("Pi exposes no eligible alternate model route for this task");
+        const judge = resolveRoutePref(cat, [policy.judge_model, ...JUDGE_PREFS, task.model], "judge", task.images.length > 0);
+        const strategist = resolveRoutePref(cat, [policy.strategist_model, ...STRAT_PREFS, task.model], "strategist");
+        const client = new PiCliChatClient(undefined, recordMeritSpend);
         const { category, benchmarks } = relevantBenchmarks(task.task);
         let benchmarkCandidates = [];
         try {
-            benchmarkCandidates = await openRouterBenchmarkCandidates(key, category, cat);
+            if (key)
+                benchmarkCandidates = await openRouterBenchmarkCandidates(key, category, new Map([...cat.values()].map((entry) => [entry.id, entry])));
         }
         catch (e) {
             console.log(`[openmerit] benchmark shortlist unavailable: ${e.message}`);
         }
-        const rubric = await ensureRubric(key, judge, task.task, new Map());
-        const tKey = taskInputKey(task.task, task.images);
+        const rubric = await ensureRubric(key ?? "", judge.route, task.task, new Map(), client);
+        const tKey = taskInputKey(task.task, task.images, task.files);
         const points = [];
         const tried = new Set();
         const failedVendors = new Set();
         const total = Math.max(1, Math.floor(policy.watch.models_per_task));
         if (settledTask !== undefined)
             emitTrialProgress({ phase: "start", taskKey: tKey,
-                model: task.model, index: 1, total });
-        const baseline = await scorePiRun(key, judge, task.task, rubric, task.model, cat.get(task.model), task.run, "trace", task.images);
+                model: routeLabel(task.route), index: 1, total });
+        const baseline = await scorePiRun(key ?? "", judge.route, task.task, rubric, task.model, activeEntry, task.run, "trace", task.images, client);
         points.push(baseline.point);
-        tried.add(task.model);
-        appendJsonl(paths.trials(), { ...baseline.point, taskKey: tKey, sessionFile: task.sessionFile });
+        tried.add(routeKey(task.route));
+        persistTrial({ ...baseline.point, taskKey: tKey, sessionFile: task.sessionFile }, sink);
         console.log(`[openmerit] task ${tKey}: A=${task.model} score=${baseline.point.score.toFixed(2)} from pi trace`);
         if (settledTask !== undefined)
             emitTrialProgress({ phase: "complete", taskKey: tKey,
-                model: task.model, index: 1, total, score: baseline.point.score,
+                model: routeLabel(task.route), index: 1, total, score: baseline.point.score,
                 price: baseline.point.price, latencyMs: baseline.point.latencyMs,
                 costUsd: baseline.costUsd, toolCalls: baseline.point.toolCalls,
                 toolErrors: baseline.point.toolErrors, changedFiles: baseline.point.changedFiles });
         for (let i = 1; i < total; i++) {
             if (!budgetOk(policy).ok)
                 break;
-            const pick = await pickNext(key, strategist, task.task, rubric, benchmarks, points, cat, tried, policy.max_usd_per_m, failedVendors, benchmarkCandidates);
+            const pick = await pickNext(key ?? "", strategist.route, task.task, rubric, benchmarks, points, cat, tried, policy.max_usd_per_m, failedVendors, benchmarkCandidates, client);
             if (!pick)
                 break;
             if (settledTask !== undefined)
                 emitTrialProgress({ phase: "start", taskKey: tKey,
-                    model: pick.model, index: i + 1, total });
-            const { point, costUsd, error } = await runPiTrial(key, judge, task.task, rubric, pick.model, cat.get(pick.model), task.cwd, task.images, task.files);
-            tried.add(pick.model);
+                    model: pick.route ? routeLabel(pick.route) : pick.model, index: i + 1, total });
+            const entry = pick.route ? cat.get(routeKey(pick.route)) : cat.get(pick.model);
+            const { point, costUsd, error } = await runPiTrial(key ?? "", judge.route, task.task, rubric, pick.model, entry, task.cwd, task.images, task.files, undefined, client);
+            tried.add(pick.route ? routeKey(pick.route) : pick.model);
             points.push(point);
             recordTrialSpend(costUsd);
-            appendJsonl(paths.trials(), { ...point, taskKey: tKey, sessionFile: task.sessionFile });
+            persistTrial({ ...point, taskKey: tKey, sessionFile: task.sessionFile }, sink);
             if (error)
-                failedVendors.add(pick.model.split("/")[0]);
+                failedVendors.add(pick.route?.provider ?? pick.model.split("/")[0]);
             console.log(`[openmerit] task ${tKey}: ${pick.model} score=${point.score.toFixed(2)} cost=$${costUsd.toFixed(4)} ` +
                 `tools=${point.toolCalls ?? 0} toolErrors=${point.toolErrors ?? 0} changedFiles=${point.changedFiles ?? 0}` +
                 `${error ? ` error=${error.slice(0, 80)}` : ""}`);
             if (settledTask !== undefined)
                 emitTrialProgress({ phase: "complete", taskKey: tKey,
-                    model: pick.model, index: i + 1, total, score: point.score, price: point.price,
+                    model: pick.route ? routeLabel(pick.route) : pick.model, index: i + 1, total, score: point.score, price: point.price,
                     latencyMs: point.latencyMs, costUsd, error: error?.slice(0, 120),
                     toolCalls: point.toolCalls, toolErrors: point.toolErrors, changedFiles: point.changedFiles });
         }
-        const rec = buildRecommendation(tKey, task.task.slice(0, 120), task.model, points, policy, task.sessionFile);
+        const rec = buildRecommendation(tKey, task.task.slice(0, 120), task.model, points, policy, task.sessionFile, task.route);
         if (rec) {
             appendJsonl(paths.recommendations(), rec);
+            sink.emit(meritEvent("recommendation.created", rec));
             console.log(`[openmerit] task ${tKey}: selected ${rec.recommended.model}; auto=${rec.policy.autoApply}`);
         }
         else
             console.log(`[openmerit] task ${tKey}: active model remains best`);
         writeJson(processedPath, { marker, status: rec ? "complete" : "no_swap",
-            updatedAt: new Date().toISOString() });
+            ownerPid: process.pid, updatedAt: new Date().toISOString() });
         return rec;
     }
     catch (error) {
         writeJson(processedPath, { marker, status: "failed",
-            updatedAt: new Date().toISOString() });
+            ownerPid: process.pid, updatedAt: new Date().toISOString() });
         throw error;
     }
 }
 /** Rebuild frontiers from trials and emit recommendations for the harness's current model. */
 export function recommendTick(policy) {
+    const sink = new LocalJsonlEventSink();
     const harness = readJson(paths.harnessState(), {});
     const currentModel = harness.currentModel ?? null;
+    const currentRoute = harness.currentRoute ?? null;
     const trials = readJsonl(paths.trials());
     const observations = readJsonl(paths.observations());
     const labelByTask = new Map();
@@ -237,16 +317,17 @@ export function recommendTick(policy) {
         latestById.set(r.id, r);
     const existing = new Set([...latestById.values()]
         .filter((r) => r.status === "pending" || r.status === "dismissed")
-        .map((r) => `${r.taskKey}:${r.recommended.model}`));
+        .map((r) => `${r.taskKey}:${r.recommended.route ? routeKey(r.recommended.route) : `openrouter:${r.recommended.model}`}`));
     const emitted = [];
     for (const [tKey, points] of byTask) {
-        const rec = buildRecommendation(tKey, labelByTask.get(tKey) ?? tKey, currentModel, points, policy);
+        const rec = buildRecommendation(tKey, labelByTask.get(tKey) ?? tKey, currentModel, points, policy, null, currentRoute);
         if (!rec)
             continue;
-        const dedupeKey = `${rec.taskKey}:${rec.recommended.model}`;
+        const dedupeKey = `${rec.taskKey}:${rec.recommended.route ? routeKey(rec.recommended.route) : `openrouter:${rec.recommended.model}`}`;
         if (existing.has(dedupeKey))
             continue;
         appendJsonl(paths.recommendations(), rec);
+        sink.emit(meritEvent("recommendation.created", rec));
         emitted.push(rec);
         console.log(`[openmerit] recommendation: ${rec.recommended.model} <- ${currentModel ?? "unknown"} ` +
             `(gain ${rec.evidence.scoreGain}, auto=${rec.policy.autoApply})`);
@@ -261,12 +342,12 @@ export async function tickOnce(policy) {
         key = loadKey();
     }
     catch (e) {
-        console.log(`[openmerit] ${e.message}; skipping provider ticks`);
+        console.log(`[openmerit] ${e.message}; OpenRouter enrichment disabled`);
     }
     if (key) {
         await catalogTick(key, policy);
-        await autoTaskTick(key, policy);
     }
+    await autoTaskTick(key, policy);
 }
 /** Run forever, honoring policy watch intervals. */
 export async function runDaemon(policy) {
@@ -291,17 +372,15 @@ export async function runDaemon(policy) {
             catch {
                 /* no key; skip provider ticks */
             }
-            if (key) {
-                try {
-                    if (wantCatalog) {
-                        await catalogTick(key, policy);
-                        lastCatalog = now;
-                    }
-                    await autoTaskTick(key, policy);
-                }
-                catch (e) {
-                    console.log(`[openmerit] provider tick failed: ${e.message}`);
+            try {
+                if (key && wantCatalog) {
+                    await catalogTick(key, policy);
+                    lastCatalog = now;
                 }
+                await autoTaskTick(key, policy);
+            }
+            catch (e) {
+                console.log(`[openmerit] provider tick failed: ${e.message}`);
             }
         }
         await new Promise((r) => setTimeout(r, tracesMs));

package/dist/diagnostics.js

@@ -0,0 +1,194 @@
+/** Read-only diagnostics plus a provider-free self-test for first-run support. */
+import { spawnSync } from "node:child_process";
+import { appendFileSync, existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, join } from "node:path";
+import { LocalJsonlEventSink, meritEvent } from "./integrations.js";
+import { piExecutable, availablePiRoutes } from "./pi-trials.js";
+import { DEFAULT_POLICY, loadPolicy, providerAllowed } from "./policy.js";
+import { buildRecommendation } from "./recommend.js";
+import { modelRoute, routeCatalog, routeKey, routeLabel } from "./routes.js";
+import { appendJsonl, paths, readJsonReport, readJsonlReport, writeJson, } from "./store.js";
+function check(id, status, message, details) {
+    return { id, status, message, ...(details?.length ? { details } : {}) };
+}
+function packageSources(output) {
+    return [...new Set(output.split("\n").filter((line) => /^ {2}\S/.test(line)).map((line) => line.trim())
+            .filter((line) => /openmerit/i.test(line) && /^(npm:|git:|https?:|\.?\.?\/|\/)/.test(line)))];
+}
+function safePackageSource(source) {
+    if (source.startsWith("npm:/"))
+        return `npm:local/${basename(source)}`;
+    if (source.startsWith("/"))
+        return `local:${basename(source)}`;
+    return source;
+}
+/** No credentials, prompts, or trace contents are included in this report. */
+export function collectDoctorReport() {
+    const checks = [];
+    const executable = piExecutable();
+    const versionResult = spawnSync(executable, ["--version"], { encoding: "utf8" });
+    const piVersion = versionResult.status === 0 ? versionResult.stdout.trim() : null;
+    checks.push(piVersion
+        ? check("pi", "pass", `Pi detected (${piVersion})`)
+        : check("pi", "fail", "Pi is not executable"));
+    try {
+        const policy = loadPolicy(paths.policy());
+        checks.push(check("policy", "pass", `Policy is valid (v${policy.version}, ${policy.mode})`));
+    }
+    catch (error) {
+        checks.push(check("policy", "fail", `Policy is invalid: ${error.message}`));
+    }
+    const stateRead = readJsonReport(paths.harnessState(), {});
+    checks.push(!stateRead.valid
+        ? check("harness-state", "fail", "harness-state.json is malformed; start Pi to replace it atomically")
+        : stateRead.exists
+            ? check("harness-state", "pass", "Pi extension state snapshot is readable")
+            : check("harness-state", "warn", "No Pi extension state yet; start Pi once with OpenMerit installed"));
+    let routes = stateRead.value.routes ?? [];
+    const liveSnapshot = routes.length > 0;
+    if (!routes.length && piVersion) {
+        try {
+            routes = availablePiRoutes();
+        }
+        catch { /* the check below explains it */ }
+    }
+    let policy = DEFAULT_POLICY;
+    try {
+        policy = loadPolicy(paths.policy());
+    }
+    catch { /* already reported */ }
+    routes = routes.filter((route) => providerAllowed(policy, route.meritId, route.provider));
+    const priced = routes.filter((route) => !!route.cost);
+    checks.push(routes.length >= 2
+        ? check("routes", "pass", `${routes.length} Pi model routes are visible`, routes.slice(0, 8).map((route) => `${routeLabel(route)}${route.cost ? "" : " (price unknown)"}`))
+        : check("routes", "fail", `Only ${routes.length} policy-eligible Pi route${routes.length === 1 ? " is" : "s are"} visible; at least two are required`));
+    checks.push(liveSnapshot
+        ? priced.length >= 2
+            ? check("pricing", "pass", `${priced.length} eligible routes expose pricing`)
+            : check("pricing", "warn", "Fewer than two eligible routes expose pricing; unknown-price routes cannot auto-apply")
+        : check("pricing", "warn", "Route prices are unavailable until the extension reports a live Pi snapshot"));
+    const session = stateRead.value.sessionFile;
+    checks.push(!session
+        ? check("session", "warn", "No active saved Pi session is reported yet")
+        : existsSync(session)
+            ? check("session", "pass", "The reported Pi session file is readable")
+            : check("session", "warn", "The reported Pi session file no longer exists"));
+    if (piVersion) {
+        const listed = spawnSync(executable, ["list"], { encoding: "utf8" });
+        const sources = listed.status === 0 ? packageSources(listed.stdout) : [];
+        const safeSources = sources.map(safePackageSource);
+        checks.push(sources.length > 1
+            ? check("installation", "fail", "OpenMerit is installed from multiple Pi package sources", safeSources)
+            : sources.length === 1
+                ? check("installation", "pass", `One Pi package source is installed`, safeSources)
+                : check("installation", "warn", "OpenMerit is not listed as an installed Pi package; use `pi install npm:openmerit`"));
+    }
+    const jsonlFiles = [
+        ["recommendations.jsonl", paths.recommendations()],
+        ["trials.jsonl", paths.trials()],
+        ["traces/observations.jsonl", paths.observations()],
+        ["events.jsonl", paths.events()],
+    ];
+    const malformed = jsonlFiles.flatMap(([label, file]) => {
+        const invalid = readJsonlReport(file).invalidLines;
+        return invalid.length ? [`${label}: lines ${invalid.join(", ")}`] : [];
+    });
+    checks.push(malformed.length
+        ? check("jsonl", "warn", "Malformed JSONL lines were skipped; valid later records remain readable", malformed)
+        : check("jsonl", "pass", "Append-only JSONL state is readable"));
+    const ledger = readJsonReport(paths.ledger(), { date: "", trials: 0, usd: 0 });
+    checks.push(ledger.valid
+        ? check("ledger", "pass", "Trial-spend ledger is readable")
+        : check("ledger", "fail", "Trial-spend ledger is malformed; spending remains disabled until repaired"));
+    let openRouter = false;
+    try {
+        const env = process.env.OPENROUTER_API_KEY;
+        if (env?.trim())
+            openRouter = true;
+        else if (existsSync(paths.envFile()))
+            openRouter = /^\s*OPENROUTER_API_KEY\s*=\s*\S+/m
+                .test(readFileSync(paths.envFile(), "utf8"));
+    }
+    catch { /* optional */ }
+    checks.push(check("openrouter-enrichment", "pass", openRouter ? "Optional OpenRouter enrichment is configured" : "Optional OpenRouter enrichment is not configured"));
+    const activeRoute = stateRead.value.currentRoute ? routeLabel(stateRead.value.currentRoute) : null;
+    return {
+        schemaVersion: 1,
+        ok: !checks.some((item) => item.status === "fail"),
+        checks,
+        summary: { piVersion, routeCount: routes.length, pricedRouteCount: priced.length, activeRoute },
+    };
+}
+export function renderDiagnosticReport(report) {
+    const icon = { pass: "PASS", warn: "WARN", fail: "FAIL" };
+    return report.checks.flatMap((item) => [
+        `${icon[item.status].padEnd(4)} ${item.id.padEnd(22)} ${item.message}`,
+        ...(item.details ?? []).map((detail) => `       - ${detail}`),
+    ]).join("\n");
+}
+/** Exercise core state, routing, recommendation, and event paths without provider calls. */
+export function runOfflineVerify() {
+    const previousHome = process.env.OPENMERIT_HOME;
+    const root = mkdtempSync(join(tmpdir(), "openmerit-verify-"));
+    const checks = [];
+    try {
+        process.env.OPENMERIT_HOME = root;
+        const active = modelRoute("native", "model-a", {
+            input: ["text"], cost: { input: 1, output: 2 }, contextWindow: 8192, maxTokens: 4096,
+        });
+        const candidate = modelRoute("local", "model-b", {
+            input: ["text"], cost: { input: 0.2, output: 0.4 }, contextWindow: 8192, maxTokens: 4096,
+        });
+        writeJson(paths.harnessState(), { schemaVersion: 1, currentRoute: active, routes: [active, candidate] });
+        const state = readJsonReport(paths.harnessState(), {});
+        checks.push(state.valid && state.value.routes?.length === 2
+            ? check("atomic-json", "pass", "Atomic JSON state round-trip passed")
+            : check("atomic-json", "fail", "Atomic JSON state round-trip failed"));
+        const entries = routeCatalog([active, candidate]);
+        const point = (route, score) => {
+            const entry = entries.get(routeKey(route));
+            return { schemaVersion: 1, model: route.meritId, route, score, price: entry.price,
+                priceKnown: entry.priceKnown, ts: new Date().toISOString(), source: "pi_trial" };
+        };
+        const recommendation = buildRecommendation("verify-task", "offline verification", active.meritId, [point(active, 0.6), point(candidate, 0.9)], DEFAULT_POLICY, "/verify/session.jsonl", active);
+        checks.push(recommendation?.recommended.route?.provider === "local" &&
+            recommendation.policy.reasons.length > 0
+            ? check("provider-neutral", "pass", "Exact provider routes and policy reasons survive recommendation")
+            : check("provider-neutral", "fail", "Provider route or policy evidence was lost"));
+        if (recommendation)
+            appendJsonl(paths.recommendations(), recommendation);
+        appendJsonl(paths.recommendations(), { schemaVersion: 0, id: "legacy", status: "dismissed" });
+        appendFileSync(paths.recommendations(), "{interrupted");
+        const history = readJsonlReport(paths.recommendations());
+        checks.push(history.records.length === 2 && history.invalidLines.length === 1
+            ? check("jsonl-recovery", "pass", "Valid records survive legacy and interrupted JSONL lines")
+            : check("jsonl-recovery", "fail", "JSONL recovery did not preserve valid history"));
+        const sink = new LocalJsonlEventSink();
+        if (recommendation)
+            sink.emit(meritEvent("recommendation.created", recommendation));
+        const events = readJsonlReport(paths.events());
+        checks.push(events.records.length === 1
+            ? check("events", "pass", "Versioned neutral event sink round-trip passed")
+            : check("events", "fail", "Neutral event sink round-trip failed"));
+        checks.push(loadPolicy(paths.policy()).mode === "recommend"
+            ? check("compatibility", "pass", "Missing 0.1.x policy fields normalize to safe defaults")
+            : check("compatibility", "fail", "Policy defaults are unsafe"));
+    }
+    catch (error) {
+        checks.push(check("verify", "fail", error.message));
+    }
+    finally {
+        if (previousHome === undefined)
+            delete process.env.OPENMERIT_HOME;
+        else
+            process.env.OPENMERIT_HOME = previousHome;
+        rmSync(root, { recursive: true, force: true });
+    }
+    return {
+        schemaVersion: 1,
+        ok: !checks.some((item) => item.status === "fail"),
+        checks,
+        summary: { piVersion: null, routeCount: 2, pricedRouteCount: 2, activeRoute: "native/model-a via native" },
+    };
+}

package/dist/frontier.js

@@ -1,10 +1,16 @@
 /** Pareto frontier over (score up, price down, latency down) + best-fit/fallback picks. */
+import { pointKey } from "./routes.js";
+function effectivePrice(point) {
+    return point.priceKnown === false ? Number.POSITIVE_INFINITY : point.price;
+}
 /** a dominates b if a is no worse on every objective and strictly better on one. */
 export function dominates(a, b) {
     const aLat = a.latencyMs ?? Number.POSITIVE_INFINITY;
     const bLat = b.latencyMs ?? Number.POSITIVE_INFINITY;
-    const noWorse = a.score >= b.score && a.price <= b.price && aLat <= bLat;
-    const better = a.score > b.score || a.price < b.price || aLat < bLat;
+    const aPrice = effectivePrice(a);
+    const bPrice = effectivePrice(b);
+    const noWorse = a.score >= b.score && aPrice <= bPrice && aLat <= bLat;
+    const better = a.score > b.score || aPrice < bPrice || aLat < bLat;
     return noWorse && better;
 }
 /**
@@ -14,11 +20,11 @@ export function dominates(a, b) {
  */
 export function paretoFrontier(points) {
     const fr = points.filter((p) => !points.some((q) => q !== p && dominates(q, p)));
-    return fr.sort((a, b) => a.price - b.price || b.score - a.score);
+    return fr.sort((a, b) => effectivePrice(a) - effectivePrice(b) || b.score - a.score);
 }
 /** Simple 2-objective chain (score vs price only), cheapest first. */
 export function scorePriceChain(points) {
-    const pts = [...points].sort((a, b) => a.price - b.price || b.score - a.score);
+    const pts = [...points].sort((a, b) => effectivePrice(a) - effectivePrice(b) || b.score - a.score);
     const chain = [];
     let best = -1;
     for (const p of pts) {
@@ -31,7 +37,8 @@ export function scorePriceChain(points) {
 }
 /** Highest score; ties broken by lower price then lower latency. */
 export function pickBest(points) {
-    return [...points].sort((a, b) => b.score - a.score || a.price - b.price || (a.latencyMs ?? 1e18) - (b.latencyMs ?? 1e18))[0];
+    return [...points].sort((a, b) => b.score - a.score || effectivePrice(a) - effectivePrice(b) ||
+        (a.latencyMs ?? 1e18) - (b.latencyMs ?? 1e18))[0];
 }
 /**
  * Fallback for `best`: prefer the cheapest frontier point that still clears
@@ -39,7 +46,7 @@ export function pickBest(points) {
  * `best` itself.
  */
 export function pickFallback(points, best, minScore) {
-    const others = points.filter((p) => p.model !== best.model);
+    const others = points.filter((p) => pointKey(p) !== pointKey(best));
     if (others.length === 0)
         return undefined;
     const fr = paretoFrontier(others).filter((p) => p.score >= minScore);

package/dist/integrations.js

@@ -0,0 +1,19 @@
+/** Stable boundaries for trace inputs and telemetry outputs. */
+import { randomUUID } from "node:crypto";
+import { appendJsonl, paths } from "./store.js";
+import { ingestNewTraces } from "./traces.js";
+export class PiTraceObservationSource {
+    sessionsDir;
+    id = "pi-jsonl";
+    constructor(sessionsDir) {
+        this.sessionsDir = sessionsDir;
+    }
+    read() { return ingestNewTraces(this.sessionsDir); }
+}
+export class LocalJsonlEventSink {
+    id = "local-jsonl";
+    emit(event) { appendJsonl(paths.events(), event); }
+}
+export function meritEvent(type, payload, source = "openmerit") {
+    return { schemaVersion: 1, id: randomUUID(), ts: new Date().toISOString(), source, type, payload };
+}

package/dist/judge.js

@@ -1,5 +1,5 @@
 /** Judge: score a model's answer against a rubric. */
-import { chat, chatWithImages } from "./llm.js";
+import { directChatClient } from "./llm.js";
 export const JUDGE_PREFS = ["openai/gpt-4o-mini", "openai/gpt-4o"];
 const JUDGE_PROMPT = `You are grading a model's answer.
 TASK: {task}
@@ -13,8 +13,8 @@ export function parseObj(txt) {
         throw new Error("no json object found");
     return JSON.parse(m[0]);
 }
-export async function judge(key, judgeModel, task, rubric, answer) {
-    const { content } = await chat(key, judgeModel, JUDGE_PROMPT.replace("{task}", task)
+export async function judge(key, judgeModel, task, rubric, answer, client = directChatClient(key)) {
+    const { content } = await client.chat(judgeModel, JUDGE_PROMPT.replace("{task}", task)
         .replace("{rubric}", rubric)
         .replace("{answer}", answer.slice(0, 16000)), 1024, 0);
     try {
@@ -26,13 +26,13 @@ export async function judge(key, judgeModel, task, rubric, answer) {
     }
 }
 /** Grade OCR answers against the actual uploaded pixels, not answer text alone. */
-export async function judgeWithImages(key, judgeModel, task, rubric, answer, images) {
+export async function judgeWithImages(key, judgeModel, task, rubric, answer, images, client = directChatClient(key)) {
     const prompt = `You are grading an answer to a visual task. Inspect the attached image carefully.\n` +
         `TASK: ${task}\nRUBRIC: ${rubric}\nANSWER: ${answer.slice(0, 16000)}\n` +
         `For invoice extraction, check every visible field and line item against the image. ` +
         `Penalize missing, invented, or mistyped values and invalid JSON. ` +
         `Return ONLY json: {"score": <0..1>, "why": "<one line>"}`;
-    const { content } = await chatWithImages(key, judgeModel, prompt, images, 1024);
+    const { content } = await client.chatWithImages(judgeModel, prompt, images, 1024);
     try {
         const j = parseObj(content);
         return { score: Number(j.score ?? 0), why: String(j.why ?? "") };