openmerit 0.1.10 → 0.1.12

package/docs/ux-reference.md

@@ -1,6 +1,6 @@
 # Pi UX behavioral reference
 
-This document records user-visible behaviors from the earlier OpenMerit implementation. It is not an implementation specification; the current implementation carries forward only the behaviors listed below, rather than its source code, storage schema, or internal contracts.
+This document is a **historical evaluation-adapter reference**, not the current Pi interface specification. The unreleased Pi 1.5.0 native Scout replacement does not carry forward the `/openmerit` commands, lifecycle, pause controls, or scheduling described here. See [the current Pi guide](pi-extension.md). The list below preserves earlier UX ideas for custom evaluation harnesses only.
 
 ## Behaviors worth preserving
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openmerit",
-  "version": "0.1.10",
+  "version": "0.1.12",
   "description": "Evidence-driven model decisions for coding harnesses.",
   "type": "module",
   "bin": {
@@ -61,7 +61,8 @@
     "access": "public"
   },
   "workspaces": [
-    "packages/*"
+    "packages/*",
+    "scout/agent"
   ],
   "engines": {
     "node": ">=22.19.0"
@@ -69,7 +70,7 @@
   "scripts": {
     "check": "npm run check --workspaces --if-present",
     "test": "npm run test --workspaces --if-present",
-    "typecheck": "tsc -p tsconfig.json && tsc -p tsconfig.scout.json",
+    "typecheck": "tsc -p tsconfig.json && tsc -p tsconfig.scout.json && tsc -p scout/agent/tsconfig.json",
     "check:typescript": "node scripts/check-typescript.ts",
     "types:workers": "node scripts/worker-types.ts",
     "check:worker-types": "node scripts/worker-types.ts --check",
@@ -81,7 +82,8 @@
     "dash": "node packages/terminal/src/cli.mjs dash",
     "build:docs": "node scripts/build-docs.mjs",
     "check:docs": "npm run build:docs && node --test scripts/docs.test.mjs",
-    "check:demo": "node --test demo/test/*.test.mjs",
+    "memo": "node demo/memo/preview.ts",
+    "check:demo": "npm ci --ignore-scripts --prefix demo && npm run build:docs && tsc -p tsconfig.memo.json && node --test demo/test/*.test.mjs demo/test/*.test.ts",
     "prepare": "npm run build",
     "verify": "npm test && npm run check && npm run typecheck && npm run check:typescript && npm run build && npm run check:demo && npm run check:scout && npm run check:docs && npm run build:scout && git diff --check",
     "prepublishOnly": "npm run verify",
@@ -89,11 +91,10 @@
     "deploy:dash-preview": "node scripts/deploy.ts terminal-preview",
     "scout": "node scout/dev.mjs",
     "scout:refresh": "node scout/refresh.mjs",
-    "check:scout": "node --test scout/test/*.test.mjs scout/test/*.test.ts && node --check scout/worker.mjs && node --check site/scout/scout.js",
+    "check:scout": "node --test scout/test/*.test.mjs scout/test/*.test.ts && node --check scout/worker.mjs",
     "build:scout": "node scripts/build-scout.mjs"
   },
   "dependencies": {
-    "@earendil-works/pi-ai": "0.87.0",
     "@iarna/toml": "2.2.5",
     "@lezer/javascript": "1.5.6",
     "@lezer/python": "1.1.19",
@@ -102,7 +103,7 @@
     "yaml": "2.9.1"
   },
   "peerDependencies": {
-    "@earendil-works/pi-coding-agent": ">=0.87.0 <0.88.0"
+    "@earendil-works/pi-coding-agent": "1.5.0"
   },
   "peerDependenciesMeta": {
     "@earendil-works/pi-coding-agent": {
@@ -110,9 +111,11 @@
     }
   },
   "devDependencies": {
+    "@earendil-works/pi-coding-agent": "1.5.0",
     "@clerk/backend": "3.26.1",
     "@clerk/shared": "4.41.0",
     "@resvg/resvg-js": "2.6.2",
+    "@types/markdown-it": "14.1.2",
     "@types/node": "^22.18.6",
     "@xyflow/react": "12.12.0",
     "elkjs": "0.12.0",
@@ -120,6 +123,7 @@
     "fflate": "0.8.2",
     "liveline": "0.0.7",
     "markdown-it": "15.0.2",
+    "miniflare": "5.20260921.0-alpha",
     "minisearch": "7.2.0",
     "morphdom": "2.7.8",
     "react": "19.3.0",

package/dist/pi/src/scheduler.d.ts

@@ -1,11 +0,0 @@
-export interface PiWakeupOptions {
-    readonly projectRoot: string;
-    readonly extensionPath: string;
-    readonly provider: string;
-    readonly modelId: string;
-}
-export declare function piWakeupLabel(projectRoot: string): string;
-export declare function piWakeupPlistPath(projectRoot: string): string;
-export declare function renderPiWakeupPlist(options: PiWakeupOptions, piPath: string, runnerPath: string): string;
-export declare function provisionPiWakeup(options: PiWakeupOptions): Promise<string>;
-export declare function removePiWakeup(projectRoot: string): Promise<boolean>;

package/dist/pi/src/scheduler.js

@@ -1,137 +0,0 @@
-import { execFile } from "node:child_process";
-import { createHash } from "node:crypto";
-import { constants } from "node:fs";
-import { access, mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises";
-import { homedir } from "node:os";
-import { basename, dirname, extname, join, resolve } from "node:path";
-import { promisify } from "node:util";
-const execFileAsync = promisify(execFile);
-const POLL_SECONDS = 300;
-function projectHash(projectRoot) {
-    return createHash("sha256").update(resolve(projectRoot)).digest("hex").slice(0, 16);
-}
-export function piWakeupLabel(projectRoot) {
-    return `com.openmerit.pi.${projectHash(projectRoot)}`;
-}
-export function piWakeupPlistPath(projectRoot) {
-    return join(homedir(), "Library", "LaunchAgents", `${piWakeupLabel(projectRoot)}.plist`);
-}
-function managedMarker(projectRoot) {
-    return `<!-- openmerit-managed:${projectHash(projectRoot)} -->`;
-}
-function xml(value) {
-    return value.replaceAll("&", "&amp;").replaceAll("<", "&lt;")
-        .replaceAll(">", "&gt;").replaceAll('"', "&quot;").replaceAll("'", "&apos;");
-}
-async function executable(name) {
-    for (const directory of (process.env.PATH ?? "").split(":")) {
-        if (!directory)
-            continue;
-        const path = join(directory, name);
-        try {
-            await access(path, constants.X_OK);
-            return path;
-        }
-        catch { /* try the next PATH entry */ }
-    }
-    throw new Error(`${name} is not on PATH`);
-}
-export function renderPiWakeupPlist(options, piPath, runnerPath) {
-    const args = [process.execPath, runnerPath, resolve(options.projectRoot), piPath,
-        resolve(options.extensionPath), options.provider, options.modelId];
-    const argumentsXml = args.map((argument) => `      <string>${xml(argument)}</string>`).join("\n");
-    const path = [dirname(process.execPath), dirname(piPath), "/usr/bin", "/bin", "/usr/sbin", "/sbin"].join(":");
-    return `<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-${managedMarker(options.projectRoot)}
-<plist version="1.0">
-<dict>
-  <key>Label</key><string>${xml(piWakeupLabel(options.projectRoot))}</string>
-  <key>ProgramArguments</key><array>
-${argumentsXml}
-  </array>
-  <key>WorkingDirectory</key><string>${xml(resolve(options.projectRoot))}</string>
-  <key>EnvironmentVariables</key><dict>
-    <key>PATH</key><string>${xml(path)}</string>
-  </dict>
-  <key>RunAtLoad</key><true/>
-  <key>StartInterval</key><integer>${POLL_SECONDS}</integer>
-</dict>
-</plist>
-`;
-}
-async function loaded(label) {
-    try {
-        await execFileAsync("launchctl", ["print", `gui/${process.getuid?.()}/${label}`]);
-        return true;
-    }
-    catch {
-        return false;
-    }
-}
-export async function provisionPiWakeup(options) {
-    if (process.platform !== "darwin" || process.getuid === undefined) {
-        throw new Error("Persistent Pi wakeups require a macOS user LaunchAgent.");
-    }
-    if (!options.provider || !options.modelId)
-        throw new Error("Select an authenticated Pi model before scheduling.");
-    const piPath = await executable("pi");
-    const auth = await execFileAsync(piPath, ["auth", "check", "--model", `${options.provider}/${options.modelId}`], {
-        env: {
-            HOME: homedir(),
-            PATH: [dirname(process.execPath), dirname(piPath), "/usr/bin", "/bin", "/usr/sbin", "/sbin"].join(":"),
-        },
-    });
-    if (auth.stdout.trim() !== "ready") {
-        throw new Error(`Pi authentication is not ready for ${options.provider}/${options.modelId} in the background environment.`);
-    }
-    const extensionPath = resolve(options.extensionPath);
-    const runnerPath = join(dirname(extensionPath), `wakeup${extname(extensionPath)}`);
-    await access(runnerPath, constants.R_OK);
-    const path = piWakeupPlistPath(options.projectRoot);
-    const content = renderPiWakeupPlist({ ...options, extensionPath }, piPath, runnerPath);
-    let prior;
-    try {
-        prior = await readFile(path, "utf8");
-    }
-    catch (error) {
-        if (error.code !== "ENOENT")
-            throw error;
-    }
-    if (prior && !prior.includes(managedMarker(options.projectRoot))) {
-        throw new Error(`Refusing to replace unmanaged LaunchAgent ${basename(path)}.`);
-    }
-    const label = piWakeupLabel(options.projectRoot);
-    if (prior === content && await loaded(label))
-        return path;
-    if (await loaded(label))
-        await execFileAsync("launchctl", ["bootout", `gui/${process.getuid()}/${label}`]);
-    await mkdir(dirname(path), { recursive: true });
-    const temporary = `${path}.${process.pid}.tmp`;
-    await writeFile(temporary, content, { mode: 0o600 });
-    await rename(temporary, path);
-    await execFileAsync("launchctl", ["bootstrap", `gui/${process.getuid()}`, path]);
-    return path;
-}
-export async function removePiWakeup(projectRoot) {
-    if (process.platform !== "darwin" || process.getuid === undefined)
-        return false;
-    const path = piWakeupPlistPath(projectRoot);
-    let content;
-    try {
-        content = await readFile(path, "utf8");
-    }
-    catch (error) {
-        if (error.code === "ENOENT")
-            return false;
-        throw error;
-    }
-    if (!content.includes(managedMarker(projectRoot))) {
-        throw new Error(`Refusing to remove unmanaged LaunchAgent ${basename(path)}.`);
-    }
-    const label = piWakeupLabel(projectRoot);
-    if (await loaded(label))
-        await execFileAsync("launchctl", ["bootout", `gui/${process.getuid()}/${label}`]);
-    await unlink(path);
-    return true;
-}

package/dist/pi/src/wakeup.d.ts

@@ -1,2 +0,0 @@
-import type { OpenMeritProjectState } from "../../protocol/src/index.ts";
-export declare function isPiWakeupDue(state: OpenMeritProjectState, now?: string): boolean;

package/dist/pi/src/wakeup.js

@@ -1,108 +0,0 @@
-import { spawn } from "node:child_process";
-import { writeFile } from "node:fs/promises";
-import { join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-import { buildAutomationPlan, OpenMeritCoordinator, ProjectStore } from "../../core/src/index.js";
-const wakeupDescriptor = {
-    id: "pi-launchagent", name: "Pi LaunchAgent", version: "0.87", protocolVersions: ["0.1"],
-    capabilities: [], executionModes: ["non_interactive"],
-    structuredOutput: { schemaDialect: "https://json-schema.org/draft/2020-12/schema", presentation: "adapter_validation", enforcement: "adapter" },
-    automation: {
-        supportedSignals: ["product_task_completed", "metric_window_available", "scheduled_tick", "model_catalog_changed", "verification_window_completed"],
-        persistentScheduling: true, backgroundExecution: true, wakeupProvisioning: "infrastructure_change",
-    },
-};
-export function isPiWakeupDue(state, now = new Date().toISOString()) {
-    const execution = state.cycle.policy?.checkPolicy.execution;
-    if (execution?.mode !== "persistent" || !execution.infrastructureChangesAllowed || state.activeIntent || state.automation.paused)
-        return false;
-    if (state.automation.pendingIntentKind)
-        return true;
-    const at = buildAutomationPlan(state, wakeupDescriptor).nextWakeupAt;
-    return at !== undefined && Date.parse(at) <= Date.parse(now);
-}
-async function run() {
-    const [projectRoot, piPath, extensionPath, provider, modelId] = process.argv.slice(2);
-    if (!projectRoot || !piPath || !extensionPath || !provider || !modelId)
-        process.exitCode = 2;
-    if (process.exitCode)
-        return;
-    const store = new ProjectStore(projectRoot);
-    const state = await store.readState();
-    if (!state || !isPiWakeupDue(state))
-        return;
-    const startedAt = new Date().toISOString();
-    const coordinator = new OpenMeritCoordinator(store, {
-        descriptor: wakeupDescriptor,
-        async dispatch() { return { accepted: false, reason: "The wakeup runner does not dispatch harness work directly." }; },
-    });
-    const check = state.automation.pendingIntentKind === "run_assessment"
-        ? await coordinator.assessBaseline(startedAt)
-        : state.automation.pendingIntentKind
-            ? undefined
-            : await coordinator.recordAutomationSignal({
-                protocolVersion: "0.1", id: `wakeup-${crypto.randomUUID()}`,
-                type: "scheduled_tick", occurredAt: startedAt, targetId: state.cycle.targetId,
-            });
-    if (check && check.decision.action !== "issue_intent") {
-        await writeFile(join(store.directory, "scheduler-status.json"), JSON.stringify({
-            schemaVersion: 1, startedAt, completedAt: new Date().toISOString(), exitCode: 0,
-            decision: check.decision.reason,
-        }, null, 2) + "\n", { mode: 0o600 });
-        return;
-    }
-    const configuredTimeout = Number(process.env.OPENMERIT_WAKEUP_TIMEOUT_MS);
-    const runLimitMs = Number.isFinite(configuredTimeout) && configuredTimeout >= 5_000
-        ? Math.min(configuredTimeout, 15 * 60_000)
-        : 15 * 60_000;
-    const deadline = Date.now() + runLimitMs;
-    const invokePi = async (prompt) => {
-        const child = spawn(piPath, [
-            "--no-extensions", "--extension", extensionPath, "--provider", provider, "--model", modelId,
-            ...(process.env.OPENMERIT_WAKEUP_DEBUG === "1" ? ["--mode", "json"] : []),
-            "--no-session", "--print", prompt,
-        ], { cwd: projectRoot, stdio: process.env.OPENMERIT_WAKEUP_DEBUG === "1" ? "inherit" : "ignore", env: process.env });
-        return new Promise((resolve, reject) => {
-            let timedOut = false;
-            const timeout = setTimeout(() => {
-                timedOut = true;
-                child.kill("SIGTERM");
-                setTimeout(() => child.kill("SIGKILL"), 5_000).unref();
-            }, Math.max(1, deadline - Date.now()));
-            child.once("error", (error) => { clearTimeout(timeout); reject(error); });
-            child.once("exit", (code) => { clearTimeout(timeout); resolve(timedOut ? 124 : code ?? 1); });
-        });
-    };
-    let exitCode = await invokePi("Run the due OpenMerit scheduled check for this configured project. " +
-        "On startup, OpenMerit may create an active intent in .openmerit/state.json. Read that intent and .openmerit/config.json, " +
-        "perform only its authorized work, and call openmerit_complete_intent exactly once with verified evidence. " +
-        "For baseline assessment, if no application metric windows exist, report insufficiency promptly; do not wait for new data or invent samples. " +
-        "If evidence is unavailable, report failure honestly. Never change Pi's model or an unauthorized application route.");
-    const issued = await store.readState();
-    if (exitCode === 0 && issued?.activeIntent && !state.activeIntent && Date.now() < deadline) {
-        const intent = issued.activeIntent;
-        exitCode = await invokePi(`Complete the already active OpenMerit ${intent.kind} intent ${intent.id}. ` +
-            "Read .openmerit/config.json and the project evidence. Follow the requested outcome and constraints below. " +
-            "Use the registered openmerit_complete_intent tool exactly once, with verified evidence; if evidence is unavailable, report failure honestly. " +
-            "Never change Pi's model or the application model unless this intent explicitly authorizes that target-bound change.\n\n" +
-            JSON.stringify(intent));
-        if ((await store.readState())?.activeIntent?.id === intent.id)
-            exitCode = 1;
-    }
-    const finalState = await store.readState();
-    if (finalState?.activeIntent && exitCode === 0)
-        exitCode = 1;
-    if (finalState?.automation.pendingIntentKind && exitCode === 0)
-        exitCode = 1;
-    await writeFile(join(store.directory, "scheduler-status.json"), JSON.stringify({
-        schemaVersion: 1, startedAt, completedAt: new Date().toISOString(), exitCode,
-        ...(finalState?.activeIntent ? { activeIntentId: finalState.activeIntent.id } : {}),
-    }, null, 2) + "\n", { mode: 0o600 });
-    process.exitCode = exitCode;
-}
-if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) {
-    run().catch((error) => {
-        process.stderr.write(`OpenMerit Pi wakeup failed: ${error instanceof Error ? error.message : String(error)}\n`);
-        process.exitCode = 1;
-    });
-}

package/docs/memo.md

@@ -1,126 +0,0 @@
-# Optimum Inference For All
-
-*Lazim*
-
-How can we accelerate the distribution of intelligence?
-
-Building and deploying more models and improving the compute landscape is where most of the energy being spent right now. I have another, less explored direction in my mind, which I call, the optimum inference problem.
-
-I see a substantial part of what limits autonomous systems is how the models are selected and assessed throughout their lifecycle. It's a harness-level infrastructure concern.
-
-#### Does this need to get solved?
-
-In companies that I had the privilege of working with, I have built or maintained or used a rudimentary version of this. When budget constraints strangled the roadmap, teams scrambled their way out by building something enough to justify why they need to pay y amount of money to solve x task. This was never a maintained system nor it was ever prioritised because it was seen as an optional optimisation issue and overlooked the power it had over the dissemination of intelligence inside the org and the products they ship.
-
-If it was ever solved, it was always at a task level and if it had room and engineers to pressure, maybe at the agent level but rarely. even rarer at the harness level. As the system develops, its inference choices should develop with it. I don't think every team should have to build and maintain that process and stack themselves. We are building OpenMerit to solve this. In the Mission of optimum inference for all.
-
-#### What optimum means
-
-Optimum depends on the work. We need to know what counts as success, how reliably it must be delivered and the customer's constraints on cost, latency and operation.
-
-A cheaper model that needs repeated attempts may make the completed job more expensive. A stronger model may add cost without improving the result. We need to account for the whole workflow, including anything a person has to fix.
-
-The range of models matters here. There are frontier models from leading labs, general-purpose open-weight models, fine-tuned and distilled variants on Hugging Face, and specialists for coding, retrieval, vision and speech. Jev, by TypeSafe, is designed for fast structured decisions. Its recent release shows there is demand for a different breed of system-one models. Smaller models can run task-specific programs optimised through tools such as DSPy. An agent may have a place for several of these. We want OpenMerit to assess all of them on the work they can actually do, to allocate as it required.
-
-The right deployment could use a direct API from the lab, a router, an inference cloud or a customer-operated model. It could combine them. A company using only commercial APIs should get a complete product. So should one operating entirely within its own environment.
-
-Hardware is equally part of the decision. For customer-operated deployments, we need to account for available compute and memory, concurrent workloads and the cost of loading or swapping models. With managed APIs, we work with measurable outcomes and the constraints the provider exposes.
-
-Local models, open weights and fine-tunes extend the possibilities. We have no preference for moving customers towards them. If the industry the customer operates in demands sovereignty then it should be able to, without sacrificing the intelligence spread.
-
-Better allocation also matters beyond the bill. Where a smaller configuration meets the requirements and the freed capacity can be reused, the same resources can support more work. That is part of what I mean by distributing intelligence.
-
-#### What we are building
-
-There's no end-to-end product or platform clarity at the moment, but on a broader spectrum, you could place OpenMerit as infrastructure for assessing, benchmarking, selecting, assigning and maintaining all model choices within your harness.
-
-It operates at task, agent and harness levels. An assignment should suit the individual task while accounting for the agent's overall result. Across a harness, those choices have to work within shared budgets and capacity.
-
-To get there, OpenMerit first needs to establish which options the customer can use. It should then coordinate comparisons on representative work using their success criteria. Production evidence tells us how the current configuration behaves. It does not establish how an untested alternative would perform.
-
-After deployment, OpenMerit should check whether the expected improvement happened and decide where further investigation is worthwhile. Keeping the current assignment is a valid decision.
-
-The interface should be agent-native. A harness should be able to request a comparison and retrieve an assignment with its evidence programmatically. Within the customer's permissions, OpenMerit should coordinate validated changes through the existing infrastructure, then retain or reverse them based on the results.
-
-We will remain harness-agnostic. The customer's harness owns execution and coordination. Its providers and serving systems run inference. We integrate with its observability and evaluation tools. We will not build those products ourselves as much as we can, and production traffic will not have to pass through us to fulfil the task, but sure can control the traffic based on the evidence.
-
-#### A continuing research problem
-
-Most of the components are still very nascent and will increasingly need newer ways of operating as models become increasingly intelligent and the compute becomes increasingly efficient. Hence we see OpenMerit as a research lab as much as a product company.
-
-To zoom out, we see optimum inference as a long-term research and deployment effort. There is no final configuration we can find, hand over and consider the problem solved.
-
-We need better methods for deciding how much evidence an assignment requires and when that evidence needs updating. Testing every option is expensive. Testing too little can leave us maintaining a poor decision. OpenMerit should spend its testing budget where additional evidence is likely to be useful.
-
-We also need to understand how individual choices affect the complete outcome. A model that performs well on one step may leave the next step with harder work. Several assignments that look sensible separately may compete for the same compute.
-
-These are some of the questions we will have to study in production. Deployment will expose cases our experiments missed and show us which improvements were worth pursuing. That experience should inform the next investigation and the methods we build.
-
-OpenMerit should support this work throughout a system's creation, iteration and ongoing operation.
-
-#### Where we start
-
-Our first customers are engineering and AI teams running repeatable production workflows with measurable outcomes and meaningful inference costs or capacity constraints.
-
-One way to think about it goes like this. A developer or their coding agent connects one workflow. OpenMerit establishes a baseline, compares eligible alternatives and returns a decision they can act on. They keep using it because maintaining that assignment is useful, then connect more work.
-
-The first line of product combines L1, organisation-wide understanding, with L2, repeatable workflows. Individual workflows provide the evidence for a wider view of which models are assigned where, why those choices were made and which ones deserve reassessment and swapping.
-
-L3, adaptive agents, is the longer-term research direction. Agents may revise their plans or create subagents while working. We want to support inference decisions during that execution without taking over the harness's responsibility.
-
-L2 should support a business on its own. It also gives us production experience for the harder questions ahead.
-
-#### A separate company, not a feature
-
-We want OpenMerit to follow the customer's work across whichever models and infrastructure are appropriate.
-
-A customer should be able to change providers, introduce a fine-tune or replace a harness without rebuilding how they make model decisions. Some evidence will need to be revalidated. The history of what they tried and why they changed should remain useful.
-
-That continuing responsibility is substantial enough to build a company around. We intend to earn it through the quality of our decisions and the maintenance we remove from the customer's team.
-
-We are thinking of building an open-core, with hosted and customer-operated options, and charge for continuous assessment and decision management. We will not mark up inference.
-
-#### We need to prove
-
-Our evaluations have to reflect the customer's actual work. We should begin with representative examples and usable success criteria, and limit autonomous changes to decisions the evidence supports. Where the criteria are inadequate, we need to identify that before automating the wrong objective.
-
-Our own costs belong in the calculation. Comparisons consume resources. Switching can introduce failures and maintenance. OpenMerit needs to recognise when an apparent improvement is too small to pursue.
-
-We also need dependable integrations. We should start with a narrow set and make them work well before extending our coverage. Supporting different infrastructure does not require supporting everything immediately.
-
-The standard is a measurable improvement after accounting for testing, switching and running OpenMerit. Depending on the workload, that means better outcomes within the same budget, lower total cost at the required quality or more completed work with available capacity.
-
-#### What is already here in some shape or form
-
-There are useful tools here, including ones that already make model decisions. OpenRouter's Auto Router selects models for incoming requests. Portkey's gateway supports conditional routing, fallbacks and controlled testing of new models in production. Not Diamond lets teams train routers on their own data and evaluation criteria, with quality, cost and latency preferences.
-
-Langfuse helps teams compare changes through evaluation scores, cost, latency and individual failures. Braintrust supports experiments and continuous production evaluation, including bringing production examples back into testing. These tools already help teams investigate and improve their systems.
-
-Many of these capabilities overlap with parts of what we want to explore. They are also components we expect to work with.
-
-We consider optimum inference far from solved. A system needs to determine when the evidence behind an assignment is no longer enough, which alternatives deserve testing and whether a change will improve the completed work after its costs are included. It needs to maintain that process across changes to its tasks, models and infrastructure.
-
-OpenMerit exists to take on that continuing responsibility. The tools for an investigation should be available to the harness, along with the means to act on the result. The customer should not inherit a permanent engineering project to keep it all working.
-
-#### Path ahead
-
-We want teams to know when the strongest model is worth paying for, when an inexpensive alternative is sufficient and when a different deployment would let them do more.
-
-We want those decisions to become part of what a system can maintain for itself. Its engineers should be able to define the work and its constraints without inheriting a permanent job of reconsidering every model assignment.
-
-How much more useful work can we make possible with the intelligence and compute available to us?
-
----
-
-**Additional Reading**
-
-1. [**"The Shift from Models to Compound AI Systems"**](https://bair.berkeley.edu/blog/2024/02/18/compound-ai-systems/) — Matei Zaharia et al., BAIR. (February 18, 2024)
-2. [**"DSPy: Program, Don't Prompt"**](https://dspy.ai/current/) — DSPy / Stanford NLP.
-3. [**"Optimas: Optimizing Compound AI Systems with Globally Aligned Local Rewards"**](https://optimas.stanford.edu/) — Shirley Wu et al., arXiv. (February 9, 2026)
-4. [**"AI Agents That Matter"**](https://arxiv.org/abs/2407.01502) — Sayash Kapoor, Benedikt Stroebl, Zachary S. Siegel, Nitya Nadgir, and Arvind Narayanan, arXiv. (July 1, 2024)
-5. [**"Training a Custom Router"**](https://docs.notdiamond.ai/docs/router-training-quickstart) — Not Diamond.
-6. [**"Auto Router — Intelligent Model Selection"**](https://openrouter.ai/docs/guides/routing/routers/auto-router) — OpenRouter.
-7. [**"AI Gateway"**](https://portkey.ai/docs/product/ai-gateway) — Portkey.
-8. [**"Compare Experiments"**](https://langfuse.com/docs/evaluation/experiments/compare-experiments) — Langfuse.
-9. [**"Evaluate Systematically"**](https://www.braintrust.dev/docs/evaluate) — Braintrust.
-10. [**"Introducing System One Models & Jev"**](https://typesafe.ai/blog/introducing-system-one-models-and-jev) — Diogo Almeida, TypeSafe AI. (September 15, 2026)