auto-model-router 0.26.0 → 0.27.1

package/.omp-plugin/marketplace.json

@@ -7,14 +7,14 @@
   },
   "metadata": {
     "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
-    "version": "0.26.0",
+    "version": "0.27.1",
     "pluginRoot": "."
   },
   "plugins": [
     {
       "name": "auto-model-router",
       "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
-      "version": "0.26.0",
+      "version": "0.27.1",
       "author": {
         "name": "drewappling",
         "email": "drewappling@gmail.com"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-model-router",
-  "version": "0.26.0",
+  "version": "0.27.1",
   "private": false,
   "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
   "type": "module",

package/src/eval/calibrate.ts

@@ -20,6 +20,25 @@ export const MIN_ANCHORS = 3;
 /** Minimum Pearson correlation between raw suite scores and AA before a fit is trusted. */
 export const MIN_R = 0.5;
 
+/**
+ * Anchor models for a run, chosen from the catalog and spread across its score range.
+ *
+ * A fit from three models that all score ~70 describes that cluster, not the scale: the
+ * slope rests on a span of noise. Sampling the extremes and the quartiles gives the least
+ * squares something to work with. Tool-capable only, since the suite calls tools, and never
+ * the target itself. Fewer than `MIN_ANCHORS` scored models available ⇒ empty, and the
+ * caller refuses rather than fitting a line through two points.
+ */
+export function pickAnchors(models: readonly { slug: string; quality: { coding?: number }; supportsTools: boolean }[], target: string): string[] {
+	const scored = models
+		.filter((m) => m.slug !== target && typeof m.quality.coding === "number" && m.supportsTools)
+		.sort((a, b) => (a.quality.coding ?? 0) - (b.quality.coding ?? 0));
+	if (scored.length < MIN_ANCHORS) return [];
+	const last = scored.length - 1;
+	const picks = [0, Math.floor(last / 4), Math.floor(last / 2), Math.floor((3 * last) / 4), last];
+	return [...new Set(picks.map((i) => scored[i]!.slug))];
+}
+
 export interface LineFit {
 	slope: number;
 	intercept: number;

package/src/server/http.ts

@@ -14,6 +14,11 @@ import { invalidateFeedCache } from "../catalog/benchmark-feeds.ts";
 import { applyRequestPolicy, resolveProfile } from "../router/index.ts";
 import { parsePolicyHeader } from "../wire/openai/request.ts";
 import { createDigester } from "./digest.ts";
+import { runEval, type Completer } from "../eval/run.ts";
+import type { QualityAxis } from "../config/types.ts";
+import { fitCalibration, pickAnchors, toLocalFeedScores, MIN_ANCHORS } from "../eval/calibrate.ts";
+import { makeJudge } from "../eval/judge.ts";
+import { loadLocalScores, saveLocalScores } from "../catalog/benchmark-feeds.ts";
 import { advise } from "./advise.ts";
 import { TIER_ORDER, type Tier } from "../router/types.ts";
 import { baselinePrices, buildUsageReport, renderUsageReport } from "../cost/report.ts";
@@ -696,6 +701,73 @@ export function startServer(cfg: RouterConfig): StartedServer {
 					if (result.deleted > 0) log.info("pruned ledger rows past retention", { deleted: result.deleted, retentionDays: cfg.ledger.retentionDays });
 					return json({ ...result, retentionDays: cfg.ledger.retentionDays });
 				}
+				if (req.method === "POST" && url.pathname === "/v1/router/benchmark") {
+					// Score one model with our OWN eval suite, for the models no feed covers: a
+					// third of a live catalog carries no published score on any axis, and a model
+					// with no score cannot clear any floor, so routing may never pick it.
+					//
+					// A raw mean is not comparable with a published index, so the run also evals
+					// ANCHOR models that do have published scores and fits raw → published per
+					// axis. Anchors are chosen from the catalog across its score range unless the
+					// caller names them, so one slug is all this needs.
+					//
+					// The result is written to `local_scores`, which only reaches routing when
+					// `benchmarks.useLocalScores` is on — measuring a model and trusting it are
+					// deliberately two decisions.
+					const body = (await req.json().catch(() => null)) as Record<string, unknown> | null;
+					const slug = typeof body?.slug === "string" ? body.slug.trim() : "";
+					if (slug === "") return wireErrorResponse({ status: 400, code: "invalid_request_error", message: "slug required" });
+					const snap = catalog.peekAll?.() ?? catalog.peek();
+					const models = snap?.models ?? [];
+					if (models.find((m) => m.slug === slug) === undefined) return wireErrorResponse({ status: 404, code: "invalid_request_error", message: `${slug} is not in the catalog` });
+					const named = Array.isArray(body?.anchors) ? (body.anchors as unknown[]).filter((a): a is string => typeof a === "string") : [];
+					const anchors = named.length > 0 ? [...new Set(named)] : pickAnchors(models, slug);
+					if (anchors.length < MIN_ANCHORS) return wireErrorResponse({ status: 422, code: "invalid_request_error", message: `need at least ${MIN_ANCHORS} scored, tool-capable anchor models to calibrate against` });
+					// A rate-limited dispatch is not a failed task. Ollama Cloud answers "too many
+					// concurrent requests" well below four models in flight, and the runner folds a
+					// throwing completion in as grade 0 — which would score a provider's throttle as
+					// the model being wrong. Retry with backoff, and keep the default concurrency
+					// low enough that the throttle is rarely reached in the first place.
+					const complete: Completer = async (target, messages) => {
+						let last: unknown = null;
+						for (let attempt = 0; attempt < 4; attempt++) {
+							if (attempt > 0) {
+								const { promise, resolve } = Promise.withResolvers<void>();
+								setTimeout(resolve, attempt * 4000);
+								await promise;
+							}
+							try {
+								const out = await upstream.complete({ model: target, stream: false, temperature: 0, max_tokens: 1024, messages }, AbortSignal.timeout(120_000));
+								return out.text;
+							} catch (err) {
+								last = err;
+							}
+						}
+						throw last instanceof Error ? last : new Error(String(last));
+					};
+					const judgeSlug = typeof body?.judge === "string" && body.judge !== "" ? body.judge : "";
+					const asked = typeof body?.concurrency === "number" ? Math.floor(body.concurrency) : 2;
+					const results = await runEval({
+						slugs: [slug, ...anchors],
+						complete,
+						concurrency: Math.min(Math.max(1, asked), 8),
+						...(judgeSlug === "" ? {} : { judge: makeJudge(complete, judgeSlug) }),
+					});
+					const target = results[0]!;
+					const anchorResults = results.slice(1);
+					const published = (s: string, axis: QualityAxis): number | undefined => models.find((m) => m.slug === s)?.quality[axis];
+					const cal = fitCalibration(anchorResults, published);
+					const authorOf = (s: string): string => models.find((m) => m.slug === s)?.author ?? "";
+					const fresh = toLocalFeedScores([target], cal, authorOf);
+					if (fresh.length === 0) {
+						return json({ slug, anchors, calibrated: null, raw: target.axes, errors: target.errors, applied: false, reason: "no axis produced a usable fit; try more or better-spread anchors" });
+					}
+					// Merge, never replace: other models' measurements are not this run's to discard.
+					const kept = loadLocalScores(db).filter((s) => s.key !== fresh[0]!.key);
+					saveLocalScores(db, [...kept, ...fresh]);
+					log.info("benchmarked a model with the local eval suite", { slug, anchors: anchors.length, errors: target.errors, useLocalScores: cfg.benchmarks.useLocalScores });
+					return json({ slug, anchors, raw: target.axes, calibrated: fresh[0], errors: target.errors, applied: cfg.benchmarks.useLocalScores });
+				}
 				if (req.method === "POST" && url.pathname === "/v1/router/feedback") {
 					// A user verdict on the newest routed turn of an omp session.
 					const body = (await req.json().catch(() => null)) as Record<string, unknown> | null;

package/test/eval.test.ts

@@ -3,7 +3,7 @@ import { describe, expect, test } from "bun:test";
 import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
 import { applyFeedScores, loadLocalScores, saveLocalScores, type FeedScore } from "../src/catalog/benchmark-feeds.ts";
 import { answerScore, extractJson, isRefusalOrEmpty, jsonField, tokenCoverage } from "../src/eval/grade.ts";
-import { applyFit, fitAxis, fitCalibration, toLocalFeedScores, MIN_ANCHORS } from "../src/eval/calibrate.ts";
+import { applyFit, fitAxis, fitCalibration, pickAnchors, toLocalFeedScores, MIN_ANCHORS } from "../src/eval/calibrate.ts";
 import { runEval, type EvalResult } from "../src/eval/run.ts";
 import { makeJudge, parseScore } from "../src/eval/judge.ts";
 import type { EvalTask, JudgedTask } from "../src/eval/tasks.ts";
@@ -54,6 +54,24 @@ describe("calibration", () => {
 		expect(fitAxis([{ raw: 0.8, aa: 40 }, { raw: 0.5, aa: 60 }, { raw: 0.2, aa: 80 }])).toBeNull();
 	});
 
+	test("pickAnchors spreads over the score range, skips the target and the unscored", () => {
+		const m = (slug: string, coding: number | undefined, supportsTools = true) => ({ slug, quality: coding === undefined ? {} : { coding }, supportsTools });
+		const catalog = [m("a/10", 10), m("a/30", 30), m("a/50", 50), m("a/70", 70), m("a/90", 90), m("a/target", undefined), m("a/notools", 60, false)];
+		const picked = pickAnchors(catalog, "a/target");
+		// Both extremes, so the fitted line spans the scale rather than a cluster.
+		expect(picked).toContain("a/10");
+		expect(picked).toContain("a/90");
+		expect(picked.length).toBeGreaterThanOrEqual(MIN_ANCHORS);
+		// An unscored model cannot anchor anything, and one that cannot call tools would fail
+		// the suite's tool tasks for a reason unrelated to its quality.
+		expect(picked).not.toContain("a/target");
+		expect(picked).not.toContain("a/notools");
+		// Refuses rather than fitting a line through too few points.
+		expect(pickAnchors([m("a/10", 10), m("a/90", 90)], "a/target")).toEqual([]);
+		// The target is excluded even when it is itself scored (a re-measurement).
+		expect(pickAnchors(catalog, "a/50")).not.toContain("a/50");
+	});
+
 	test("fitCalibration + toLocalFeedScores place a target on the AA scale", () => {
 		expect(MIN_ANCHORS).toBe(3);
 		const anchors: EvalResult[] = [