auto-model-router 0.30.3 → 0.31.0

package/.omp-plugin/marketplace.json

@@ -7,14 +7,14 @@
   },
   "metadata": {
     "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
-    "version": "0.30.3",
+    "version": "0.31.0",
     "pluginRoot": "."
   },
   "plugins": [
     {
       "name": "auto-model-router",
       "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
-      "version": "0.30.3",
+      "version": "0.31.0",
       "author": {
         "name": "drewappling",
         "email": "drewappling@gmail.com"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-model-router",
-  "version": "0.30.3",
+  "version": "0.31.0",
   "private": false,
   "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
   "type": "module",

package/src/eval/calibrate.ts

@@ -58,13 +58,26 @@ export interface AnchorPoint {
 }
 
 /**
- * OLS fit, or null when the calibration cannot be trusted: too few points, no
- * spread, a non-positive slope, or weak correlation. A suite that does not track
- * AA positively and with real correlation would turn a target's score into noise
- * dressed as signal, so we refuse it and emit nothing for that axis.
+ * The raw range the anchors must actually cover before a line through them means anything.
+ *
+ * Correlation alone does NOT catch a compressed fit: three anchors published 20/50/80 that
+ * our suite scores 0.96/0.97/0.98 correlate at r = 1.0, and the line they define has a slope
+ * of ~3000 index points per unit of raw score. That fit is arithmetically perfect and
+ * completely useless — it is how a model published at 39.5 was calibrated to 22.8. If the
+ * anchors barely differ on our suite, our suite cannot place anything between them.
+ */
+export const MIN_RAW_SPREAD = 0.15;
+
+/**
+ * OLS fit, or null when the calibration cannot be trusted: too few points, too narrow a raw
+ * range, no spread, a non-positive slope, or weak correlation. A suite that does not track
+ * AA positively, with real correlation, over a real range would turn a target's score into
+ * noise dressed as signal, so we refuse it and emit nothing for that axis.
  */
 export function fitAxis(points: readonly AnchorPoint[]): LineFit | null {
 	if (points.length < MIN_ANCHORS) return null;
+	const raws = points.map((p) => p.raw);
+	if (Math.max(...raws) - Math.min(...raws) < MIN_RAW_SPREAD) return null;
 	const n = points.length;
 	let sx = 0;
 	let sy = 0;
@@ -101,19 +114,32 @@ export function applyFit(fit: LineFit, raw: number): number {
 const AXES: readonly QualityAxis[] = ["coding", "intelligence", "agentic"];
 
 /**
- * Fit every axis from the anchors' raw suite scores paired with their known AA
- * scores. `anchorAa` supplies the AA index per slug+axis (absent ⇒ that anchor
- * is not used on that axis).
+ * The correlation a fit must reach before its numbers are published as scores. `MIN_R` (0.5)
+ * is the bar for a fit being computable at all; this is the bar for TRUSTING one. The fit's
+ * `r` and `n` were previously computed and then discarded, so an r of 0.51 and one of 0.99
+ * produced indistinguishable output — and a shallow fit silently compressed every target.
+ */
+export const PUBLISH_MIN_R = 0.8;
+
+/**
+ * Fit every axis from the anchors' raw suite scores paired with their known AA scores.
+ * `anchorAa` supplies the AA index per slug+axis (absent ⇒ that anchor is not used there).
+ *
+ * `rawOf` selects which observations the fit is built from. It defaults to every band, but
+ * callers should pass the HARD band: easy and moderate sit at ~1.0 for every model worth
+ * ranking, so including them leaves the regression almost no variation in x against a wide
+ * spread in published y, and the slope collapses toward flat.
  */
 export function fitCalibration(
 	anchors: readonly EvalResult[],
 	anchorAa: (slug: string, axis: QualityAxis) => number | undefined,
+	rawOf: (result: EvalResult, axis: QualityAxis) => number | null = (r, axis) => axisMean(r.axes[axis]),
 ): Calibration {
 	const cal: Calibration = {};
 	for (const axis of AXES) {
 		const points: AnchorPoint[] = [];
 		for (const r of anchors) {
-			const raw = axisMean(r.axes[axis]);
+			const raw = rawOf(r, axis);
 			const aa = anchorAa(r.slug, axis);
 			if (raw !== null && aa !== undefined) points.push({ raw, aa });
 		}
@@ -123,15 +149,24 @@ export function fitCalibration(
 	return cal;
 }
 
-function axisMean(a: AxisScore | undefined): number | null {
+export function axisMean(a: AxisScore | undefined): number | null {
 	return a === undefined || a.n === 0 ? null : a.sum / a.n;
 }
 
-/** Calibrated local FeedScores for the targets, one axis at a time, skipping axes with no fit. */
+/** The hard band alone, for calibration. */
+export const hardRaw = (r: EvalResult, axis: QualityAxis): number | null => axisMean(r.axesHard[axis]);
+
+/**
+ * Calibrated local FeedScores for the targets, one axis at a time. An axis is skipped when it
+ * has no fit, when the fit is weaker than `minR`, or when the target made no observation on
+ * it — honest silence rather than a number nobody should act on.
+ */
 export function toLocalFeedScores(
 	targets: readonly EvalResult[],
 	cal: Calibration,
 	authorOf: (slug: string) => string,
+	rawOf: (result: EvalResult, axis: QualityAxis) => number | null = (r, axis) => axisMean(r.axes[axis]),
+	minR = PUBLISH_MIN_R,
 ): FeedScore[] {
 	const out: FeedScore[] = [];
 	for (const r of targets) {
@@ -139,8 +174,8 @@ export function toLocalFeedScores(
 		let any = false;
 		for (const axis of AXES) {
 			const fit = cal[axis];
-			const raw = axisMean(r.axes[axis]);
-			if (fit === undefined || raw === null) continue;
+			const raw = rawOf(r, axis);
+			if (fit === undefined || raw === null || fit.r < minR) continue;
 			entry[axis] = applyFit(fit, raw);
 			any = true;
 		}

package/src/eval/run.ts

@@ -39,6 +39,14 @@ export interface EvalResult {
 	 * questions were easy.
 	 */
 	byComplexity: Partial<Record<Complexity, AxisScore>>;
+	/**
+	 * Per-axis scores from the HARD band alone. Calibration fits against these: the easy and
+	 * moderate bands sit at ~1.0 for every model worth ranking, so including them gives the
+	 * regression almost no variation in x against a wide spread in published y — the fitted
+	 * slope goes shallow and every target is dragged toward the middle. Measured: a model
+	 * published at intelligence 39.5 calibrated to 22.8 across 10 passes.
+	 */
+	axesHard: Record<QualityAxis, AxisScore>;
 	/**
 	 * Per-axis spread across passes: max pass mean minus min pass mean, or null under two
 	 * passes. A wide spread means the headline is one sample of a noisy quantity, and is the
@@ -144,6 +152,7 @@ async function scorePass(slug: string, args: RunEvalArgs): Promise<EvalResult> {
 		}
 	}
 	const byComplexity: Partial<Record<Complexity, AxisScore>> = {};
+	const axesHard: Record<QualityAxis, AxisScore> = { coding: { sum: 0, n: 0 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } };
 	for (const o of [...objective, ...judgedOutcomes, ...scenarioOutcomes]) {
 		if (!o.ok) {
 			// An unobserved task is NOT a zero: a provider's throttle or outage would otherwise
@@ -154,11 +163,15 @@ async function scorePass(slug: string, args: RunEvalArgs): Promise<EvalResult> {
 		}
 		axes[o.axis].sum += o.grade;
 		axes[o.axis].n += 1;
+		if (o.complexity === "hard") {
+			axesHard[o.axis].sum += o.grade;
+			axesHard[o.axis].n += 1;
+		}
 		const band = (byComplexity[o.complexity] ??= { sum: 0, n: 0 });
 		band.sum += o.grade;
 		band.n += 1;
 	}
-	return { slug, axes, errors, repeats: 1, spread: {}, byComplexity };
+	return { slug, axes, errors, repeats: 1, spread: {}, byComplexity, axesHard };
 }
 
 const AXES: readonly QualityAxis[] = ["coding", "intelligence", "agentic"];
@@ -174,6 +187,7 @@ async function scoreModel(slug: string, args: RunEvalArgs): Promise<EvalResult>
 	const axes: Record<QualityAxis, AxisScore> = { coding: { sum: 0, n: 0 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } };
 	const means: Record<QualityAxis, number[]> = { coding: [], intelligence: [], agentic: [] };
 	const byComplexity: Partial<Record<Complexity, AxisScore>> = {};
+	const axesHard: Record<QualityAxis, AxisScore> = { coding: { sum: 0, n: 0 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } };
 	let errors = 0;
 	for (let i = 0; i < passes; i++) {
 		const pass = await scorePass(slug, args);
@@ -182,6 +196,8 @@ async function scoreModel(slug: string, args: RunEvalArgs): Promise<EvalResult>
 		for (const axis of AXES) {
 			axes[axis].sum += pass.axes[axis].sum;
 			axes[axis].n += pass.axes[axis].n;
+			axesHard[axis].sum += pass.axesHard[axis].sum;
+			axesHard[axis].n += pass.axesHard[axis].n;
 			if (pass.axes[axis].n > 0) means[axis].push(pass.axes[axis].sum / pass.axes[axis].n);
 		}
 		for (const [band, score] of Object.entries(pass.byComplexity) as [Complexity, AxisScore][]) {
@@ -195,7 +211,7 @@ async function scoreModel(slug: string, args: RunEvalArgs): Promise<EvalResult>
 		const m = means[axis];
 		if (m.length > 1) spread[axis] = Math.max(...m) - Math.min(...m);
 	}
-	return { slug, axes, errors, repeats: passes, spread, byComplexity };
+	return { slug, axes, errors, repeats: passes, spread, byComplexity, axesHard };
 }
 
 export async function runEval(args: RunEvalArgs): Promise<EvalResult[]> {

package/src/server/http.ts

@@ -18,7 +18,7 @@ import { runEval, type Completer } from "../eval/run.ts";
 import type { ToolSpec } from "../eval/agentic.ts";
 import type { ToolCall } from "../upstream/types.ts";
 import type { QualityAxis } from "../config/types.ts";
-import { fitCalibration, pickAnchors, toLocalFeedScores, MIN_ANCHORS } from "../eval/calibrate.ts";
+import { fitCalibration, hardRaw, pickAnchors, toLocalFeedScores, MIN_ANCHORS, PUBLISH_MIN_R } from "../eval/calibrate.ts";
 import { makeJudge } from "../eval/judge.ts";
 import { loadLocalScores, saveLocalScores } from "../catalog/benchmark-feeds.ts";
 import { advise } from "./advise.ts";
@@ -812,12 +812,15 @@ export function startServer(cfg: RouterConfig): StartedServer {
 							});
 							const target = results[0]!;
 							const published = (s: string, axis: QualityAxis): number | undefined => models.find((m) => m.slug === s)?.quality[axis];
-							const cal = fitCalibration(results.slice(1), published);
+							const cal = fitCalibration(results.slice(1), published, hardRaw);
 							const authorOf = (s: string): string => models.find((m) => m.slug === s)?.author ?? "";
-							const fresh = toLocalFeedScores([target], cal, authorOf);
-							const shared = { slug, anchors, raw: target.axes, byComplexity: target.byComplexity, repeats: target.repeats, spread: target.spread, errors: target.errors, tookMs: Date.now() - started };
+							const fresh = toLocalFeedScores([target], cal, authorOf, hardRaw);
+							// The fit is reported, not just used: `r` and `n` say whether the numbers deserve
+							// belief, and an axis dropped for a weak fit should say so rather than vanish.
+							const fitDetail = Object.fromEntries(Object.entries(cal).map(([axis, f]) => [axis, { r: Number(f.r.toFixed(3)), n: f.n, published: f.r >= PUBLISH_MIN_R }]));
+							const shared = { slug, anchors, raw: target.axes, rawHard: target.axesHard, byComplexity: target.byComplexity, repeats: target.repeats, spread: target.spread, errors: target.errors, fit: fitDetail, publishMinR: PUBLISH_MIN_R, tookMs: Date.now() - started };
 							if (fresh.length === 0) {
-								benchmarkJobs.set(jobId, { ...benchmarkJobs.get(jobId)!, state: "done", result: { ...shared, calibrated: null, applied: false, reason: "no axis produced a usable fit; try more or better-spread anchors" } });
+								benchmarkJobs.set(jobId, { ...benchmarkJobs.get(jobId)!, state: "done", result: { ...shared, calibrated: null, applied: false, reason: `no axis produced a fit at r >= ${PUBLISH_MIN_R} on the hard band; try more or better-spread anchors` } });
 								return;
 							}
 							// Merge, never replace: other models' measurements are not this run's to discard.

package/test/eval.test.ts

@@ -3,9 +3,10 @@ import { describe, expect, test } from "bun:test";
 import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
 import { applyFeedScores, loadLocalScores, saveLocalScores, type FeedScore } from "../src/catalog/benchmark-feeds.ts";
 import { answerScore, extractJson, isRefusalOrEmpty, jsonField, tokenCoverage } from "../src/eval/grade.ts";
-import { applyFit, fitAxis, fitCalibration, pickAnchors, toLocalFeedScores, MIN_ANCHORS } from "../src/eval/calibrate.ts";
+import { applyFit, fitAxis, fitCalibration, hardRaw, pickAnchors, toLocalFeedScores, MIN_ANCHORS, MIN_R, PUBLISH_MIN_R } from "../src/eval/calibrate.ts";
 import { runEval, type EvalResult } from "../src/eval/run.ts";
 import { EVAL_TASKS } from "../src/eval/tasks.ts";
+import type { QualityAxis } from "../src/config/types.ts";
 import { makeJudge, parseScore } from "../src/eval/judge.ts";
 import type { EvalTask, JudgedTask } from "../src/eval/tasks.ts";
 import { openDb } from "../src/util/sqlite.ts";
@@ -130,9 +131,9 @@ describe("calibration", () => {
 	test("fitCalibration + toLocalFeedScores place a target on the AA scale", () => {
 		expect(MIN_ANCHORS).toBe(3);
 		const anchors: EvalResult[] = [
-			{ slug: "a/one", axes: { coding: { sum: 0.2, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {} },
-			{ slug: "a/two", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {} },
-			{ slug: "a/three", axes: { coding: { sum: 0.8, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {} },
+			{ slug: "a/one", axes: { coding: { sum: 0.2, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {}, axesHard: { coding: { sum: 0, n: 0 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } } },
+			{ slug: "a/two", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {}, axesHard: { coding: { sum: 0, n: 0 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } } },
+			{ slug: "a/three", axes: { coding: { sum: 0.8, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {}, axesHard: { coding: { sum: 0, n: 0 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } } },
 		];
 		const aaOf: Record<string, number> = { "a/one": 40, "a/two": 60, "a/three": 80 };
 		const cal = fitCalibration(anchors, (slug, axis) => (axis === "coding" ? aaOf[slug] : undefined));
@@ -140,7 +141,7 @@ describe("calibration", () => {
 		expect(cal.intelligence).toBeUndefined(); // no anchor data on that axis
 
 		const targets: EvalResult[] = [
-			{ slug: "z/gap", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0.9, n: 1 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {} },
+			{ slug: "z/gap", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0.9, n: 1 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {}, axesHard: { coding: { sum: 0, n: 0 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } } },
 		];
 		const local = toLocalFeedScores(targets, cal, (s) => s.slice(0, s.indexOf("/")));
 		expect(local).toHaveLength(1);
@@ -148,6 +149,62 @@ describe("calibration", () => {
 		expect(local[0]!.coding).toBeCloseTo(60, 5); // calibrated from raw 0.5
 		expect(local[0]!.intelligence).toBeUndefined(); // axis had no fit, so not emitted
 	});
+
+	test("calibrating on the hard band beats calibrating on everything", () => {
+		// Three anchors published 20/50/80 apart. On the FULL suite they all score ~0.97
+		// because easy and moderate pin everyone at the ceiling; on the hard band alone they
+		// separate. Same models, same publishing, different x — and only one of them can fit.
+		const mk = (slug: string, full: number, hard: number): EvalResult => ({
+			slug,
+			axes: { coding: { sum: full, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } },
+			axesHard: { coding: { sum: hard, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } },
+			errors: 0,
+			repeats: 1,
+			spread: {},
+			byComplexity: {},
+		});
+		const anchors = [mk("a/low", 0.96, 0.2), mk("a/mid", 0.97, 0.5), mk("a/high", 0.98, 0.8)];
+		const aa: Record<string, number> = { "a/low": 20, "a/mid": 50, "a/high": 80 };
+		const published = (slug: string, axis: QualityAxis) => (axis === "coding" ? aa[slug] : undefined);
+		const target = mk("z/target", 0.97, 0.5);
+
+		const onHard = toLocalFeedScores([target], fitCalibration(anchors, published, hardRaw), () => "z", hardRaw);
+		// The hard band spans 0.2-0.8 against 20-80, so the fit is a real line: raw 0.5 ⇒ ~50.
+		expect(onHard[0]!.coding).toBeCloseTo(50, 0);
+
+		// On the full suite the anchors span 0.96-0.98: the same published spread compressed into
+		// a fiftieth of the range. That is the shallow slope that produced 22.8 for a model
+		// published at 39.5, so the fit must be refused rather than published.
+		const pooledFit = fitCalibration(anchors, published);
+		const pooled = toLocalFeedScores([target], pooledFit, () => "z");
+		expect(pooled).toEqual([]);
+	});
+
+	test("a weak fit is refused, not published", () => {
+		// Points with a real but noisy relationship: computable (r >= MIN_R) yet not worth
+		// acting on. `r` and `n` used to be computed and then thrown away.
+		const noisy = [
+			{ raw: 0.1, aa: 20 },
+			{ raw: 0.5, aa: 70 },
+			{ raw: 0.6, aa: 30 },
+			{ raw: 0.9, aa: 60 },
+		];
+		const fit = fitAxis(noisy)!;
+		expect(fit.r).toBeGreaterThanOrEqual(MIN_R);
+		expect(fit.r).toBeLessThan(PUBLISH_MIN_R);
+		const target: EvalResult = {
+			slug: "z/t",
+			axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } },
+			axesHard: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } },
+			errors: 0,
+			repeats: 1,
+			spread: {},
+			byComplexity: {},
+		};
+		expect(toLocalFeedScores([target], { coding: fit }, () => "z", hardRaw)).toEqual([]);
+		// A caller that deliberately lowers the bar still can, so the gate is policy not dogma.
+		expect(toLocalFeedScores([target], { coding: fit }, () => "z", hardRaw, MIN_R)[0]!.coding).toBeGreaterThan(0);
+	});
 });
 
 describe("runEval", () => {