auto-model-router 0.29.0 → 0.30.0

package/.omp-plugin/marketplace.json

@@ -7,14 +7,14 @@
   },
   "metadata": {
     "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
-    "version": "0.29.0",
+    "version": "0.30.0",
     "pluginRoot": "."
   },
   "plugins": [
     {
       "name": "auto-model-router",
       "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
-      "version": "0.29.0",
+      "version": "0.30.0",
       "author": {
         "name": "drewappling",
         "email": "drewappling@gmail.com"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-model-router",
-  "version": "0.29.0",
+  "version": "0.30.0",
   "private": false,
   "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
   "type": "module",

package/src/eval/run.ts

@@ -8,7 +8,7 @@
 
 import type { QualityAxis } from "../config/types.ts";
 import type { Judge } from "./judge.ts";
-import { EVAL_TASKS, JUDGED_TASKS, type EvalTask, type JudgedTask } from "./tasks.ts";
+import { EVAL_TASKS, JUDGED_TASKS, type Complexity, type EvalTask, type JudgedTask } from "./tasks.ts";
 import { AGENTIC_SCENARIOS, runScenario, type Scenario, type ToolSpec } from "./agentic.ts";
 import type { ToolCall } from "../upstream/types.ts";
 
@@ -31,6 +31,20 @@ export interface EvalResult {
 	axes: Record<QualityAxis, AxisScore>;
 	/** Tasks whose completion threw (dispatch failure). Excluded from `axes`. */
 	errors: number;
+	/** Passes actually completed. 1 unless `repeats` was given. */
+	repeats: number;
+	/**
+	 * Mean grade per difficulty band. This is where a model's ceiling shows: a suite of one
+	 * difficulty reports a single number and cannot say whether a model is strong or the
+	 * questions were easy.
+	 */
+	byComplexity: Partial<Record<Complexity, AxisScore>>;
+	/**
+	 * Per-axis spread across passes: max pass mean minus min pass mean, or null under two
+	 * passes. A wide spread means the headline is one sample of a noisy quantity, and is the
+	 * honest counterpart to reporting a score at all.
+	 */
+	spread: Partial<Record<QualityAxis, number>>;
 }
 
 function messagesFor(task: { system?: string; user: string }): ChatMessage[] {
@@ -60,9 +74,17 @@ export interface RunEvalArgs {
 	toolComplete?: (slug: string, messages: Record<string, unknown>[], tools: ToolSpec[]) => Promise<{ text: string; toolCalls: ToolCall[] }>;
 	/** Agentic tool-loop scenarios. Defaults to the built-in set when `toolComplete` is given. */
 	scenarios?: readonly Scenario[];
+	/**
+	 * How many times to run the whole suite per model, default 1. A single pass cannot tell a
+	 * real difference from sampling noise — Artificial Analysis runs 3-5 repeats and spends
+	 * >10 to claim a confidence interval. Every attempt is an independent observation, so the
+	 * axis mean is over `tasks x repeats` and `spread` reports how much the passes disagreed.
+	 */
+	repeats?: number;
 }
 
-async function scoreModel(slug: string, args: RunEvalArgs): Promise<EvalResult> {
+/** One pass of the whole suite. Repeats call this and the observations are pooled. */
+async function scorePass(slug: string, args: RunEvalArgs): Promise<EvalResult> {
 	const tasks = args.tasks ?? EVAL_TASKS;
 	const judge = args.judge;
 	const judged = judge !== undefined ? (args.judged ?? JUDGED_TASKS) : [];
@@ -72,7 +94,7 @@ async function scoreModel(slug: string, args: RunEvalArgs): Promise<EvalResult>
 		agentic: { sum: 0, n: 0 },
 	};
 	let errors = 0;
-	type Outcome = { axis: QualityAxis; grade: number; ok: boolean };
+	type Outcome = { axis: QualityAxis; grade: number; ok: boolean; complexity: Complexity };
 	// A THROW (or an unscorable judge reply) means the turn produced no usable
 	// observation — NOT a score of 0, which would poison an anchor whose model is
 	// merely unavailable. Such turns are tallied as errors and excluded.
@@ -80,9 +102,9 @@ async function scoreModel(slug: string, args: RunEvalArgs): Promise<EvalResult>
 		tasks.map(async (task) => {
 			try {
 				const text = await args.complete(slug, messagesFor(task));
-				return { axis: task.axis, grade: task.grade(text), ok: true };
+				return { axis: task.axis, grade: task.grade(text), ok: true, complexity: task.complexity ?? "easy" };
 			} catch {
-				return { axis: task.axis, grade: 0, ok: false };
+				return { axis: task.axis, grade: 0, ok: false, complexity: task.complexity ?? "easy" };
 			}
 		}),
 	);
@@ -94,9 +116,9 @@ async function scoreModel(slug: string, args: RunEvalArgs): Promise<EvalResult>
 						try {
 							const answer = await args.complete(slug, messagesFor(task));
 							const score = await judge(task, answer);
-							return score === null ? { axis: task.axis, grade: 0, ok: false } : { axis: task.axis, grade: score, ok: true };
+							return score === null ? { axis: task.axis, grade: 0, ok: false, complexity: "hard" } : { axis: task.axis, grade: score, ok: true, complexity: "hard" };
 						} catch {
-							return { axis: task.axis, grade: 0, ok: false };
+							return { axis: task.axis, grade: 0, ok: false, complexity: "hard" };
 						}
 					}),
 				);
@@ -109,21 +131,64 @@ async function scoreModel(slug: string, args: RunEvalArgs): Promise<EvalResult>
 		for (const scenario of scenarios) {
 			try {
 				const run = await runScenario(scenario, (messages, tools) => args.toolComplete!(slug, messages, tools));
-				scenarioOutcomes.push({ axis: "agentic", grade: scenario.grade(run), ok: true });
+				scenarioOutcomes.push({ axis: "agentic", grade: scenario.grade(run), ok: true, complexity: "moderate" });
 			} catch {
-				scenarioOutcomes.push({ axis: "agentic", grade: 0, ok: false });
+				scenarioOutcomes.push({ axis: "agentic", grade: 0, ok: false, complexity: "moderate" });
 			}
 		}
 	}
+	const byComplexity: Partial<Record<Complexity, AxisScore>> = {};
 	for (const o of [...objective, ...judgedOutcomes, ...scenarioOutcomes]) {
 		if (!o.ok) {
+			// An unobserved task is NOT a zero: a provider's throttle or outage would otherwise
+			// be recorded as the model answering wrongly. Artificial Analysis goes further and
+			// withholds a result whose failures persisted; we at least never score one.
 			errors += 1;
 			continue;
 		}
 		axes[o.axis].sum += o.grade;
 		axes[o.axis].n += 1;
+		const band = (byComplexity[o.complexity] ??= { sum: 0, n: 0 });
+		band.sum += o.grade;
+		band.n += 1;
+	}
+	return { slug, axes, errors, repeats: 1, spread: {}, byComplexity };
+}
+
+const AXES: readonly QualityAxis[] = ["coding", "intelligence", "agentic"];
+
+/**
+ * A model's score over `repeats` passes. Observations are POOLED rather than averaged over
+ * pass means, so a pass that lost tasks to dispatch errors weighs only what it observed.
+ * Passes run one after another: they are the same model, and firing them concurrently just
+ * trips a provider's throttle and buys errors instead of data.
+ */
+async function scoreModel(slug: string, args: RunEvalArgs): Promise<EvalResult> {
+	const passes = Math.max(1, Math.floor(args.repeats ?? 1));
+	const axes: Record<QualityAxis, AxisScore> = { coding: { sum: 0, n: 0 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } };
+	const means: Record<QualityAxis, number[]> = { coding: [], intelligence: [], agentic: [] };
+	const byComplexity: Partial<Record<Complexity, AxisScore>> = {};
+	let errors = 0;
+	for (let i = 0; i < passes; i++) {
+		const pass = await scorePass(slug, args);
+		errors += pass.errors;
+		for (const axis of AXES) {
+			axes[axis].sum += pass.axes[axis].sum;
+			axes[axis].n += pass.axes[axis].n;
+			if (pass.axes[axis].n > 0) means[axis].push(pass.axes[axis].sum / pass.axes[axis].n);
+		}
+		for (const [band, score] of Object.entries(pass.byComplexity) as [Complexity, AxisScore][]) {
+			const acc = (byComplexity[band] ??= { sum: 0, n: 0 });
+			acc.sum += score.sum;
+			acc.n += score.n;
+		}
+	}
+	const spread: Partial<Record<QualityAxis, number>> = {};
+	for (const axis of AXES) {
+		const m = means[axis];
+		if (m.length > 1) spread[axis] = Math.max(...m) - Math.min(...m);
 	}
-	return { slug, axes, errors };
+	return { slug, axes, errors, repeats: passes, spread, byComplexity };
 }
 
 export async function runEval(args: RunEvalArgs): Promise<EvalResult[]> {

package/src/eval/tasks.ts

@@ -12,9 +12,19 @@
 import type { QualityAxis } from "../config/types.ts";
 import { answerScore, extractJson, jsonField, multiAnswerCoverage, tokenCoverage } from "./grade.ts";
 
+/**
+ * How hard a task is, reported separately so a model's score says WHERE it falls off
+ * rather than only how far. A suite of one difficulty cannot separate competent models:
+ * measured on a live catalog, every model above the floor scored 7/7 on the original set,
+ * which is why calibration could not fit the coding axis at all.
+ */
+export type Complexity = "easy" | "moderate" | "hard";
+
 export interface EvalTask {
 	id: string;
 	axis: QualityAxis;
+	/** Absent ⇒ `easy`: the original suite was uniformly easy, and saying so is the point. */
+	complexity?: Complexity;
 	system?: string;
 	user: string;
 	/** Deterministic proxy grade in [0, 1]. */
@@ -231,6 +241,72 @@ export const EVAL_TASKS: readonly EvalTask[] = [
 			return hit / want.length;
 		},
 	},
+
+	// ---- hard: items competent models actually get wrong, so the axis has a top end.
+	// Every answer below is hand-derived; multi-part items give partial credit, which is what
+	// produces spread instead of a wall of 1.0s.
+	{
+		id: "coding/event-loop-order",
+		axis: "coding",
+		complexity: "hard",
+		system: JSON_ONLY,
+		user: "Given:\nconsole.log('a');\nsetTimeout(() => console.log('b'), 0);\nPromise.resolve().then(() => console.log('c'));\nconsole.log('d');\nReply with the four letters in the order they are printed, comma separated.",
+		// Microtasks drain before timers: a, d, c, b.
+		grade: (o) => multiAnswerCoverage(o, ["a", "d", "c", "b"]) === 1 ? (/a\W+d\W+c\W+b/i.test(o) ? 1 : 0.5) : 0,
+	},
+	{
+		id: "coding/sort-lexicographic",
+		axis: "coding",
+		complexity: "hard",
+		system: JSON_ONLY,
+		user: "What does `[10, 9, 80].sort()` return in JavaScript? Reply with only the array.",
+		// Default sort compares as STRINGS: "10" < "80" < "9".
+		grade: (o) => (answerScore(o, "[10, 80, 9]") === 1 ? 1 : answerScore(o, "[10,80,9]")),
+	},
+	{
+		id: "coding/trace-hard",
+		axis: "coding",
+		complexity: "hard",
+		system: JSON_ONLY,
+		user: "Give the result of each, one per line, in order:\n(1) [1,[2,[3,[4]]]].flat(2).length\n(2) 'abc'.padStart(5,'xy')\n(3) [...'aab'].filter((c,i,a)=>a.indexOf(c)===i).join('')\n(4) Number('')\n(5) [1,2,3].at(-1)",
+		// flat(2) leaves [1,2,3,[4]] ⇒ 4; padStart cycles the pad ⇒ xyabc; dedupe ⇒ ab; 0; 3.
+		grade: (o) => multiAnswerCoverage(o, ["4", "xyabc", "ab", "0", "3"]),
+	},
+	{
+		id: "intel/collatz-steps",
+		axis: "intelligence",
+		complexity: "hard",
+		system: JSON_ONLY,
+		user: "Start with x = 7. Repeat exactly 7 times: if x is even, x = x / 2; otherwise x = 3x + 1. Reply with the final value of x alone.",
+		// 7 → 22 → 11 → 34 → 17 → 52 → 26 → 13
+		grade: (o) => answerScore(o, "13"),
+	},
+	{
+		id: "intel/arith-hard",
+		axis: "intelligence",
+		complexity: "hard",
+		system: JSON_ONLY,
+		user: "Answer each, one per line, in order:\n(1) 47 * 53\n(2) 2^13\n(3) the 17th prime number\n(4) LCM(12, 18)\n(5) how many 1 bits are in the binary form of 1000",
+		// 2491; 8192; 59; 36; 1000 = 1111101000 ⇒ six 1 bits.
+		grade: (o) => multiAnswerCoverage(o, ["2491", "8192", "59", "36", "6"]),
+	},
+	{
+		id: "intel/strict-format",
+		axis: "intelligence",
+		complexity: "hard",
+		system: "Follow the output constraints exactly. Any extra text is a failure.",
+		user: "Name three primary colours. Reply with exactly three words, all lowercase, separated by single spaces, with no punctuation and no other text.",
+		// Instruction adherence under a negative constraint — what IFBench measures.
+		grade: (o) => {
+			const text = o.trim();
+			if (text === "" || /[.,;:!?"'`\n]/.test(text)) return 0;
+			const words = text.split(" ");
+			if (words.length !== 3) return 0;
+			if (words.some((w) => w !== w.toLowerCase())) return 0.5;
+			const known = ["red", "blue", "yellow", "green"];
+			return words.every((w) => known.includes(w)) ? 1 : 0.5;
+		},
+	},
 ];
 
 /**

package/src/server/http.ts

@@ -703,6 +703,18 @@ export function startServer(cfg: RouterConfig): StartedServer {
 					if (result.deleted > 0) log.info("pruned ledger rows past retention", { deleted: result.deleted, retentionDays: cfg.ledger.retentionDays });
 					return json({ ...result, retentionDays: cfg.ledger.retentionDays });
 				}
+				if (req.method === "DELETE" && url.pathname === "/v1/router/benchmark") {
+					// Purge local measurements. A stored score carries no provenance, so one taken
+					// under a broken harness is indistinguishable from a good one — measured: an
+					// `agentic: 0` for a model published at 41.7, left over from a run whose
+					// dispatch errors were being graded as wrong answers. `?key=` drops one.
+					const key = url.searchParams.get("key");
+					const before = loadLocalScores(db);
+					const kept = key === null ? [] : before.filter((s) => s.key !== key);
+					saveLocalScores(db, kept);
+					log.info("purged local eval scores", { removed: before.length - kept.length, kept: kept.length });
+					return json({ removed: before.length - kept.length, kept: kept.map((s) => s.key) });
+				}
 				if (req.method === "POST" && url.pathname === "/v1/router/benchmark") {
 					// Score one model with our OWN eval suite, for the models no feed covers: a
 					// third of a live catalog carries no published score on any axis, and a model
@@ -760,6 +772,7 @@ export function startServer(cfg: RouterConfig): StartedServer {
 						slugs: [slug, ...anchors],
 						complete,
 						toolComplete,
+						repeats: Math.min(Math.max(1, typeof body?.repeats === "number" ? Math.floor(body.repeats) : 1), 20),
 						concurrency: Math.min(Math.max(1, asked), 8),
 						...(judgeSlug === "" ? {} : { judge: makeJudge(complete, judgeSlug) }),
 					});
@@ -770,13 +783,13 @@ export function startServer(cfg: RouterConfig): StartedServer {
 					const authorOf = (s: string): string => models.find((m) => m.slug === s)?.author ?? "";
 					const fresh = toLocalFeedScores([target], cal, authorOf);
 					if (fresh.length === 0) {
-						return json({ slug, anchors, calibrated: null, raw: target.axes, errors: target.errors, applied: false, reason: "no axis produced a usable fit; try more or better-spread anchors" });
+						return json({ slug, anchors, calibrated: null, raw: target.axes, byComplexity: target.byComplexity, repeats: target.repeats, spread: target.spread, errors: target.errors, applied: false, reason: "no axis produced a usable fit; try more or better-spread anchors" });
 					}
 					// Merge, never replace: other models' measurements are not this run's to discard.
 					const kept = loadLocalScores(db).filter((s) => s.key !== fresh[0]!.key);
 					saveLocalScores(db, [...kept, ...fresh]);
 					log.info("benchmarked a model with the local eval suite", { slug, anchors: anchors.length, errors: target.errors, useLocalScores: cfg.benchmarks.useLocalScores });
-					return json({ slug, anchors, raw: target.axes, calibrated: fresh[0], errors: target.errors, applied: cfg.benchmarks.useLocalScores });
+					return json({ slug, anchors, raw: target.axes, byComplexity: target.byComplexity, repeats: target.repeats, spread: target.spread, calibrated: fresh[0], errors: target.errors, applied: cfg.benchmarks.useLocalScores });
 				}
 				if (req.method === "POST" && url.pathname === "/v1/router/feedback") {
 					// A user verdict on the newest routed turn of an omp session.

package/test/eval.test.ts

@@ -5,6 +5,7 @@ import { applyFeedScores, loadLocalScores, saveLocalScores, type FeedScore } fro
 import { answerScore, extractJson, isRefusalOrEmpty, jsonField, tokenCoverage } from "../src/eval/grade.ts";
 import { applyFit, fitAxis, fitCalibration, pickAnchors, toLocalFeedScores, MIN_ANCHORS } from "../src/eval/calibrate.ts";
 import { runEval, type EvalResult } from "../src/eval/run.ts";
+import { EVAL_TASKS } from "../src/eval/tasks.ts";
 import { makeJudge, parseScore } from "../src/eval/judge.ts";
 import type { EvalTask, JudgedTask } from "../src/eval/tasks.ts";
 import { openDb } from "../src/util/sqlite.ts";
@@ -72,12 +73,58 @@ describe("calibration", () => {
 		expect(pickAnchors(catalog, "a/50")).not.toContain("a/50");
 	});
 
+	test("repeats pool observations, report spread, and split scores by complexity", async () => {
+		// A flaky model: the strict-format task passes on odd calls only. One pass cannot tell
+		// that apart from a model that always passes or always fails.
+		let call = 0;
+		const flaky = async (_slug: string, messages: { role: string; content: string }[]) => {
+			call += 1;
+			const user = messages[messages.length - 1]!.content;
+			if (user.includes("primary colours")) return call % 2 === 0 ? "red blue yellow" : "Red, Blue, and Yellow!";
+			return "";
+		};
+		const [single] = await runEval({ slugs: ["a/flaky"], complete: flaky, tasks: EVAL_TASKS.filter((t) => t.id === "intel/strict-format") });
+		expect(single!.repeats).toBe(1);
+		expect(single!.spread).toEqual({});
+
+		call = 0;
+		const [many] = await runEval({ slugs: ["a/flaky"], complete: flaky, tasks: EVAL_TASKS.filter((t) => t.id === "intel/strict-format"), repeats: 10 });
+		expect(many!.repeats).toBe(10);
+		expect(many!.axes.intelligence.n).toBe(10);
+		// Half the passes score 1 and half 0, so the pooled mean sits mid-range and the spread
+		// says plainly that the headline is one sample of something noisy.
+		expect(many!.axes.intelligence.sum / many!.axes.intelligence.n).toBeCloseTo(0.5, 1);
+		expect(many!.spread.intelligence).toBe(1);
+		// That task is in the hard band, so the breakdown attributes it there and nowhere else.
+		expect(many!.byComplexity.hard?.n).toBe(10);
+		expect(many!.byComplexity.easy).toBeUndefined();
+	});
+
+	test("the suite spans complexities, and hard items are not all pinned at the ceiling", () => {
+		const bands = new Set(EVAL_TASKS.map((t) => t.complexity ?? "easy"));
+		expect(bands.has("easy")).toBe(true);
+		expect(bands.has("hard")).toBe(true);
+		// A perfect model must still score 1 on every hard task: a task nobody can pass
+		// measures the grader, not the model.
+		const hard = EVAL_TASKS.filter((t) => t.complexity === "hard");
+		expect(hard.length).toBeGreaterThanOrEqual(6);
+		expect(hard.find((t) => t.id === "coding/sort-lexicographic")!.grade("[10, 80, 9]")).toBe(1);
+		expect(hard.find((t) => t.id === "coding/event-loop-order")!.grade("a, d, c, b")).toBe(1);
+		expect(hard.find((t) => t.id === "intel/collatz-steps")!.grade("13")).toBe(1);
+		expect(hard.find((t) => t.id === "intel/arith-hard")!.grade("2491\n8192\n59\n36\n6")).toBe(1);
+		expect(hard.find((t) => t.id === "coding/trace-hard")!.grade("4\nxyabc\nab\n0\n3")).toBe(1);
+		// And a plausible wrong answer must NOT score 1, or the task adds no signal.
+		expect(hard.find((t) => t.id === "coding/sort-lexicographic")!.grade("[9, 10, 80]")).toBeLessThan(1);
+		expect(hard.find((t) => t.id === "intel/collatz-steps")!.grade("1")).toBeLessThan(1);
+		expect(hard.find((t) => t.id === "intel/strict-format")!.grade("Red, blue, and yellow.")).toBeLessThan(1);
+	});
+
 	test("fitCalibration + toLocalFeedScores place a target on the AA scale", () => {
 		expect(MIN_ANCHORS).toBe(3);
 		const anchors: EvalResult[] = [
-			{ slug: "a/one", axes: { coding: { sum: 0.2, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0 },
-			{ slug: "a/two", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0 },
-			{ slug: "a/three", axes: { coding: { sum: 0.8, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0 },
+			{ slug: "a/one", axes: { coding: { sum: 0.2, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {} },
+			{ slug: "a/two", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {} },
+			{ slug: "a/three", axes: { coding: { sum: 0.8, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {} },
 		];
 		const aaOf: Record<string, number> = { "a/one": 40, "a/two": 60, "a/three": 80 };
 		const cal = fitCalibration(anchors, (slug, axis) => (axis === "coding" ? aaOf[slug] : undefined));
@@ -85,7 +132,7 @@ describe("calibration", () => {
 		expect(cal.intelligence).toBeUndefined(); // no anchor data on that axis
 
 		const targets: EvalResult[] = [
-			{ slug: "z/gap", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0.9, n: 1 }, agentic: { sum: 0, n: 0 } }, errors: 0 },
+			{ slug: "z/gap", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0.9, n: 1 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {} },
 		];
 		const local = toLocalFeedScores(targets, cal, (s) => s.slice(0, s.indexOf("/")));
 		expect(local).toHaveLength(1);