auto-model-router 0.1.4 → 0.2.0

package/src/util/sqlite.ts

@@ -18,7 +18,7 @@ import { mkdirSync } from "node:fs";
 import { dirname } from "node:path";
 
 /** Bump when a migration is added; guarded below so reopening never regresses it. */
-const USER_VERSION = 5;
+const USER_VERSION = 10;
 
 const MIGRATIONS = `
 CREATE TABLE IF NOT EXISTS catalog_cache (
@@ -29,6 +29,18 @@ CREATE TABLE IF NOT EXISTS catalog_cache (
   key_scoped INTEGER NOT NULL DEFAULT 0
 );
 
+CREATE TABLE IF NOT EXISTS benchmark_cache (
+  id INTEGER PRIMARY KEY CHECK (id = 1),
+  payload TEXT NOT NULL,
+  fetched_at_ms INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS local_scores (
+  id INTEGER PRIMARY KEY CHECK (id = 1),
+  payload TEXT NOT NULL,
+  fetched_at_ms INTEGER NOT NULL
+);
+
 CREATE TABLE IF NOT EXISTS ledger (
   id TEXT PRIMARY KEY,
   created_at_ms INTEGER NOT NULL,
@@ -128,6 +140,58 @@ const MIGRATE_V5 = `
 ALTER TABLE ledger ADD COLUMN omp_session_id TEXT NOT NULL DEFAULT '';
 `;
 
+// v6: classifier INPUTS. Before this the ledger recorded only what routing
+// decided (tier, slug, cost, escalation) and never what it decided FROM, so a
+// turn could not be replayed, and no weight could be fit offline. `features`
+// is the verbatim feature vector as JSON; `score` and `confidence` are the
+// classifier's own outputs, kept alongside because they are cheap and make
+// weight drift detectable when the scorer changes under a fixed feature set.
+// `classifier_reasons` is the per-feature breakdown, which `select.ts` drops
+// from the decision trail on every path except a hysteresis hold.
+//
+// All nullable with no backfill: pre-v6 rows genuinely lack these inputs and
+// NULL says so honestly. A DEFAULT would invent data that was never observed.
+const MIGRATE_V6 = `
+ALTER TABLE ledger ADD COLUMN features TEXT;
+ALTER TABLE ledger ADD COLUMN score REAL;
+ALTER TABLE ledger ADD COLUMN confidence REAL;
+ALTER TABLE ledger ADD COLUMN task TEXT;
+ALTER TABLE ledger ADD COLUMN classifier_reasons TEXT;
+`;
+
+// v7: records epsilon-greedy exploration. `explored_from` is the tier the
+// classifier actually chose on a turn we deliberately routed one step
+// cheaper; NULL means the turn was routed normally.
+//
+// This is the counterfactual the ledger could never observe before. Natural
+// traffic only reveals UNDER-routing, because a tier that was too low
+// escalates and leaves a trace, while a tier that was too high looks
+// indistinguishable from a tier that was exactly right.
+const MIGRATE_V7 = `
+ALTER TABLE ledger ADD COLUMN explored_from TEXT;
+`;
+
+// v8: records the hold-length arm a conversation was assigned by hold
+// exploration. NULL means the conversation was not part of the experiment.
+//
+// Recorded on EVERY turn of the conversation, not only the turns a hold
+// actually affects, so arms can be compared on total conversation cost
+// rather than on the subset the treatment happened to touch.
+const MIGRATE_V8 = `
+ALTER TABLE ledger ADD COLUMN hold_arm INTEGER;
+`;
+
+// v9: benchmark_cache holds the external benchmark feeds (Artificial Analysis,
+// BenchLM) that backfill quality scores OpenRouter leaves unpublished. It is a
+// whole new table, created idempotently by the MIGRATIONS block above, so there
+// is no ALTER guard here — the version bump alone records that the schema now
+// includes it.
+
+// v10: local_scores holds calibrated scores from our OWN eval harness
+// (src/eval), a `local`-source feed applied only when benchmarks.useLocalScores
+// is on. Another new table via the idempotent MIGRATIONS block; version bump
+// only, no ALTER guard.
+
 export function openDb(path: string): Database {
 	// ":memory:" has no parent directory to create.
 	if (path !== ":memory:") mkdirSync(dirname(path), { recursive: true });
@@ -145,6 +209,9 @@ export function openDb(path: string): Database {
 		if (!ledgerCols.some((c) => c.name === "harness_id")) db.exec(MIGRATE_V3);
 		if (!ledgerCols.some((c) => c.name === "error_kind")) db.exec(MIGRATE_V4);
 		if (!ledgerCols.some((c) => c.name === "omp_session_id")) db.exec(MIGRATE_V5);
+		if (!ledgerCols.some((c) => c.name === "features")) db.exec(MIGRATE_V6);
+		if (!ledgerCols.some((c) => c.name === "explored_from")) db.exec(MIGRATE_V7);
+		if (!ledgerCols.some((c) => c.name === "hold_arm")) db.exec(MIGRATE_V8);
 		db.exec(`PRAGMA user_version = ${USER_VERSION}`);
 	}
 	return db;

package/test/benchmark-feeds.test.ts

@@ -0,0 +1,222 @@
+import { describe, expect, test } from "bun:test";
+
+import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
+import {
+	applyFeedScores,
+	fetchBenchlmScores,
+	normalizeModelKey,
+	parseAaModels,
+	parseBenchlmModels,
+	refreshFeedScores,
+	type FeedScore,
+	type FetchLike,
+} from "../src/catalog/benchmark-feeds.ts";
+import { loadConfig } from "../src/config/load.ts";
+import type { RouterConfig } from "../src/config/types.ts";
+import { openDb } from "../src/util/sqlite.ts";
+
+/** A bare OpenRouter `/models` record, optionally pre-scored. */
+function raw(id: string, benchmarks?: Record<string, unknown>): Record<string, unknown> {
+	const record: Record<string, unknown> = {
+		id,
+		canonical_slug: id,
+		name: id,
+		context_length: 131_072,
+		pricing: { prompt: "0.0000003", completion: "0.0000011" },
+		supported_parameters: ["tools"],
+		architecture: { input_modalities: ["text"], tokenizer: "Other" },
+		created: 1_700_000_000,
+	};
+	if (benchmarks !== undefined) record.benchmarks = benchmarks;
+	return record;
+}
+
+function aaScore(over: Partial<FeedScore> & { key: string }): FeedScore {
+	return { creator: "", source: "artificial_analysis", ...over };
+}
+function blScore(over: Partial<FeedScore> & { key: string }): FeedScore {
+	return { creator: "", source: "benchlm", ...over };
+}
+
+describe("normalizeModelKey", () => {
+	test("strips provider, tilde, and release words but keeps the parameter size", () => {
+		expect(normalizeModelKey("z-ai/glm-5.3-flash")).toBe("glm-5-3-flash");
+		expect(normalizeModelKey("~deepseek/deepseek-v4-flash-latest")).toBe("deepseek-v4-flash");
+		expect(normalizeModelKey("meta/muse-glimmer-30b")).toBe("muse-glimmer-30b");
+		// The feed's own display spelling collapses onto the same key.
+		expect(normalizeModelKey("Muse Glimmer 30B")).toBe("muse-glimmer-30b");
+		expect(normalizeModelKey("MiniMax M3")).toBe("minimax-m3");
+	});
+});
+
+describe("parseAaModels", () => {
+	test("reads the three indices, keeps in-range values, and skips empty rows", () => {
+		const body = {
+			data: [
+				{
+					slug: "glm-5.3-flash",
+					model_creator: { slug: "z-ai" },
+					evaluations: {
+						artificial_analysis_coding_index: 61.2,
+						artificial_analysis_intelligence_index: 58.4,
+						artificial_analysis_agentic_index: 150, // out of range → dropped
+					},
+				},
+				{ slug: "no-evals", model_creator: { slug: "x" }, evaluations: {} },
+			],
+		};
+		const parsed = parseAaModels(body);
+		expect(parsed).toHaveLength(1);
+		expect(parsed[0]).toMatchObject({ key: "glm-5-3-flash", creator: "z-ai", coding: 61.2, intelligence: 58.4 });
+		expect(parsed[0]?.agentic).toBeUndefined();
+	});
+});
+
+describe("parseBenchlmModels", () => {
+	test("maps categories to axes, drops estimated rows, and ignores out-of-range", () => {
+		const body = {
+			models: [
+				{
+					model: "Muse Glimmer 30B",
+					creator: "Meta",
+					evidenceStatus: "supported",
+					categoryScores: { coding: 55, reasoning: 52, agentic: 48 },
+				},
+				{
+					model: "Guessed Model",
+					creator: "x",
+					evidenceStatus: "estimated",
+					categoryScores: { coding: 90 },
+				},
+			],
+		};
+		const parsed = parseBenchlmModels(body);
+		expect(parsed).toHaveLength(1);
+		expect(parsed[0]).toMatchObject({ key: "muse-glimmer-30b", coding: 55, intelligence: 52, agentic: 48 });
+	});
+});
+
+describe("applyFeedScores", () => {
+	test("fills the real gap models and reaches normalizeCatalogModel", () => {
+		const catalog = [
+			raw("meta/muse-glimmer-30b"),
+			raw("z-ai/glm-5.3-flash"),
+			// Already scored by OpenRouter on coding; a feed must not overwrite it.
+			raw("google/gemini-3.7-flash", { artificial_analysis: { coding_index: 76.1 } }),
+		];
+		const feeds: FeedScore[] = [
+			aaScore({ key: "glm-5-3-flash", creator: "z-ai", coding: 61, intelligence: 58 }),
+			aaScore({ key: "gemini-3-7-flash", creator: "google", coding: 40, intelligence: 63 }),
+			blScore({ key: "muse-glimmer-30b", creator: "meta", coding: 55, agentic: 48 }),
+			blScore({ key: "glm-5-3-flash", creator: "z-ai", agentic: 44 }),
+		];
+
+		const result = applyFeedScores(catalog, feeds);
+
+		// muse-glimmer: was empty, gains coding + agentic from BenchLM.
+		const muse = normalizeCatalogModel(catalog[0]);
+		expect(muse?.quality).toEqual({ coding: 55, agentic: 48 });
+
+		// glm: coding + intelligence from AA (stronger), agentic from BenchLM.
+		const glm = normalizeCatalogModel(catalog[1]);
+		expect(glm?.quality).toEqual({ coding: 61, intelligence: 58, agentic: 44 });
+
+		// gemini: published coding survives untouched; intelligence filled from AA.
+		const gemini = normalizeCatalogModel(catalog[2]);
+		expect(gemini?.quality.coding).toBe(76.1);
+		expect(gemini?.quality.intelligence).toBe(63);
+
+		// Provenance recorded, counts add up.
+		const museBench = catalog[0]?.benchmarks;
+		expect(museBench).toMatchObject({ fill_sources: { coding: "benchlm", agentic: "benchlm" } });
+		expect(result.modelsFilled).toBe(3);
+		expect(result.sources.artificial_analysis).toBe(3); // glm coding+intel, gemini intel
+		expect(result.sources.benchlm).toBe(3); // muse coding+agentic, glm agentic
+	});
+
+	test("AA wins over BenchLM for the same axis", () => {
+		const catalog = [raw("z-ai/glm-5.3-flash")];
+		const feeds: FeedScore[] = [
+			blScore({ key: "glm-5-3-flash", creator: "z-ai", coding: 10 }),
+			aaScore({ key: "glm-5-3-flash", creator: "z-ai", coding: 61 }),
+		];
+		applyFeedScores(catalog, feeds);
+		expect(normalizeCatalogModel(catalog[0])?.quality.coding).toBe(61);
+	});
+
+	test("never fuzzy-matches a different model", () => {
+		const catalog = [raw("meta/muse-glimmer-30b")];
+		// Same family, different model — must not lend its score.
+		const feeds: FeedScore[] = [aaScore({ key: "muse-spark-1-2", creator: "meta", coding: 72 })];
+		const result = applyFeedScores(catalog, feeds);
+		expect(result.modelsFilled).toBe(0);
+		expect(normalizeCatalogModel(catalog[0])?.quality).toEqual({});
+	});
+
+	test("a shared key with conflicting creators fills only the creator that matches", () => {
+		const catalog = [raw("z-ai/glm-5.3-flash")];
+		const feeds: FeedScore[] = [
+			aaScore({ key: "glm-5-3-flash", creator: "someone-else", coding: 5 }),
+			aaScore({ key: "glm-5-3-flash", creator: "z-ai", coding: 61 }),
+		];
+		applyFeedScores(catalog, feeds);
+		expect(normalizeCatalogModel(catalog[0])?.quality.coding).toBe(61);
+	});
+});
+
+describe("refreshFeedScores", () => {
+	function cfgWith(over: Partial<RouterConfig["benchmarks"]>): RouterConfig {
+		const base = loadConfig({});
+		return { ...base, benchmarks: { ...base.benchmarks, ...over } };
+	}
+
+	test("fetches once, then serves the cache within the TTL", async () => {
+		const db = openDb(":memory:");
+		let calls = 0;
+		const fakeFetch: FetchLike = async (url) => {
+			calls += 1;
+			const u = String(url);
+			if (u.includes("benchlm")) {
+				return Response.json({
+					models: [
+						{ model: "MiniMax M3", creator: "MiniMax", evidenceStatus: "supported", categoryScores: { coding: 58 } },
+					],
+				});
+			}
+			return Response.json({ data: [] });
+		};
+
+		const cfg = cfgWith({ enabled: true, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 1_000_000 });
+		const first = await refreshFeedScores(cfg, db, { fetchImpl: fakeFetch, now: 1000 });
+		expect(first).toHaveLength(1);
+		expect(first[0]).toMatchObject({ key: "minimax-m3", coding: 58, source: "benchlm" });
+		expect(calls).toBe(1); // AA skipped (no key), BenchLM fetched once
+
+		const second = await refreshFeedScores(cfg, db, { fetchImpl: fakeFetch, now: 2000 });
+		expect(second).toHaveLength(1);
+		expect(calls).toBe(1); // within TTL → no new fetch
+		db.close();
+	});
+
+	test("falls back to the stale cache when a refresh returns nothing", async () => {
+		const db = openDb(":memory:");
+		const seed = [{ key: "minimax-m3", creator: "minimax", coding: 58, source: "benchlm" }];
+		db.query("INSERT INTO benchmark_cache (id, payload, fetched_at_ms) VALUES (1, ?, ?)").run(JSON.stringify(seed), 0);
+		const emptyFetch: FetchLike = async () => Response.json({ models: [] });
+		const cfg = cfgWith({ enabled: true, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 10 });
+		const got = await refreshFeedScores(cfg, db, { fetchImpl: emptyFetch, now: 1_000_000 });
+		expect(got).toHaveLength(1);
+		expect(got[0]).toMatchObject({ key: "minimax-m3", coding: 58 });
+		db.close();
+	});
+});
+
+// fetchBenchlmScores over a fake fetch: the keyless path parses end to end.
+test("fetchBenchlmScores parses a keyless leaderboard response", async () => {
+	const fake: FetchLike = async () =>
+		Response.json({
+			models: [{ model: "GLM 5.3 Flash", creator: "Z-AI", evidenceStatus: "supported", categoryScores: { coding: 61, reasoning: 58 } }],
+		});
+	const scores = await fetchBenchlmScores({ fetchImpl: fake });
+	expect(scores).toEqual([{ key: "glm-5-3-flash", creator: "z-ai", coding: 61, intelligence: 58, source: "benchlm" }]);
+});

package/test/eval.test.ts

@@ -0,0 +1,184 @@
+import { describe, expect, test } from "bun:test";
+
+import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
+import { applyFeedScores, loadLocalScores, saveLocalScores, type FeedScore } from "../src/catalog/benchmark-feeds.ts";
+import { answerScore, extractJson, isRefusalOrEmpty, jsonField, tokenCoverage } from "../src/eval/grade.ts";
+import { applyFit, fitAxis, fitCalibration, toLocalFeedScores, MIN_ANCHORS } from "../src/eval/calibrate.ts";
+import { runEval, type EvalResult } from "../src/eval/run.ts";
+import { makeJudge, parseScore } from "../src/eval/judge.ts";
+import type { EvalTask, JudgedTask } from "../src/eval/tasks.ts";
+import { openDb } from "../src/util/sqlite.ts";
+
+describe("grade helpers", () => {
+	test("answerScore matches whole reply, last line, or a standalone token", () => {
+		expect(answerScore("9.9", "9.9")).toBe(1);
+		expect(answerScore("The answer is 9.9", "9.9")).toBe(1);
+		expect(answerScore("reasoning...\n9.9", "9.9")).toBe(1);
+		expect(answerScore("19.99", "9.9")).toBe(0); // not a substring match
+		expect(answerScore("", "9.9")).toBe(0);
+	});
+	test("tokenCoverage is the fraction of tokens present", () => {
+		expect(tokenCoverage("return a + b;", ["a + b"])).toBe(1);
+		expect(tokenCoverage("n * 2", ["n", "*", "2"])).toBe(1);
+		expect(tokenCoverage("n plus two", ["n", "*", "2"])).toBeCloseTo(1 / 3);
+	});
+	test("extractJson tolerates fences and prose; jsonField reads a key", () => {
+		expect(extractJson('here: {"answer": 8} ok')).toEqual({ answer: 8 });
+		expect(extractJson("```json\n[2,3,5]\n```")).toEqual([2, 3, 5]);
+		expect(extractJson("no json here")).toBeUndefined();
+		expect(jsonField({ tool: "read_file" }, "tool")).toBe("read_file");
+		expect(jsonField([1, 2], "tool")).toBeUndefined();
+	});
+	test("isRefusalOrEmpty flags empties and refusals", () => {
+		expect(isRefusalOrEmpty("")).toBe(true);
+		expect(isRefusalOrEmpty("I cannot help with that")).toBe(true);
+		expect(isRefusalOrEmpty("sure, here")).toBe(false);
+	});
+});
+
+describe("calibration", () => {
+	test("fitAxis is OLS, needs MIN_ANCHORS points and some spread", () => {
+		const fit = fitAxis([
+			{ raw: 0.2, aa: 40 },
+			{ raw: 0.5, aa: 60 },
+			{ raw: 0.8, aa: 80 },
+		]);
+		expect(fit).not.toBeNull();
+		expect(fit!.slope).toBeCloseTo(66.67, 1);
+		expect(fit!.r).toBeCloseTo(1, 5);
+		expect(applyFit(fit!, 0.5)).toBeCloseTo(60, 5);
+		expect(applyFit(fit!, 5)).toBe(100); // clamped
+		expect(fitAxis([{ raw: 0.2, aa: 40 }, { raw: 0.5, aa: 60 }])).toBeNull(); // < MIN_ANCHORS
+		expect(fitAxis([{ raw: 0.5, aa: 40 }, { raw: 0.5, aa: 60 }, { raw: 0.5, aa: 80 }])).toBeNull(); // no spread
+		// Negative correlation (suite ranks models opposite to AA) is refused.
+		expect(fitAxis([{ raw: 0.8, aa: 40 }, { raw: 0.5, aa: 60 }, { raw: 0.2, aa: 80 }])).toBeNull();
+	});
+
+	test("fitCalibration + toLocalFeedScores place a target on the AA scale", () => {
+		expect(MIN_ANCHORS).toBe(3);
+		const anchors: EvalResult[] = [
+			{ slug: "a/one", axes: { coding: { sum: 0.2, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0 },
+			{ slug: "a/two", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0 },
+			{ slug: "a/three", axes: { coding: { sum: 0.8, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0 },
+		];
+		const aaOf: Record<string, number> = { "a/one": 40, "a/two": 60, "a/three": 80 };
+		const cal = fitCalibration(anchors, (slug, axis) => (axis === "coding" ? aaOf[slug] : undefined));
+		expect(cal.coding).toBeDefined();
+		expect(cal.intelligence).toBeUndefined(); // no anchor data on that axis
+
+		const targets: EvalResult[] = [
+			{ slug: "z/gap", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0.9, n: 1 }, agentic: { sum: 0, n: 0 } }, errors: 0 },
+		];
+		const local = toLocalFeedScores(targets, cal, (s) => s.slice(0, s.indexOf("/")));
+		expect(local).toHaveLength(1);
+		expect(local[0]).toMatchObject({ key: "gap", creator: "z", source: "local" });
+		expect(local[0]!.coding).toBeCloseTo(60, 5); // calibrated from raw 0.5
+		expect(local[0]!.intelligence).toBeUndefined(); // axis had no fit, so not emitted
+	});
+});
+
+describe("runEval", () => {
+	test("aggregates grades into per-axis means", async () => {
+		const tasks: EvalTask[] = [
+			{ id: "c1", axis: "coding", user: "x", grade: (o) => (o === "good" ? 1 : 0) },
+			{ id: "c2", axis: "coding", user: "y", grade: () => 0.5 },
+			{ id: "a1", axis: "agentic", user: "z", grade: (o) => (o === "good" ? 1 : 0) },
+		];
+		const results = await runEval({ slugs: ["good", "bad"], tasks, complete: async (slug) => slug });
+		const good = results.find((r) => r.slug === "good")!;
+		expect(good.axes.coding.sum).toBe(1.5); // 1 + 0.5
+		expect(good.axes.coding.n).toBe(2);
+		expect(good.axes.agentic.sum).toBe(1);
+		const bad = results.find((r) => r.slug === "bad")!;
+		expect(bad.axes.coding.sum).toBe(0.5); // 0 + 0.5
+		expect(bad.axes.agentic.sum).toBe(0);
+	});
+
+	test("a throwing completion is excluded, not scored 0", async () => {
+		const tasks: EvalTask[] = [{ id: "a", axis: "coding", user: "x", grade: () => 1 }];
+		const results = await runEval({
+			slugs: ["m"],
+			tasks,
+			complete: async () => {
+				throw new Error("boom");
+			},
+		});
+		expect(results[0]!.axes.coding.n).toBe(0); // no observation
+		expect(results[0]!.errors).toBe(1);
+	});
+});
+
+describe("local source integration", () => {
+	function raw(id: string): Record<string, unknown> {
+		return {
+			id,
+			canonical_slug: id,
+			name: id,
+			context_length: 131072,
+			pricing: { prompt: "0.0000003", completion: "0.0000011" },
+			supported_parameters: ["tools"],
+			architecture: { input_modalities: ["text"], tokenizer: "Other" },
+			created: 1_700_000_000,
+		};
+	}
+
+	test("local fills only where no stronger source has the axis", () => {
+		const catalog = [raw("z-ai/glm-5.3-flash")];
+		const feeds: FeedScore[] = [
+			{ key: "glm-5-3-flash", creator: "z-ai", source: "artificial_analysis", coding: 61 },
+			{ key: "glm-5-3-flash", creator: "z-ai", source: "local", coding: 20, intelligence: 55 },
+		];
+		const result = applyFeedScores(catalog, feeds);
+		const q = normalizeCatalogModel(catalog[0])?.quality;
+		expect(q?.coding).toBe(61); // AA wins over local
+		expect(q?.intelligence).toBe(55); // local fills the axis nobody else had
+		expect(result.sources.local).toBe(1);
+		expect(result.sources.artificial_analysis).toBe(1);
+	});
+
+	test("saveLocalScores / loadLocalScores round-trip", () => {
+		const db = openDb(":memory:");
+		const scores: FeedScore[] = [{ key: "muse-glimmer-30b", creator: "meta", source: "local", coding: 42, agentic: 39 }];
+		saveLocalScores(db, scores, 123);
+		expect(loadLocalScores(db)).toEqual(scores);
+		db.close();
+	});
+});
+
+describe("llm judge", () => {
+	test("parseScore takes the last standalone 0-10 and scales to 0-1", () => {
+		expect(parseScore("8")).toBeCloseTo(0.8, 5);
+		expect(parseScore("Score: 10/10")).toBeCloseTo(1, 5);
+		expect(parseScore("I count 3 issues, so 7")).toBeCloseTo(0.7, 5); // last wins
+		expect(parseScore("no number here")).toBeNull();
+	});
+
+	test("makeJudge parses a score, and returns null on a thrown completion", async () => {
+		const task: JudgedTask = { id: "j", axis: "coding", user: "do a thing" };
+		const good = makeJudge(async () => "the answer earns 8", "judge/model");
+		expect(await good(task, "some answer")).toBeCloseTo(0.8, 5);
+		const bad = makeJudge(async () => {
+			throw new Error("judge down");
+		}, "judge/model");
+		expect(await bad(task, "some answer")).toBeNull();
+	});
+
+	test("runEval folds judged scores into the axis mean, and drops unscorable ones", async () => {
+		const judged: JudgedTask[] = [
+			{ id: "j1", axis: "coding", user: "a" },
+			{ id: "j2", axis: "coding", user: "b" },
+		];
+		// j1 scores 0.6; j2 is unscorable (null) → excluded as an error.
+		const judge = async (t: JudgedTask) => (t.id === "j1" ? 0.6 : null);
+		const results = await runEval({ slugs: ["m"], tasks: [], judged, judge, complete: async () => "ans" });
+		expect(results[0]!.axes.coding).toEqual({ sum: 0.6, n: 1 });
+		expect(results[0]!.errors).toBe(1);
+	});
+
+	test("judged tasks are skipped entirely when no judge is supplied", async () => {
+		const judged: JudgedTask[] = [{ id: "j1", axis: "coding", user: "a" }];
+		const results = await runEval({ slugs: ["m"], tasks: [], judged, complete: async () => "ans" });
+		expect(results[0]!.axes.coding).toEqual({ sum: 0, n: 0 });
+		expect(results[0]!.errors).toBe(0);
+	});
+});