auto-model-router 0.30.2 → 0.31.0

package/.omp-plugin/marketplace.json

@@ -7,14 +7,14 @@
   },
   "metadata": {
     "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
-    "version": "0.30.2",
+    "version": "0.31.0",
     "pluginRoot": "."
   },
   "plugins": [
     {
       "name": "auto-model-router",
       "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
-      "version": "0.30.2",
+      "version": "0.31.0",
       "author": {
         "name": "drewappling",
         "email": "drewappling@gmail.com"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-model-router",
-  "version": "0.30.2",
+  "version": "0.31.0",
   "private": false,
   "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
   "type": "module",

package/src/eval/calibrate.ts

@@ -58,13 +58,26 @@ export interface AnchorPoint {
 }
 
 /**
- * OLS fit, or null when the calibration cannot be trusted: too few points, no
- * spread, a non-positive slope, or weak correlation. A suite that does not track
- * AA positively and with real correlation would turn a target's score into noise
- * dressed as signal, so we refuse it and emit nothing for that axis.
+ * The raw range the anchors must actually cover before a line through them means anything.
+ *
+ * Correlation alone does NOT catch a compressed fit: three anchors published 20/50/80 that
+ * our suite scores 0.96/0.97/0.98 correlate at r = 1.0, and the line they define has a slope
+ * of ~3000 index points per unit of raw score. That fit is arithmetically perfect and
+ * completely useless — it is how a model published at 39.5 was calibrated to 22.8. If the
+ * anchors barely differ on our suite, our suite cannot place anything between them.
+ */
+export const MIN_RAW_SPREAD = 0.15;
+
+/**
+ * OLS fit, or null when the calibration cannot be trusted: too few points, too narrow a raw
+ * range, no spread, a non-positive slope, or weak correlation. A suite that does not track
+ * AA positively, with real correlation, over a real range would turn a target's score into
+ * noise dressed as signal, so we refuse it and emit nothing for that axis.
  */
 export function fitAxis(points: readonly AnchorPoint[]): LineFit | null {
 	if (points.length < MIN_ANCHORS) return null;
+	const raws = points.map((p) => p.raw);
+	if (Math.max(...raws) - Math.min(...raws) < MIN_RAW_SPREAD) return null;
 	const n = points.length;
 	let sx = 0;
 	let sy = 0;
@@ -101,19 +114,32 @@ export function applyFit(fit: LineFit, raw: number): number {
 const AXES: readonly QualityAxis[] = ["coding", "intelligence", "agentic"];
 
 /**
- * Fit every axis from the anchors' raw suite scores paired with their known AA
- * scores. `anchorAa` supplies the AA index per slug+axis (absent ⇒ that anchor
- * is not used on that axis).
+ * The correlation a fit must reach before its numbers are published as scores. `MIN_R` (0.5)
+ * is the bar for a fit being computable at all; this is the bar for TRUSTING one. The fit's
+ * `r` and `n` were previously computed and then discarded, so an r of 0.51 and one of 0.99
+ * produced indistinguishable output — and a shallow fit silently compressed every target.
+ */
+export const PUBLISH_MIN_R = 0.8;
+
+/**
+ * Fit every axis from the anchors' raw suite scores paired with their known AA scores.
+ * `anchorAa` supplies the AA index per slug+axis (absent ⇒ that anchor is not used there).
+ *
+ * `rawOf` selects which observations the fit is built from. It defaults to every band, but
+ * callers should pass the HARD band: easy and moderate sit at ~1.0 for every model worth
+ * ranking, so including them leaves the regression almost no variation in x against a wide
+ * spread in published y, and the slope collapses toward flat.
  */
 export function fitCalibration(
 	anchors: readonly EvalResult[],
 	anchorAa: (slug: string, axis: QualityAxis) => number | undefined,
+	rawOf: (result: EvalResult, axis: QualityAxis) => number | null = (r, axis) => axisMean(r.axes[axis]),
 ): Calibration {
 	const cal: Calibration = {};
 	for (const axis of AXES) {
 		const points: AnchorPoint[] = [];
 		for (const r of anchors) {
-			const raw = axisMean(r.axes[axis]);
+			const raw = rawOf(r, axis);
 			const aa = anchorAa(r.slug, axis);
 			if (raw !== null && aa !== undefined) points.push({ raw, aa });
 		}
@@ -123,15 +149,24 @@ export function fitCalibration(
 	return cal;
 }
 
-function axisMean(a: AxisScore | undefined): number | null {
+export function axisMean(a: AxisScore | undefined): number | null {
 	return a === undefined || a.n === 0 ? null : a.sum / a.n;
 }
 
-/** Calibrated local FeedScores for the targets, one axis at a time, skipping axes with no fit. */
+/** The hard band alone, for calibration. */
+export const hardRaw = (r: EvalResult, axis: QualityAxis): number | null => axisMean(r.axesHard[axis]);
+
+/**
+ * Calibrated local FeedScores for the targets, one axis at a time. An axis is skipped when it
+ * has no fit, when the fit is weaker than `minR`, or when the target made no observation on
+ * it — honest silence rather than a number nobody should act on.
+ */
 export function toLocalFeedScores(
 	targets: readonly EvalResult[],
 	cal: Calibration,
 	authorOf: (slug: string) => string,
+	rawOf: (result: EvalResult, axis: QualityAxis) => number | null = (r, axis) => axisMean(r.axes[axis]),
+	minR = PUBLISH_MIN_R,
 ): FeedScore[] {
 	const out: FeedScore[] = [];
 	for (const r of targets) {
@@ -139,8 +174,8 @@ export function toLocalFeedScores(
 		let any = false;
 		for (const axis of AXES) {
 			const fit = cal[axis];
-			const raw = axisMean(r.axes[axis]);
-			if (fit === undefined || raw === null) continue;
+			const raw = rawOf(r, axis);
+			if (fit === undefined || raw === null || fit.r < minR) continue;
 			entry[axis] = applyFit(fit, raw);
 			any = true;
 		}

package/src/eval/run.ts

@@ -39,6 +39,14 @@ export interface EvalResult {
 	 * questions were easy.
 	 */
 	byComplexity: Partial<Record<Complexity, AxisScore>>;
+	/**
+	 * Per-axis scores from the HARD band alone. Calibration fits against these: the easy and
+	 * moderate bands sit at ~1.0 for every model worth ranking, so including them gives the
+	 * regression almost no variation in x against a wide spread in published y — the fitted
+	 * slope goes shallow and every target is dragged toward the middle. Measured: a model
+	 * published at intelligence 39.5 calibrated to 22.8 across 10 passes.
+	 */
+	axesHard: Record<QualityAxis, AxisScore>;
 	/**
 	 * Per-axis spread across passes: max pass mean minus min pass mean, or null under two
 	 * passes. A wide spread means the headline is one sample of a noisy quantity, and is the
@@ -67,6 +75,12 @@ export interface RunEvalArgs {
 	concurrency?: number;
 	/** Called as each model finishes, for progress logging. */
 	onProgress?: (result: EvalResult, done: number, total: number) => void;
+	/**
+	 * Called as each PASS of each model finishes. A run is `models x repeats` passes and takes
+	 * the better part of an hour at ten repeats, so per-model progress is too coarse to show
+	 * anyone: without this the only observable states are "running" and "finished".
+	 */
+	onPass?: (slug: string, pass: number, of: number) => void;
 	/**
 	 * A tool-capable completer. Absent ⇒ the agentic SCENARIOS are skipped and the axis falls
 	 * back to the text tasks, which only ever measured whether a model can format JSON.
@@ -138,6 +152,7 @@ async function scorePass(slug: string, args: RunEvalArgs): Promise<EvalResult> {
 		}
 	}
 	const byComplexity: Partial<Record<Complexity, AxisScore>> = {};
+	const axesHard: Record<QualityAxis, AxisScore> = { coding: { sum: 0, n: 0 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } };
 	for (const o of [...objective, ...judgedOutcomes, ...scenarioOutcomes]) {
 		if (!o.ok) {
 			// An unobserved task is NOT a zero: a provider's throttle or outage would otherwise
@@ -148,11 +163,15 @@ async function scorePass(slug: string, args: RunEvalArgs): Promise<EvalResult> {
 		}
 		axes[o.axis].sum += o.grade;
 		axes[o.axis].n += 1;
+		if (o.complexity === "hard") {
+			axesHard[o.axis].sum += o.grade;
+			axesHard[o.axis].n += 1;
+		}
 		const band = (byComplexity[o.complexity] ??= { sum: 0, n: 0 });
 		band.sum += o.grade;
 		band.n += 1;
 	}
-	return { slug, axes, errors, repeats: 1, spread: {}, byComplexity };
+	return { slug, axes, errors, repeats: 1, spread: {}, byComplexity, axesHard };
 }
 
 const AXES: readonly QualityAxis[] = ["coding", "intelligence", "agentic"];
@@ -168,13 +187,17 @@ async function scoreModel(slug: string, args: RunEvalArgs): Promise<EvalResult>
 	const axes: Record<QualityAxis, AxisScore> = { coding: { sum: 0, n: 0 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } };
 	const means: Record<QualityAxis, number[]> = { coding: [], intelligence: [], agentic: [] };
 	const byComplexity: Partial<Record<Complexity, AxisScore>> = {};
+	const axesHard: Record<QualityAxis, AxisScore> = { coding: { sum: 0, n: 0 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } };
 	let errors = 0;
 	for (let i = 0; i < passes; i++) {
 		const pass = await scorePass(slug, args);
+		args.onPass?.(slug, i + 1, passes);
 		errors += pass.errors;
 		for (const axis of AXES) {
 			axes[axis].sum += pass.axes[axis].sum;
 			axes[axis].n += pass.axes[axis].n;
+			axesHard[axis].sum += pass.axesHard[axis].sum;
+			axesHard[axis].n += pass.axesHard[axis].n;
 			if (pass.axes[axis].n > 0) means[axis].push(pass.axes[axis].sum / pass.axes[axis].n);
 		}
 		for (const [band, score] of Object.entries(pass.byComplexity) as [Complexity, AxisScore][]) {
@@ -188,7 +211,7 @@ async function scoreModel(slug: string, args: RunEvalArgs): Promise<EvalResult>
 		const m = means[axis];
 		if (m.length > 1) spread[axis] = Math.max(...m) - Math.min(...m);
 	}
-	return { slug, axes, errors, repeats: passes, spread, byComplexity };
+	return { slug, axes, errors, repeats: passes, spread, byComplexity, axesHard };
 }
 
 export async function runEval(args: RunEvalArgs): Promise<EvalResult[]> {

package/src/server/http.ts

@@ -18,7 +18,7 @@ import { runEval, type Completer } from "../eval/run.ts";
 import type { ToolSpec } from "../eval/agentic.ts";
 import type { ToolCall } from "../upstream/types.ts";
 import type { QualityAxis } from "../config/types.ts";
-import { fitCalibration, pickAnchors, toLocalFeedScores, MIN_ANCHORS } from "../eval/calibrate.ts";
+import { fitCalibration, hardRaw, pickAnchors, toLocalFeedScores, MIN_ANCHORS, PUBLISH_MIN_R } from "../eval/calibrate.ts";
 import { makeJudge } from "../eval/judge.ts";
 import { loadLocalScores, saveLocalScores } from "../catalog/benchmark-feeds.ts";
 import { advise } from "./advise.ts";
@@ -182,6 +182,19 @@ export function ollamaRunway(
 	return { dailyBurnUsd, creditsLeftUsd, days: dailyBurnUsd > 0 ? creditsLeftUsd / dailyBurnUsd : null };
 }
 
+/**
+ * Progress for a local benchmark run, as the dashboard needs it. The ETA is measured from
+ * this run's own completed passes rather than assumed — passes vary with the model's speed,
+ * and a guess from a fixed per-pass constant would be wrong by minutes on a slow upstream.
+ * Null until a pass has finished, because until then there is nothing to extrapolate from.
+ */
+function benchmarkProgress(job: { state: string; startedAtMs: number; donePasses: number; totalPasses: number; current: string }): Record<string, unknown> {
+	const elapsedMs = Date.now() - job.startedAtMs;
+	const fraction = job.totalPasses > 0 ? job.donePasses / job.totalPasses : 0;
+	const etaMs = job.state === "running" && job.donePasses > 0 ? Math.round((elapsedMs / job.donePasses) * (job.totalPasses - job.donePasses)) : null;
+	return { donePasses: job.donePasses, totalPasses: job.totalPasses, fraction, current: job.current, elapsedMs, etaMs };
+}
+
 function json(data: unknown, status = 200): Response {
 	return new Response(JSON.stringify(data), {
 		status,
@@ -252,7 +265,7 @@ export function startServer(cfg: RouterConfig): StartedServer {
 	 * Background local-benchmark runs. In memory and not persisted: a run is an explicit,
 	 * attended admin action, and its lasting output is the `local_scores` row it writes.
 	 */
-	const benchmarkJobs = new Map<string, { state: "running" | "done" | "error"; slug: string; startedAtMs: number; result?: Record<string, unknown>; error?: string }>();
+	const benchmarkJobs = new Map<string, { state: "running" | "done" | "error"; slug: string; startedAtMs: number; donePasses: number; totalPasses: number; current: string; result?: Record<string, unknown>; error?: string }>();
 	const digester = createDigester({ cfg, catalog, ledger, upstream, log });
 	const turnDeps = { config: cfg, router, upstream, ledger, conversations, catalog, context, overrides, ollamaCostScale, digester };
 
@@ -779,46 +792,56 @@ export function startServer(cfg: RouterConfig): StartedServer {
 					// lost. The caller polls instead.
 					const jobId = `bench_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
 					const started = Date.now();
-					benchmarkJobs.set(jobId, { state: "running", slug, startedAtMs: started });
+					const repeats = Math.min(Math.max(1, typeof body?.repeats === "number" ? Math.floor(body.repeats) : 1), 20);
+					// A pass of one model is the unit of progress: models x repeats, known up front.
+					const totalPasses = repeats * (1 + anchors.length);
+					benchmarkJobs.set(jobId, { state: "running", slug, startedAtMs: started, donePasses: 0, totalPasses, current: slug });
 					void (async () => {
 						try {
 							const results = await runEval({
 								slugs: [slug, ...anchors],
 								complete,
 								toolComplete,
-								repeats: Math.min(Math.max(1, typeof body?.repeats === "number" ? Math.floor(body.repeats) : 1), 20),
+								repeats,
 								concurrency: Math.min(Math.max(1, asked), 8),
+								onPass: (passSlug, _pass, _of) => {
+									const job = benchmarkJobs.get(jobId);
+									if (job !== undefined) benchmarkJobs.set(jobId, { ...job, donePasses: job.donePasses + 1, current: passSlug });
+								},
 								...(judgeSlug === "" ? {} : { judge: makeJudge(complete, judgeSlug) }),
 							});
 							const target = results[0]!;
 							const published = (s: string, axis: QualityAxis): number | undefined => models.find((m) => m.slug === s)?.quality[axis];
-							const cal = fitCalibration(results.slice(1), published);
+							const cal = fitCalibration(results.slice(1), published, hardRaw);
 							const authorOf = (s: string): string => models.find((m) => m.slug === s)?.author ?? "";
-							const fresh = toLocalFeedScores([target], cal, authorOf);
-							const shared = { slug, anchors, raw: target.axes, byComplexity: target.byComplexity, repeats: target.repeats, spread: target.spread, errors: target.errors, tookMs: Date.now() - started };
+							const fresh = toLocalFeedScores([target], cal, authorOf, hardRaw);
+							// The fit is reported, not just used: `r` and `n` say whether the numbers deserve
+							// belief, and an axis dropped for a weak fit should say so rather than vanish.
+							const fitDetail = Object.fromEntries(Object.entries(cal).map(([axis, f]) => [axis, { r: Number(f.r.toFixed(3)), n: f.n, published: f.r >= PUBLISH_MIN_R }]));
+							const shared = { slug, anchors, raw: target.axes, rawHard: target.axesHard, byComplexity: target.byComplexity, repeats: target.repeats, spread: target.spread, errors: target.errors, fit: fitDetail, publishMinR: PUBLISH_MIN_R, tookMs: Date.now() - started };
 							if (fresh.length === 0) {
-								benchmarkJobs.set(jobId, { state: "done", slug, startedAtMs: started, result: { ...shared, calibrated: null, applied: false, reason: "no axis produced a usable fit; try more or better-spread anchors" } });
+								benchmarkJobs.set(jobId, { ...benchmarkJobs.get(jobId)!, state: "done", result: { ...shared, calibrated: null, applied: false, reason: `no axis produced a fit at r >= ${PUBLISH_MIN_R} on the hard band; try more or better-spread anchors` } });
 								return;
 							}
 							// Merge, never replace: other models' measurements are not this run's to discard.
 							const kept = loadLocalScores(db).filter((s) => s.key !== fresh[0]!.key);
 							saveLocalScores(db, [...kept, ...fresh]);
 							log.info("benchmarked a model with the local eval suite", { slug, anchors: anchors.length, repeats: target.repeats, errors: target.errors, useLocalScores: cfg.benchmarks.useLocalScores });
-							benchmarkJobs.set(jobId, { state: "done", slug, startedAtMs: started, result: { ...shared, calibrated: fresh[0], applied: cfg.benchmarks.useLocalScores } });
+							benchmarkJobs.set(jobId, { ...benchmarkJobs.get(jobId)!, state: "done", result: { ...shared, calibrated: fresh[0], applied: cfg.benchmarks.useLocalScores } });
 						} catch (err) {
 							const message = err instanceof Error ? err.message : String(err);
 							log.warn("local benchmark run failed", { slug, error: message });
-							benchmarkJobs.set(jobId, { state: "error", slug, startedAtMs: started, error: message });
+							benchmarkJobs.set(jobId, { ...benchmarkJobs.get(jobId)!, state: "error", error: message });
 						}
 					})();
-					return json({ jobId, state: "running", slug, anchors });
+					return json({ jobId, state: "running", slug, anchors, totalPasses });
 				}
 				if (req.method === "GET" && url.pathname === "/v1/router/benchmark") {
 					const jobId = url.searchParams.get("job");
-					if (jobId === null) return json({ jobs: [...benchmarkJobs.entries()].map(([id, j]) => ({ id, slug: j.slug, state: j.state, startedAtMs: j.startedAtMs })) });
+					if (jobId === null) return json({ jobs: [...benchmarkJobs.entries()].map(([id, j]) => ({ id, slug: j.slug, state: j.state, startedAtMs: j.startedAtMs, ...benchmarkProgress(j) })) });
 					const job = benchmarkJobs.get(jobId);
 					if (job === undefined) return wireErrorResponse({ status: 404, code: "invalid_request_error", message: `no benchmark job ${jobId}` });
-					return json({ id: jobId, slug: job.slug, state: job.state, startedAtMs: job.startedAtMs, ...(job.result === undefined ? {} : { result: job.result }), ...(job.error === undefined ? {} : { error: job.error }) });
+					return json({ id: jobId, slug: job.slug, state: job.state, startedAtMs: job.startedAtMs, ...benchmarkProgress(job), ...(job.result === undefined ? {} : { result: job.result }), ...(job.error === undefined ? {} : { error: job.error }) });
 				}
 				if (req.method === "POST" && url.pathname === "/v1/router/feedback") {
 					// A user verdict on the newest routed turn of an omp session.

package/test/eval.test.ts

@@ -3,9 +3,10 @@ import { describe, expect, test } from "bun:test";
 import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
 import { applyFeedScores, loadLocalScores, saveLocalScores, type FeedScore } from "../src/catalog/benchmark-feeds.ts";
 import { answerScore, extractJson, isRefusalOrEmpty, jsonField, tokenCoverage } from "../src/eval/grade.ts";
-import { applyFit, fitAxis, fitCalibration, pickAnchors, toLocalFeedScores, MIN_ANCHORS } from "../src/eval/calibrate.ts";
+import { applyFit, fitAxis, fitCalibration, hardRaw, pickAnchors, toLocalFeedScores, MIN_ANCHORS, MIN_R, PUBLISH_MIN_R } from "../src/eval/calibrate.ts";
 import { runEval, type EvalResult } from "../src/eval/run.ts";
 import { EVAL_TASKS } from "../src/eval/tasks.ts";
+import type { QualityAxis } from "../src/config/types.ts";
 import { makeJudge, parseScore } from "../src/eval/judge.ts";
 import type { EvalTask, JudgedTask } from "../src/eval/tasks.ts";
 import { openDb } from "../src/util/sqlite.ts";
@@ -130,9 +131,9 @@ describe("calibration", () => {
 	test("fitCalibration + toLocalFeedScores place a target on the AA scale", () => {
 		expect(MIN_ANCHORS).toBe(3);
 		const anchors: EvalResult[] = [
-			{ slug: "a/one", axes: { coding: { sum: 0.2, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {} },
-			{ slug: "a/two", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {} },
-			{ slug: "a/three", axes: { coding: { sum: 0.8, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {} },
+			{ slug: "a/one", axes: { coding: { sum: 0.2, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {}, axesHard: { coding: { sum: 0, n: 0 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } } },
+			{ slug: "a/two", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {}, axesHard: { coding: { sum: 0, n: 0 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } } },
+			{ slug: "a/three", axes: { coding: { sum: 0.8, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {}, axesHard: { coding: { sum: 0, n: 0 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } } },
 		];
 		const aaOf: Record<string, number> = { "a/one": 40, "a/two": 60, "a/three": 80 };
 		const cal = fitCalibration(anchors, (slug, axis) => (axis === "coding" ? aaOf[slug] : undefined));
@@ -140,7 +141,7 @@ describe("calibration", () => {
 		expect(cal.intelligence).toBeUndefined(); // no anchor data on that axis
 
 		const targets: EvalResult[] = [
-			{ slug: "z/gap", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0.9, n: 1 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {} },
+			{ slug: "z/gap", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0.9, n: 1 }, agentic: { sum: 0, n: 0 } }, errors: 0, repeats: 1, spread: {}, byComplexity: {}, axesHard: { coding: { sum: 0, n: 0 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } } },
 		];
 		const local = toLocalFeedScores(targets, cal, (s) => s.slice(0, s.indexOf("/")));
 		expect(local).toHaveLength(1);
@@ -148,6 +149,62 @@ describe("calibration", () => {
 		expect(local[0]!.coding).toBeCloseTo(60, 5); // calibrated from raw 0.5
 		expect(local[0]!.intelligence).toBeUndefined(); // axis had no fit, so not emitted
 	});
+
+	test("calibrating on the hard band beats calibrating on everything", () => {
+		// Three anchors published 20/50/80 apart. On the FULL suite they all score ~0.97
+		// because easy and moderate pin everyone at the ceiling; on the hard band alone they
+		// separate. Same models, same publishing, different x — and only one of them can fit.
+		const mk = (slug: string, full: number, hard: number): EvalResult => ({
+			slug,
+			axes: { coding: { sum: full, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } },
+			axesHard: { coding: { sum: hard, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } },
+			errors: 0,
+			repeats: 1,
+			spread: {},
+			byComplexity: {},
+		});
+		const anchors = [mk("a/low", 0.96, 0.2), mk("a/mid", 0.97, 0.5), mk("a/high", 0.98, 0.8)];
+		const aa: Record<string, number> = { "a/low": 20, "a/mid": 50, "a/high": 80 };
+		const published = (slug: string, axis: QualityAxis) => (axis === "coding" ? aa[slug] : undefined);
+		const target = mk("z/target", 0.97, 0.5);
+
+		const onHard = toLocalFeedScores([target], fitCalibration(anchors, published, hardRaw), () => "z", hardRaw);
+		// The hard band spans 0.2-0.8 against 20-80, so the fit is a real line: raw 0.5 ⇒ ~50.
+		expect(onHard[0]!.coding).toBeCloseTo(50, 0);
+
+		// On the full suite the anchors span 0.96-0.98: the same published spread compressed into
+		// a fiftieth of the range. That is the shallow slope that produced 22.8 for a model
+		// published at 39.5, so the fit must be refused rather than published.
+		const pooledFit = fitCalibration(anchors, published);
+		const pooled = toLocalFeedScores([target], pooledFit, () => "z");
+		expect(pooled).toEqual([]);
+	});
+
+	test("a weak fit is refused, not published", () => {
+		// Points with a real but noisy relationship: computable (r >= MIN_R) yet not worth
+		// acting on. `r` and `n` used to be computed and then thrown away.
+		const noisy = [
+			{ raw: 0.1, aa: 20 },
+			{ raw: 0.5, aa: 70 },
+			{ raw: 0.6, aa: 30 },
+			{ raw: 0.9, aa: 60 },
+		];
+		const fit = fitAxis(noisy)!;
+		expect(fit.r).toBeGreaterThanOrEqual(MIN_R);
+		expect(fit.r).toBeLessThan(PUBLISH_MIN_R);
+		const target: EvalResult = {
+			slug: "z/t",
+			axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } },
+			axesHard: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } },
+			errors: 0,
+			repeats: 1,
+			spread: {},
+			byComplexity: {},
+		};
+		expect(toLocalFeedScores([target], { coding: fit }, () => "z", hardRaw)).toEqual([]);
+		// A caller that deliberately lowers the bar still can, so the gate is policy not dogma.
+		expect(toLocalFeedScores([target], { coding: fit }, () => "z", hardRaw, MIN_R)[0]!.coding).toBeGreaterThan(0);
+	});
 });
 
 describe("runEval", () => {