auto-model-router 0.1.3 → 0.2.0

package/test/eval.test.ts

@@ -0,0 +1,184 @@
+import { describe, expect, test } from "bun:test";
+
+import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
+import { applyFeedScores, loadLocalScores, saveLocalScores, type FeedScore } from "../src/catalog/benchmark-feeds.ts";
+import { answerScore, extractJson, isRefusalOrEmpty, jsonField, tokenCoverage } from "../src/eval/grade.ts";
+import { applyFit, fitAxis, fitCalibration, toLocalFeedScores, MIN_ANCHORS } from "../src/eval/calibrate.ts";
+import { runEval, type EvalResult } from "../src/eval/run.ts";
+import { makeJudge, parseScore } from "../src/eval/judge.ts";
+import type { EvalTask, JudgedTask } from "../src/eval/tasks.ts";
+import { openDb } from "../src/util/sqlite.ts";
+
+describe("grade helpers", () => {
+	test("answerScore matches whole reply, last line, or a standalone token", () => {
+		expect(answerScore("9.9", "9.9")).toBe(1);
+		expect(answerScore("The answer is 9.9", "9.9")).toBe(1);
+		expect(answerScore("reasoning...\n9.9", "9.9")).toBe(1);
+		expect(answerScore("19.99", "9.9")).toBe(0); // not a substring match
+		expect(answerScore("", "9.9")).toBe(0);
+	});
+	test("tokenCoverage is the fraction of tokens present", () => {
+		expect(tokenCoverage("return a + b;", ["a + b"])).toBe(1);
+		expect(tokenCoverage("n * 2", ["n", "*", "2"])).toBe(1);
+		expect(tokenCoverage("n plus two", ["n", "*", "2"])).toBeCloseTo(1 / 3);
+	});
+	test("extractJson tolerates fences and prose; jsonField reads a key", () => {
+		expect(extractJson('here: {"answer": 8} ok')).toEqual({ answer: 8 });
+		expect(extractJson("```json\n[2,3,5]\n```")).toEqual([2, 3, 5]);
+		expect(extractJson("no json here")).toBeUndefined();
+		expect(jsonField({ tool: "read_file" }, "tool")).toBe("read_file");
+		expect(jsonField([1, 2], "tool")).toBeUndefined();
+	});
+	test("isRefusalOrEmpty flags empties and refusals", () => {
+		expect(isRefusalOrEmpty("")).toBe(true);
+		expect(isRefusalOrEmpty("I cannot help with that")).toBe(true);
+		expect(isRefusalOrEmpty("sure, here")).toBe(false);
+	});
+});
+
+describe("calibration", () => {
+	test("fitAxis is OLS, needs MIN_ANCHORS points and some spread", () => {
+		const fit = fitAxis([
+			{ raw: 0.2, aa: 40 },
+			{ raw: 0.5, aa: 60 },
+			{ raw: 0.8, aa: 80 },
+		]);
+		expect(fit).not.toBeNull();
+		expect(fit!.slope).toBeCloseTo(66.67, 1);
+		expect(fit!.r).toBeCloseTo(1, 5);
+		expect(applyFit(fit!, 0.5)).toBeCloseTo(60, 5);
+		expect(applyFit(fit!, 5)).toBe(100); // clamped
+		expect(fitAxis([{ raw: 0.2, aa: 40 }, { raw: 0.5, aa: 60 }])).toBeNull(); // < MIN_ANCHORS
+		expect(fitAxis([{ raw: 0.5, aa: 40 }, { raw: 0.5, aa: 60 }, { raw: 0.5, aa: 80 }])).toBeNull(); // no spread
+		// Negative correlation (suite ranks models opposite to AA) is refused.
+		expect(fitAxis([{ raw: 0.8, aa: 40 }, { raw: 0.5, aa: 60 }, { raw: 0.2, aa: 80 }])).toBeNull();
+	});
+
+	test("fitCalibration + toLocalFeedScores place a target on the AA scale", () => {
+		expect(MIN_ANCHORS).toBe(3);
+		const anchors: EvalResult[] = [
+			{ slug: "a/one", axes: { coding: { sum: 0.2, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0 },
+			{ slug: "a/two", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0 },
+			{ slug: "a/three", axes: { coding: { sum: 0.8, n: 1 }, intelligence: { sum: 0, n: 0 }, agentic: { sum: 0, n: 0 } }, errors: 0 },
+		];
+		const aaOf: Record<string, number> = { "a/one": 40, "a/two": 60, "a/three": 80 };
+		const cal = fitCalibration(anchors, (slug, axis) => (axis === "coding" ? aaOf[slug] : undefined));
+		expect(cal.coding).toBeDefined();
+		expect(cal.intelligence).toBeUndefined(); // no anchor data on that axis
+
+		const targets: EvalResult[] = [
+			{ slug: "z/gap", axes: { coding: { sum: 0.5, n: 1 }, intelligence: { sum: 0.9, n: 1 }, agentic: { sum: 0, n: 0 } }, errors: 0 },
+		];
+		const local = toLocalFeedScores(targets, cal, (s) => s.slice(0, s.indexOf("/")));
+		expect(local).toHaveLength(1);
+		expect(local[0]).toMatchObject({ key: "gap", creator: "z", source: "local" });
+		expect(local[0]!.coding).toBeCloseTo(60, 5); // calibrated from raw 0.5
+		expect(local[0]!.intelligence).toBeUndefined(); // axis had no fit, so not emitted
+	});
+});
+
+describe("runEval", () => {
+	test("aggregates grades into per-axis means", async () => {
+		const tasks: EvalTask[] = [
+			{ id: "c1", axis: "coding", user: "x", grade: (o) => (o === "good" ? 1 : 0) },
+			{ id: "c2", axis: "coding", user: "y", grade: () => 0.5 },
+			{ id: "a1", axis: "agentic", user: "z", grade: (o) => (o === "good" ? 1 : 0) },
+		];
+		const results = await runEval({ slugs: ["good", "bad"], tasks, complete: async (slug) => slug });
+		const good = results.find((r) => r.slug === "good")!;
+		expect(good.axes.coding.sum).toBe(1.5); // 1 + 0.5
+		expect(good.axes.coding.n).toBe(2);
+		expect(good.axes.agentic.sum).toBe(1);
+		const bad = results.find((r) => r.slug === "bad")!;
+		expect(bad.axes.coding.sum).toBe(0.5); // 0 + 0.5
+		expect(bad.axes.agentic.sum).toBe(0);
+	});
+
+	test("a throwing completion is excluded, not scored 0", async () => {
+		const tasks: EvalTask[] = [{ id: "a", axis: "coding", user: "x", grade: () => 1 }];
+		const results = await runEval({
+			slugs: ["m"],
+			tasks,
+			complete: async () => {
+				throw new Error("boom");
+			},
+		});
+		expect(results[0]!.axes.coding.n).toBe(0); // no observation
+		expect(results[0]!.errors).toBe(1);
+	});
+});
+
+describe("local source integration", () => {
+	function raw(id: string): Record<string, unknown> {
+		return {
+			id,
+			canonical_slug: id,
+			name: id,
+			context_length: 131072,
+			pricing: { prompt: "0.0000003", completion: "0.0000011" },
+			supported_parameters: ["tools"],
+			architecture: { input_modalities: ["text"], tokenizer: "Other" },
+			created: 1_700_000_000,
+		};
+	}
+
+	test("local fills only where no stronger source has the axis", () => {
+		const catalog = [raw("z-ai/glm-5.3-flash")];
+		const feeds: FeedScore[] = [
+			{ key: "glm-5-3-flash", creator: "z-ai", source: "artificial_analysis", coding: 61 },
+			{ key: "glm-5-3-flash", creator: "z-ai", source: "local", coding: 20, intelligence: 55 },
+		];
+		const result = applyFeedScores(catalog, feeds);
+		const q = normalizeCatalogModel(catalog[0])?.quality;
+		expect(q?.coding).toBe(61); // AA wins over local
+		expect(q?.intelligence).toBe(55); // local fills the axis nobody else had
+		expect(result.sources.local).toBe(1);
+		expect(result.sources.artificial_analysis).toBe(1);
+	});
+
+	test("saveLocalScores / loadLocalScores round-trip", () => {
+		const db = openDb(":memory:");
+		const scores: FeedScore[] = [{ key: "muse-glimmer-30b", creator: "meta", source: "local", coding: 42, agentic: 39 }];
+		saveLocalScores(db, scores, 123);
+		expect(loadLocalScores(db)).toEqual(scores);
+		db.close();
+	});
+});
+
+describe("llm judge", () => {
+	test("parseScore takes the last standalone 0-10 and scales to 0-1", () => {
+		expect(parseScore("8")).toBeCloseTo(0.8, 5);
+		expect(parseScore("Score: 10/10")).toBeCloseTo(1, 5);
+		expect(parseScore("I count 3 issues, so 7")).toBeCloseTo(0.7, 5); // last wins
+		expect(parseScore("no number here")).toBeNull();
+	});
+
+	test("makeJudge parses a score, and returns null on a thrown completion", async () => {
+		const task: JudgedTask = { id: "j", axis: "coding", user: "do a thing" };
+		const good = makeJudge(async () => "the answer earns 8", "judge/model");
+		expect(await good(task, "some answer")).toBeCloseTo(0.8, 5);
+		const bad = makeJudge(async () => {
+			throw new Error("judge down");
+		}, "judge/model");
+		expect(await bad(task, "some answer")).toBeNull();
+	});
+
+	test("runEval folds judged scores into the axis mean, and drops unscorable ones", async () => {
+		const judged: JudgedTask[] = [
+			{ id: "j1", axis: "coding", user: "a" },
+			{ id: "j2", axis: "coding", user: "b" },
+		];
+		// j1 scores 0.6; j2 is unscorable (null) → excluded as an error.
+		const judge = async (t: JudgedTask) => (t.id === "j1" ? 0.6 : null);
+		const results = await runEval({ slugs: ["m"], tasks: [], judged, judge, complete: async () => "ans" });
+		expect(results[0]!.axes.coding).toEqual({ sum: 0.6, n: 1 });
+		expect(results[0]!.errors).toBe(1);
+	});
+
+	test("judged tasks are skipped entirely when no judge is supplied", async () => {
+		const judged: JudgedTask[] = [{ id: "j1", axis: "coding", user: "a" }];
+		const results = await runEval({ slugs: ["m"], tasks: [], judged, complete: async () => "ans" });
+		expect(results[0]!.axes.coding).toEqual({ sum: 0, n: 0 });
+		expect(results[0]!.errors).toBe(0);
+	});
+});

package/test/exploration.test.ts

@@ -0,0 +1,251 @@
+import { describe, expect, test } from "bun:test";
+
+import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
+import type { CatalogModel, CatalogSnapshot } from "../src/catalog/types.ts";
+import { loadConfig } from "../src/config/load.ts";
+import type { ExplorationConfig, ProfileConfig, RouterConfig } from "../src/config/types.ts";
+import { scoreHeuristic } from "../src/router/classify.ts";
+import { extractFeatures } from "../src/router/features.ts";
+import { select } from "../src/router/select.ts";
+import type { ClassificationSource, ConversationState, Tier } from "../src/router/types.ts";
+import { parseChatRequest } from "../src/wire/openai/request.ts";
+import type { NormRequest } from "../src/wire/types.ts";
+
+const FIXTURE = (await Bun.file("test/fixtures/openrouter-models.json").json()) as { data: unknown[] };
+const MODELS: CatalogModel[] = FIXTURE.data.map(normalizeCatalogModel).filter((m): m is CatalogModel => m !== null);
+const SNAPSHOT: CatalogSnapshot = { models: MODELS, fetchedAtMs: Date.now() };
+
+const BASE = loadConfig({});
+const PROFILE: ProfileConfig = {
+	id: "auto",
+	name: "Auto",
+	minTier: "trivial",
+	maxTier: "hard",
+	contextWindow: 400_000,
+	maxTokens: 32_000,
+};
+const NOW = Date.now();
+
+function request(userText: string): NormRequest {
+	return parseChatRequest(
+		{
+			model: "auto",
+			messages: [
+				{ role: "system", content: "You are a coding agent." },
+				{ role: "user", content: userText },
+			],
+		},
+		new Headers(),
+	);
+}
+
+function state(over: Partial<ConversationState> = {}): ConversationState {
+	return {
+		key: "abc123",
+		sessionId: "omp-abc123",
+		turn: 1,
+		currentSlug: null,
+		currentTier: null,
+		stickyUntilTurn: 0,
+		escalations: 0,
+		spentUsd: 0,
+		lastPromptTokens: 0,
+		cacheWarmSlug: null,
+		cacheWarmAtMs: 0,
+		updatedAtMs: NOW,
+		...over,
+	};
+}
+
+/** A hysteresis hold at the given tier, with the prompt cache warm or expired. */
+function heldState(held: Tier, cache: "warm" | "cold"): ConversationState {
+	return state({
+		turn: 1,
+		stickyUntilTurn: 5,
+		currentTier: held,
+		currentSlug: "vendor/model",
+		cacheWarmSlug: cache === "warm" ? "vendor/model" : null,
+		cacheWarmAtMs: cache === "warm" ? NOW : 0,
+	});
+}
+
+function withExploration(over: Partial<ExplorationConfig>): RouterConfig {
+	return { ...BASE, exploration: { ...BASE.exploration, ...over } };
+}
+
+function run(opts: {
+	userText?: string;
+	cfg?: RouterConfig;
+	st?: ConversationState;
+	tier?: Tier;
+	source?: ClassificationSource;
+	profile?: ProfileConfig;
+	excludeSlugs?: readonly string[];
+}) {
+	const cfg = opts.cfg ?? BASE;
+	const req = request(opts.userText ?? "tidy the retry helper");
+	const features = extractFeatures(req, 4000);
+	const heuristic = scoreHeuristic(features, cfg);
+	const classification = {
+		...heuristic,
+		...(opts.tier === undefined ? {} : { tier: opts.tier }),
+		...(opts.source === undefined ? {} : { source: opts.source }),
+	};
+	return select({
+		req,
+		features,
+		classification,
+		profile: opts.profile ?? PROFILE,
+		state: opts.st ?? state(),
+		snapshot: SNAPSHOT,
+		ledger: null,
+		cfg,
+		nowMs: NOW,
+		...(opts.excludeSlugs === undefined ? {} : { excludeSlugs: opts.excludeSlugs }),
+	});
+}
+
+const ALWAYS = { simple: 1, moderate: 1, hard: 1 };
+
+describe("exploration is opt-in", () => {
+	test("never fires under the shipped defaults", () => {
+		expect(BASE.exploration.enabled).toBe(false);
+		for (const tier of ["simple", "moderate", "hard"] as Tier[]) {
+			expect(run({ tier }).explored).toBeNull();
+		}
+	});
+
+	test("enabled with no rates configured still never fires", () => {
+		const cfg = withExploration({ enabled: true, rates: {} });
+		for (const tier of ["simple", "moderate", "hard"] as Tier[]) {
+			expect(run({ tier, cfg }).explored).toBeNull();
+		}
+	});
+});
+
+describe("per-tier rates", () => {
+	test("each tier is governed by its own rate, not one global one", () => {
+		const cfg = withExploration({ enabled: true, rates: { simple: 0, moderate: 0, hard: 1 } });
+		expect(run({ tier: "simple", cfg }).explored).toBeNull();
+		expect(run({ tier: "moderate", cfg }).explored).toBeNull();
+		expect(run({ tier: "hard", cfg }).explored).toEqual({ from: "hard", to: "moderate" });
+	});
+
+	test("a tier absent from the rates map is never explored", () => {
+		const cfg = withExploration({ enabled: true, rates: { hard: 1 } });
+		expect(run({ tier: "moderate", cfg }).explored).toBeNull();
+		expect(run({ tier: "hard", cfg }).explored).not.toBeNull();
+	});
+
+	test("the shipped defaults weight expensive tiers far above cheap ones", () => {
+		const r = BASE.exploration.rates;
+		expect(r.hard ?? 0).toBeGreaterThan(r.simple ?? 0);
+		expect(r.moderate ?? 0).toBeGreaterThan(r.simple ?? 0);
+	});
+});
+
+describe("exploration drops exactly one tier", () => {
+	const cfg = withExploration({ enabled: true, rates: ALWAYS });
+
+	test("moderate explores down to simple", () => {
+		expect(run({ tier: "moderate", cfg }).explored).toEqual({ from: "moderate", to: "simple" });
+	});
+
+	test("hard explores down to moderate, never further", () => {
+		expect(run({ tier: "hard", cfg }).explored).toEqual({ from: "hard", to: "moderate" });
+	});
+
+	test("trivial is the floor and cannot be explored below", () => {
+		expect(run({ tier: "trivial", cfg }).explored).toBeNull();
+	});
+
+	test("the decision trail says so out loud", () => {
+		expect(run({ tier: "moderate", cfg }).reasons.some((r) => r.startsWith("exploration:"))).toBe(true);
+	});
+});
+
+describe("hysteresis holds are explored only once the cache is cold", () => {
+	const cfg = withExploration({ enabled: true, rates: ALWAYS, stickyPolicy: "cold-cache" });
+
+	test("a held tier with a WARM cache is left alone", () => {
+		expect(run({ tier: "simple", cfg, st: heldState("hard", "warm") }).explored).toBeNull();
+	});
+
+	test("a held tier with a COLD cache is explorable", () => {
+		// This is the population that carries most of the spend: turns that
+		// reach hard by hold rather than by classification.
+		expect(run({ tier: "simple", cfg, st: heldState("hard", "cold") }).explored).toEqual({
+			from: "hard",
+			to: "moderate",
+		});
+	});
+
+	test("the reason names the hold and the cache state, for later analysis", () => {
+		const d = run({ tier: "simple", cfg, st: heldState("hard", "cold") });
+		expect(d.reasons.some((r) => r.includes("held tier (cold cache)"))).toBe(true);
+	});
+
+	test("stickyPolicy never leaves holds alone entirely", () => {
+		const off = withExploration({ enabled: true, rates: ALWAYS, stickyPolicy: "never" });
+		expect(run({ tier: "simple", cfg: off, st: heldState("hard", "cold") }).explored).toBeNull();
+		expect(run({ tier: "simple", cfg: off, st: heldState("hard", "warm") }).explored).toBeNull();
+	});
+
+	test("stickyPolicy always reaches held turns even with a live cache", () => {
+		// The only setting that samples the population carrying most of the
+		// spend, at the price of a forfeited cache read.
+		const always = withExploration({ enabled: true, rates: ALWAYS, stickyPolicy: "always" });
+		expect(run({ tier: "simple", cfg: always, st: heldState("hard", "warm") }).explored).toEqual({
+			from: "hard",
+			to: "moderate",
+		});
+		expect(
+			run({ tier: "simple", cfg: always, st: heldState("hard", "warm") }).reasons.some((r) =>
+				r.includes("held tier (warm cache)"),
+			),
+		).toBe(true);
+	});
+});
+
+describe("exploration respects the remaining guards", () => {
+	const cfg = withExploration({ enabled: true, rates: ALWAYS });
+
+	test("never routes below the profile floor", () => {
+		const floored: ProfileConfig = { ...PROFILE, minTier: "moderate" };
+		expect(run({ tier: "moderate", cfg, profile: floored }).explored).toBeNull();
+	});
+
+	test("skips forced escalations, which already proved the cheap tier failed", () => {
+		expect(run({ tier: "moderate", cfg, source: "escalation" }).explored).toBeNull();
+	});
+
+	test("skips failover retries so a second confound is not introduced", () => {
+		expect(run({ tier: "moderate", cfg, excludeSlugs: ["vendor/broken"] }).explored).toBeNull();
+	});
+});
+
+describe("exploration is deterministic", () => {
+	const cfg = withExploration({ enabled: true, rates: { simple: 0.5, moderate: 0.5, hard: 0.5 } });
+
+	test("the same turn always draws the same way, so explain can replay it", () => {
+		for (const text of ["alpha task", "beta task", "gamma task"]) {
+			const first = run({ userText: text, tier: "moderate", cfg });
+			for (let i = 0; i < 5; i++) {
+				expect(run({ userText: text, tier: "moderate", cfg }).explored).toEqual(first.explored);
+			}
+		}
+	});
+
+	test("the draw honours the configured rate across many turns", () => {
+		const cfg25 = withExploration({ enabled: true, rates: { moderate: 0.25 } });
+		let explored = 0;
+		const N = 400;
+		for (let i = 0; i < N; i++) {
+			if (run({ userText: "distinct task " + i, tier: "moderate", cfg: cfg25 }).explored !== null) explored++;
+		}
+		// Deterministic hash, so this cannot flake; the band is wide enough that
+		// only a genuinely biased draw would fail it.
+		expect(explored).toBeGreaterThan(N * 0.25 - 40);
+		expect(explored).toBeLessThan(N * 0.25 + 40);
+	});
+});

package/test/failover.test.ts

@@ -29,6 +29,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 	return {
 		server: { host: "127.0.0.1", port: 8787 },
 		openrouter: { baseUrl: "https://openrouter.ai/api/v1", apiKey: "", title: "test", timeoutMs: 30_000, catalogTtlMs: 3_600_000, catalogRefreshMs: 0 },
+		benchmarks: { enabled: false, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 86_400_000, timeoutMs: 30_000, useLocalScores: false },
 		tiers: {
 			trivial: { minQuality: 0, maxInputPerMtok: 0.3, qualityExponent: 0, pin: [] },
 			simple: { minQuality: 40, maxInputPerMtok: 1.5, qualityExponent: 0, pin: [] },
@@ -65,11 +66,13 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 			...escalation,
 		},
 		hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1 },
+		exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
 		cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024 },
 		budget: { onExceeded: "downgrade" },
 		profiles: [],
 		ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 },
 		adaptiveTierFloors: true,
+		adaptivePriceCeilings: false,
 		logLevel: "silent",
 	};
 }
@@ -79,6 +82,7 @@ function mkReq(): NormRequest {
 		protocol: "openai-chat",
 		conversationKey: "conv-test",
 		harnessId: "",
+		ompSessionId: "",
 		requestedModel: "auto",
 		messages: [{ role: "user", text: "hi", images: 0, textBytes: 2, toolCalls: [] }],
 		tools: [],
@@ -140,6 +144,7 @@ function mkDecision(tier: Tier, slug: string, probe: Partial<ProbePlan> = {}): D
 		considered: [],
 		rejected: [],
 		reasons: ["test decision"],
+		explored: null,
 		budgetDowngraded: false,
 	};
 }

package/test/hold-exploration.test.ts

@@ -0,0 +1,124 @@
+import { describe, expect, test } from "bun:test";
+
+import { loadConfig } from "../src/config/load.ts";
+import type { RouterConfig } from "../src/config/types.ts";
+import { explorationDraw, resolveHoldTurns } from "../src/router/explore.ts";
+
+const BASE = loadConfig({});
+
+function withHold(over: { enabled?: boolean; values?: number[] }, enabled = true): RouterConfig {
+	return {
+		...BASE,
+		exploration: {
+			...BASE.exploration,
+			enabled,
+			holdTurns: { enabled: over.enabled ?? true, values: over.values ?? [2, 3, 4] },
+		},
+	};
+}
+
+describe("hold exploration is opt-in", () => {
+	test("shipped defaults leave the hold alone", () => {
+		expect(BASE.exploration.holdTurns.enabled).toBe(false);
+		const got = resolveHoldTurns(BASE, "conv-1", true);
+		expect(got.turns).toBe(BASE.hysteresis.holdTurnsAfterEscalation);
+		expect(got.arm).toBeNull();
+	});
+
+	test("hold exploration stays off while exploration as a whole is off", () => {
+		const cfg = withHold({ values: [1] }, false);
+		const got = resolveHoldTurns(cfg, "conv-1", true);
+		expect(got.turns).toBe(BASE.hysteresis.holdTurnsAfterEscalation);
+		expect(got.arm).toBeNull();
+	});
+
+	test("an explicitly disabled hold experiment is inert", () => {
+		const cfg = withHold({ enabled: false, values: [1] });
+		expect(resolveHoldTurns(cfg, "conv-1", true).arm).toBeNull();
+	});
+});
+
+describe("only the post-escalation hold is randomised", () => {
+	const cfg = withHold({ values: [1] });
+
+	test("an escalated turn uses the drawn arm", () => {
+		expect(resolveHoldTurns(cfg, "conv-1", true).turns).toBe(1);
+	});
+
+	test("an ordinary turn keeps the configured hold", () => {
+		// The experiment targets the hold that governs expensive spend; leaving
+		// the ordinary hold fixed keeps the comparison narrow enough to read.
+		expect(resolveHoldTurns(cfg, "conv-1", false).turns).toBe(BASE.hysteresis.holdTurns);
+	});
+
+	test("but the arm is still recorded on non-escalated turns", () => {
+		// Intention-to-treat: arms are compared on whole-conversation cost, so
+		// every turn of an assigned conversation has to carry its arm.
+		expect(resolveHoldTurns(cfg, "conv-1", false).arm).toBe(1);
+	});
+});
+
+describe("assignment is per conversation", () => {
+	const cfg = withHold({ values: [2, 3, 4] });
+
+	test("the same conversation always draws the same arm", () => {
+		const first = resolveHoldTurns(cfg, "conv-stable", true).arm;
+		for (let i = 0; i < 20; i++) {
+			expect(resolveHoldTurns(cfg, "conv-stable", true).arm).toBe(first);
+		}
+	});
+
+	test("the arm never changes mid-hold, whatever the escalation state", () => {
+		const a: number | null = resolveHoldTurns(cfg, "conv-x", true).arm;
+		const b = resolveHoldTurns(cfg, "conv-x", false).arm;
+		expect(a).toEqual(b);
+	});
+
+	test("every drawn arm comes from the configured set", () => {
+		for (let i = 0; i < 200; i++) {
+			const arm = resolveHoldTurns(cfg, `conv-${i}`, true).arm;
+			expect(arm).not.toBeNull();
+			expect([2, 3, 4]).toContain(arm ?? -1);
+		}
+	});
+
+	test("arms are spread across conversations rather than collapsing to one", () => {
+		const counts = new Map<number, number>();
+		const N = 600;
+		for (let i = 0; i < N; i++) {
+			const arm = resolveHoldTurns(cfg, `spread-${i}`, true).arm ?? -1;
+			counts.set(arm, (counts.get(arm) ?? 0) + 1);
+		}
+		expect(counts.size).toBe(3);
+		// Deterministic hash, so this cannot flake. Each of 3 arms should land
+		// near N/3; the band is wide enough that only real bias would fail.
+		for (const [, n] of counts) {
+			expect(n).toBeGreaterThan(N / 3 - 60);
+			expect(n).toBeLessThan(N / 3 + 60);
+		}
+	});
+
+	test("a single-value set assigns everyone the same arm", () => {
+		const one = withHold({ values: [3] });
+		for (let i = 0; i < 20; i++) {
+			expect(resolveHoldTurns(one, `conv-${i}`, true).turns).toBe(3);
+		}
+	});
+});
+
+describe("the draw itself", () => {
+	test("is uniform in [0,1) and stable for a seed", () => {
+		expect(explorationDraw("seed-a")).toBe(explorationDraw("seed-a"));
+		expect(explorationDraw("seed-a")).not.toBe(explorationDraw("seed-b"));
+		for (const seed of ["a", "b", "c", "d", "e"]) {
+			const d = explorationDraw(seed);
+			expect(d).toBeGreaterThanOrEqual(0);
+			expect(d).toBeLessThan(1);
+		}
+	});
+
+	test("the tier draw and the hold draw are independent seeds", () => {
+		// Sharing a seed would correlate the two experiments and confound both.
+		expect(explorationDraw("hold:conv-1")).not.toBe(explorationDraw("explore:conv-1:1"));
+	});
+});

package/test/tier-plan.test.ts

@@ -5,7 +5,7 @@ import type { CatalogModel, CatalogSnapshot } from "../src/catalog/types.ts";
 import { loadConfig } from "../src/config/load.ts";
 import { buildCandidates } from "../src/router/candidates.ts";
 import { extractFeatures } from "../src/router/features.ts";
-import { computeTierPlan, effectiveQualityFloor, tierPlanFor } from "../src/router/tier-plan.ts";
+import { computeTierPlan, effectivePriceCeiling, effectiveQualityFloor, tierPlanFor } from "../src/router/tier-plan.ts";
 import { TIER_ORDER } from "../src/router/types.ts";
 import { parseChatRequest } from "../src/wire/openai/request.ts";
 
@@ -300,3 +300,57 @@ describe("adaptive floors in candidate selection", () => {
 		expect(rejected.some((r) => r.slug === target && r.reason === "failed_this_turn")).toBe(true);
 	});
 });
+
+describe("adaptive price ceilings", () => {
+	const priced = models([
+		["a/1", 80, 1],
+		["a/2", 80, 2],
+		["a/3", 80, 3],
+		["a/4", 80, 4],
+	]);
+	const req = parseChatRequest(
+		{
+			model: "auto",
+			tools: [{ type: "function", function: { name: "read", description: "Read", parameters: { type: "object", properties: {} } } }],
+			messages: [{ role: "user", content: "refactor the auth module" }],
+		},
+		new Headers(),
+	);
+	const features = extractFeatures(req, 100);
+
+	test("computeTierPlan derives per-tier price bands from the catalog", () => {
+		const plan = computeTierPlan(priced, BASE);
+		expect(plan.priceCeilings).toEqual({ trivial: 1, simple: 2, moderate: 3, hard: 4 });
+	});
+
+	test("effectivePriceCeiling: band when on, tighter of config/band, config when off", () => {
+		const plan = computeTierPlan(priced, BASE);
+		expect(effectivePriceCeiling(undefined, "moderate", plan, true)).toBe(3); // band
+		expect(effectivePriceCeiling(2, "moderate", plan, true)).toBe(2); // config tightens
+		expect(effectivePriceCeiling(10, "moderate", plan, true)).toBe(3); // band tightens
+		expect(effectivePriceCeiling(2, "moderate", plan, false)).toBe(2); // off ⇒ config
+		expect(effectivePriceCeiling(undefined, "hard", plan, false)).toBeUndefined();
+	});
+
+	test("a model above the adaptive band is dropped in candidate selection", () => {
+		const snap = snapshot(priced);
+		const run = (adaptivePriceCeilings: boolean) =>
+			buildCandidates({
+				req,
+				features,
+				tier: "moderate",
+				task: "coding",
+				snapshot: snap,
+				ledger: null,
+				cfg: { ...BASE, adaptivePriceCeilings },
+				expectedCompletionTokens: 512,
+				warmSlug: null,
+			});
+		// Off: the fixed moderate ceiling ($4) admits a/4 at $4.
+		expect(run(false).candidates.map((c) => c.model.slug)).toContain("a/4");
+		// On: the catalog band tightens moderate to $3, so a/4 is over-ceiling.
+		const on = run(true);
+		expect(on.candidates.map((c) => c.model.slug)).not.toContain("a/4");
+		expect(on.rejected.some((r) => r.slug === "a/4" && r.reason === "over_price_ceiling")).toBe(true);
+	});
+});