claudeup 6.3.2 → 6.4.0

package/src/__tests__/models-core.test.ts

@@ -0,0 +1,640 @@
+/**
+ * Contract tests for the pure routing core.
+ *
+ * The behaviour asserted here is not invented: each rule cites the run in
+ * `benches/agent-model-routing/` (AMR-1) that measured it on real Claude Code, interactive,
+ * n=5. A test that drifts from those numbers is a bug in the code or a signal to re-run the
+ * bench — never a reason to relax the assertion.
+ */
+import { describe, expect, test } from "bun:test";
+import {
+	BUILT_IN_PRESETS,
+	DEFAULT_AGENT_GRADES,
+	findPreset,
+	presetNames,
+} from "../data/models-presets.js";
+import {
+	EFFORTS,
+	type Effort,
+	GRADES,
+	type ModelsConfig,
+	baseAlias,
+	buildSettingsPatch,
+	computeModelsStatus,
+	evaluateAgentHook,
+	isRoutableModel,
+	resolveAgentRouting,
+	validateModelsConfig,
+} from "../services/models-core.js";
+
+const base = (): ModelsConfig => ({
+	version: 1,
+	preset: "test",
+	main: { model: "opus", effort: "medium" },
+	grades: {
+		smart: { model: "fable", effort: "xhigh" },
+		normal: { model: "opus", effort: "medium" },
+		cheap: { model: "sonnet", effort: "low" },
+	},
+	agents: {
+		"dev:architect": "smart",
+		Explore: "cheap",
+		"seo:editor": { model: "inherit" },
+	},
+	fallback: "normal",
+});
+
+const noFullId = () => null;
+
+describe("validateModelsConfig", () => {
+	test("accepts a well-formed config", () => {
+		expect(validateModelsConfig(base())).toEqual([]);
+	});
+
+	test("every built-in preset validates", () => {
+		for (const p of BUILT_IN_PRESETS) {
+			expect({ preset: p.preset, errors: validateModelsConfig(p) }).toEqual({
+				preset: p.preset,
+				errors: [],
+			});
+		}
+	});
+
+	test("rejects a full model id, and says why", () => {
+		const c = base();
+		(c.main as { model: string }).model = "claude-opus-5";
+		const errors = validateModelsConfig(c);
+		expect(errors).toHaveLength(1);
+		expect(errors[0]?.path).toBe("main.model");
+		expect(errors[0]?.message).toContain("go stale");
+	});
+
+	test("rejects a non-alias on a grade — AMR-1 bogus-model: it refuses the spawn outright", () => {
+		const c = base();
+		(c.grades.cheap as { model: string }).model = "kimi-k2-thinking";
+		const errors = validateModelsConfig(c);
+		expect(errors.map((e) => e.path)).toEqual(["grades.cheap.model"]);
+	});
+
+	test("allows [1m] on main but not on a grade", () => {
+		const ok = base();
+		(ok.main as { model: string }).model = "opus[1m]";
+		expect(validateModelsConfig(ok)).toEqual([]);
+
+		const bad = base();
+		(bad.grades.smart as { model: string }).model = "opus[1m]";
+		expect(validateModelsConfig(bad).map((e) => e.path)).toEqual([
+			"grades.smart.model",
+		]);
+	});
+
+	test("rejects `inherit` on main — the main thread needs a real model", () => {
+		const c = base();
+		(c.main as { model: string }).model = "inherit";
+		expect(validateModelsConfig(c)[0]?.path).toBe("main.model");
+	});
+
+	test("rejects two efforts for one model, naming both sites", () => {
+		// modelSettings is keyed by MODEL, so one of the two would silently lose.
+		const c = base();
+		c.grades.normal = { model: "opus", effort: "xhigh" }; // main already claims opus/medium
+		const errors = validateModelsConfig(c);
+		expect(errors).toHaveLength(1);
+		expect(errors[0]?.path).toBe("grades.normal");
+		expect(errors[0]?.message).toContain("main");
+		expect(errors[0]?.message).toContain("medium");
+	});
+
+	test("the same model with the SAME effort twice is fine", () => {
+		const c = base();
+		c.grades.normal = { model: "opus", effort: "medium" };
+		expect(validateModelsConfig(c)).toEqual([]);
+	});
+
+	test("rejects effort on `inherit`, which names no model to key it by", () => {
+		const c = base();
+		c.agents["seo:editor"] = { model: "inherit", effort: "low" } as never;
+		expect(validateModelsConfig(c).map((e) => e.path)).toEqual([
+			"agents.seo:editor.effort",
+		]);
+	});
+
+	test("rejects an unknown grade name in agents", () => {
+		const c = base();
+		c.agents.whatever = "turbo" as never;
+		expect(validateModelsConfig(c).map((e) => e.path)).toEqual([
+			"agents.whatever",
+		]);
+	});
+
+	test("a wrong version stops immediately rather than emitting noise", () => {
+		const errors = validateModelsConfig({ version: 2, main: {}, grades: {} });
+		expect(errors).toHaveLength(1);
+		expect(errors[0]?.path).toBe("version");
+	});
+
+	test("a non-object is one clear error", () => {
+		expect(validateModelsConfig("nope")).toEqual([
+			{ path: "", message: "must be a JSON object" },
+		]);
+	});
+});
+
+describe("resolveAgentRouting", () => {
+	test("an explicit agent grade wins", () => {
+		expect(resolveAgentRouting(base(), "dev:architect")).toEqual({
+			model: "fable",
+			effort: "xhigh",
+			source: "agent-grade",
+			grade: "smart",
+		});
+	});
+
+	test("a per-agent override beats the grade table", () => {
+		expect(resolveAgentRouting(base(), "seo:editor")).toEqual({
+			model: "inherit",
+			source: "agent-override",
+		});
+	});
+
+	test("an unknown agent falls to the fallback grade, so a new plugin routes on day one", () => {
+		expect(resolveAgentRouting(base(), "brand-new:agent")).toEqual({
+			model: "opus",
+			effort: "medium",
+			source: "default-grade",
+			grade: "normal",
+		});
+	});
+
+	test("a missing subagent_type still resolves", () => {
+		expect(resolveAgentRouting(base(), undefined).source).toBe("default-grade");
+	});
+});
+
+describe("evaluateAgentHook", () => {
+	const agentCall = (input: Record<string, unknown> = {}) => ({
+		tool_name: "Agent",
+		tool_input: { subagent_type: "dev:architect", prompt: "go", ...input },
+	});
+
+	test("rewrites the model and carries the whole input through", () => {
+		const d = evaluateAgentHook(agentCall(), base());
+		expect(d.kind).toBe("update");
+		if (d.kind !== "update") return;
+		expect(d.output.hookSpecificOutput.updatedInput).toEqual({
+			subagent_type: "dev:architect",
+			prompt: "go",
+			model: "fable",
+		});
+	});
+
+	test("ALWAYS emits hookEventName — AMR-1 hook-minimal fired 5/5 and was ignored without it", () => {
+		const d = evaluateAgentHook(agentCall(), base());
+		if (d.kind !== "update") throw new Error("expected an update");
+		expect(d.output.hookSpecificOutput.hookEventName).toBe("PreToolUse");
+	});
+
+	test("never emits permissionDecision — measured neither required nor harmful, so it only widens permission", () => {
+		const d = evaluateAgentHook(agentCall(), base());
+		if (d.kind !== "update") throw new Error("expected an update");
+		expect(Object.keys(d.output.hookSpecificOutput).sort()).toEqual([
+			"hookEventName",
+			"updatedInput",
+		]);
+	});
+
+	test("passes through a tool that is not Agent", () => {
+		expect(
+			evaluateAgentHook({ tool_name: "Bash", tool_input: {} }, base()),
+		).toEqual({
+			kind: "passthrough",
+			reason: "not-agent",
+		});
+	});
+
+	test("passes through with no config — fail open", () => {
+		expect(evaluateAgentHook(agentCall(), null)).toEqual({
+			kind: "passthrough",
+			reason: "no-config",
+		});
+	});
+
+	test("respects a model the caller already chose", () => {
+		expect(evaluateAgentHook(agentCall({ model: "haiku" }), base())).toEqual({
+			kind: "passthrough",
+			reason: "explicit-model",
+		});
+	});
+
+	test("`inherit` is the opt-out and touches nothing", () => {
+		expect(
+			evaluateAgentHook(agentCall({ subagent_type: "seo:editor" }), base()),
+		).toEqual({
+			kind: "passthrough",
+			reason: "inherit",
+		});
+	});
+
+	test("refuses to inject an unroutable value even if a hand-edited config carries one", () => {
+		// AMR-1 bogus-model: injecting a non-alias refuses the spawn entirely, so passing
+		// through costs the routing while injecting costs the user every subagent.
+		const c = base();
+		(c.grades.smart as { model: string }).model = "kimi-k2-thinking";
+		expect(evaluateAgentHook(agentCall(), c)).toEqual({
+			kind: "passthrough",
+			reason: "unroutable-model",
+		});
+	});
+
+	test("an empty model string is not treated as a caller's choice", () => {
+		const d = evaluateAgentHook(agentCall({ model: "" }), base());
+		expect(d.kind).toBe("update");
+	});
+});
+
+describe("buildSettingsPatch", () => {
+	test("writes main model and effort", () => {
+		const { set } = buildSettingsPatch(base(), noFullId);
+		expect(set.model).toBe("opus");
+		expect(set.effortLevel).toBe("medium");
+	});
+
+	test("writes BOTH key forms with the same value — AMR-1 effort-conflict: the full id wins", () => {
+		const resolve = (a: string) =>
+			a === "fable" ? "claude-fable-5-1" : `claude-${a}-5`;
+		const { set } = buildSettingsPatch(base(), resolve as never);
+		expect(set.modelSettings).toEqual({
+			opus: { effortLevel: "medium" },
+			"claude-opus-5": { effortLevel: "medium" },
+			fable: { effortLevel: "xhigh" },
+			"claude-fable-5-1": { effortLevel: "xhigh" },
+			sonnet: { effortLevel: "low" },
+			"claude-sonnet-5": { effortLevel: "low" },
+		});
+	});
+
+	test("a resolver that cannot answer leaves the alias key alone rather than inventing an id", () => {
+		const { set } = buildSettingsPatch(base(), noFullId);
+		expect(set.modelSettings).toEqual({
+			opus: { effortLevel: "medium" },
+			fable: { effortLevel: "xhigh" },
+			sonnet: { effortLevel: "low" },
+		});
+	});
+
+	test("omits modelSettings entirely when no effort is declared", () => {
+		const c = base();
+		c.main = { model: "opus" };
+		c.grades = {
+			smart: { model: "fable" },
+			normal: { model: "opus" },
+			cheap: { model: "sonnet" },
+		};
+		c.agents = {};
+		const { set } = buildSettingsPatch(c, noFullId);
+		expect(set.modelSettings).toBeUndefined();
+	});
+
+	test("a [1m] main model keys its effort by the base alias", () => {
+		const c = base();
+		(c.main as { model: string }).model = "opus[1m]";
+		const { set } = buildSettingsPatch(c, noFullId);
+		expect(set.model).toBe("opus[1m]");
+		expect((set.modelSettings as Record<string, unknown>).opus).toEqual({
+			effortLevel: "medium",
+		});
+	});
+});
+
+describe("computeModelsStatus", () => {
+	const applied = () => buildSettingsPatch(base(), noFullId).set;
+
+	test("off when there is no config", () => {
+		expect(
+			computeModelsStatus({
+				config: null,
+				errors: [],
+				hookRegistered: true,
+				settings: {},
+				resolveFullId: noFullId,
+			}).state,
+		).toBe("off");
+	});
+
+	test("invalid reports the validation errors as drift", () => {
+		const s = computeModelsStatus({
+			config: null,
+			errors: [{ path: "main.model", message: "bad" }],
+			hookRegistered: true,
+			settings: {},
+			resolveFullId: noFullId,
+		});
+		expect(s.state).toBe("invalid");
+		expect(s.drift[0]).toContain("main.model");
+	});
+
+	test("unhooked outranks stale — nothing routes, so that is the headline", () => {
+		const s = computeModelsStatus({
+			config: base(),
+			errors: [],
+			hookRegistered: false,
+			settings: {},
+			resolveFullId: noFullId,
+		});
+		expect(s.state).toBe("unhooked");
+	});
+
+	test("applied when settings already match", () => {
+		const s = computeModelsStatus({
+			config: base(),
+			errors: [],
+			hookRegistered: true,
+			settings: applied(),
+			resolveFullId: noFullId,
+		});
+		expect(s.state).toBe("on");
+		expect(s.drift).toEqual([]);
+	});
+
+	test("stale names the key that differs", () => {
+		const s = computeModelsStatus({
+			config: base(),
+			errors: [],
+			hookRegistered: true,
+			settings: { ...applied(), effortLevel: "high" },
+			resolveFullId: noFullId,
+		});
+		expect(s.state).toBe("stale");
+		expect(s.drift.join(" ")).toContain("effortLevel");
+	});
+	/**
+	 * Drift text is READ BY A PERSON, and these three assertions each pin a defect that
+	 * shipped because the only existing test checked `.toContain("effortLevel")` — which the
+	 * broken copy satisfied too.
+	 */
+	test("an absent key is said in words, never as the string `undefined`", () => {
+		// `JSON.stringify(undefined)` returns the STRING "undefined", so the old line read
+		// `model: settings has undefined` — a JavaScript value leaking into user-facing text.
+		const s = computeModelsStatus({
+			config: base(),
+			errors: [],
+			hookRegistered: true,
+			settings: { model: "sonnet" },
+			resolveFullId: noFullId,
+		});
+		expect(s.state).toBe("stale");
+		expect(s.drift.join(" ")).not.toContain("undefined");
+	});
+
+	/**
+	 * A drift line may never say that X differs from X.
+	 *
+	 * This is the defect a code review caught by EXECUTING the function rather than reading
+	 * it: `modelSettings` was summarised on each side with the full-id keys filtered out to
+	 * keep the line short, so when only a full id differed — the overwhelmingly common case,
+	 * since the patch writes every model twice — both sides rendered identical text while
+	 * `JSON.stringify` correctly said they differed.
+	 *
+	 * It matters beyond legibility. Per AMR-1, a full id BEATS an alias when the two disagree,
+	 * so the suppressed key was the one actually serving subagents. And it arrives on its own
+	 * schedule: apply a preset, wait for the next model release, and the stored full id stops
+	 * matching the resolved one for every project.
+	 */
+	test("a modelSettings difference in a full-id key is named, not collapsed", () => {
+		const settings = {
+			...applied(),
+			modelSettings: {
+				opus: { effortLevel: "medium" },
+				"claude-opus-4-1-stale": { effortLevel: "low" },
+			},
+		};
+		const s = computeModelsStatus({
+			config: base(),
+			errors: [],
+			hookRegistered: true,
+			settings,
+			resolveFullId: noFullId,
+		});
+		const line = s.drift.find((d) => d.startsWith("modelSettings"));
+		expect(line).toBeDefined();
+		// The stale user-scope key is the one that wins at runtime, so it has to be visible.
+		expect(line).toContain("claude-opus-4-1-stale");
+		// And the line must state a CHANGE, never two identical halves.
+		expect(line).not.toMatch(/settings has (.+), config wants \1/);
+	});
+
+	test("only the models that actually differ are listed", () => {
+		// The control for the test above: a diff that named every key would satisfy it while
+		// burying the one that changed.
+		const s = computeModelsStatus({
+			config: base(),
+			errors: [],
+			hookRegistered: true,
+			settings: {
+				...applied(),
+				modelSettings: {
+					...(applied().modelSettings as Record<string, unknown>),
+					opus: { effortLevel: "low" },
+				},
+			},
+			resolveFullId: noFullId,
+		});
+		const line = s.drift.find((d) => d.startsWith("modelSettings"));
+		expect(line).toContain("opus low");
+		// `fable` agrees on both sides, so it has no business in a line about what differs.
+		expect(line).not.toContain("fable");
+	});
+	test("modelSettings drift is summarised, not dumped as JSON", () => {
+		// It is a map of model → effort and prints as one. Raw JSON was a 200-character wall
+		// that wrapped over four lines and told the reader nothing they could act on.
+		const s = computeModelsStatus({
+			config: base(),
+			errors: [],
+			hookRegistered: true,
+			settings: {
+				...applied(),
+				modelSettings: { opus: { effortLevel: "low" } },
+			},
+			resolveFullId: noFullId,
+		});
+		const line = s.drift.find((d) => d.startsWith("modelSettings"));
+		expect(line).toBeDefined();
+		expect(line).not.toContain("{");
+		expect(line).toContain("opus low");
+	});
+
+	test("nothing written at all is ONE line, not one per key", () => {
+		// Three lines that all say "absent" read as three distinct problems. The wording states
+		// effect, not history — an unparseable settings.json also reads as all-absent here.
+		const s = computeModelsStatus({
+			config: base(),
+			errors: [],
+			hookRegistered: true,
+			settings: {},
+			resolveFullId: noFullId,
+		});
+		expect(s.state).toBe("stale");
+		expect(s.drift).toHaveLength(1);
+		expect(s.drift[0]).toContain("carry none of the config");
+	});
+
+	test("but a partial write still names each key that differs", () => {
+		// The negative control for the collapse above: without it, a status that ALWAYS
+		// returned one line would pass that test and hide every real per-key difference.
+		const s = computeModelsStatus({
+			config: base(),
+			errors: [],
+			hookRegistered: true,
+			settings: { model: "opus" },
+			resolveFullId: noFullId,
+		});
+		expect(s.drift.length).toBeGreaterThan(1);
+		expect(s.drift.join(" ")).toContain("effortLevel");
+	});
+
+	test("warns that CLAUDE_CODE_SUBAGENT_MODEL is now inert", () => {
+		const s = computeModelsStatus({
+			config: base(),
+			errors: [],
+			hookRegistered: true,
+			settings: applied(),
+			resolveFullId: noFullId,
+			subagentModelEnv: "claude-haiku-4-5",
+		});
+		expect(s.warnings.join(" ")).toContain("CLAUDE_CODE_SUBAGENT_MODEL");
+		expect(s.state).toBe("on");
+	});
+});
+
+describe("presets", () => {
+	/**
+	 * The first request shape still ships verbatim: fable on the orchestrator at `medium`.
+	 *
+	 * The second — "fable main, opus for other tasks, sonnet for quick tasks" — does NOT,
+	 * because the revised preset spec took opus out of `fable-lead` entirely and made it
+	 * fable in every seat but `cheap`. That is a deliberate change to what the preset means,
+	 * not a regression, and it is asserted here so the removal stays visible rather than
+	 * being discovered later as a surprise.
+	 */
+	test("fable-lead puts fable on the orchestrator at medium", () => {
+		const fable = findPreset("fable-lead");
+		expect(fable?.main).toEqual({ model: "fable", effort: "medium" });
+		// opus is gone from this preset; the only non-fable seat is `cheap`.
+		expect(
+			GRADES.map((grade) => baseAlias(fable?.grades[grade].model ?? "")),
+		).toEqual(["fable", "fable", "sonnet"]);
+	});
+
+	/**
+	 * The orchestrator never gets the most spend.
+	 *
+	 * This replaced a test that ranked the models against each other — `haiku < sonnet <
+	 * opus < fable` — and asserted `smart` held the top-ranked one. That is a CAPABILITY
+	 * claim, it was written from memory, and it went stale the moment the presets started
+	 * treating fable as the fast model rather than the strongest. Effort is the axis this
+	 * config actually controls, so the principle is stated in effort: the thread that
+	 * dispatches the work is never strictly the biggest spend in the preset.
+	 *
+	 * A tie is allowed. `sonnet-economy` runs `main` and `smart` both at `xhigh`, because
+	 * its saving is the MODEL rather than a throttled effort on top of one.
+	 */
+	test("no preset spends more on the orchestrator than on the work it dispatches", () => {
+		const level = (effort: string | undefined) =>
+			effort ? EFFORTS.indexOf(effort as Effort) : -1;
+		for (const p of BUILT_IN_PRESETS) {
+			const highestTier = Math.max(
+				...Object.values(p.grades).map((g) => level(g.effort)),
+			);
+			expect({
+				preset: p.preset,
+				mainIsNotTheBiggestSpend: level(p.main.effort) <= highestTier,
+			}).toEqual({ preset: p.preset, mainIsNotTheBiggestSpend: true });
+		}
+	});
+
+	/**
+	 * One model, one effort — the only cross-seat rule there is.
+	 *
+	 * `modelSettings` is keyed by model, so a preset naming one model in two seats at two
+	 * efforts cannot be written to settings at all: one value silently wins. This replaced a
+	 * test that fixed each model's effort across EVERY preset (`fable` always `medium`, and
+	 * so on). That was a convention invented to satisfy this constraint, not the constraint
+	 * itself, and it blocked legitimate presets — `fable-lead` and `fable-top` differ by
+	 * nothing except fable's effort, which is the whole point of having both.
+	 *
+	 * Presets never collide with each other; only one is applied to a project at a time.
+	 */
+	test("within one preset, a model appears at exactly one effort", () => {
+		for (const p of BUILT_IN_PRESETS) {
+			const efforts = new Map<string, Set<string>>();
+			const seats = [
+				{ seat: "main", model: p.main.model, effort: p.main.effort },
+				...GRADES.map((grade) => ({
+					seat: grade,
+					model: p.grades[grade].model,
+					effort: p.grades[grade].effort,
+				})),
+			];
+			for (const { model, effort } of seats) {
+				const alias = baseAlias(model);
+				if (alias === "inherit" || effort === undefined) continue;
+				const seen = efforts.get(alias) ?? new Set<string>();
+				seen.add(effort);
+				efforts.set(alias, seen);
+			}
+			for (const [alias, seen] of efforts) {
+				expect({ preset: p.preset, model: alias, efforts: [...seen] }).toEqual({
+					preset: p.preset,
+					model: alias,
+					efforts: [...seen].slice(0, 1),
+				});
+			}
+		}
+	});
+
+	test("no preset uses haiku", () => {
+		// Excluded deliberately, not by oversight — asserted so re-adding it is a decision.
+		for (const p of BUILT_IN_PRESETS) {
+			const models = [
+				p.main.model,
+				...GRADES.map((grade) => p.grades[grade].model),
+			].map(baseAlias);
+			expect({ preset: p.preset, usesHaiku: models.includes("haiku") }).toEqual(
+				{
+					preset: p.preset,
+					usesHaiku: false,
+				},
+			);
+		}
+	});
+
+	test("preset names are unique", () => {
+		expect(new Set(presetNames()).size).toBe(presetNames().length);
+	});
+
+	test("the shared agent table only names real grades", () => {
+		for (const [agent, grade] of Object.entries(DEFAULT_AGENT_GRADES)) {
+			expect({
+				agent,
+				ok: ["smart", "normal", "cheap"].includes(grade),
+			}).toEqual({ agent, ok: true });
+		}
+	});
+});
+
+describe("isRoutableModel", () => {
+	test("accepts the four aliases and nothing else", () => {
+		for (const a of ["opus", "sonnet", "haiku", "fable"])
+			expect(isRoutableModel(a)).toBe(true);
+		for (const bad of [
+			"inherit",
+			"claude-opus-5",
+			"opus[1m]",
+			"",
+			"kimi-k2-thinking",
+			null,
+			5,
+		])
+			expect(isRoutableModel(bad)).toBe(false);
+	});
+});