claudeup 6.3.2 → 6.5.0

package/src/__tests__/models-manager.test.ts

@@ -0,0 +1,497 @@
+/**
+ * models-manager — the filesystem half of per-subagent model routing.
+ *
+ * Every test works in a temp project AND a temp Claude config dir. The config
+ * dir matters: `applyModelPreset` registers the hook at user scope, so a test
+ * that did not isolate `CLAUDE_CONFIG_DIR` would write into the operator's real
+ * `~/.claude/settings.json`. `hook-registration.ts` refuses to resolve a path
+ * under `bun test` without it, so forgetting is loud rather than damaging.
+ */
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import fs from "fs-extra";
+import { findPreset } from "../data/models-presets.js";
+import { getManifestPath, writeManifest } from "../services/manifest.js";
+import type { ModelsConfig } from "../services/models-core.js";
+import {
+	applyModelPreset,
+	clearModels,
+	configPath,
+	readModelsConfig,
+	readModelsStatus,
+	resolveFullModelId,
+} from "../services/models-manager.js";
+import type { ProfileManifest } from "../types/index.js";
+
+let project: string;
+let configDir: string;
+let previousConfigDir: string | undefined;
+
+beforeEach(async () => {
+	project = await mkdtemp(join(tmpdir(), "models-project-"));
+	configDir = await mkdtemp(join(tmpdir(), "models-config-"));
+	previousConfigDir = process.env.CLAUDE_CONFIG_DIR;
+	process.env.CLAUDE_CONFIG_DIR = configDir;
+});
+
+afterEach(async () => {
+	// biome-ignore lint/performance/noDelete: absence is the intent, not a shortcut
+	if (previousConfigDir === undefined) delete process.env.CLAUDE_CONFIG_DIR;
+	else process.env.CLAUDE_CONFIG_DIR = previousConfigDir;
+	await rm(project, { recursive: true, force: true });
+	await rm(configDir, { recursive: true, force: true });
+});
+
+const PRESET = "fable-advisor";
+
+function manifestWith(profile: string, extra = {}): ProfileManifest {
+	return {
+		version: 2,
+		profiles: { [profile]: { name: profile, ...extra } },
+	};
+}
+
+/**
+ * Stage a project that already has a profile ACTIVE, the way `claudeup install`
+ * leaves one: a materialized dir and `.claude/settings.json` symlinked into it.
+ * Built by hand rather than by running install, which needs the claude CLI.
+ */
+async function stageActiveProfile(
+	name: string,
+	entry: Record<string, unknown> = {},
+	settings: Record<string, unknown> = {},
+): Promise<void> {
+	await writeManifest(manifestWith(name, entry), project);
+	const dir = join(project, ".claude", "_profiles", name);
+	await fs.outputJson(join(dir, "settings.json"), settings, { spaces: 2 });
+	await fs.ensureDir(join(project, ".claude"));
+	await fs.symlink(
+		join("_profiles", name, "settings.json"),
+		join(project, ".claude", "settings.json"),
+	);
+}
+
+async function readSettings(): Promise<Record<string, unknown>> {
+	return fs.readJson(join(project, ".claude", "settings.json"));
+}
+
+describe("applyModelPreset", () => {
+	test("writes the config, the settings, and the manifest entry", async () => {
+		await stageActiveProfile("team");
+
+		const result = await applyModelPreset({
+			projectPath: project,
+			preset: PRESET,
+		});
+
+		expect(result.profile).toBe("team");
+		expect(result.recordedInManifest).toBe(true);
+
+		// The generated config the hook reads.
+		const written = await fs.readJson(configPath(project));
+		expect(written.preset).toBe(PRESET);
+		expect(written.grades.smart.model).toBe("fable");
+
+		// The settings the main thread reads.
+		const settings = await readSettings();
+		expect(settings.model).toBe("opus");
+		expect(settings.effortLevel).toBe("xhigh");
+		expect((settings.modelSettings as Record<string, unknown>).fable).toEqual({
+			effortLevel: "medium",
+		});
+
+		// The committed record — without this, the next `claudeup install`
+		// rewrites settings.json from the manifest and reverts all of the above.
+		const manifest = await fs.readJson(getManifestPath(project));
+		expect(manifest.profiles.team.models.preset).toBe(PRESET);
+		expect(manifest.profiles.team.settings.model).toBe("opus");
+	});
+
+	test("registers the agent-model hook at user scope", async () => {
+		await stageActiveProfile("team");
+		const result = await applyModelPreset({
+			projectPath: project,
+			preset: PRESET,
+		});
+		expect(result.hook).toBe("registered");
+
+		const userSettings = await fs.readJson(join(configDir, "settings.json"));
+		expect(userSettings.hooks.PreToolUse[0].matcher).toBe("Agent");
+		expect(userSettings.hooks.PreToolUse[0].hooks[0].command).toBe(
+			"claudeup hook agent-model",
+		);
+
+		// Second apply finds it already there.
+		const again = await applyModelPreset({
+			projectPath: project,
+			preset: PRESET,
+		});
+		expect(again.hook).toBe("already");
+	});
+
+	// The symlink is how the active profile is wired. `updateSettingsFile` writes
+	// through it on purpose; a remove-then-create would silently detach the
+	// project from its profile and leave the next read looking at a fresh file.
+	test("keeps the profile symlink and updates profiles.json under it", async () => {
+		await stageActiveProfile("team", { plugins: { "dev@magus": "latest" } });
+
+		await applyModelPreset({ projectPath: project, preset: PRESET });
+
+		const link = join(project, ".claude", "settings.json");
+		expect((await fs.lstat(link)).isSymbolicLink()).toBe(true);
+		expect(await fs.readlink(link)).toBe(
+			join("_profiles", "team", "settings.json"),
+		);
+
+		// The write landed in the profile's copy, and re-materializing did not
+		// cost the profile the plugins its manifest entry declares.
+		const inProfile = await fs.readJson(
+			join(project, ".claude", "_profiles", "team", "settings.json"),
+		);
+		expect(inProfile.model).toBe("opus");
+		expect(inProfile.enabledPlugins["dev@magus"]).toBe(true);
+
+		// And .claude/models.json is a link into the same profile dir, not a copy.
+		const modelsLink = join(project, ".claude", "models.json");
+		expect((await fs.lstat(modelsLink)).isSymbolicLink()).toBe(true);
+		expect(await fs.readlink(modelsLink)).toBe(
+			join("_profiles", "team", "models.json"),
+		);
+	});
+
+	test("without a profile it writes a plain models.json and records nothing", async () => {
+		const result = await applyModelPreset({
+			projectPath: project,
+			preset: PRESET,
+		});
+
+		expect(result.profile).toBeNull();
+		expect(result.recordedInManifest).toBe(false);
+		expect((await fs.lstat(configPath(project))).isSymbolicLink()).toBe(false);
+		expect((await fs.readJson(configPath(project))).preset).toBe(PRESET);
+		expect((await readSettings()).model).toBe("opus");
+	});
+
+	// updateSettingsFile refuses a file it cannot parse — a human is probably
+	// mid-edit, and writing our keys back over `{}` is how a whole config
+	// disappears.
+	test("refuses a settings.json it cannot parse, and changes nothing", async () => {
+		const settingsPath = join(project, ".claude", "settings.json");
+		await fs.outputFile(settingsPath, "{ not json");
+
+		await expect(
+			applyModelPreset({ projectPath: project, preset: PRESET }),
+		).rejects.toThrow(/not valid JSON/);
+
+		expect(await fs.readFile(settingsPath, "utf8")).toBe("{ not json");
+	});
+
+	test("refuses an invalid config and names every error", async () => {
+		const bad = {
+			...(findPreset(PRESET) as ModelsConfig),
+			grades: {
+				smart: { model: "gpt-5" },
+				normal: { model: "opus" },
+				cheap: { model: "sonnet" },
+			},
+		} as unknown as ModelsConfig;
+
+		await expect(
+			applyModelPreset({ projectPath: project, preset: bad }),
+		).rejects.toThrow(/grades.smart.model/);
+		expect(await fs.pathExists(configPath(project))).toBe(false);
+	});
+
+	test("an unknown preset name lists the valid ones", async () => {
+		await expect(
+			applyModelPreset({ projectPath: project, preset: "fable-leed" }),
+		).rejects.toThrow(/Available: .*fable-lead/);
+	});
+});
+
+describe("clearModels", () => {
+	test("removes the config, the link, and the manifest entry", async () => {
+		await stageActiveProfile("team");
+		await applyModelPreset({ projectPath: project, preset: PRESET });
+
+		await clearModels(project);
+
+		expect(await fs.pathExists(configPath(project))).toBe(false);
+		// lstat, not pathExists: a DANGLING symlink is the failure this guards.
+		await expect(fs.lstat(configPath(project))).rejects.toThrow();
+		expect(
+			await fs.pathExists(
+				join(project, ".claude", "_profiles", "team", "models.json"),
+			),
+		).toBe(false);
+
+		const manifest = await fs.readJson(getManifestPath(project));
+		expect(manifest.profiles.team.models).toBeUndefined();
+		expect(manifest.profiles.team.settings.model).toBeUndefined();
+	});
+
+	// The user's own `/effort` entries live in the same modelSettings object.
+	// Deleting the object would take their settings with ours.
+	test("leaves unrelated modelSettings keys and unrelated settings intact", async () => {
+		await applyModelPreset({ projectPath: project, preset: PRESET });
+
+		const settingsPath = join(project, ".claude", "settings.json");
+		const before = await fs.readJson(settingsPath);
+		await fs.writeJson(
+			settingsPath,
+			{
+				...before,
+				outputStyle: "composed-team",
+				modelSettings: {
+					...(before.modelSettings as Record<string, unknown>),
+					"claude-something-else": { effortLevel: "low" },
+				},
+			},
+			{ spaces: 2 },
+		);
+
+		await clearModels(project);
+
+		const after = await fs.readJson(settingsPath);
+		expect(after.model).toBeUndefined();
+		expect(after.effortLevel).toBeUndefined();
+		expect(after.outputStyle).toBe("composed-team");
+		expect(after.modelSettings).toEqual({
+			"claude-something-else": { effortLevel: "low" },
+		});
+	});
+
+	test("removes modelSettings entirely once nothing is left in it", async () => {
+		await applyModelPreset({ projectPath: project, preset: PRESET });
+		await clearModels(project);
+		expect(
+			(await fs.readJson(join(project, ".claude", "settings.json")))
+				.modelSettings,
+		).toBeUndefined();
+	});
+
+	test("is a no-op on a project that never had routing", async () => {
+		await clearModels(project);
+		expect(await fs.pathExists(configPath(project))).toBe(false);
+	});
+});
+
+describe("readModelsConfig", () => {
+	test("a missing file is off, not an error", async () => {
+		const result = await readModelsConfig(project);
+		expect(result.config).toBeNull();
+		expect(result.errors).toEqual([]);
+	});
+
+	test("an invalid file reports errors and yields no config", async () => {
+		await fs.outputJson(configPath(project), { version: 1, preset: "x" });
+		const result = await readModelsConfig(project);
+		expect(result.config).toBeNull();
+		expect(result.errors.map((e) => e.path)).toContain("main");
+	});
+
+	test("unparseable JSON is an error, not an empty config", async () => {
+		await fs.outputFile(configPath(project), "{ nope");
+		const result = await readModelsConfig(project);
+		expect(result.config).toBeNull();
+		expect(result.errors[0]?.message).toMatch(/not valid JSON/);
+	});
+});
+
+describe("readModelsStatus", () => {
+	test("reports applied once the preset is written and the hook is on", async () => {
+		await applyModelPreset({ projectPath: project, preset: PRESET });
+		const status = await readModelsStatus(project);
+		expect(status.state).toBe("on");
+		expect(status.preset).toBe(PRESET);
+	});
+
+	test("reports stale when settings drift from the config", async () => {
+		await applyModelPreset({ projectPath: project, preset: PRESET });
+		const settingsPath = join(project, ".claude", "settings.json");
+		await fs.writeJson(settingsPath, {
+			...(await fs.readJson(settingsPath)),
+			model: "haiku",
+		});
+
+		const status = await readModelsStatus(project);
+		expect(status.state).toBe("stale");
+		expect(status.drift.join(" ")).toContain("model");
+	});
+
+	// `unhooked` outranks `stale`: a config with no hook routes NOTHING, so
+	// leading with a drifted effort value would bury the reason.
+	test("reports unhooked when the hook is not registered", async () => {
+		await applyModelPreset({ projectPath: project, preset: PRESET });
+		await fs.writeJson(join(configDir, "settings.json"), {});
+
+		const status = await readModelsStatus(project);
+		expect(status.state).toBe("unhooked");
+	});
+
+	test("warns that CLAUDE_CODE_SUBAGENT_MODEL is now inert", async () => {
+		await applyModelPreset({ projectPath: project, preset: PRESET });
+		const previous = process.env.CLAUDE_CODE_SUBAGENT_MODEL;
+		process.env.CLAUDE_CODE_SUBAGENT_MODEL = "haiku";
+		try {
+			const status = await readModelsStatus(project);
+			expect(status.warnings.join(" ")).toContain("CLAUDE_CODE_SUBAGENT_MODEL");
+		} finally {
+			// biome-ignore lint/performance/noDelete: absence is the intent, not a shortcut
+			if (previous === undefined) delete process.env.CLAUDE_CODE_SUBAGENT_MODEL;
+			else process.env.CLAUDE_CODE_SUBAGENT_MODEL = previous;
+		}
+	});
+});
+
+describe("resolveFullModelId", () => {
+	/** Write a transcript line the way Claude Code records one. */
+	async function transcript(
+		home: string,
+		name: string,
+		models: string[],
+	): Promise<void> {
+		await fs.outputFile(
+			join(home, ".claude", "projects", "-tmp-x", `${name}.jsonl`),
+			`${models
+				.map((m) => JSON.stringify({ message: { model: m } }))
+				.join("\n")}\n`,
+		);
+	}
+
+	test("returns null when the home has no transcripts at all", async () => {
+		const home = await mkdtemp(join(tmpdir(), "models-home-"));
+		try {
+			expect(resolveFullModelId("opus", { home })).toBeNull();
+		} finally {
+			await rm(home, { recursive: true, force: true });
+		}
+	});
+
+	test("returns null when nothing in the transcripts names that alias", async () => {
+		const home = await mkdtemp(join(tmpdir(), "models-home-"));
+		try {
+			await transcript(home, "a", ["claude-sonnet-9-20260101"]);
+			expect(resolveFullModelId("opus", { home })).toBeNull();
+		} finally {
+			await rm(home, { recursive: true, force: true });
+		}
+	});
+
+	// The plainest spelling wins: transcripts record `claude-opus-5` while usage
+	// envelopes record `claude-opus-5[1m]`, and a context-window suffix is a
+	// DIFFERENT modelSettings key — writing it would set effort on a model the
+	// session is not running.
+	test("finds the id and prefers the spelling with no bracketed suffix", async () => {
+		const home = await mkdtemp(join(tmpdir(), "models-home-"));
+		try {
+			await transcript(home, "a", [
+				"claude-opus-7-20260401[1m]",
+				"claude-opus-7-20260401[1m]",
+				"claude-opus-7-20260401[1m]",
+				"claude-opus-7-20260401",
+			]);
+			expect(resolveFullModelId("opus", { home })).toBe(
+				"claude-opus-7-20260401",
+			);
+		} finally {
+			await rm(home, { recursive: true, force: true });
+		}
+	});
+
+	test("takes the most frequent spelling, not the first seen", async () => {
+		const home = await mkdtemp(join(tmpdir(), "models-home-"));
+		try {
+			await transcript(home, "a", [
+				"claude-haiku-4-5-20251001",
+				"claude-haiku-9-20260701",
+				"claude-haiku-9-20260701",
+			]);
+			expect(resolveFullModelId("haiku", { home })).toBe(
+				"claude-haiku-9-20260701",
+			);
+		} finally {
+			await rm(home, { recursive: true, force: true });
+		}
+	});
+
+	/**
+	 * Write a transcript of roughly `mb` megabytes that names `model` ONCE, at the very
+	 * start, followed by padding. Shaped to defeat a tail-only reader on purpose.
+	 */
+	async function bigTranscript(
+		home: string,
+		name: string,
+		model: string | null,
+		mb: number,
+	): Promise<void> {
+		const head = model
+			? `${JSON.stringify({ message: { model } })}\n`
+			: `${JSON.stringify({ message: { note: "no model here" } })}\n`;
+		// Padding carries no `"model"` key, so a reader that only sees it finds nothing.
+		const line = `${JSON.stringify({ message: { text: "x".repeat(900) } })}\n`;
+		const pad = line.repeat(Math.ceil((mb * 1024 * 1024) / line.length));
+		await fs.outputFile(
+			join(home, ".claude", "projects", "-tmp-big", `${name}.jsonl`),
+			head + pad,
+		);
+	}
+
+	/**
+	 * MEASURED, and the reason the window is a byte budget rather than a file count: this
+	 * machine holds 2995 transcripts, and the newest SIXTY spanned fifty minutes, because
+	 * every live session appends to its own file. `claude-sonnet-5` sat at rank 85 and so
+	 * could not resolve at all — while `claude-opus-5`, running right then, resolved fine.
+	 * An alias is not gone because it is an hour old.
+	 */
+	test("resolves an alias whose newest mention is far down the ranking", async () => {
+		const home = await mkdtemp(join(tmpdir(), "models-home-"));
+		try {
+			// 120 newer transcripts that never name sonnet, then the one that does.
+			for (let i = 0; i < 120; i += 1) {
+				await transcript(home, `newer-${String(i).padStart(3, "0")}`, [
+					"claude-opus-5",
+				]);
+			}
+			await transcript(home, "older", ["claude-sonnet-5"]);
+			// Make the sonnet one the OLDEST, so only a wide window reaches it.
+			const file = join(home, ".claude", "projects", "-tmp-x", "older.jsonl");
+			const old = new Date(Date.now() - 86_400_000);
+			await fs.utimes(file, old, old);
+
+			expect(resolveFullModelId("sonnet", { home })).toBe("claude-sonnet-5");
+		} finally {
+			await rm(home, { recursive: true, force: true });
+		}
+	});
+
+	/**
+	 * The other half of the same defect. Reading whole files here was a memory bomb: the
+	 * largest single transcript on this machine is 597 MB, and `readFileSync(file, "utf8")`
+	 * on the newest sixty took 6.5 s per `models use`. Capping only the TOTAL was not enough
+	 * either — one huge decoy consumed the whole budget before the scan reached the file
+	 * that had the answer, so the cap is per file as well.
+	 */
+	test("a huge transcript with no match does not hide a later one", async () => {
+		const home = await mkdtemp(join(tmpdir(), "models-home-"));
+		try {
+			await bigTranscript(home, "decoy", null, 12);
+			await transcript(home, "real", ["claude-fable-5-1"]);
+			const older = join(home, ".claude", "projects", "-tmp-x", "real.jsonl");
+			const old = new Date(Date.now() - 86_400_000);
+			await fs.utimes(older, old, old);
+
+			const started = performance.now();
+			const id = resolveFullModelId("fable", { home });
+			const elapsed = performance.now() - started;
+
+			expect(id).toBe("claude-fable-5-1");
+			// Generous: the point is that it does not read 12 MB looking for nothing.
+			expect(elapsed).toBeLessThan(2000);
+		} finally {
+			await rm(home, { recursive: true, force: true });
+		}
+	});
+});