@selesai/code 0.2.1 → 0.3.1

package/dist/extensions/pi-subagents/test/unit/profiles.test.ts

@@ -0,0 +1,299 @@
+import assert from "node:assert/strict";
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { afterEach, beforeEach, describe, it } from "node:test";
+
+import {
+	applySubagentProfile,
+	checkSubagentProfile,
+	DEFAULT_PROVIDER_MODELS_MAX_AGE_DAYS,
+	generateProfilesForProvider,
+	getProviderModelsPath,
+	getSubagentProfilesDir,
+	listSubagentProfiles,
+	refreshProviderModelCatalog,
+	readProviderModelCatalog,
+} from "../../src/profiles/profiles.ts";
+
+let homeDir = "";
+const previousHome = process.env.HOME;
+const previousUserProfile = process.env.USERPROFILE;
+
+function makeCtx(cwd: string, models: Array<Record<string, unknown>>) {
+	return {
+		cwd,
+		modelRegistry: {
+			getAvailable: () => models,
+		},
+	};
+}
+
+describe("profiles helpers", () => {
+	beforeEach(() => {
+		homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "pi-subprofiles-home-"));
+		process.env.HOME = homeDir;
+		process.env.USERPROFILE = homeDir;
+	});
+
+	afterEach(() => {
+		if (previousHome === undefined) delete process.env.HOME;
+		else process.env.HOME = previousHome;
+		if (previousUserProfile === undefined) delete process.env.USERPROFILE;
+		else process.env.USERPROFILE = previousUserProfile;
+		fs.rmSync(homeDir, { recursive: true, force: true });
+	});
+
+	it("lists no profiles when the directory is empty", () => {
+		assert.deepEqual(listSubagentProfiles(), []);
+		assert.equal(fs.existsSync(getSubagentProfilesDir()), true);
+	});
+
+	it("applies a saved profile by replacing only settings.subagents", () => {
+		const profilesDir = getSubagentProfilesDir();
+		fs.mkdirSync(profilesDir, { recursive: true });
+		fs.writeFileSync(path.join(profilesDir, "openai-codex.quota.json"), JSON.stringify({
+			subagents: {
+				agentOverrides: {
+					scout: { model: "openai-codex/gpt-5.3-codex-spark" },
+				},
+			},
+		}, null, 2));
+		const settingsPath = path.join(homeDir, ".selesai", "agent", "settings.json");
+		fs.mkdirSync(path.dirname(settingsPath), { recursive: true });
+		fs.writeFileSync(settingsPath, JSON.stringify({ defaultModel: "openai/gpt-5", subagents: { agentOverrides: { scout: { model: "old" } } } }, null, 2));
+
+		const result = applySubagentProfile("openai-codex.quota");
+		const written = JSON.parse(fs.readFileSync(settingsPath, "utf-8"));
+		assert.equal(result.settingsPath, settingsPath);
+		assert.equal(result.filePath.endsWith("openai-codex.quota.json"), true);
+		assert.equal(written.defaultModel, "openai/gpt-5");
+		assert.equal(written.subagents.agentOverrides.scout.model, "openai-codex/gpt-5.3-codex-spark");
+	});
+
+	it("rejects profile and provider path traversal names", async () => {
+		assert.throws(() => applySubagentProfile("../escape"), /safe file name/);
+		assert.throws(() => getProviderModelsPath("../escape"), /safe file name/);
+		await assert.rejects(
+			refreshProviderModelCatalog({ exec: async () => ({ stdout: "OK\n", stderr: "", code: 0, killed: false }) }, makeCtx(process.cwd(), []) as never, "../escape"),
+			/safe file name/,
+		);
+	});
+
+	it("refreshes a provider model catalog and writes a cache file", async () => {
+		const execCalls: string[] = [];
+		const execCwds: unknown[] = [];
+		const pi = {
+			exec: async (_command: string, args: string[], options?: Record<string, unknown>) => {
+				execCalls.push(args.join(" "));
+				execCwds.push(options?.cwd);
+				return { stdout: "OK\n", stderr: "", code: 0, killed: false };
+			},
+		};
+		const ctx = makeCtx(process.cwd(), [
+			{ provider: "openai-codex", id: "gpt-5.3-codex-spark", reasoning: true, name: "Spark" },
+			{ provider: "openai-codex", id: "gpt-5.4-mini", reasoning: true, name: "Mini" },
+			{ provider: "openai-codex", id: "gpt-5.4", reasoning: true, name: "Base" },
+			{ provider: "openai-codex", id: "gpt-5.5", reasoning: true, name: "Best" },
+		]);
+
+		const result = await refreshProviderModelCatalog(pi, ctx as never, "openai-codex");
+		assert.equal(result.reused, false);
+		assert.equal(fs.existsSync(result.filePath), true);
+		assert.equal(result.catalog.models.length, 4);
+		assert.equal(execCalls.length, 4);
+		assert.equal(execCalls.every((call) => call.includes("--no-tools")), true);
+		assert.equal(execCalls.some((call) => call.includes("--tools read")), false);
+		assert.deepEqual(execCwds, [os.tmpdir(), os.tmpdir(), os.tmpdir(), os.tmpdir()]);
+		assert.deepEqual(result.catalog.models.map((entry) => entry.fullId), [
+			"openai-codex/gpt-5.3-codex-spark",
+			"openai-codex/gpt-5.4-mini",
+			"openai-codex/gpt-5.4",
+			"openai-codex/gpt-5.5",
+		]);
+		assert.equal(readProviderModelCatalog("openai-codex")?.provider, "openai-codex");
+		assert.equal(result.heuristicFallbackCount, 4);
+		assert.deepEqual(result.catalog.models[0]?.derived.classificationSources, ["heuristic-name"]);
+		assert.deepEqual(result.catalog.models[0]?.warnings, ["Classification fell back to name heuristics."]);
+	});
+
+	it("does not count heuristic fallback when official metadata is present", async () => {
+		const pi = {
+			exec: async () => ({ stdout: "OK\n", stderr: "", code: 0, killed: false }),
+		};
+		const ctx = makeCtx(process.cwd(), [
+			{
+				provider: "openai",
+				id: "gpt-5-mini",
+				name: "GPT-5 Mini",
+				reasoning: true,
+				contextWindow: 128000,
+				maxTokens: 16000,
+				cost: { input: 0.25, output: 2 },
+			},
+			{
+				provider: "openai",
+				id: "gpt-5",
+				name: "GPT-5",
+				reasoning: true,
+				contextWindow: 200000,
+				maxTokens: 32000,
+				cost: { input: 1.25, output: 10 },
+			},
+		]);
+		const result = await refreshProviderModelCatalog(pi, ctx as never, "openai");
+		assert.equal(result.heuristicFallbackCount, 0);
+		assert.deepEqual(result.catalog.models[0]?.derived.classificationSources, ["official-metadata", "heuristic-name"]);
+		assert.deepEqual(result.catalog.models[0]?.warnings, []);
+	});
+
+	it("reuses a fresh provider model catalog", async () => {
+		const pi = {
+			exec: async () => {
+				throw new Error("should not probe fresh cache");
+			},
+		};
+		const filePath = getProviderModelsPath("openai-codex");
+		fs.mkdirSync(path.dirname(filePath), { recursive: true });
+		fs.writeFileSync(filePath, JSON.stringify({
+			provider: "openai-codex",
+			refreshedAt: new Date().toISOString(),
+			maxAgeDays: DEFAULT_PROVIDER_MODELS_MAX_AGE_DAYS,
+			sources: ["runtime-registry"],
+			models: [],
+		}, null, 2));
+		const result = await refreshProviderModelCatalog(pi, makeCtx(process.cwd(), []) as never, "openai-codex");
+		assert.equal(result.reused, true);
+	});
+
+	it("generates quota and quality profiles from sorted provider models", async () => {
+		const pi = {
+			exec: async () => ({ stdout: "OK\n", stderr: "", code: 0, killed: false }),
+		};
+		const ctx = makeCtx(process.cwd(), [
+			{ provider: "openai-codex", id: "gpt-5.3-codex-spark", reasoning: true },
+			{ provider: "openai-codex", id: "gpt-5.4-mini", reasoning: true },
+			{ provider: "openai-codex", id: "gpt-5.4", reasoning: true },
+			{ provider: "openai-codex", id: "gpt-5.5", reasoning: true },
+		]);
+
+		const result = await generateProfilesForProvider(pi, ctx as never, "openai-codex");
+		assert.equal(result.heuristicFallbackCount, 4);
+		assert.equal(result.selectedHeuristicFallbackCount, 4);
+		assert.equal(result.quotaModels.cheap, "openai-codex/gpt-5.3-codex-spark");
+		assert.equal(result.quotaModels.medium, "openai-codex/gpt-5.4-mini");
+		assert.equal(result.quotaModels.strong, "openai-codex/gpt-5.4-mini");
+		assert.equal(result.qualityModels.cheap, "openai-codex/gpt-5.4-mini");
+		assert.equal(result.qualityModels.medium, "openai-codex/gpt-5.4");
+		assert.equal(result.qualityModels.strong, "openai-codex/gpt-5.5");
+	});
+
+	it("does not treat provider names like MiniMax as mini-tier tokens and prefers M3 over M2.7-highspeed for quality", async () => {
+		const pi = {
+			exec: async () => ({ stdout: "OK\n", stderr: "", code: 0, killed: false }),
+		};
+		const ctx = makeCtx(process.cwd(), [
+			{
+				provider: "minimax",
+				id: "MiniMax-M2.7",
+				name: "MiniMax-M2.7",
+				reasoning: true,
+				contextWindow: 204800,
+				maxTokens: 131072,
+				cost: { input: 0.3, output: 1.2, cacheRead: 0.06, cacheWrite: 0.375 },
+			},
+			{
+				provider: "minimax",
+				id: "MiniMax-M3",
+				name: "MiniMax-M3",
+				reasoning: true,
+				contextWindow: 512000,
+				maxTokens: 128000,
+				cost: { input: 0.6, output: 2.4, cacheRead: 0.12, cacheWrite: 0 },
+			},
+			{
+				provider: "minimax",
+				id: "MiniMax-M2.7-highspeed",
+				name: "MiniMax-M2.7-highspeed",
+				reasoning: true,
+				contextWindow: 204800,
+				maxTokens: 131072,
+				cost: { input: 0.6, output: 2.4, cacheRead: 0.06, cacheWrite: 0.375 },
+			},
+		]);
+
+		const refreshed = await refreshProviderModelCatalog(pi, ctx as never, "minimax");
+		const highspeed = refreshed.catalog.models.find((entry) => entry.fullId === "minimax/MiniMax-M2.7-highspeed");
+		const m3 = refreshed.catalog.models.find((entry) => entry.fullId === "minimax/MiniMax-M3");
+		assert.equal(highspeed?.derived.latencyTier, "fast");
+		assert.equal(typeof highspeed?.derived.profileRank, "number");
+		assert.equal(typeof m3?.derived.profileRank, "number");
+		assert.equal((m3?.derived.profileRank ?? 0) > (highspeed?.derived.profileRank ?? 0), true);
+		const result = await generateProfilesForProvider(pi, ctx as never, "minimax", { forceRefresh: true });
+		assert.equal(result.quotaModels.cheap, "minimax/MiniMax-M2.7");
+		assert.equal(result.quotaModels.medium, "minimax/MiniMax-M2.7");
+		assert.equal(result.quotaModels.strong, "minimax/MiniMax-M2.7");
+		assert.equal(result.qualityModels.strong, "minimax/MiniMax-M3");
+		assert.equal(Object.values(result.quotaModels).includes("minimax/MiniMax-M2.7-highspeed"), false);
+		assert.equal(Object.values(result.qualityModels).includes("minimax/MiniMax-M2.7-highspeed"), false);
+	});
+
+	it("checks a profile against the registry and live probe", async () => {
+		const profilesDir = getSubagentProfilesDir();
+		fs.mkdirSync(profilesDir, { recursive: true });
+		fs.writeFileSync(path.join(profilesDir, "demo.json"), JSON.stringify({
+			subagents: {
+				agentOverrides: {
+					scout: { model: "openai-codex/gpt-5.3-codex-spark" },
+					worker: { model: "openai-codex/gpt-5.9" },
+				},
+			},
+		}, null, 2));
+		const pi = {
+			exec: async (_command: string, args: string[]) => {
+				const model = args[2];
+				if (model === "openai-codex/gpt-5.9") {
+					return { stdout: "", stderr: "model unavailable", code: 1, killed: false };
+				}
+				return { stdout: "OK\n", stderr: "", code: 0, killed: false };
+			},
+		};
+		const ctx = makeCtx(process.cwd(), [{ provider: "openai-codex", id: "gpt-5.3-codex-spark" }]);
+		const result = await checkSubagentProfile(pi, ctx as never, "demo");
+		assert.deepEqual(result.results, [
+			{
+				agent: "scout",
+				model: "openai-codex/gpt-5.3-codex-spark",
+				inRegistry: true,
+				probe: { status: "ok", message: "OK" },
+			},
+			{
+				agent: "worker",
+				model: "openai-codex/gpt-5.9",
+				inRegistry: false,
+				probe: { status: "unavailable", message: "model unavailable" },
+			},
+		]);
+	});
+
+	it("checks short model ids and thinking suffixes against the registry", async () => {
+		const profilesDir = getSubagentProfilesDir();
+		fs.mkdirSync(profilesDir, { recursive: true });
+		fs.writeFileSync(path.join(profilesDir, "demo.json"), JSON.stringify({
+			subagents: { agentOverrides: { scout: { model: "gpt-5.3-codex-spark:high" } } },
+		}, null, 2));
+		const probedModels: unknown[] = [];
+		const pi = {
+			exec: async (_command: string, args: string[]) => {
+				probedModels.push(args[2]);
+				assert.equal(args.includes("--no-tools"), true);
+				assert.equal(args.includes("--tools"), false);
+				return { stdout: "OK\n", stderr: "", code: 0, killed: false };
+			},
+		};
+		const ctx = makeCtx(process.cwd(), [{ provider: "openai-codex", id: "gpt-5.3-codex-spark" }]);
+		const result = await checkSubagentProfile(pi, ctx as never, "demo");
+		assert.equal(result.results[0]?.inRegistry, true);
+		assert.deepEqual(probedModels, ["openai-codex/gpt-5.3-codex-spark:high"]);
+	});
+});