magus-cli 0.0.0-placeholder → 7.0.0

package/src/__tests__/blackbox-tool-state.test.ts

@@ -0,0 +1,669 @@
+/**
+ * BLACK-BOX: R3 (one tool probe), R4 (no unconditional upgrade), R5 (pins are
+ * checked). Written from requirements.md, baseline-tools.md and §8.2/§8.3 only.
+ *
+ * Probe IO is a scripted fake `ToolProbeIO` that answers commands BY PATTERN
+ * (the exact argv the probe issues is an implementation detail). Output bodies
+ * are the real captures in fixtures/tool-probe/ and baseline-tools.md.
+ */
+import { describe, expect, test } from "bun:test";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import {
+	type ExecResult,
+	type FetchJsonResult,
+	type ProbeProgress,
+	type ToolFacts,
+	type ToolProbeIO,
+	classifyTool,
+	compareToolVersions,
+	probeTool,
+	probeTools,
+} from "../services/tool-probe.js";
+import { planWorkItems } from "../services/update-engine.js";
+import {
+	type BinUpdateItem,
+	type PluginUpdateItem,
+	type UpdatePlan,
+	binNeedsWork,
+	planBinUpdates,
+	planHasWork,
+	planIsBehind,
+	summarizePlan,
+} from "../services/update-plan.js";
+import type { BinRequirement, ResolvedBin } from "../types/index.js";
+import type { Command } from "../utils/command-utils.js";
+
+// ── helpers ────────────────────────────────────────────────────────────────
+
+function facts(name: string, over: Partial<ToolFacts> = {}): ToolFacts {
+	return {
+		name,
+		present: true,
+		resolvedPath: `/usr/local/bin/${name}`,
+		installed: "1.0.0",
+		latest: null,
+		behind: null,
+		managed: true,
+		sources: ["path"],
+		...over,
+	};
+}
+
+function bin(name: string, over: Partial<ResolvedBin> = {}): ResolvedBin {
+	return { name, via: "npm", sources: ["test"], ...over };
+}
+
+function binItem(name: string, action: BinUpdateItem["action"]): BinUpdateItem {
+	return {
+		name,
+		action,
+		reason: action === "unknown" ? "latest-unavailable" : "up-to-date",
+		installed: "1.0.0",
+		latest: null,
+		target: null,
+		sources: ["test"],
+	};
+}
+
+function planOf(
+	bins: BinUpdateItem[],
+	plugins: PluginUpdateItem[] = [],
+): UpdatePlan {
+	return {
+		profileId: "default",
+		plugins,
+		bins,
+		skills: [],
+		binSpecs: new Map(bins.map((b) => [b.name, bin(b.name)])),
+	};
+}
+
+const FIXTURES = join(import.meta.dir, "fixtures", "tool-probe");
+function capture(file: string): string {
+	return readFileSync(join(FIXTURES, file), "utf8")
+		.split("\n")
+		.filter((l) => !l.startsWith("#"))
+		.join("\n");
+}
+
+const OK = (stdout: string): ExecResult => ({
+	code: 0,
+	stdout,
+	stderr: "",
+	timedOut: false,
+	missing: false,
+});
+const NOPE: ExecResult = {
+	code: 1,
+	stdout: "",
+	stderr: "fake: no scripted answer",
+	timedOut: false,
+	missing: false,
+};
+const MISSING: ExecResult = {
+	code: null,
+	stdout: "",
+	stderr: "",
+	timedOut: false,
+	missing: true,
+};
+const TIMED_OUT: ExecResult = {
+	code: null,
+	stdout: "",
+	stderr: "",
+	timedOut: true,
+	missing: false,
+};
+
+interface FakeIOSpec {
+	exec?: (argv: string[]) => ExecResult | Promise<ExecResult> | undefined;
+	paths?: Record<string, string>; // executable name → PATH location
+	realpaths?: Record<string, string>;
+	texts?: Record<string, string>;
+	env?: Record<string, string>;
+	fetch?: (url: string) => FetchJsonResult | undefined;
+}
+
+function fakeIO(spec: FakeIOSpec): ToolProbeIO & { calls: string[] } {
+	const calls: string[] = [];
+	return {
+		calls,
+		async exec(command: Command) {
+			const argv = [command.cmd, ...command.args];
+			calls.push(argv.join(" "));
+			const base = command.cmd.split("/").pop() ?? command.cmd;
+			if (spec.paths && !(base in spec.paths) && !command.cmd.includes("/"))
+				return MISSING;
+			return (await spec.exec?.([base, ...command.args])) ?? NOPE;
+		},
+		async fetchJson(url: string) {
+			calls.push(`fetch ${url}`);
+			return spec.fetch?.(url) ?? { ok: false, reason: "fake: offline" };
+		},
+		async readJson() {
+			return null;
+		},
+		async readText(file: string) {
+			return spec.texts?.[file] ?? null;
+		},
+		async readlink() {
+			return null;
+		},
+		async realpath(file: string) {
+			return spec.realpaths?.[file] ?? file;
+		},
+		async resolveExecutable(name: string) {
+			return spec.paths?.[name] ?? null;
+		},
+		env: {
+			PATH: "/opt/homebrew/bin:/usr/bin:/bin",
+			HOME: "/Users/test",
+			...spec.env,
+		},
+		homedir: "/Users/test",
+	};
+}
+
+// ── classifyTool ───────────────────────────────────────────────────────────
+
+describe("classifyTool — unpinned", () => {
+	const spec: BinRequirement = { name: "tmux", via: "brew" };
+
+	test("not on PATH: install", () => {
+		const got = classifyTool(
+			spec,
+			facts("tmux", {
+				present: false,
+				resolvedPath: null,
+				installed: null,
+				managed: null,
+			}),
+		);
+		expect(got.state).toBe("install");
+	});
+
+	test("installer says not behind (baseline tmux 3.7c): current", () => {
+		const got = classifyTool(
+			spec,
+			facts("tmux", { installed: "3.7c", behind: false }),
+		);
+		expect(got.state).toBe("current");
+	});
+
+	test("installer says behind, and the upgrade would replace the file: upgrade with a reason note", () => {
+		const got = classifyTool(
+			spec,
+			facts("tmux", { installed: "3.7b", latest: "3.7c", behind: true }),
+		);
+		expect({ state: got.state, reason: got.reason }).toEqual({
+			state: "upgrade",
+			reason: "behind",
+		});
+		expect(got.note).toBeTruthy();
+	});
+
+	test("installed version below the latest with no installer verdict: upgrade to latest", () => {
+		const got = classifyTool(
+			{ name: "codex", via: "npm" },
+			facts("codex", { installed: "0.40.0", latest: "0.41.0" }),
+		);
+		expect({ state: got.state, target: got.target }).toEqual({
+			state: "upgrade",
+			target: "0.41.0",
+		});
+	});
+
+	test("installed equal to latest: current", () => {
+		const got = classifyTool(
+			{ name: "claudish", via: "bun" },
+			facts("claudish", { installed: "9.3.0", latest: "9.3.0" }),
+		);
+		expect(got.state).toBe("current");
+	});
+
+	test("no latest and no verdict: unknown with a reason — not current, not upgrade", () => {
+		const got = classifyTool(
+			{ name: "gemini", via: "npm" },
+			facts("gemini", { installed: "0.5.0" }),
+		);
+		expect(got.state).toBe("unknown");
+		expect(got.note).toBeTruthy();
+	});
+
+	test("installed version unreadable: unknown / version-unreadable", () => {
+		const got = classifyTool(
+			{ name: "x", via: "npm" },
+			facts("x", { installed: null, latest: "2.0.0" }),
+		);
+		expect({ state: got.state, reason: got.reason }).toEqual({
+			state: "unknown",
+			reason: "version-unreadable",
+		});
+	});
+
+	test("probe timed out: unknown, never upgrade", () => {
+		const got = classifyTool(
+			{ name: "gemini", via: "npm" },
+			facts("gemini", { installed: "0.5.0", failure: "timed out after 8s" }),
+		);
+		expect(got.state).toBe("unknown");
+	});
+
+	test("behind, but the upgrade would not replace the file PATH runs: not upgrade / not-managed", () => {
+		const got = classifyTool(
+			{ name: "crush", via: "npm" },
+			facts("crush", {
+				installed: "0.7.0",
+				latest: "0.8.0",
+				behind: true,
+				managed: false,
+			}),
+		);
+		expect(got.state).not.toBe("upgrade");
+		expect(got.reason).toBe("not-managed");
+	});
+
+	test("behind, but held back by the installer: not upgrade / held-back", () => {
+		const got = classifyTool(
+			spec,
+			facts("tmux", {
+				installed: "3.7b",
+				latest: "3.7c",
+				behind: true,
+				heldBack: "held by brew pin",
+			}),
+		);
+		expect(got.state).not.toBe("upgrade");
+		expect(got.reason).toBe("held-back");
+	});
+});
+
+describe("classifyTool — pinned (R5)", () => {
+	const pinned = (version: string): BinRequirement => ({
+		name: "mnemex",
+		via: "bun",
+		version,
+	});
+
+	test("installed at the pin: current / at-pin", () => {
+		const got = classifyTool(
+			pinned("1.0.0"),
+			facts("mnemex", { installed: "1.0.0", latest: "1.2.0" }),
+		);
+		expect({ state: got.state, reason: got.reason }).toEqual({
+			state: "current",
+			reason: "at-pin",
+		});
+	});
+
+	test("installed behind the pin: upgrade / behind-pin, aiming at the pin not at latest", () => {
+		const got = classifyTool(
+			pinned("1.0.0"),
+			facts("mnemex", { installed: "0.9.0", latest: "1.2.0" }),
+		);
+		expect({
+			state: got.state,
+			reason: got.reason,
+			target: got.target,
+		}).toEqual({
+			state: "upgrade",
+			reason: "behind-pin",
+			target: "1.0.0",
+		});
+	});
+
+	test("installed ahead of the pin: not current, and no upgrade planned", () => {
+		const got = classifyTool(
+			pinned("1.0.0"),
+			facts("mnemex", { installed: "1.1.0", latest: "1.2.0" }),
+		);
+		expect(got.state).not.toBe("current");
+		expect(got.state).not.toBe("upgrade");
+	});
+
+	test("installed version unorderable against the pin: not current", () => {
+		const got = classifyTool(
+			pinned("1.0.0"),
+			facts("mnemex", { installed: "3ea32df27be7" }),
+		);
+		expect(got.state).not.toBe("current");
+	});
+
+	test("go tool pinned v2.0.0 with v2.0.0 installed (tmux-mcp): current / at-pin", () => {
+		const got = classifyTool(
+			{
+				name: "tmux-mcp",
+				via: "go",
+				module: "github.com/MadAppGang/tmux-mcp/v2",
+				version: "v2.0.0",
+			},
+			facts("tmux-mcp", { installed: "v2.0.0" }),
+		);
+		expect({ state: got.state, reason: got.reason }).toEqual({
+			state: "current",
+			reason: "at-pin",
+		});
+	});
+});
+
+describe("compareToolVersions", () => {
+	test("orders semver pairs and refuses a commit sha against a semver", () => {
+		expect(compareToolVersions("1.0.0", "1.0.1")).toBe(-1);
+		expect(compareToolVersions("2.0.0", "1.9.9")).toBe(1);
+		expect(compareToolVersions("1.2.3", "1.2.3")).toBe(0);
+		expect(compareToolVersions("3ea32df27be7", "1.0.0")).toBeNull();
+	});
+});
+
+// ── plan predicates ────────────────────────────────────────────────────────
+
+describe("planBinUpdates and the one work predicate", () => {
+	test("a bin with no facts is unknown / not-probed, never upgrade", () => {
+		const [item] = planBinUpdates([bin("orphan-tool")], []);
+		expect({ action: item?.action, reason: item?.reason }).toEqual({
+			action: "unknown",
+			reason: "not-probed",
+		});
+	});
+
+	test("the exact pin is carried on the item; the three baseline tools plan current", () => {
+		const items = planBinUpdates(
+			[
+				bin("tmux", { via: "brew" }),
+				bin("browser-use", { via: "pip" }),
+				bin("mcp", { via: "pip" }),
+				bin("mnemex", { via: "bun", version: "1.0.0" }),
+			],
+			[
+				facts("tmux", { installed: "3.7c", behind: false }),
+				facts("browser-use", { installed: "0.13.10", behind: false }),
+				facts("mcp", { installed: "2.2.0", behind: false }),
+				facts("mnemex", { installed: "1.0.0" }),
+			],
+		);
+		expect(items.map((i) => [i.name, i.action])).toEqual([
+			["tmux", "current"],
+			["browser-use", "current"],
+			["mcp", "current"],
+			["mnemex", "current"],
+		]);
+		expect(items.find((i) => i.name === "mnemex")?.version).toBe("1.0.0");
+	});
+
+	test("binNeedsWork, planWorkItems, planHasWork and planIsBehind agree for every action", () => {
+		const table: Array<[BinUpdateItem["action"], boolean]> = [
+			["install", true],
+			["upgrade", true],
+			["current", false],
+			["unknown", false],
+		];
+		for (const [action, work] of table) {
+			const it = binItem(`t-${action}`, action);
+			const plan = planOf([it]);
+			expect({ action, needsWork: binNeedsWork(it) }).toEqual({
+				action,
+				needsWork: work,
+			});
+			expect({ action, inWork: planWorkItems(plan).includes(it.name) }).toEqual(
+				{ action, inWork: work },
+			);
+			expect({ action, hasWork: planHasWork(plan) }).toEqual({
+				action,
+				hasWork: work,
+			});
+			expect({ action, behind: planIsBehind(plan) }).toEqual({
+				action,
+				behind: work,
+			});
+		}
+	});
+
+	test("summarizePlan splits unknown into plugins and tools and sums them", () => {
+		const unknownPlugin: PluginUpdateItem = {
+			pluginId: "code-review@claude-plugins-official",
+			pinned: "latest",
+			installed: "3ea32df27be7",
+			available: null,
+			target: null,
+			action: "unknown",
+			reason: "no-published-version",
+			scopes: [],
+			beforeVersions: {},
+			note: "n",
+		};
+		const s = summarizePlan(
+			planOf(
+				[
+					binItem("gemini", "unknown"),
+					binItem("x", "unknown"),
+					binItem("tmux", "current"),
+				],
+				[unknownPlugin],
+			),
+		);
+		expect({
+			unknownPlugins: s.unknownPlugins,
+			unknownTools: s.unknownTools,
+			unknown: s.unknown,
+		}).toEqual({
+			unknownPlugins: 1,
+			unknownTools: 2,
+			unknown: 3,
+		});
+	});
+});
+
+// ── the probe, through injected IO ─────────────────────────────────────────
+
+function brewIO(opts: { installed: string; outdatedJson: string }) {
+	return fakeIO({
+		paths: { tmux: "/opt/homebrew/bin/tmux", brew: "/opt/homebrew/bin/brew" },
+		realpaths: {
+			"/opt/homebrew/bin/tmux": `/opt/homebrew/Cellar/tmux/${opts.installed}/bin/tmux`,
+		},
+		env: {
+			HOMEBREW_PREFIX: "/opt/homebrew",
+			HOMEBREW_CELLAR: "/opt/homebrew/Cellar",
+		},
+		exec: ([cmd, ...args]) => {
+			if (cmd === "brew") {
+				if (args.includes("outdated")) return OK(opts.outdatedJson);
+				if (args.includes("list")) return OK(`tmux ${opts.installed}\n`);
+				if (args.includes("--cellar")) return OK("/opt/homebrew/Cellar\n");
+				if (args.includes("--prefix"))
+					return OK(
+						args.length > 1 ? "/opt/homebrew/opt/tmux\n" : "/opt/homebrew\n",
+					);
+				if (args.includes("--repository")) return OK("/opt/homebrew\n");
+			}
+			if (cmd === "tmux") return OK(`tmux ${opts.installed}\n`);
+			return undefined;
+		},
+	});
+}
+
+describe("probeTool through injected IO (no real brew/uv, no network)", () => {
+	test("real brew output: tmux 3.7c, `brew outdated` empty → current (V5 baseline)", async () => {
+		const io = brewIO({
+			installed: "3.7c",
+			outdatedJson: capture("brew-outdated-tmux-current.txt"),
+		});
+		const spec: BinRequirement = {
+			name: "tmux",
+			via: "brew",
+			check: "tmux -V",
+		};
+
+		const f = await probeTool(spec, io, { timeoutMs: 500, deadlineMs: 2000 });
+		const got = classifyTool(spec, f);
+
+		expect({ state: got.state, facts: f, calls: io.calls }).toMatchObject({
+			state: "current",
+		});
+		expect(f.present).toBe(true);
+	});
+
+	test("brew outdated lists tmux 3.7b → 3.7c: upgrade", async () => {
+		const outdated = JSON.stringify({
+			formulae: [
+				{
+					name: "tmux",
+					installed_versions: ["3.7b"],
+					current_version: "3.7c",
+					pinned: false,
+					pinned_version: null,
+				},
+			],
+			casks: [],
+		});
+		const io = brewIO({ installed: "3.7b", outdatedJson: outdated });
+		const spec: BinRequirement = {
+			name: "tmux",
+			via: "brew",
+			check: "tmux -V",
+		};
+
+		const f = await probeTool(spec, io, { timeoutMs: 500, deadlineMs: 2000 });
+		const got = classifyTool(spec, f);
+
+		expect({ state: got.state, facts: f, calls: io.calls }).toMatchObject({
+			state: "upgrade",
+		});
+	});
+
+	test("real uv output: browser-use absent from `uv tool list --outdated` → current (V5 baseline)", async () => {
+		const toolDir = "/Users/test/.local/share/uv/tools";
+		const io = fakeIO({
+			paths: {
+				"browser-use": "/Users/test/.local/bin/browser-use",
+				uv: "/opt/homebrew/bin/uv",
+			},
+			realpaths: {
+				"/Users/test/.local/bin/browser-use": `${toolDir}/browser-use/bin/browser-use`,
+			},
+			texts: {
+				[`${toolDir}/browser-use/uv-receipt.toml`]: capture(
+					"uv-receipt-browser-use.toml",
+				),
+			},
+			exec: ([cmd, ...args]) => {
+				if (cmd !== "uv") return undefined;
+				if (args[0] === "--version")
+					return OK("uv 0.11.15 (3cffe97c2 2026-05-18 aarch64-apple-darwin)\n");
+				if (args[0] === "tool" && args[1] === "dir") return OK(`${toolDir}\n`);
+				if (
+					args[0] === "tool" &&
+					args[1] === "list" &&
+					args.includes("--outdated")
+				)
+					return OK(capture("uv-tool-list-outdated.txt"));
+				if (args[0] === "tool" && args[1] === "list")
+					return OK(capture("uv-tool-list.txt"));
+				return undefined;
+			},
+		});
+		const spec: BinRequirement = { name: "browser-use", via: "pip" };
+
+		const f = await probeTool(spec, io, { timeoutMs: 500, deadlineMs: 2000 });
+		const got = classifyTool(spec, f);
+
+		expect({ state: got.state, facts: f, calls: io.calls }).toMatchObject({
+			state: "current",
+		});
+	});
+
+	test("every command times out: facts resolve and the tool is unknown, never upgrade", async () => {
+		const io = fakeIO({
+			paths: { tmux: "/opt/homebrew/bin/tmux", brew: "/opt/homebrew/bin/brew" },
+			exec: () => TIMED_OUT,
+		});
+		const spec: BinRequirement = { name: "tmux", via: "brew" };
+
+		const f = await probeTool(spec, io, { timeoutMs: 50, deadlineMs: 500 });
+		const got = classifyTool(spec, f);
+
+		expect(got.state).toBe("unknown");
+	});
+
+	test("a command that never settles is bounded by the deadline; probeTools never rejects", async () => {
+		const io = fakeIO({
+			paths: { codex: "/usr/local/bin/codex", npm: "/usr/local/bin/npm" },
+			exec: () => new Promise<ExecResult>(() => {}),
+		});
+		const spec: BinRequirement = {
+			name: "codex",
+			via: "npm",
+			package: "@openai/codex",
+		};
+		const started = Date.now();
+
+		const guard = new Promise<"guard">((r) =>
+			setTimeout(() => r("guard"), 3000),
+		);
+		const result = await Promise.race([
+			probeTools([spec], io, { timeoutMs: 50, deadlineMs: 200 }),
+			guard,
+		]);
+
+		expect(result).not.toBe("guard");
+		const [f] = result as ToolFacts[];
+		expect(Date.now() - started).toBeLessThan(3000);
+		expect(f?.name).toBe("codex");
+		expect(f?.failure).toBeTruthy();
+		expect(classifyTool(spec, f as ToolFacts).state).toBe("unknown");
+	});
+
+	test("probeTools returns results in spec order; onResult reports a monotonic settled count", async () => {
+		const delays: Record<string, number> = { slow: 60, mid: 30, fast: 0 };
+		const io = fakeIO({
+			paths: {
+				slow: "/usr/local/bin/slow",
+				mid: "/usr/local/bin/mid",
+				fast: "/usr/local/bin/fast",
+			},
+			// `undefined` ("no scripted answer") is synchronous only — the contract
+			// has no async undefined — so the delay wraps just the scripted reply.
+			exec: ([cmd]) => {
+				const d = delays[cmd ?? ""];
+				if (d === undefined) return undefined;
+				return new Promise<ExecResult>((r) =>
+					setTimeout(() => r(OK(`${cmd} 1.0.0\n`)), d),
+				);
+			},
+		});
+		const specs: BinRequirement[] = ["slow", "mid", "fast"].map((n) => ({
+			name: n,
+			via: "npm",
+			check: `${n} --version`,
+		}));
+		const seen: Array<{ name: string; p: ProbeProgress }> = [];
+
+		const out = await probeTools(specs, io, {
+			timeoutMs: 1000,
+			deadlineMs: 3000,
+			onResult: (f, p) => seen.push({ name: f.name, p }),
+		});
+
+		expect(out.map((f) => f.name)).toEqual(["slow", "mid", "fast"]);
+		expect(seen.map((s) => s.p.settled)).toEqual([1, 2, 3]);
+		for (const s of seen) {
+			expect(s.p.total).toBe(3);
+			expect(specs[s.p.index]?.name).toBe(s.name);
+		}
+	});
+
+	test("a tool not on PATH is reported absent and classifies install", async () => {
+		const io = fakeIO({ paths: {}, exec: () => MISSING });
+		const spec: BinRequirement = {
+			name: "qwen",
+			via: "npm",
+			package: "@qwen-code/qwen-code",
+		};
+
+		const f = await probeTool(spec, io, { timeoutMs: 50, deadlineMs: 500 });
+
+		expect(f.present).toBe(false);
+		expect(classifyTool(spec, f).state).toBe("install");
+	});
+});