@ag-eco/agentplate-cli 0.13.0

package/src/doctor/providers.test.ts

@@ -0,0 +1,409 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { AgentplateConfig } from "../types.ts";
+import { checkProviders } from "./providers.ts";
+
+/** Build a minimal valid AgentplateConfig for testing. */
+function makeConfig(overrides: Partial<AgentplateConfig> = {}): AgentplateConfig {
+	const tmp = tmpdir();
+	return {
+		project: {
+			name: "test-project",
+			root: tmp,
+			canonicalBranch: "main",
+		},
+		agents: {
+			manifestPath: join(tmp, ".agentplate", "agent-manifest.json"),
+			baseDir: join(tmp, ".agentplate", "agents"),
+			maxConcurrent: 5,
+			staggerDelayMs: 1000,
+			maxDepth: 2,
+			maxSessionsPerRun: 0,
+			maxAgentsPerLead: 5,
+		},
+		worktrees: {
+			baseDir: join(tmp, ".agentplate", "worktrees"),
+		},
+		taskTracker: {
+			backend: "auto",
+			enabled: false,
+		},
+		loam: {
+			enabled: false,
+			domains: [],
+			primeFormat: "markdown",
+		},
+		merge: {
+			aiResolveEnabled: false,
+			reimagineEnabled: false,
+		},
+		providers: {
+			anthropic: { type: "native" },
+		},
+		watchdog: {
+			tier0Enabled: false,
+			tier0IntervalMs: 30000,
+			tier1Enabled: false,
+			tier2Enabled: false,
+			staleThresholdMs: 300000,
+			zombieThresholdMs: 600000,
+			nudgeIntervalMs: 60000,
+		},
+		models: {},
+		logging: {
+			verbose: false,
+			redactSecrets: true,
+		},
+		...overrides,
+	};
+}
+
+// Dummy agentplateDir — provider checks don't use the filesystem
+const AGENTPLATE_DIR = join(tmpdir(), ".agentplate");
+
+describe("checkProviders", () => {
+	test("all checks have required DoctorCheck fields", async () => {
+		const config = makeConfig();
+		const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+		expect(checks).toBeArray();
+		for (const check of checks) {
+			expect(typeof check.name).toBe("string");
+			expect(check.category).toBe("providers");
+			expect(["pass", "warn", "fail"]).toContain(check.status);
+			expect(typeof check.message).toBe("string");
+			if (check.details !== undefined) {
+				expect(check.details).toBeArray();
+			}
+		}
+	});
+
+	describe("providers-configured check", () => {
+		test("native-only config (no gateway) returns pass for providers-configured", async () => {
+			const config = makeConfig({
+				providers: { anthropic: { type: "native" } },
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const check = checks.find((c) => c.name === "providers-configured");
+			expect(check).toBeDefined();
+			expect(check?.status).toBe("pass");
+		});
+
+		test("empty providers returns warn for providers-configured", async () => {
+			const config = makeConfig({ providers: {} });
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const check = checks.find((c) => c.name === "providers-configured");
+			expect(check).toBeDefined();
+			expect(check?.status).toBe("warn");
+		});
+
+		test("providers-configured details list provider names and types", async () => {
+			const config = makeConfig({
+				providers: {
+					anthropic: { type: "native" },
+					openrouter: { type: "gateway", baseUrl: "http://127.0.0.1:19873" },
+				},
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const check = checks.find((c) => c.name === "providers-configured");
+			expect(check?.status).toBe("pass");
+			expect(check?.details).toBeDefined();
+			expect(check?.details?.some((d) => d.includes("anthropic"))).toBe(true);
+			expect(check?.details?.some((d) => d.includes("openrouter"))).toBe(true);
+		});
+	});
+
+	describe("provider-reachable-{name} check", () => {
+		test("gateway config triggers reachability check (warn path — no real server)", async () => {
+			const config = makeConfig({
+				providers: {
+					fake: {
+						type: "gateway",
+						// Use a port that is almost certainly not listening
+						baseUrl: "http://127.0.0.1:19873",
+					},
+				},
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const check = checks.find((c) => c.name === "provider-reachable-fake");
+			expect(check).toBeDefined();
+			expect(check?.status).toBe("warn");
+			expect(check?.message).toContain("fake");
+		});
+
+		test("reachability pass path — local HTTP server", async () => {
+			// Start a minimal Bun HTTP server on an ephemeral port
+			const server = Bun.serve({
+				port: 0, // OS assigns a free port
+				fetch() {
+					return new Response("ok");
+				},
+			});
+
+			try {
+				const config = makeConfig({
+					providers: {
+						localtest: {
+							type: "gateway",
+							baseUrl: `http://127.0.0.1:${server.port}`,
+						},
+					},
+				});
+				const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+				const check = checks.find((c) => c.name === "provider-reachable-localtest");
+				expect(check).toBeDefined();
+				expect(check?.status).toBe("pass");
+			} finally {
+				await server.stop();
+			}
+		});
+
+		test("gateway without baseUrl skips reachability check", async () => {
+			const config = makeConfig({
+				providers: {
+					nourl: { type: "gateway" }, // no baseUrl
+				},
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const reachCheck = checks.find((c) => c.name === "provider-reachable-nourl");
+			expect(reachCheck).toBeUndefined();
+		});
+	});
+
+	describe("provider-auth-token-{name} check", () => {
+		const ENV_KEY = "AGENTPLATE_TEST_FAKE_PROVIDER_TOKEN_XYZ";
+
+		beforeAll(() => {
+			// Ensure env var is unset before tests
+			delete process.env[ENV_KEY];
+		});
+
+		afterAll(() => {
+			delete process.env[ENV_KEY];
+		});
+
+		test("gateway with authTokenEnv warns when env var missing", async () => {
+			const config = makeConfig({
+				providers: {
+					testgateway: {
+						type: "gateway",
+						authTokenEnv: ENV_KEY,
+					},
+				},
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const check = checks.find((c) => c.name === "provider-auth-token-testgateway");
+			expect(check).toBeDefined();
+			expect(check?.status).toBe("warn");
+			// Details must include the env var NAME, never a value
+			expect(check?.details?.some((d) => d.includes(ENV_KEY))).toBe(true);
+		});
+
+		test("gateway with authTokenEnv passes when env var is set", async () => {
+			process.env[ENV_KEY] = "test-token-value";
+
+			const config = makeConfig({
+				providers: {
+					testgateway: {
+						type: "gateway",
+						authTokenEnv: ENV_KEY,
+					},
+				},
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const check = checks.find((c) => c.name === "provider-auth-token-testgateway");
+			expect(check).toBeDefined();
+			expect(check?.status).toBe("pass");
+			// Details include the var name, not the value
+			expect(check?.details?.some((d) => d.includes(ENV_KEY))).toBe(true);
+			// Value must NOT appear in details
+			expect(check?.details?.some((d) => d.includes("test-token-value"))).toBe(false);
+		});
+
+		test("native provider with no authTokenEnv skips auth-token check", async () => {
+			const config = makeConfig({
+				providers: { anthropic: { type: "native" } },
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const authCheck = checks.find((c) => c.name?.startsWith("provider-auth-token-"));
+			expect(authCheck).toBeUndefined();
+		});
+	});
+
+	describe("tool-use-compat check", () => {
+		// Local HTTP server to avoid network calls to real openrouter.ai
+		let localServer: ReturnType<typeof Bun.serve>;
+		let localBaseUrl: string;
+
+		beforeAll(() => {
+			localServer = Bun.serve({
+				port: 0,
+				fetch() {
+					return new Response("ok");
+				},
+			});
+			localBaseUrl = `http://127.0.0.1:${localServer.port}`;
+		});
+
+		afterAll(async () => {
+			await localServer.stop();
+		});
+
+		test("tool-heavy role with provider-prefixed model warns", async () => {
+			const config = makeConfig({
+				models: { builder: "openrouter/openai/gpt-4o" },
+				providers: {
+					openrouter: { type: "gateway", baseUrl: localBaseUrl },
+				},
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const check = checks.find((c) => c.name === "tool-use-compat" && c.status === "warn");
+			expect(check).toBeDefined();
+			expect(check?.message).toContain("builder");
+		});
+
+		test("non-tool-heavy role with provider-prefixed model does not warn", async () => {
+			// "lead" is not a tool-heavy role
+			const config = makeConfig({
+				models: { lead: "openrouter/openai/gpt-4o" },
+				providers: {
+					openrouter: { type: "gateway", baseUrl: localBaseUrl },
+				},
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const warnChecks = checks.filter((c) => c.name === "tool-use-compat" && c.status === "warn");
+			expect(warnChecks.length).toBe(0);
+		});
+
+		test("no tool-heavy roles with prefixed models emits single pass", async () => {
+			const config = makeConfig({
+				models: { builder: "sonnet" }, // alias, not provider-prefixed
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const passCheck = checks.find((c) => c.name === "tool-use-compat" && c.status === "pass");
+			expect(passCheck).toBeDefined();
+		});
+
+		test("all three tool-heavy roles can trigger separate warns", async () => {
+			const config = makeConfig({
+				models: {
+					builder: "openrouter/openai/gpt-4o",
+					scout: "openrouter/openai/gpt-4o",
+					merger: "openrouter/openai/gpt-4o",
+				},
+				providers: {
+					openrouter: { type: "gateway", baseUrl: localBaseUrl },
+				},
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const warnChecks = checks.filter((c) => c.name === "tool-use-compat" && c.status === "warn");
+			expect(warnChecks.length).toBe(3);
+		});
+	});
+
+	describe("model-provider-ref(s) check", () => {
+		// Local HTTP server to avoid network calls to real openrouter.ai
+		let localServer: ReturnType<typeof Bun.serve>;
+		let localBaseUrl: string;
+
+		beforeAll(() => {
+			localServer = Bun.serve({
+				port: 0,
+				fetch() {
+					return new Response("ok");
+				},
+			});
+			localBaseUrl = `http://127.0.0.1:${localServer.port}`;
+		});
+
+		afterAll(async () => {
+			await localServer.stop();
+		});
+
+		test("model referencing unknown provider fails", async () => {
+			const config = makeConfig({
+				models: { builder: "unknownprovider/some-model" },
+				// unknownprovider not in providers
+				providers: { anthropic: { type: "native" } },
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const check = checks.find((c) => c.name === "model-provider-ref" && c.status === "fail");
+			expect(check).toBeDefined();
+			expect(check?.message).toContain("unknownprovider");
+		});
+
+		test("model referencing defined provider passes", async () => {
+			const config = makeConfig({
+				models: { builder: "openrouter/openai/gpt-4o" },
+				providers: {
+					openrouter: { type: "gateway", baseUrl: localBaseUrl },
+				},
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const check = checks.find((c) => c.name === "model-provider-ref" && c.status === "pass");
+			expect(check).toBeDefined();
+		});
+
+		test("no provider-prefixed models emits single pass named model-provider-refs", async () => {
+			const config = makeConfig({
+				models: { builder: "sonnet", scout: "haiku" },
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const check = checks.find((c) => c.name === "model-provider-refs");
+			expect(check).toBeDefined();
+			expect(check?.status).toBe("pass");
+		});
+
+		test("empty models emits single pass named model-provider-refs", async () => {
+			const config = makeConfig({ models: {} });
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const check = checks.find((c) => c.name === "model-provider-refs");
+			expect(check).toBeDefined();
+			expect(check?.status).toBe("pass");
+		});
+	});
+
+	describe("gateway-api-key-reminder check", () => {
+		test("gateway present triggers api-key reminder warn", async () => {
+			const config = makeConfig({
+				providers: {
+					openrouter: { type: "gateway", baseUrl: "http://127.0.0.1:19873" },
+				},
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const check = checks.find((c) => c.name === "gateway-api-key-reminder");
+			expect(check).toBeDefined();
+			expect(check?.status).toBe("warn");
+			expect(check?.message).toContain("ANTHROPIC_API_KEY");
+		});
+
+		test("no gateway providers — reminder is absent", async () => {
+			const config = makeConfig({
+				providers: { anthropic: { type: "native" } },
+			});
+			const checks = await checkProviders(config, AGENTPLATE_DIR);
+
+			const check = checks.find((c) => c.name === "gateway-api-key-reminder");
+			expect(check).toBeUndefined();
+		});
+	});
+});

package/src/doctor/providers.ts

@@ -0,0 +1,250 @@
+import type { AgentplateConfig, ProviderConfig } from "../types.ts";
+import type { DoctorCheck, DoctorCheckFn } from "./types.ts";
+
+/** Roles that rely heavily on tool-use (function calling). */
+const TOOL_HEAVY_ROLES = new Set(["builder", "scout", "merger"]);
+
+/**
+ * Provider and multi-runtime configuration checks.
+ *
+ * Validates gateway provider reachability, auth tokens, model-provider references,
+ * and tool-use compatibility across configured runtimes.
+ */
+export const checkProviders: DoctorCheckFn = async (
+	config,
+	_agentplateDir,
+): Promise<DoctorCheck[]> => {
+	const checks: DoctorCheck[] = [];
+
+	// Base check: at least one provider configured
+	checks.push(buildProvidersConfigured(config));
+
+	// Identify gateway providers
+	const gatewayEntries = Object.entries(config.providers).filter(([, p]) => p.type === "gateway");
+
+	// Check 1: provider-reachable-{name} — one per gateway provider with baseUrl
+	for (const [name, provider] of gatewayEntries) {
+		if (provider.baseUrl) {
+			checks.push(await checkProviderReachable(name, provider));
+		}
+	}
+
+	// Check 2: provider-auth-token-{name} — one per gateway provider with authTokenEnv
+	for (const [name, provider] of gatewayEntries) {
+		if (provider.authTokenEnv) {
+			checks.push(buildProviderAuthToken(name, provider));
+		}
+	}
+
+	// Check 3: tool-use-compat — one warn per tool-heavy role using a provider-prefixed model
+	checks.push(...buildToolUseCompat(config));
+
+	// Check 4: model-provider-ref(s) — one per provider-prefixed model, or single pass
+	checks.push(...buildModelProviderRefs(config));
+
+	// Check 5: gateway-api-key-reminder — only when gateway providers exist
+	if (gatewayEntries.length > 0) {
+		checks.push(buildGatewayApiKeyReminder());
+	}
+
+	return checks;
+};
+
+/**
+ * Base check: verifies at least one provider is configured.
+ */
+function buildProvidersConfigured(config: AgentplateConfig): DoctorCheck {
+	const entries = Object.entries(config.providers);
+
+	if (entries.length > 0) {
+		return {
+			name: "providers-configured",
+			category: "providers",
+			status: "pass",
+			message: `${entries.length} provider${entries.length === 1 ? "" : "s"} configured`,
+			details: entries.map(([name, p]) => `${name} (${p.type})`),
+		};
+	}
+
+	return {
+		name: "providers-configured",
+		category: "providers",
+		status: "warn",
+		message: "No providers configured — add providers to config.yaml",
+		details: ["At least one native or gateway provider should be configured."],
+	};
+}
+
+/**
+ * Check 1: HTTP reachability of a gateway provider's baseUrl.
+ *
+ * Uses fetch() with a 5-second timeout. Any HTTP response (any status code)
+ * counts as reachable — only network errors or timeouts produce a warn.
+ */
+async function checkProviderReachable(
+	name: string,
+	provider: ProviderConfig,
+): Promise<DoctorCheck> {
+	const baseUrl = provider.baseUrl as string; // caller guards baseUrl is defined
+
+	try {
+		await fetch(baseUrl, {
+			method: "HEAD",
+			signal: AbortSignal.timeout(5000),
+		});
+
+		return {
+			name: `provider-reachable-${name}`,
+			category: "providers",
+			status: "pass",
+			message: `Gateway provider '${name}' is reachable`,
+			details: [baseUrl],
+		};
+	} catch (error) {
+		const errorMsg = error instanceof Error ? error.message : String(error);
+		return {
+			name: `provider-reachable-${name}`,
+			category: "providers",
+			status: "warn",
+			message: `Gateway provider '${name}' is unreachable`,
+			details: [baseUrl, errorMsg],
+		};
+	}
+}
+
+/**
+ * Check 2: Validate that the auth token env var for a gateway provider is set.
+ *
+ * Reports the env var NAME in details — never the value.
+ */
+function buildProviderAuthToken(name: string, provider: ProviderConfig): DoctorCheck {
+	const envVar = provider.authTokenEnv as string; // caller guards authTokenEnv is defined
+	const value = process.env[envVar];
+
+	if (value && value.length > 0) {
+		return {
+			name: `provider-auth-token-${name}`,
+			category: "providers",
+			status: "pass",
+			message: `Auth token for provider '${name}' is set`,
+			details: [`Env var: ${envVar}`],
+		};
+	}
+
+	return {
+		name: `provider-auth-token-${name}`,
+		category: "providers",
+		status: "warn",
+		message: `Auth token for provider '${name}' is missing`,
+		details: [`Env var: ${envVar}`, `Set ${envVar} to authenticate with this provider.`],
+	};
+}
+
+/**
+ * Check 3: Tool-use compatibility for tool-heavy roles using non-Anthropic models.
+ *
+ * Tool-heavy roles (builder, scout, merger) rely on structured tool-use (function
+ * calling). Non-Anthropic models accessed via gateway providers may have different
+ * tool-use behavior. Emits one warn per affected role, or a single pass if none.
+ */
+function buildToolUseCompat(config: AgentplateConfig): DoctorCheck[] {
+	const checks: DoctorCheck[] = [];
+
+	for (const [role, model] of Object.entries(config.models)) {
+		if (!TOOL_HEAVY_ROLES.has(role)) continue;
+		if (model === undefined) continue;
+		if (!model.includes("/")) continue;
+
+		checks.push({
+			name: "tool-use-compat",
+			category: "providers",
+			status: "warn",
+			message: `models.${role} uses non-Anthropic model — tool-use compatibility not guaranteed`,
+			details: [
+				`Model: ${model}`,
+				"Tool use (function calling) behavior varies across providers.",
+				"Test agent behavior thoroughly before using in production.",
+			],
+		});
+	}
+
+	if (checks.length === 0) {
+		checks.push({
+			name: "tool-use-compat",
+			category: "providers",
+			status: "pass",
+			message: "No tool-heavy roles use non-Anthropic models",
+		});
+	}
+
+	return checks;
+}
+
+/**
+ * Check 4: Validate that provider-prefixed model references point to configured providers.
+ *
+ * For each config.models entry containing '/' (provider-qualified), extracts the
+ * provider name and verifies it exists in config.providers. Emits one check per
+ * provider-prefixed model, or a single pass if no such models exist.
+ */
+function buildModelProviderRefs(config: AgentplateConfig): DoctorCheck[] {
+	const checks: DoctorCheck[] = [];
+
+	for (const [role, model] of Object.entries(config.models)) {
+		if (model === undefined) continue;
+		if (!model.includes("/")) continue;
+
+		const providerName = model.split("/")[0];
+		if (!providerName) continue;
+
+		if (config.providers[providerName]) {
+			checks.push({
+				name: "model-provider-ref",
+				category: "providers",
+				status: "pass",
+				message: `models.${role} references defined provider '${providerName}'`,
+				details: [`Model: ${model}`],
+			});
+		} else {
+			checks.push({
+				name: "model-provider-ref",
+				category: "providers",
+				status: "fail",
+				message: `models.${role} references undefined provider '${providerName}'`,
+				details: [
+					`Model: ${model}`,
+					`Provider '${providerName}' is not defined in config.yaml providers section.`,
+					`Add it: providers:\n  ${providerName}:\n    type: gateway\n    baseUrl: https://...`,
+				],
+			});
+		}
+	}
+
+	if (checks.length === 0) {
+		checks.push({
+			name: "model-provider-refs",
+			category: "providers",
+			status: "pass",
+			message: "No provider-prefixed model references",
+		});
+	}
+
+	return checks;
+}
+
+/**
+ * Check 5: Reminder about ANTHROPIC_API_KEY when gateway providers are configured.
+ *
+ * Agents spawned via gateway routes receive ANTHROPIC_API_KEY="" so they use the
+ * gateway instead of Anthropic directly. Any direct Anthropic API calls (e.g.,
+ * from merge/resolver.ts) require a separate key.
+ */
+function buildGatewayApiKeyReminder(): DoctorCheck {
+	return {
+		name: "gateway-api-key-reminder",
+		category: "providers",
+		status: "warn",
+		message:
+			"Gateway providers configured — agents using gateway routes will have ANTHROPIC_API_KEY set to empty string. Direct Anthropic API calls require a separate key.",
+	};
+}