@elyracode/doctor 0.9.6 → 0.9.8

package/README.md

@@ -18,6 +18,7 @@ elyra install npm:@elyracode/doctor
 | Tool | Description |
 |------|-------------|
 | `project_health_check` | Run health checks. Can filter by category: security, dependencies, config, code-debt, code-quality, git, project |
+| `probe_models` | Live-verify configured LLM providers against the model registry. Optional filters: `provider`, `model_pattern`, `max_models` |
 
 ## Checks
 
@@ -42,6 +43,23 @@ elyra install npm:@elyracode/doctor
 
 The agent runs the checks automatically and suggests fixes based on findings.
 
+## Model Probes
+
+`probe_models` live-verifies that configured LLM providers actually behave the way the model registry claims (registry metadata can drift from provider reality):
+
+- Detects providers with credentials via environment API keys (e.g. `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`); providers without credentials are skipped
+- Probes at most 2-3 cheap representative models per provider with a minimal live completion
+- For models with `thinkingType` metadata (`adaptive` | `budget`), runs the probe with reasoning enabled at the lowest level and checks whether thinking content is actually produced
+- Reports per model: provider, id, ok/failed, latency, and any mismatch between registry claims and observed behavior
+- With no credentials configured it returns a clear "no providers configured" result instead of failing
+
+Note: probes make real (paid) API calls.
+
+```
+> Probe my configured models
+> Verify that Bedrock thinking models actually support thinking
+```
+
 ### Auto-Heal Mode
 ```
 /doctor --heal

package/extensions/index.ts

@@ -1,6 +1,7 @@
 import type { ExtensionAPI } from "@elyracode/coding-agent";
 import { Type } from "typebox";
 import { type Finding, runAllChecks } from "./checks.js";
+import { DEFAULT_MAX_MODELS, formatProbeReport, runModelProbes } from "./probes.js";
 
 export default function (elyra: ExtensionAPI): void {
 	// ── Command: /doctor ──
@@ -85,6 +86,56 @@ export default function (elyra: ExtensionAPI): void {
 			};
 		},
 	});
+
+	// ── Tool: probe_models ──
+	elyra.registerTool({
+		name: "probe_models",
+		label: "Probe Models",
+		description:
+			"Live-verify that configured LLM providers behave the way the model registry claims. " +
+			"Probes a small set of cheap representative models per provider with credentials (env API keys) " +
+			"using a minimal completion, and verifies thinking mode for models with thinkingType metadata. " +
+			"Reports ok/failed, latency, and mismatches between registry claims and observed behavior. " +
+			"Note: makes real (paid) API calls.",
+		parameters: Type.Object({
+			provider: Type.Optional(
+				Type.String({ description: "Only probe this provider (e.g. openai, anthropic, amazon-bedrock)" }),
+			),
+			model_pattern: Type.Optional(
+				Type.String({ description: "Only probe models whose id contains this substring (case-insensitive)" }),
+			),
+			max_models: Type.Optional(
+				Type.Number({ description: `Maximum total number of models to probe (default: ${DEFAULT_MAX_MODELS})` }),
+			),
+		}),
+		execute: async (_toolCallId, params, signal) => {
+			try {
+				const report = await runModelProbes({
+					provider: params.provider,
+					modelPattern: params.model_pattern,
+					maxModels: params.max_models,
+					signal,
+				});
+
+				return {
+					content: [{ type: "text", text: formatProbeReport(report) }],
+					details: {
+						probed: report.probed.length,
+						ok: report.probed.filter((r) => r.ok).length,
+						failed: report.probed.filter((r) => !r.ok).length,
+						mismatches: report.probed.reduce((sum, r) => sum + r.mismatches.length, 0),
+						skippedProviders: report.skippedProviders,
+					},
+				};
+			} catch (error) {
+				const msg = error instanceof Error ? error.message : String(error);
+				return {
+					content: [{ type: "text", text: `Model probes failed: ${msg}` }],
+					details: { probed: 0, ok: 0, failed: 0, mismatches: 0, skippedProviders: [] },
+				};
+			}
+		},
+	});
 }
 
 function formatReport(findings: Finding[]): string {

package/extensions/probes.ts

@@ -0,0 +1,311 @@
+/**
+ * Live model probes for elyra doctor.
+ * Verifies that configured LLM providers actually behave the way the model
+ * registry claims (availability, reasoning support, thinking mode), since
+ * registry metadata can drift from provider reality.
+ */
+
+import type { Api, KnownProvider, Model, ThinkingLevel } from "@elyracode/ai";
+import { completeSimple, getEnvApiKey, getModels, getProviders, getSupportedThinkingLevels } from "@elyracode/ai";
+
+export const DEFAULT_MAX_MODELS = 10;
+export const MODELS_PER_PROVIDER = 2;
+export const DEFAULT_PROBE_TIMEOUT_MS = 60000;
+
+export interface ProbeOptions {
+	/** Only probe this provider (case-insensitive). */
+	provider?: string;
+	/** Only probe models whose id contains this substring (case-insensitive). */
+	modelPattern?: string;
+	/** Maximum total number of models to probe. */
+	maxModels?: number;
+	/** Abort signal from the tool execution. */
+	signal?: AbortSignal;
+	/** Per-probe timeout in milliseconds. */
+	timeoutMs?: number;
+}
+
+export interface ProbeResult {
+	provider: string;
+	modelId: string;
+	ok: boolean;
+	latencyMs: number;
+	error?: string;
+	/** Set when the probe ran with reasoning enabled. */
+	reasoningProbe?: { level: ThinkingLevel; observedThinking: boolean };
+	/** Discrepancies between registry claims and observed behavior. */
+	mismatches: string[];
+	/** Informational observations that are not necessarily mismatches. */
+	notes: string[];
+}
+
+export interface ProbeReport {
+	probed: ProbeResult[];
+	/** Providers in the registry that were skipped because no credentials were found. */
+	skippedProviders: string[];
+	/** Set when no probes could run at all (e.g. no credentials configured). */
+	message?: string;
+}
+
+export interface ThinkingObservation {
+	/** Reasoning level the probe requested, if any. */
+	requestedLevel?: ThinkingLevel;
+	/** Whether the response contained thinking content. Undefined if the probe failed. */
+	observedThinking?: boolean;
+	/** Error message when the probe failed. */
+	error?: string;
+}
+
+// ── Pure logic: model selection ──
+
+function probeCost(model: Model<Api>): number {
+	return model.cost.input + model.cost.output;
+}
+
+/**
+ * Pick a small representative set of models for a provider:
+ * the cheapest two, plus the cheapest model with thinkingType metadata
+ * if none of the cheap picks have it (at most 3 total).
+ */
+export function selectModelsForProvider(models: Model<Api>[], modelPattern?: string): Model<Api>[] {
+	const pattern = modelPattern?.toLowerCase();
+	const filtered = pattern ? models.filter((m) => m.id.toLowerCase().includes(pattern)) : models;
+	const byCost = [...filtered].sort((a, b) => probeCost(a) - probeCost(b));
+
+	const selected = byCost.slice(0, MODELS_PER_PROVIDER);
+	if (!selected.some((m) => m.thinkingType !== undefined)) {
+		const thinkingModel = byCost.find((m) => m.thinkingType !== undefined && !selected.includes(m));
+		if (thinkingModel) selected.push(thinkingModel);
+	}
+	return selected;
+}
+
+// ── Pure logic: thinking claims ──
+
+/** Lowest supported thinking level for a reasoning probe, or undefined when not applicable. */
+export function pickReasoningLevel(model: Model<Api>): ThinkingLevel | undefined {
+	if (model.thinkingType === undefined || !model.reasoning) return undefined;
+	const levels = getSupportedThinkingLevels(model).filter((l): l is ThinkingLevel => l !== "off");
+	return levels[0];
+}
+
+/** Compare registry thinking claims against observed probe behavior. */
+export function evaluateThinkingClaims(
+	model: Model<Api>,
+	observation: ThinkingObservation,
+): { mismatches: string[]; notes: string[] } {
+	const mismatches: string[] = [];
+	const notes: string[] = [];
+
+	if (model.thinkingType === undefined) return { mismatches, notes };
+
+	if (!model.reasoning) {
+		mismatches.push(`registry claims thinkingType=${model.thinkingType} but reasoning=false`);
+		return { mismatches, notes };
+	}
+
+	if (observation.requestedLevel === undefined) {
+		mismatches.push(
+			`registry claims thinkingType=${model.thinkingType} but thinkingLevelMap disables all thinking levels`,
+		);
+		return { mismatches, notes };
+	}
+
+	if (observation.error !== undefined) {
+		mismatches.push(
+			`request with thinking enabled (level: ${observation.requestedLevel}) failed: ${observation.error}`,
+		);
+		return { mismatches, notes };
+	}
+
+	if (observation.observedThinking === false) {
+		if (model.thinkingType === "budget") {
+			mismatches.push("no thinking content observed despite thinkingType=budget");
+		} else {
+			notes.push("adaptive thinking produced no thinking content (model may skip thinking for trivial prompts)");
+		}
+	}
+
+	return { mismatches, notes };
+}
+
+// ── Live probing ──
+
+function combineSignals(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal {
+	const timeout = AbortSignal.timeout(timeoutMs);
+	return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+
+async function probeModel(model: Model<Api>, apiKey: string, options: ProbeOptions): Promise<ProbeResult> {
+	const result: ProbeResult = {
+		provider: model.provider,
+		modelId: model.id,
+		ok: false,
+		latencyMs: 0,
+		mismatches: [],
+		notes: [],
+	};
+
+	const reasoningLevel = pickReasoningLevel(model);
+	const start = Date.now();
+
+	try {
+		const message = await completeSimple(
+			model,
+			{ messages: [{ role: "user", content: "Reply with the single word: OK", timestamp: Date.now() }] },
+			{
+				apiKey,
+				maxTokens: 64,
+				reasoning: reasoningLevel,
+				signal: combineSignals(options.signal, options.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS),
+			},
+		);
+		result.latencyMs = Date.now() - start;
+
+		if (message.stopReason === "error" || message.stopReason === "aborted") {
+			result.error = message.errorMessage ?? `request ended with stopReason=${message.stopReason}`;
+		} else {
+			result.ok = true;
+			if (reasoningLevel !== undefined) {
+				result.reasoningProbe = {
+					level: reasoningLevel,
+					observedThinking: message.content.some((c) => c.type === "thinking"),
+				};
+			}
+		}
+	} catch (error) {
+		result.latencyMs = Date.now() - start;
+		result.error = error instanceof Error ? error.message : String(error);
+	}
+
+	const claims = evaluateThinkingClaims(model, {
+		requestedLevel: reasoningLevel,
+		observedThinking: result.reasoningProbe?.observedThinking,
+		error: result.error,
+	});
+	result.mismatches.push(...claims.mismatches);
+	result.notes.push(...claims.notes);
+
+	return result;
+}
+
+/** Run live probes for all providers with credentials. Never throws. */
+export async function runModelProbes(options: ProbeOptions = {}): Promise<ProbeReport> {
+	const providerFilter = options.provider?.toLowerCase();
+	const providers = getProviders().filter((p) => !providerFilter || p.toLowerCase() === providerFilter);
+
+	const available: { provider: string; apiKey: string }[] = [];
+	const skipped: string[] = [];
+	for (const provider of providers) {
+		const apiKey = getEnvApiKey(provider);
+		if (apiKey) {
+			available.push({ provider, apiKey });
+		} else {
+			skipped.push(provider);
+		}
+	}
+
+	if (available.length === 0) {
+		return {
+			probed: [],
+			skippedProviders: skipped,
+			message: options.provider
+				? `No credentials found for provider "${options.provider}". Set the provider's API key environment variable to enable probing.`
+				: "No providers configured. Set an API key environment variable (e.g. OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY) to enable model probes.",
+		};
+	}
+
+	const maxModels = Math.max(1, Math.floor(options.maxModels ?? DEFAULT_MAX_MODELS));
+	const groups: { models: Model<Api>[]; apiKey: string }[] = [];
+	let budget = maxModels;
+	for (const { provider, apiKey } of available) {
+		if (budget <= 0) break;
+		const all = getModels(provider as KnownProvider) as Model<Api>[];
+		const models = selectModelsForProvider(all, options.modelPattern).slice(0, budget);
+		if (models.length > 0) {
+			groups.push({ models, apiKey });
+			budget -= models.length;
+		}
+	}
+
+	if (groups.length === 0) {
+		return {
+			probed: [],
+			skippedProviders: skipped,
+			message: options.modelPattern
+				? `No registry models match pattern "${options.modelPattern}" for the configured providers.`
+				: "No registry models found for the configured providers.",
+		};
+	}
+
+	// Probe providers in parallel, models within a provider sequentially (rate-limit friendly).
+	const results = await Promise.all(
+		groups.map(async ({ models, apiKey }) => {
+			const groupResults: ProbeResult[] = [];
+			for (const model of models) {
+				if (options.signal?.aborted) break;
+				groupResults.push(await probeModel(model, apiKey, options));
+			}
+			return groupResults;
+		}),
+	);
+
+	return { probed: results.flat(), skippedProviders: skipped };
+}
+
+// ── Pure logic: report formatting ──
+
+export function formatProbeReport(report: ProbeReport): string {
+	if (report.message) {
+		return `# Model Probe Report\n\n${report.message}`;
+	}
+
+	const ok = report.probed.filter((r) => r.ok);
+	const failed = report.probed.filter((r) => !r.ok);
+	const mismatchCount = report.probed.reduce((sum, r) => sum + r.mismatches.length, 0);
+
+	const lines: string[] = [
+		"# Model Probe Report",
+		"",
+		`Probed: ${report.probed.length} | OK: ${ok.length} | Failed: ${failed.length} | Mismatches: ${mismatchCount}`,
+		"",
+		"## Results",
+	];
+
+	for (const r of report.probed) {
+		const status = r.ok ? `OK (${r.latencyMs}ms)` : `FAILED (${r.latencyMs}ms): ${r.error ?? "unknown error"}`;
+		const thinking = r.reasoningProbe
+			? `, thinking(${r.reasoningProbe.level}): ${r.reasoningProbe.observedThinking ? "observed" : "not observed"}`
+			: "";
+		lines.push(`- [${r.provider}] ${r.modelId} -- ${status}${thinking}`);
+	}
+	lines.push("");
+
+	if (mismatchCount > 0) {
+		lines.push("## Mismatches");
+		for (const r of report.probed) {
+			for (const m of r.mismatches) {
+				lines.push(`- [${r.provider}] ${r.modelId}: ${m}`);
+			}
+		}
+		lines.push("");
+	}
+
+	const noteCount = report.probed.reduce((sum, r) => sum + r.notes.length, 0);
+	if (noteCount > 0) {
+		lines.push("## Notes");
+		for (const r of report.probed) {
+			for (const n of r.notes) {
+				lines.push(`- [${r.provider}] ${r.modelId}: ${n}`);
+			}
+		}
+		lines.push("");
+	}
+
+	if (report.skippedProviders.length > 0) {
+		lines.push("## Skipped Providers (no credentials)");
+		lines.push(report.skippedProviders.join(", "));
+	}
+
+	return lines.join("\n").trimEnd();
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elyracode/doctor",
-  "version": "0.9.6",
+  "version": "0.9.8",
   "description": "Elyra extension for project health analysis -- dependency audit, security checks, code quality, and more",
   "type": "module",
   "keywords": [
@@ -23,6 +23,7 @@
     ]
   },
   "peerDependencies": {
+    "@elyracode/ai": "*",
     "@elyracode/coding-agent": "*",
     "typebox": "*"
   },

package/test/probes.test.ts

@@ -0,0 +1,216 @@
+import type { Api, Model } from "@elyracode/ai";
+import { describe, expect, it } from "vitest";
+import {
+	evaluateThinkingClaims,
+	formatProbeReport,
+	type ProbeReport,
+	pickReasoningLevel,
+	selectModelsForProvider,
+} from "../extensions/probes.js";
+
+function makeModel(overrides: Partial<Model<Api>> & { id: string }): Model<Api> {
+	return {
+		name: overrides.id,
+		api: "openai-completions",
+		provider: "test-provider",
+		baseUrl: "https://example.invalid",
+		reasoning: false,
+		input: ["text"],
+		cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 },
+		contextWindow: 128000,
+		maxTokens: 8192,
+		...overrides,
+	};
+}
+
+describe("selectModelsForProvider", () => {
+	it("picks the two cheapest models", () => {
+		const models = [
+			makeModel({ id: "expensive", cost: { input: 10, output: 30, cacheRead: 0, cacheWrite: 0 } }),
+			makeModel({ id: "cheap", cost: { input: 0.1, output: 0.4, cacheRead: 0, cacheWrite: 0 } }),
+			makeModel({ id: "mid", cost: { input: 1, output: 4, cacheRead: 0, cacheWrite: 0 } }),
+		];
+		const selected = selectModelsForProvider(models);
+		expect(selected.map((m) => m.id)).toEqual(["cheap", "mid"]);
+	});
+
+	it("adds the cheapest thinkingType model when the cheap picks have none", () => {
+		const models = [
+			makeModel({ id: "cheap-a", cost: { input: 0.1, output: 0.2, cacheRead: 0, cacheWrite: 0 } }),
+			makeModel({ id: "cheap-b", cost: { input: 0.2, output: 0.4, cacheRead: 0, cacheWrite: 0 } }),
+			makeModel({
+				id: "thinker-pricey",
+				reasoning: true,
+				thinkingType: "budget",
+				cost: { input: 15, output: 75, cacheRead: 0, cacheWrite: 0 },
+			}),
+			makeModel({
+				id: "thinker-cheap",
+				reasoning: true,
+				thinkingType: "adaptive",
+				cost: { input: 3, output: 15, cacheRead: 0, cacheWrite: 0 },
+			}),
+		];
+		const selected = selectModelsForProvider(models);
+		expect(selected.map((m) => m.id)).toEqual(["cheap-a", "cheap-b", "thinker-cheap"]);
+	});
+
+	it("does not add a third model when a cheap pick already has thinkingType", () => {
+		const models = [
+			makeModel({ id: "cheap-thinker", reasoning: true, thinkingType: "budget" }),
+			makeModel({ id: "cheap-plain", cost: { input: 2, output: 4, cacheRead: 0, cacheWrite: 0 } }),
+			makeModel({
+				id: "other-thinker",
+				reasoning: true,
+				thinkingType: "adaptive",
+				cost: { input: 5, output: 10, cacheRead: 0, cacheWrite: 0 },
+			}),
+		];
+		const selected = selectModelsForProvider(models);
+		expect(selected.map((m) => m.id)).toEqual(["cheap-thinker", "cheap-plain"]);
+	});
+
+	it("filters by model pattern (case-insensitive substring)", () => {
+		const models = [makeModel({ id: "gpt-5-mini" }), makeModel({ id: "claude-haiku" })];
+		const selected = selectModelsForProvider(models, "HAIKU");
+		expect(selected.map((m) => m.id)).toEqual(["claude-haiku"]);
+	});
+
+	it("returns empty for an empty registry", () => {
+		expect(selectModelsForProvider([])).toEqual([]);
+	});
+});
+
+describe("pickReasoningLevel", () => {
+	it("returns undefined when the model has no thinkingType", () => {
+		expect(pickReasoningLevel(makeModel({ id: "plain" }))).toBeUndefined();
+	});
+
+	it("returns undefined when thinkingType is set but reasoning is false", () => {
+		expect(pickReasoningLevel(makeModel({ id: "broken", thinkingType: "budget" }))).toBeUndefined();
+	});
+
+	it("returns the lowest supported level", () => {
+		expect(pickReasoningLevel(makeModel({ id: "thinker", reasoning: true, thinkingType: "budget" }))).toBe("minimal");
+	});
+
+	it("respects thinkingLevelMap exclusions", () => {
+		const model = makeModel({
+			id: "no-minimal",
+			reasoning: true,
+			thinkingType: "adaptive",
+			thinkingLevelMap: { minimal: null },
+		});
+		expect(pickReasoningLevel(model)).toBe("low");
+	});
+});
+
+describe("evaluateThinkingClaims", () => {
+	it("returns nothing for models without thinkingType", () => {
+		const result = evaluateThinkingClaims(makeModel({ id: "plain" }), {});
+		expect(result.mismatches).toEqual([]);
+		expect(result.notes).toEqual([]);
+	});
+
+	it("flags thinkingType set on a non-reasoning model", () => {
+		const result = evaluateThinkingClaims(makeModel({ id: "broken", thinkingType: "budget" }), {});
+		expect(result.mismatches).toEqual(["registry claims thinkingType=budget but reasoning=false"]);
+	});
+
+	it("flags thinkingType with all levels disabled", () => {
+		const model = makeModel({ id: "no-levels", reasoning: true, thinkingType: "adaptive" });
+		const result = evaluateThinkingClaims(model, { requestedLevel: undefined });
+		expect(result.mismatches).toEqual([
+			"registry claims thinkingType=adaptive but thinkingLevelMap disables all thinking levels",
+		]);
+	});
+
+	it("flags a failed thinking-enabled request", () => {
+		const model = makeModel({ id: "thinker", reasoning: true, thinkingType: "budget" });
+		const result = evaluateThinkingClaims(model, { requestedLevel: "minimal", error: "400 invalid thinking config" });
+		expect(result.mismatches).toEqual([
+			"request with thinking enabled (level: minimal) failed: 400 invalid thinking config",
+		]);
+	});
+
+	it("flags missing thinking content for budget models", () => {
+		const model = makeModel({ id: "thinker", reasoning: true, thinkingType: "budget" });
+		const result = evaluateThinkingClaims(model, { requestedLevel: "minimal", observedThinking: false });
+		expect(result.mismatches).toEqual(["no thinking content observed despite thinkingType=budget"]);
+		expect(result.notes).toEqual([]);
+	});
+
+	it("notes (not flags) missing thinking content for adaptive models", () => {
+		const model = makeModel({ id: "thinker", reasoning: true, thinkingType: "adaptive" });
+		const result = evaluateThinkingClaims(model, { requestedLevel: "minimal", observedThinking: false });
+		expect(result.mismatches).toEqual([]);
+		expect(result.notes).toEqual([
+			"adaptive thinking produced no thinking content (model may skip thinking for trivial prompts)",
+		]);
+	});
+
+	it("reports no issues when thinking is observed", () => {
+		const model = makeModel({ id: "thinker", reasoning: true, thinkingType: "budget" });
+		const result = evaluateThinkingClaims(model, { requestedLevel: "minimal", observedThinking: true });
+		expect(result.mismatches).toEqual([]);
+		expect(result.notes).toEqual([]);
+	});
+});
+
+describe("formatProbeReport", () => {
+	it("returns the message when no probes ran", () => {
+		const report: ProbeReport = {
+			probed: [],
+			skippedProviders: ["openai", "anthropic"],
+			message: "No providers configured.",
+		};
+		const text = formatProbeReport(report);
+		expect(text).toContain("# Model Probe Report");
+		expect(text).toContain("No providers configured.");
+	});
+
+	it("formats results, mismatches, notes, and skipped providers", () => {
+		const report: ProbeReport = {
+			probed: [
+				{
+					provider: "openai",
+					modelId: "gpt-5-mini",
+					ok: true,
+					latencyMs: 812,
+					mismatches: [],
+					notes: [],
+				},
+				{
+					provider: "amazon-bedrock",
+					modelId: "claude-x",
+					ok: true,
+					latencyMs: 1500,
+					reasoningProbe: { level: "minimal", observedThinking: false },
+					mismatches: ["no thinking content observed despite thinkingType=budget"],
+					notes: [],
+				},
+				{
+					provider: "groq",
+					modelId: "llama-y",
+					ok: false,
+					latencyMs: 230,
+					error: "401 unauthorized",
+					mismatches: [],
+					notes: ["some note"],
+				},
+			],
+			skippedProviders: ["mistral"],
+		};
+		const text = formatProbeReport(report);
+		expect(text).toContain("Probed: 3 | OK: 2 | Failed: 1 | Mismatches: 1");
+		expect(text).toContain("- [openai] gpt-5-mini -- OK (812ms)");
+		expect(text).toContain("- [amazon-bedrock] claude-x -- OK (1500ms), thinking(minimal): not observed");
+		expect(text).toContain("- [groq] llama-y -- FAILED (230ms): 401 unauthorized");
+		expect(text).toContain("## Mismatches");
+		expect(text).toContain("- [amazon-bedrock] claude-x: no thinking content observed despite thinkingType=budget");
+		expect(text).toContain("## Notes");
+		expect(text).toContain("- [groq] llama-y: some note");
+		expect(text).toContain("## Skipped Providers (no credentials)");
+		expect(text).toContain("mistral");
+	});
+});