@elyracode/doctor 0.9.7 → 0.9.9

package/README.md

@@ -18,6 +18,7 @@ elyra install npm:@elyracode/doctor
 | Tool | Description |
 |------|-------------|
 | `project_health_check` | Run health checks. Can filter by category: security, dependencies, config, code-debt, code-quality, git, project |
+| `probe_models` | Live-verify configured LLM providers against the model registry. Optional filters: `provider`, `model_pattern`, `max_models` |
 
 ## Checks
 
@@ -42,6 +43,23 @@ elyra install npm:@elyracode/doctor
 
 The agent runs the checks automatically and suggests fixes based on findings.
 
+## Model Probes
+
+`probe_models` live-verifies that configured LLM providers actually behave the way the model registry claims (registry metadata can drift from provider reality):
+
+- Detects providers with credentials via environment API keys (e.g. `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`); providers without credentials are skipped
+- Probes at most 2-3 cheap representative models per provider with a minimal live completion
+- For models with `thinkingType` metadata (`adaptive` | `budget`), runs the probe with reasoning enabled at the lowest level and checks whether thinking content is actually produced
+- Reports per model: provider, id, ok/failed, latency, and any mismatch between registry claims and observed behavior
+- With no credentials configured it returns a clear "no providers configured" result instead of failing
+
+Note: probes make real (paid) API calls.
+
+```
+> Probe my configured models
+> Verify that Bedrock thinking models actually support thinking
+```
+
 ### Auto-Heal Mode
 ```
 /doctor --heal

package/extensions/index.ts

@@ -1,6 +1,7 @@
 import type { ExtensionAPI } from "@elyracode/coding-agent";
 import { Type } from "typebox";
 import { type Finding, runAllChecks } from "./checks.js";
+import { DEFAULT_MAX_MODELS, formatProbeReport, runModelProbes } from "./probes.js";
 
 export default function (elyra: ExtensionAPI): void {
 	// ── Command: /doctor ──
@@ -85,6 +86,56 @@ export default function (elyra: ExtensionAPI): void {
 			};
 		},
 	});
+
+	// ── Tool: probe_models ──
+	elyra.registerTool({
+		name: "probe_models",
+		label: "Probe Models",
+		description:
+			"Live-verify that configured LLM providers behave the way the model registry claims. " +
+			"Probes a small set of cheap representative models per provider with credentials (env API keys) " +
+			"using a minimal completion, and verifies thinking mode for models with thinkingType metadata. " +
+			"Reports ok/failed, latency, and mismatches between registry claims and observed behavior. " +
+			"Note: makes real (paid) API calls.",
+		parameters: Type.Object({
+			provider: Type.Optional(
+				Type.String({ description: "Only probe this provider (e.g. openai, anthropic, amazon-bedrock)" }),
+			),
+			model_pattern: Type.Optional(
+				Type.String({ description: "Only probe models whose id contains this substring (case-insensitive)" }),
+			),
+			max_models: Type.Optional(
+				Type.Number({ description: `Maximum total number of models to probe (default: ${DEFAULT_MAX_MODELS})` }),
+			),
+		}),
+		execute: async (_toolCallId, params, signal) => {
+			try {
+				const report = await runModelProbes({
+					provider: params.provider,
+					modelPattern: params.model_pattern,
+					maxModels: params.max_models,
+					signal,
+				});
+
+				return {
+					content: [{ type: "text", text: formatProbeReport(report) }],
+					details: {
+						probed: report.probed.length,
+						ok: report.probed.filter((r) => r.ok).length,
+						failed: report.probed.filter((r) => !r.ok).length,
+						mismatches: report.probed.reduce((sum, r) => sum + r.mismatches.length, 0),
+						skippedProviders: report.skippedProviders,
+					},
+				};
+			} catch (error) {
+				const msg = error instanceof Error ? error.message : String(error);
+				return {
+					content: [{ type: "text", text: `Model probes failed: ${msg}` }],
+					details: { probed: 0, ok: 0, failed: 0, mismatches: 0, skippedProviders: [] },
+				};
+			}
+		},
+	});
 }
 
 function formatReport(findings: Finding[]): string {

package/extensions/probes.ts

@@ -0,0 +1,20 @@
+/**
+ * Live model probes for elyra doctor.
+ * The probe engine lives in @elyracode/ai (so the coding-agent CLI can use
+ * it too); this module re-exports it for the doctor extension and tests.
+ */
+
+export {
+	DEFAULT_MAX_MODELS,
+	DEFAULT_PROBE_TIMEOUT_MS,
+	evaluateThinkingClaims,
+	formatProbeReport,
+	MODELS_PER_PROVIDER,
+	pickReasoningLevel,
+	type ProbeOptions,
+	type ProbeReport,
+	type ProbeResult,
+	runModelProbes,
+	selectModelsForProvider,
+	type ThinkingObservation,
+} from "@elyracode/ai";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elyracode/doctor",
-  "version": "0.9.7",
+  "version": "0.9.9",
   "description": "Elyra extension for project health analysis -- dependency audit, security checks, code quality, and more",
   "type": "module",
   "keywords": [
@@ -23,6 +23,7 @@
     ]
   },
   "peerDependencies": {
+    "@elyracode/ai": "*",
     "@elyracode/coding-agent": "*",
     "typebox": "*"
   },

package/test/probes.test.ts

@@ -0,0 +1,216 @@
+import type { Api, Model } from "@elyracode/ai";
+import { describe, expect, it } from "vitest";
+import {
+	evaluateThinkingClaims,
+	formatProbeReport,
+	type ProbeReport,
+	pickReasoningLevel,
+	selectModelsForProvider,
+} from "../extensions/probes.js";
+
+function makeModel(overrides: Partial<Model<Api>> & { id: string }): Model<Api> {
+	return {
+		name: overrides.id,
+		api: "openai-completions",
+		provider: "test-provider",
+		baseUrl: "https://example.invalid",
+		reasoning: false,
+		input: ["text"],
+		cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 },
+		contextWindow: 128000,
+		maxTokens: 8192,
+		...overrides,
+	};
+}
+
+describe("selectModelsForProvider", () => {
+	it("picks the two cheapest models", () => {
+		const models = [
+			makeModel({ id: "expensive", cost: { input: 10, output: 30, cacheRead: 0, cacheWrite: 0 } }),
+			makeModel({ id: "cheap", cost: { input: 0.1, output: 0.4, cacheRead: 0, cacheWrite: 0 } }),
+			makeModel({ id: "mid", cost: { input: 1, output: 4, cacheRead: 0, cacheWrite: 0 } }),
+		];
+		const selected = selectModelsForProvider(models);
+		expect(selected.map((m) => m.id)).toEqual(["cheap", "mid"]);
+	});
+
+	it("adds the cheapest thinkingType model when the cheap picks have none", () => {
+		const models = [
+			makeModel({ id: "cheap-a", cost: { input: 0.1, output: 0.2, cacheRead: 0, cacheWrite: 0 } }),
+			makeModel({ id: "cheap-b", cost: { input: 0.2, output: 0.4, cacheRead: 0, cacheWrite: 0 } }),
+			makeModel({
+				id: "thinker-pricey",
+				reasoning: true,
+				thinkingType: "budget",
+				cost: { input: 15, output: 75, cacheRead: 0, cacheWrite: 0 },
+			}),
+			makeModel({
+				id: "thinker-cheap",
+				reasoning: true,
+				thinkingType: "adaptive",
+				cost: { input: 3, output: 15, cacheRead: 0, cacheWrite: 0 },
+			}),
+		];
+		const selected = selectModelsForProvider(models);
+		expect(selected.map((m) => m.id)).toEqual(["cheap-a", "cheap-b", "thinker-cheap"]);
+	});
+
+	it("does not add a third model when a cheap pick already has thinkingType", () => {
+		const models = [
+			makeModel({ id: "cheap-thinker", reasoning: true, thinkingType: "budget" }),
+			makeModel({ id: "cheap-plain", cost: { input: 2, output: 4, cacheRead: 0, cacheWrite: 0 } }),
+			makeModel({
+				id: "other-thinker",
+				reasoning: true,
+				thinkingType: "adaptive",
+				cost: { input: 5, output: 10, cacheRead: 0, cacheWrite: 0 },
+			}),
+		];
+		const selected = selectModelsForProvider(models);
+		expect(selected.map((m) => m.id)).toEqual(["cheap-thinker", "cheap-plain"]);
+	});
+
+	it("filters by model pattern (case-insensitive substring)", () => {
+		const models = [makeModel({ id: "gpt-5-mini" }), makeModel({ id: "claude-haiku" })];
+		const selected = selectModelsForProvider(models, "HAIKU");
+		expect(selected.map((m) => m.id)).toEqual(["claude-haiku"]);
+	});
+
+	it("returns empty for an empty registry", () => {
+		expect(selectModelsForProvider([])).toEqual([]);
+	});
+});
+
+describe("pickReasoningLevel", () => {
+	it("returns undefined when the model has no thinkingType", () => {
+		expect(pickReasoningLevel(makeModel({ id: "plain" }))).toBeUndefined();
+	});
+
+	it("returns undefined when thinkingType is set but reasoning is false", () => {
+		expect(pickReasoningLevel(makeModel({ id: "broken", thinkingType: "budget" }))).toBeUndefined();
+	});
+
+	it("returns the lowest supported level", () => {
+		expect(pickReasoningLevel(makeModel({ id: "thinker", reasoning: true, thinkingType: "budget" }))).toBe("minimal");
+	});
+
+	it("respects thinkingLevelMap exclusions", () => {
+		const model = makeModel({
+			id: "no-minimal",
+			reasoning: true,
+			thinkingType: "adaptive",
+			thinkingLevelMap: { minimal: null },
+		});
+		expect(pickReasoningLevel(model)).toBe("low");
+	});
+});
+
+describe("evaluateThinkingClaims", () => {
+	it("returns nothing for models without thinkingType", () => {
+		const result = evaluateThinkingClaims(makeModel({ id: "plain" }), {});
+		expect(result.mismatches).toEqual([]);
+		expect(result.notes).toEqual([]);
+	});
+
+	it("flags thinkingType set on a non-reasoning model", () => {
+		const result = evaluateThinkingClaims(makeModel({ id: "broken", thinkingType: "budget" }), {});
+		expect(result.mismatches).toEqual(["registry claims thinkingType=budget but reasoning=false"]);
+	});
+
+	it("flags thinkingType with all levels disabled", () => {
+		const model = makeModel({ id: "no-levels", reasoning: true, thinkingType: "adaptive" });
+		const result = evaluateThinkingClaims(model, { requestedLevel: undefined });
+		expect(result.mismatches).toEqual([
+			"registry claims thinkingType=adaptive but thinkingLevelMap disables all thinking levels",
+		]);
+	});
+
+	it("flags a failed thinking-enabled request", () => {
+		const model = makeModel({ id: "thinker", reasoning: true, thinkingType: "budget" });
+		const result = evaluateThinkingClaims(model, { requestedLevel: "minimal", error: "400 invalid thinking config" });
+		expect(result.mismatches).toEqual([
+			"request with thinking enabled (level: minimal) failed: 400 invalid thinking config",
+		]);
+	});
+
+	it("flags missing thinking content for budget models", () => {
+		const model = makeModel({ id: "thinker", reasoning: true, thinkingType: "budget" });
+		const result = evaluateThinkingClaims(model, { requestedLevel: "minimal", observedThinking: false });
+		expect(result.mismatches).toEqual(["no thinking content observed despite thinkingType=budget"]);
+		expect(result.notes).toEqual([]);
+	});
+
+	it("notes (not flags) missing thinking content for adaptive models", () => {
+		const model = makeModel({ id: "thinker", reasoning: true, thinkingType: "adaptive" });
+		const result = evaluateThinkingClaims(model, { requestedLevel: "minimal", observedThinking: false });
+		expect(result.mismatches).toEqual([]);
+		expect(result.notes).toEqual([
+			"adaptive thinking produced no thinking content (model may skip thinking for trivial prompts)",
+		]);
+	});
+
+	it("reports no issues when thinking is observed", () => {
+		const model = makeModel({ id: "thinker", reasoning: true, thinkingType: "budget" });
+		const result = evaluateThinkingClaims(model, { requestedLevel: "minimal", observedThinking: true });
+		expect(result.mismatches).toEqual([]);
+		expect(result.notes).toEqual([]);
+	});
+});
+
+describe("formatProbeReport", () => {
+	it("returns the message when no probes ran", () => {
+		const report: ProbeReport = {
+			probed: [],
+			skippedProviders: ["openai", "anthropic"],
+			message: "No providers configured.",
+		};
+		const text = formatProbeReport(report);
+		expect(text).toContain("# Model Probe Report");
+		expect(text).toContain("No providers configured.");
+	});
+
+	it("formats results, mismatches, notes, and skipped providers", () => {
+		const report: ProbeReport = {
+			probed: [
+				{
+					provider: "openai",
+					modelId: "gpt-5-mini",
+					ok: true,
+					latencyMs: 812,
+					mismatches: [],
+					notes: [],
+				},
+				{
+					provider: "amazon-bedrock",
+					modelId: "claude-x",
+					ok: true,
+					latencyMs: 1500,
+					reasoningProbe: { level: "minimal", observedThinking: false },
+					mismatches: ["no thinking content observed despite thinkingType=budget"],
+					notes: [],
+				},
+				{
+					provider: "groq",
+					modelId: "llama-y",
+					ok: false,
+					latencyMs: 230,
+					error: "401 unauthorized",
+					mismatches: [],
+					notes: ["some note"],
+				},
+			],
+			skippedProviders: ["mistral"],
+		};
+		const text = formatProbeReport(report);
+		expect(text).toContain("Probed: 3 | OK: 2 | Failed: 1 | Mismatches: 1");
+		expect(text).toContain("- [openai] gpt-5-mini -- OK (812ms)");
+		expect(text).toContain("- [amazon-bedrock] claude-x -- OK (1500ms), thinking(minimal): not observed");
+		expect(text).toContain("- [groq] llama-y -- FAILED (230ms): 401 unauthorized");
+		expect(text).toContain("## Mismatches");
+		expect(text).toContain("- [amazon-bedrock] claude-x: no thinking content observed despite thinkingType=budget");
+		expect(text).toContain("## Notes");
+		expect(text).toContain("- [groq] llama-y: some note");
+		expect(text).toContain("## Skipped Providers (no credentials)");
+		expect(text).toContain("mistral");
+	});
+});

package/vitest.config.ts

@@ -0,0 +1,15 @@
+import { fileURLToPath } from "node:url";
+import { defineConfig } from "vitest/config";
+
+const aiSrcIndex = fileURLToPath(new URL("../ai/src/index.ts", import.meta.url));
+
+export default defineConfig({
+	test: {
+		globals: true,
+		environment: "node",
+		testTimeout: 30000,
+	},
+	resolve: {
+		alias: [{ find: /^@elyracode\/ai$/, replacement: aiSrcIndex }],
+	},
+});