@elyracode/doctor 0.9.8 → 0.9.9

package/extensions/probes.ts

@@ -1,311 +1,20 @@
 /**
  * Live model probes for elyra doctor.
- * Verifies that configured LLM providers actually behave the way the model
- * registry claims (availability, reasoning support, thinking mode), since
- * registry metadata can drift from provider reality.
+ * The probe engine lives in @elyracode/ai (so the coding-agent CLI can use
+ * it too); this module re-exports it for the doctor extension and tests.
  */
 
-import type { Api, KnownProvider, Model, ThinkingLevel } from "@elyracode/ai";
-import { completeSimple, getEnvApiKey, getModels, getProviders, getSupportedThinkingLevels } from "@elyracode/ai";
-
-export const DEFAULT_MAX_MODELS = 10;
-export const MODELS_PER_PROVIDER = 2;
-export const DEFAULT_PROBE_TIMEOUT_MS = 60000;
-
-export interface ProbeOptions {
-	/** Only probe this provider (case-insensitive). */
-	provider?: string;
-	/** Only probe models whose id contains this substring (case-insensitive). */
-	modelPattern?: string;
-	/** Maximum total number of models to probe. */
-	maxModels?: number;
-	/** Abort signal from the tool execution. */
-	signal?: AbortSignal;
-	/** Per-probe timeout in milliseconds. */
-	timeoutMs?: number;
-}
-
-export interface ProbeResult {
-	provider: string;
-	modelId: string;
-	ok: boolean;
-	latencyMs: number;
-	error?: string;
-	/** Set when the probe ran with reasoning enabled. */
-	reasoningProbe?: { level: ThinkingLevel; observedThinking: boolean };
-	/** Discrepancies between registry claims and observed behavior. */
-	mismatches: string[];
-	/** Informational observations that are not necessarily mismatches. */
-	notes: string[];
-}
-
-export interface ProbeReport {
-	probed: ProbeResult[];
-	/** Providers in the registry that were skipped because no credentials were found. */
-	skippedProviders: string[];
-	/** Set when no probes could run at all (e.g. no credentials configured). */
-	message?: string;
-}
-
-export interface ThinkingObservation {
-	/** Reasoning level the probe requested, if any. */
-	requestedLevel?: ThinkingLevel;
-	/** Whether the response contained thinking content. Undefined if the probe failed. */
-	observedThinking?: boolean;
-	/** Error message when the probe failed. */
-	error?: string;
-}
-
-// ── Pure logic: model selection ──
-
-function probeCost(model: Model<Api>): number {
-	return model.cost.input + model.cost.output;
-}
-
-/**
- * Pick a small representative set of models for a provider:
- * the cheapest two, plus the cheapest model with thinkingType metadata
- * if none of the cheap picks have it (at most 3 total).
- */
-export function selectModelsForProvider(models: Model<Api>[], modelPattern?: string): Model<Api>[] {
-	const pattern = modelPattern?.toLowerCase();
-	const filtered = pattern ? models.filter((m) => m.id.toLowerCase().includes(pattern)) : models;
-	const byCost = [...filtered].sort((a, b) => probeCost(a) - probeCost(b));
-
-	const selected = byCost.slice(0, MODELS_PER_PROVIDER);
-	if (!selected.some((m) => m.thinkingType !== undefined)) {
-		const thinkingModel = byCost.find((m) => m.thinkingType !== undefined && !selected.includes(m));
-		if (thinkingModel) selected.push(thinkingModel);
-	}
-	return selected;
-}
-
-// ── Pure logic: thinking claims ──
-
-/** Lowest supported thinking level for a reasoning probe, or undefined when not applicable. */
-export function pickReasoningLevel(model: Model<Api>): ThinkingLevel | undefined {
-	if (model.thinkingType === undefined || !model.reasoning) return undefined;
-	const levels = getSupportedThinkingLevels(model).filter((l): l is ThinkingLevel => l !== "off");
-	return levels[0];
-}
-
-/** Compare registry thinking claims against observed probe behavior. */
-export function evaluateThinkingClaims(
-	model: Model<Api>,
-	observation: ThinkingObservation,
-): { mismatches: string[]; notes: string[] } {
-	const mismatches: string[] = [];
-	const notes: string[] = [];
-
-	if (model.thinkingType === undefined) return { mismatches, notes };
-
-	if (!model.reasoning) {
-		mismatches.push(`registry claims thinkingType=${model.thinkingType} but reasoning=false`);
-		return { mismatches, notes };
-	}
-
-	if (observation.requestedLevel === undefined) {
-		mismatches.push(
-			`registry claims thinkingType=${model.thinkingType} but thinkingLevelMap disables all thinking levels`,
-		);
-		return { mismatches, notes };
-	}
-
-	if (observation.error !== undefined) {
-		mismatches.push(
-			`request with thinking enabled (level: ${observation.requestedLevel}) failed: ${observation.error}`,
-		);
-		return { mismatches, notes };
-	}
-
-	if (observation.observedThinking === false) {
-		if (model.thinkingType === "budget") {
-			mismatches.push("no thinking content observed despite thinkingType=budget");
-		} else {
-			notes.push("adaptive thinking produced no thinking content (model may skip thinking for trivial prompts)");
-		}
-	}
-
-	return { mismatches, notes };
-}
-
-// ── Live probing ──
-
-function combineSignals(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal {
-	const timeout = AbortSignal.timeout(timeoutMs);
-	return signal ? AbortSignal.any([signal, timeout]) : timeout;
-}
-
-async function probeModel(model: Model<Api>, apiKey: string, options: ProbeOptions): Promise<ProbeResult> {
-	const result: ProbeResult = {
-		provider: model.provider,
-		modelId: model.id,
-		ok: false,
-		latencyMs: 0,
-		mismatches: [],
-		notes: [],
-	};
-
-	const reasoningLevel = pickReasoningLevel(model);
-	const start = Date.now();
-
-	try {
-		const message = await completeSimple(
-			model,
-			{ messages: [{ role: "user", content: "Reply with the single word: OK", timestamp: Date.now() }] },
-			{
-				apiKey,
-				maxTokens: 64,
-				reasoning: reasoningLevel,
-				signal: combineSignals(options.signal, options.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS),
-			},
-		);
-		result.latencyMs = Date.now() - start;
-
-		if (message.stopReason === "error" || message.stopReason === "aborted") {
-			result.error = message.errorMessage ?? `request ended with stopReason=${message.stopReason}`;
-		} else {
-			result.ok = true;
-			if (reasoningLevel !== undefined) {
-				result.reasoningProbe = {
-					level: reasoningLevel,
-					observedThinking: message.content.some((c) => c.type === "thinking"),
-				};
-			}
-		}
-	} catch (error) {
-		result.latencyMs = Date.now() - start;
-		result.error = error instanceof Error ? error.message : String(error);
-	}
-
-	const claims = evaluateThinkingClaims(model, {
-		requestedLevel: reasoningLevel,
-		observedThinking: result.reasoningProbe?.observedThinking,
-		error: result.error,
-	});
-	result.mismatches.push(...claims.mismatches);
-	result.notes.push(...claims.notes);
-
-	return result;
-}
-
-/** Run live probes for all providers with credentials. Never throws. */
-export async function runModelProbes(options: ProbeOptions = {}): Promise<ProbeReport> {
-	const providerFilter = options.provider?.toLowerCase();
-	const providers = getProviders().filter((p) => !providerFilter || p.toLowerCase() === providerFilter);
-
-	const available: { provider: string; apiKey: string }[] = [];
-	const skipped: string[] = [];
-	for (const provider of providers) {
-		const apiKey = getEnvApiKey(provider);
-		if (apiKey) {
-			available.push({ provider, apiKey });
-		} else {
-			skipped.push(provider);
-		}
-	}
-
-	if (available.length === 0) {
-		return {
-			probed: [],
-			skippedProviders: skipped,
-			message: options.provider
-				? `No credentials found for provider "${options.provider}". Set the provider's API key environment variable to enable probing.`
-				: "No providers configured. Set an API key environment variable (e.g. OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY) to enable model probes.",
-		};
-	}
-
-	const maxModels = Math.max(1, Math.floor(options.maxModels ?? DEFAULT_MAX_MODELS));
-	const groups: { models: Model<Api>[]; apiKey: string }[] = [];
-	let budget = maxModels;
-	for (const { provider, apiKey } of available) {
-		if (budget <= 0) break;
-		const all = getModels(provider as KnownProvider) as Model<Api>[];
-		const models = selectModelsForProvider(all, options.modelPattern).slice(0, budget);
-		if (models.length > 0) {
-			groups.push({ models, apiKey });
-			budget -= models.length;
-		}
-	}
-
-	if (groups.length === 0) {
-		return {
-			probed: [],
-			skippedProviders: skipped,
-			message: options.modelPattern
-				? `No registry models match pattern "${options.modelPattern}" for the configured providers.`
-				: "No registry models found for the configured providers.",
-		};
-	}
-
-	// Probe providers in parallel, models within a provider sequentially (rate-limit friendly).
-	const results = await Promise.all(
-		groups.map(async ({ models, apiKey }) => {
-			const groupResults: ProbeResult[] = [];
-			for (const model of models) {
-				if (options.signal?.aborted) break;
-				groupResults.push(await probeModel(model, apiKey, options));
-			}
-			return groupResults;
-		}),
-	);
-
-	return { probed: results.flat(), skippedProviders: skipped };
-}
-
-// ── Pure logic: report formatting ──
-
-export function formatProbeReport(report: ProbeReport): string {
-	if (report.message) {
-		return `# Model Probe Report\n\n${report.message}`;
-	}
-
-	const ok = report.probed.filter((r) => r.ok);
-	const failed = report.probed.filter((r) => !r.ok);
-	const mismatchCount = report.probed.reduce((sum, r) => sum + r.mismatches.length, 0);
-
-	const lines: string[] = [
-		"# Model Probe Report",
-		"",
-		`Probed: ${report.probed.length} | OK: ${ok.length} | Failed: ${failed.length} | Mismatches: ${mismatchCount}`,
-		"",
-		"## Results",
-	];
-
-	for (const r of report.probed) {
-		const status = r.ok ? `OK (${r.latencyMs}ms)` : `FAILED (${r.latencyMs}ms): ${r.error ?? "unknown error"}`;
-		const thinking = r.reasoningProbe
-			? `, thinking(${r.reasoningProbe.level}): ${r.reasoningProbe.observedThinking ? "observed" : "not observed"}`
-			: "";
-		lines.push(`- [${r.provider}] ${r.modelId} -- ${status}${thinking}`);
-	}
-	lines.push("");
-
-	if (mismatchCount > 0) {
-		lines.push("## Mismatches");
-		for (const r of report.probed) {
-			for (const m of r.mismatches) {
-				lines.push(`- [${r.provider}] ${r.modelId}: ${m}`);
-			}
-		}
-		lines.push("");
-	}
-
-	const noteCount = report.probed.reduce((sum, r) => sum + r.notes.length, 0);
-	if (noteCount > 0) {
-		lines.push("## Notes");
-		for (const r of report.probed) {
-			for (const n of r.notes) {
-				lines.push(`- [${r.provider}] ${r.modelId}: ${n}`);
-			}
-		}
-		lines.push("");
-	}
-
-	if (report.skippedProviders.length > 0) {
-		lines.push("## Skipped Providers (no credentials)");
-		lines.push(report.skippedProviders.join(", "));
-	}
-
-	return lines.join("\n").trimEnd();
-}
+export {
+	DEFAULT_MAX_MODELS,
+	DEFAULT_PROBE_TIMEOUT_MS,
+	evaluateThinkingClaims,
+	formatProbeReport,
+	MODELS_PER_PROVIDER,
+	pickReasoningLevel,
+	type ProbeOptions,
+	type ProbeReport,
+	type ProbeResult,
+	runModelProbes,
+	selectModelsForProvider,
+	type ThinkingObservation,
+} from "@elyracode/ai";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elyracode/doctor",
-  "version": "0.9.8",
+  "version": "0.9.9",
   "description": "Elyra extension for project health analysis -- dependency audit, security checks, code quality, and more",
   "type": "module",
   "keywords": [

package/vitest.config.ts

@@ -0,0 +1,15 @@
+import { fileURLToPath } from "node:url";
+import { defineConfig } from "vitest/config";
+
+const aiSrcIndex = fileURLToPath(new URL("../ai/src/index.ts", import.meta.url));
+
+export default defineConfig({
+	test: {
+		globals: true,
+		environment: "node",
+		testTimeout: 30000,
+	},
+	resolve: {
+		alias: [{ find: /^@elyracode\/ai$/, replacement: aiSrcIndex }],
+	},
+});