@elyracode/ai 0.9.7 → 0.9.9

package/dist/probes.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Live model probes.
+ * Verifies that configured LLM providers actually behave the way the model
+ * registry claims (availability, reasoning support, thinking mode), since
+ * registry metadata can drift from provider reality.
+ */
+import type { Api, Model, ThinkingLevel } from "./types.js";
+export declare const DEFAULT_MAX_MODELS = 10;
+export declare const MODELS_PER_PROVIDER = 2;
+export declare const DEFAULT_PROBE_TIMEOUT_MS = 60000;
+export interface ProbeOptions {
+    /** Only probe this provider (case-insensitive). */
+    provider?: string;
+    /** Only probe models whose id contains this substring (case-insensitive). */
+    modelPattern?: string;
+    /** Maximum total number of models to probe. */
+    maxModels?: number;
+    /** Abort signal from the tool execution. */
+    signal?: AbortSignal;
+    /** Per-probe timeout in milliseconds. */
+    timeoutMs?: number;
+    /**
+     * Credential resolver per provider. Defaults to environment variables
+     * (getEnvApiKey). Callers with richer credential stores (e.g. auth.json)
+     * can supply their own resolver.
+     */
+    resolveApiKey?: (provider: string) => Promise<string | undefined> | string | undefined;
+}
+export interface ProbeResult {
+    provider: string;
+    modelId: string;
+    ok: boolean;
+    latencyMs: number;
+    error?: string;
+    /** Set when the probe ran with reasoning enabled. */
+    reasoningProbe?: {
+        level: ThinkingLevel;
+        observedThinking: boolean;
+    };
+    /** Discrepancies between registry claims and observed behavior. */
+    mismatches: string[];
+    /** Informational observations that are not necessarily mismatches. */
+    notes: string[];
+}
+export interface ProbeReport {
+    probed: ProbeResult[];
+    /** Providers in the registry that were skipped because no credentials were found. */
+    skippedProviders: string[];
+    /** Set when no probes could run at all (e.g. no credentials configured). */
+    message?: string;
+}
+export interface ThinkingObservation {
+    /** Reasoning level the probe requested, if any. */
+    requestedLevel?: ThinkingLevel;
+    /** Whether the response contained thinking content. Undefined if the probe failed. */
+    observedThinking?: boolean;
+    /** Error message when the probe failed. */
+    error?: string;
+}
+/**
+ * Pick a small representative set of models for a provider:
+ * the cheapest two, plus the cheapest model with thinkingType metadata
+ * if none of the cheap picks have it (at most 3 total).
+ */
+export declare function selectModelsForProvider(models: Model<Api>[], modelPattern?: string): Model<Api>[];
+/** Lowest supported thinking level for a reasoning probe, or undefined when not applicable. */
+export declare function pickReasoningLevel(model: Model<Api>): ThinkingLevel | undefined;
+/** Compare registry thinking claims against observed probe behavior. */
+export declare function evaluateThinkingClaims(model: Model<Api>, observation: ThinkingObservation): {
+    mismatches: string[];
+    notes: string[];
+};
+/** Run live probes for all providers with credentials. Never throws. */
+export declare function runModelProbes(options?: ProbeOptions): Promise<ProbeReport>;
+export declare function formatProbeReport(report: ProbeReport): string;
+//# sourceMappingURL=probes.d.ts.map

package/dist/probes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"probes.d.ts","sourceRoot":"","sources":["../src/probes.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,GAAG,EAAiB,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3E,eAAO,MAAM,kBAAkB,KAAK,CAAC;AACrC,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,eAAO,MAAM,wBAAwB,QAAQ,CAAC;AAE9C,MAAM,WAAW,YAAY;IAC5B,mDAAmD;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6EAA6E;IAC7E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4CAA4C;IAC5C,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,yCAAyC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,MAAM,GAAG,SAAS,CAAC;CACvF;AAED,MAAM,WAAW,WAAW;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,OAAO,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,cAAc,CAAC,EAAE;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,gBAAgB,EAAE,OAAO,CAAA;KAAE,CAAC;IACrE,mEAAmE;IACnE,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,sEAAsE;IACtE,KAAK,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,MAAM,WAAW,WAAW;IAC3B,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,qFAAqF;IACrF,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,4EAA4E;IAC5E,OAAO,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,mBAAmB;IACnC,mDAAmD;IACnD,cAAc,CAAC,EAAE,aAAa,CAAC;IAC/B,sFAAsF;IACtF,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,2CAA2C;IAC3C,KAAK,CAAC,EAAE,MAAM,CAAC;CACf;AAQD;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,EAAE,CAWjG;AAID,+FAA+F;AAC/F,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,aAAa,GAAG,SAAS,CAI/E;AAED,wEAAwE;AACxE,wBAAgB,sBAAsB,CACrC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,EACjB,WAAW,EAAE,mBAAmB,GAC9B;IAAE,UAAU,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAA;CAAE,CAkC3C;AA8DD,wEAAwE;AACxE,wBAAsB,cAAc,CAAC,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CA8DrF;AAID,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,WAAW,GAAG,MAAM,CAqD7D"}

package/dist/probes.js

@@ -0,0 +1,227 @@
+/**
+ * Live model probes.
+ * Verifies that configured LLM providers actually behave the way the model
+ * registry claims (availability, reasoning support, thinking mode), since
+ * registry metadata can drift from provider reality.
+ */
+import { getEnvApiKey } from "./env-api-keys.js";
+import { getModels, getProviders, getSupportedThinkingLevels } from "./models.js";
+import { completeSimple } from "./stream.js";
+export const DEFAULT_MAX_MODELS = 10;
+export const MODELS_PER_PROVIDER = 2;
+export const DEFAULT_PROBE_TIMEOUT_MS = 60000;
+// ── Pure logic: model selection ──
+function probeCost(model) {
+    return model.cost.input + model.cost.output;
+}
+/**
+ * Pick a small representative set of models for a provider:
+ * the cheapest two, plus the cheapest model with thinkingType metadata
+ * if none of the cheap picks have it (at most 3 total).
+ */
+export function selectModelsForProvider(models, modelPattern) {
+    const pattern = modelPattern?.toLowerCase();
+    const filtered = pattern ? models.filter((m) => m.id.toLowerCase().includes(pattern)) : models;
+    const byCost = [...filtered].sort((a, b) => probeCost(a) - probeCost(b));
+    const selected = byCost.slice(0, MODELS_PER_PROVIDER);
+    if (!selected.some((m) => m.thinkingType !== undefined)) {
+        const thinkingModel = byCost.find((m) => m.thinkingType !== undefined && !selected.includes(m));
+        if (thinkingModel)
+            selected.push(thinkingModel);
+    }
+    return selected;
+}
+// ── Pure logic: thinking claims ──
+/** Lowest supported thinking level for a reasoning probe, or undefined when not applicable. */
+export function pickReasoningLevel(model) {
+    if (model.thinkingType === undefined || !model.reasoning)
+        return undefined;
+    const levels = getSupportedThinkingLevels(model).filter((l) => l !== "off");
+    return levels[0];
+}
+/** Compare registry thinking claims against observed probe behavior. */
+export function evaluateThinkingClaims(model, observation) {
+    const mismatches = [];
+    const notes = [];
+    if (model.thinkingType === undefined)
+        return { mismatches, notes };
+    if (!model.reasoning) {
+        mismatches.push(`registry claims thinkingType=${model.thinkingType} but reasoning=false`);
+        return { mismatches, notes };
+    }
+    if (observation.requestedLevel === undefined) {
+        mismatches.push(`registry claims thinkingType=${model.thinkingType} but thinkingLevelMap disables all thinking levels`);
+        return { mismatches, notes };
+    }
+    if (observation.error !== undefined) {
+        mismatches.push(`request with thinking enabled (level: ${observation.requestedLevel}) failed: ${observation.error}`);
+        return { mismatches, notes };
+    }
+    if (observation.observedThinking === false) {
+        if (model.thinkingType === "budget") {
+            mismatches.push("no thinking content observed despite thinkingType=budget");
+        }
+        else {
+            notes.push("adaptive thinking produced no thinking content (model may skip thinking for trivial prompts)");
+        }
+    }
+    return { mismatches, notes };
+}
+// ── Live probing ──
+function combineSignals(signal, timeoutMs) {
+    const timeout = AbortSignal.timeout(timeoutMs);
+    return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+async function probeModel(model, apiKey, options) {
+    const result = {
+        provider: model.provider,
+        modelId: model.id,
+        ok: false,
+        latencyMs: 0,
+        mismatches: [],
+        notes: [],
+    };
+    const reasoningLevel = pickReasoningLevel(model);
+    const start = Date.now();
+    try {
+        const message = await completeSimple(model, { messages: [{ role: "user", content: "Reply with the single word: OK", timestamp: Date.now() }] }, {
+            apiKey,
+            maxTokens: 64,
+            reasoning: reasoningLevel,
+            signal: combineSignals(options.signal, options.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS),
+        });
+        result.latencyMs = Date.now() - start;
+        if (message.stopReason === "error" || message.stopReason === "aborted") {
+            result.error = message.errorMessage ?? `request ended with stopReason=${message.stopReason}`;
+        }
+        else {
+            result.ok = true;
+            if (reasoningLevel !== undefined) {
+                result.reasoningProbe = {
+                    level: reasoningLevel,
+                    observedThinking: message.content.some((c) => c.type === "thinking"),
+                };
+            }
+        }
+    }
+    catch (error) {
+        result.latencyMs = Date.now() - start;
+        result.error = error instanceof Error ? error.message : String(error);
+    }
+    const claims = evaluateThinkingClaims(model, {
+        requestedLevel: reasoningLevel,
+        observedThinking: result.reasoningProbe?.observedThinking,
+        error: result.error,
+    });
+    result.mismatches.push(...claims.mismatches);
+    result.notes.push(...claims.notes);
+    return result;
+}
+/** Run live probes for all providers with credentials. Never throws. */
+export async function runModelProbes(options = {}) {
+    const providerFilter = options.provider?.toLowerCase();
+    const providers = getProviders().filter((p) => !providerFilter || p.toLowerCase() === providerFilter);
+    const resolveApiKey = options.resolveApiKey ?? getEnvApiKey;
+    const available = [];
+    const skipped = [];
+    for (const provider of providers) {
+        const apiKey = await resolveApiKey(provider);
+        if (apiKey) {
+            available.push({ provider, apiKey });
+        }
+        else {
+            skipped.push(provider);
+        }
+    }
+    if (available.length === 0) {
+        return {
+            probed: [],
+            skippedProviders: skipped,
+            message: options.provider
+                ? `No credentials found for provider "${options.provider}". Set the provider's API key environment variable to enable probing.`
+                : "No providers configured. Set an API key environment variable (e.g. OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY) to enable model probes.",
+        };
+    }
+    const maxModels = Math.max(1, Math.floor(options.maxModels ?? DEFAULT_MAX_MODELS));
+    const groups = [];
+    let budget = maxModels;
+    for (const { provider, apiKey } of available) {
+        if (budget <= 0)
+            break;
+        const all = getModels(provider);
+        const models = selectModelsForProvider(all, options.modelPattern).slice(0, budget);
+        if (models.length > 0) {
+            groups.push({ models, apiKey });
+            budget -= models.length;
+        }
+    }
+    if (groups.length === 0) {
+        return {
+            probed: [],
+            skippedProviders: skipped,
+            message: options.modelPattern
+                ? `No registry models match pattern "${options.modelPattern}" for the configured providers.`
+                : "No registry models found for the configured providers.",
+        };
+    }
+    // Probe providers in parallel, models within a provider sequentially (rate-limit friendly).
+    const results = await Promise.all(groups.map(async ({ models, apiKey }) => {
+        const groupResults = [];
+        for (const model of models) {
+            if (options.signal?.aborted)
+                break;
+            groupResults.push(await probeModel(model, apiKey, options));
+        }
+        return groupResults;
+    }));
+    return { probed: results.flat(), skippedProviders: skipped };
+}
+// ── Pure logic: report formatting ──
+export function formatProbeReport(report) {
+    if (report.message) {
+        return `# Model Probe Report\n\n${report.message}`;
+    }
+    const ok = report.probed.filter((r) => r.ok);
+    const failed = report.probed.filter((r) => !r.ok);
+    const mismatchCount = report.probed.reduce((sum, r) => sum + r.mismatches.length, 0);
+    const lines = [
+        "# Model Probe Report",
+        "",
+        `Probed: ${report.probed.length} | OK: ${ok.length} | Failed: ${failed.length} | Mismatches: ${mismatchCount}`,
+        "",
+        "## Results",
+    ];
+    for (const r of report.probed) {
+        const status = r.ok ? `OK (${r.latencyMs}ms)` : `FAILED (${r.latencyMs}ms): ${r.error ?? "unknown error"}`;
+        const thinking = r.reasoningProbe
+            ? `, thinking(${r.reasoningProbe.level}): ${r.reasoningProbe.observedThinking ? "observed" : "not observed"}`
+            : "";
+        lines.push(`- [${r.provider}] ${r.modelId} -- ${status}${thinking}`);
+    }
+    lines.push("");
+    if (mismatchCount > 0) {
+        lines.push("## Mismatches");
+        for (const r of report.probed) {
+            for (const m of r.mismatches) {
+                lines.push(`- [${r.provider}] ${r.modelId}: ${m}`);
+            }
+        }
+        lines.push("");
+    }
+    const noteCount = report.probed.reduce((sum, r) => sum + r.notes.length, 0);
+    if (noteCount > 0) {
+        lines.push("## Notes");
+        for (const r of report.probed) {
+            for (const n of r.notes) {
+                lines.push(`- [${r.provider}] ${r.modelId}: ${n}`);
+            }
+        }
+        lines.push("");
+    }
+    if (report.skippedProviders.length > 0) {
+        lines.push("## Skipped Providers (no credentials)");
+        lines.push(report.skippedProviders.join(", "));
+    }
+    return lines.join("\n").trimEnd();
+}
+//# sourceMappingURL=probes.js.map

package/dist/probes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"probes.js","sourceRoot":"","sources":["../src/probes.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAClF,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAG7C,MAAM,CAAC,MAAM,kBAAkB,GAAG,EAAE,CAAC;AACrC,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC;AACrC,MAAM,CAAC,MAAM,wBAAwB,GAAG,KAAK,CAAC;AAoD9C,oCAAoC;AAEpC,SAAS,SAAS,CAAC,KAAiB;IACnC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;AAC7C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,MAAoB,EAAE,YAAqB;IAClF,MAAM,OAAO,GAAG,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAC/F,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzE,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;IACtD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS,CAAC,EAAE,CAAC;QACzD,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAChG,IAAI,aAAa;YAAE,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,oCAAoC;AAEpC,+FAA+F;AAC/F,MAAM,UAAU,kBAAkB,CAAC,KAAiB;IACnD,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS;QAAE,OAAO,SAAS,CAAC;IAC3E,MAAM,MAAM,GAAG,0BAA0B,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAsB,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC;IAChG,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,sBAAsB,CACrC,KAAiB,EACjB,WAAgC;IAEhC,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS;QAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;IAEnE,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACtB,UAAU,CAAC,IAAI,CAAC,gCAAgC,KAAK,CAAC,YAAY,sBAAsB,CAAC,CAAC;QAC1F,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;IAC9B,CAAC;IAED,IAAI,WAAW,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QAC9C,UAAU,CAAC,IAAI,CACd,gCAAgC,KAAK,CAAC,YAAY,oDAAoD,CACtG,CAAC;QACF,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;IAC9B,CAAC;IAED,IAAI,WAAW,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QACrC,UAAU,CAAC,IAAI,CACd,yCAAyC,WAAW,CAAC,cAAc,aAAa,WAAW,CAAC,KAAK,EAAE,CACnG,CAAC;QACF,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;IAC9B,CAAC;IAED,IAAI,WAAW,CAAC,gBAAgB,KAAK,KAAK,EAAE,CAAC;QAC5C,IAAI,KAAK,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;YACrC,UAAU,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;QAC7E,CAAC;aAAM,CAAC;YACP,KAAK,CAAC,IAAI,CAAC,8FAA8F,CAAC,CAAC;QAC5G,CAAC;IACF,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;AAC9B,CAAC;AAED,qBAAqB;AAErB,SAAS,cAAc,CAAC,MAA+B,EAAE,SAAiB;IACzE,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/C,OAAO,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;AAC9D,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,KAAiB,EAAE,MAAc,EAAE,OAAqB;IACjF,MAAM,MAAM,GAAgB;QAC3B,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,OAAO,EAAE,KAAK,CAAC,EAAE;QACjB,EAAE,EAAE,KAAK;QACT,SAAS,EAAE,CAAC;QACZ,UAAU,EAAE,EAAE;QACd,KAAK,EAAE,EAAE;KACT,CAAC;IAEF,MAAM,cAAc,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,MAAM,cAAc,CACnC,KAAK,EACL,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gCAAgC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAClG;YACC,MAAM;YACN,SAAS,EAAE,EAAE;YACb,SAAS,EAAE,cAAc;YACzB,MAAM,EAAE,cAAc,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,IAAI,wBAAwB,CAAC;SACrF,CACD,CAAC;QACF,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAEtC,IAAI,OAAO,CAAC,UAAU,KAAK,OAAO,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACxE,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC,YAAY,IAAI,iCAAiC,OAAO,CAAC,UAAU,EAAE,CAAC;QAC9F,CAAC;aAAM,CAAC;YACP,MAAM,CAAC,EAAE,GAAG,IAAI,CAAC;YACjB,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;gBAClC,MAAM,CAAC,cAAc,GAAG;oBACvB,KAAK,EAAE,cAAc;oBACrB,gBAAgB,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC;iBACpE,CAAC;YACH,CAAC;QACF,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,MAAM,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,MAAM,GAAG,sBAAsB,CAAC,KAAK,EAAE;QAC5C,cAAc,EAAE,cAAc;QAC9B,gBAAgB,EAAE,MAAM,CAAC,cAAc,EAAE,gBAAgB;QACzD,KAAK,EAAE,MAAM,CAAC,KAAK;KACnB,CAAC,CAAC;IACH,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IAC7C,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAEnC,OAAO,MAAM,CAAC;AACf,CAAC;AAED,wEAAwE;AACxE,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAO,GAAiB,EAAE;IAC9D,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC;IACvD,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,cAAc,CAAC,CAAC;IAEtG,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,YAAY,CAAC;IAC5D,MAAM,SAAS,GAA2C,EAAE,CAAC;IAC7D,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC7C,IAAI,MAAM,EAAE,CAAC;YACZ,SAAS,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;QACtC,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxB,CAAC;IACF,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO;YACN,MAAM,EAAE,EAAE;YACV,gBAAgB,EAAE,OAAO;YACzB,OAAO,EAAE,OAAO,CAAC,QAAQ;gBACxB,CAAC,CAAC,sCAAsC,OAAO,CAAC,QAAQ,uEAAuE;gBAC/H,CAAC,CAAC,+IAA+I;SAClJ,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC,CAAC;IACnF,MAAM,MAAM,GAA+C,EAAE,CAAC;IAC9D,IAAI,MAAM,GAAG,SAAS,CAAC;IACvB,KAAK,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;QAC9C,IAAI,MAAM,IAAI,CAAC;YAAE,MAAM;QACvB,MAAM,GAAG,GAAG,SAAS,CAAC,QAAyB,CAAiB,CAAC;QACjE,MAAM,MAAM,GAAG,uBAAuB,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACnF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YAChC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC;QACzB,CAAC;IACF,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACN,MAAM,EAAE,EAAE;YACV,gBAAgB,EAAE,OAAO;YACzB,OAAO,EAAE,OAAO,CAAC,YAAY;gBAC5B,CAAC,CAAC,qCAAqC,OAAO,CAAC,YAAY,iCAAiC;gBAC5F,CAAC,CAAC,wDAAwD;SAC3D,CAAC;IACH,CAAC;IAED,4FAA4F;IAC5F,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAChC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE;QACvC,MAAM,YAAY,GAAkB,EAAE,CAAC;QACvC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;gBAAE,MAAM;YACnC,YAAY,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,YAAY,CAAC;IACrB,CAAC,CAAC,CACF,CAAC;IAEF,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,gBAAgB,EAAE,OAAO,EAAE,CAAC;AAC9D,CAAC;AAED,sCAAsC;AAEtC,MAAM,UAAU,iBAAiB,CAAC,MAAmB;IACpD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,2BAA2B,MAAM,CAAC,OAAO,EAAE,CAAC;IACpD,CAAC;IAED,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAClD,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAErF,MAAM,KAAK,GAAa;QACvB,sBAAsB;QACtB,EAAE;QACF,WAAW,MAAM,CAAC,MAAM,CAAC,MAAM,UAAU,EAAE,CAAC,MAAM,cAAc,MAAM,CAAC,MAAM,kBAAkB,aAAa,EAAE;QAC9G,EAAE;QACF,YAAY;KACZ,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,SAAS,QAAQ,CAAC,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC;QAC3G,MAAM,QAAQ,GAAG,CAAC,CAAC,cAAc;YAChC,CAAC,CAAC,cAAc,CAAC,CAAC,cAAc,CAAC,KAAK,MAAM,CAAC,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,EAAE;YAC7G,CAAC,CAAC,EAAE,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,OAAO,OAAO,MAAM,GAAG,QAAQ,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC5B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC,CAAC;YACpD,CAAC;QACF,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChB,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC5E,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACzB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC,CAAC;YACpD,CAAC;QACF,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChB,CAAC;IAED,IAAI,MAAM,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;AACnC,CAAC","sourcesContent":["/**\n * Live model probes.\n * Verifies that configured LLM providers actually behave the way the model\n * registry claims (availability, reasoning support, thinking mode), since\n * registry metadata can drift from provider reality.\n */\n\nimport { getEnvApiKey } from \"./env-api-keys.js\";\nimport { getModels, getProviders, getSupportedThinkingLevels } from \"./models.js\";\nimport { completeSimple } from \"./stream.js\";\nimport type { Api, KnownProvider, Model, ThinkingLevel } from \"./types.js\";\n\nexport const DEFAULT_MAX_MODELS = 10;\nexport const MODELS_PER_PROVIDER = 2;\nexport const DEFAULT_PROBE_TIMEOUT_MS = 60000;\n\nexport interface ProbeOptions {\n\t/** Only probe this provider (case-insensitive). */\n\tprovider?: string;\n\t/** Only probe models whose id contains this substring (case-insensitive). */\n\tmodelPattern?: string;\n\t/** Maximum total number of models to probe. */\n\tmaxModels?: number;\n\t/** Abort signal from the tool execution. */\n\tsignal?: AbortSignal;\n\t/** Per-probe timeout in milliseconds. */\n\ttimeoutMs?: number;\n\t/**\n\t * Credential resolver per provider. Defaults to environment variables\n\t * (getEnvApiKey). Callers with richer credential stores (e.g. auth.json)\n\t * can supply their own resolver.\n\t */\n\tresolveApiKey?: (provider: string) => Promise<string | undefined> | string | undefined;\n}\n\nexport interface ProbeResult {\n\tprovider: string;\n\tmodelId: string;\n\tok: boolean;\n\tlatencyMs: number;\n\terror?: string;\n\t/** Set when the probe ran with reasoning enabled. */\n\treasoningProbe?: { level: ThinkingLevel; observedThinking: boolean };\n\t/** Discrepancies between registry claims and observed behavior. */\n\tmismatches: string[];\n\t/** Informational observations that are not necessarily mismatches. */\n\tnotes: string[];\n}\n\nexport interface ProbeReport {\n\tprobed: ProbeResult[];\n\t/** Providers in the registry that were skipped because no credentials were found. */\n\tskippedProviders: string[];\n\t/** Set when no probes could run at all (e.g. no credentials configured). */\n\tmessage?: string;\n}\n\nexport interface ThinkingObservation {\n\t/** Reasoning level the probe requested, if any. */\n\trequestedLevel?: ThinkingLevel;\n\t/** Whether the response contained thinking content. Undefined if the probe failed. */\n\tobservedThinking?: boolean;\n\t/** Error message when the probe failed. */\n\terror?: string;\n}\n\n// ── Pure logic: model selection ──\n\nfunction probeCost(model: Model<Api>): number {\n\treturn model.cost.input + model.cost.output;\n}\n\n/**\n * Pick a small representative set of models for a provider:\n * the cheapest two, plus the cheapest model with thinkingType metadata\n * if none of the cheap picks have it (at most 3 total).\n */\nexport function selectModelsForProvider(models: Model<Api>[], modelPattern?: string): Model<Api>[] {\n\tconst pattern = modelPattern?.toLowerCase();\n\tconst filtered = pattern ? models.filter((m) => m.id.toLowerCase().includes(pattern)) : models;\n\tconst byCost = [...filtered].sort((a, b) => probeCost(a) - probeCost(b));\n\n\tconst selected = byCost.slice(0, MODELS_PER_PROVIDER);\n\tif (!selected.some((m) => m.thinkingType !== undefined)) {\n\t\tconst thinkingModel = byCost.find((m) => m.thinkingType !== undefined && !selected.includes(m));\n\t\tif (thinkingModel) selected.push(thinkingModel);\n\t}\n\treturn selected;\n}\n\n// ── Pure logic: thinking claims ──\n\n/** Lowest supported thinking level for a reasoning probe, or undefined when not applicable. */\nexport function pickReasoningLevel(model: Model<Api>): ThinkingLevel | undefined {\n\tif (model.thinkingType === undefined || !model.reasoning) return undefined;\n\tconst levels = getSupportedThinkingLevels(model).filter((l): l is ThinkingLevel => l !== \"off\");\n\treturn levels[0];\n}\n\n/** Compare registry thinking claims against observed probe behavior. */\nexport function evaluateThinkingClaims(\n\tmodel: Model<Api>,\n\tobservation: ThinkingObservation,\n): { mismatches: string[]; notes: string[] } {\n\tconst mismatches: string[] = [];\n\tconst notes: string[] = [];\n\n\tif (model.thinkingType === undefined) return { mismatches, notes };\n\n\tif (!model.reasoning) {\n\t\tmismatches.push(`registry claims thinkingType=${model.thinkingType} but reasoning=false`);\n\t\treturn { mismatches, notes };\n\t}\n\n\tif (observation.requestedLevel === undefined) {\n\t\tmismatches.push(\n\t\t\t`registry claims thinkingType=${model.thinkingType} but thinkingLevelMap disables all thinking levels`,\n\t\t);\n\t\treturn { mismatches, notes };\n\t}\n\n\tif (observation.error !== undefined) {\n\t\tmismatches.push(\n\t\t\t`request with thinking enabled (level: ${observation.requestedLevel}) failed: ${observation.error}`,\n\t\t);\n\t\treturn { mismatches, notes };\n\t}\n\n\tif (observation.observedThinking === false) {\n\t\tif (model.thinkingType === \"budget\") {\n\t\t\tmismatches.push(\"no thinking content observed despite thinkingType=budget\");\n\t\t} else {\n\t\t\tnotes.push(\"adaptive thinking produced no thinking content (model may skip thinking for trivial prompts)\");\n\t\t}\n\t}\n\n\treturn { mismatches, notes };\n}\n\n// ── Live probing ──\n\nfunction combineSignals(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal {\n\tconst timeout = AbortSignal.timeout(timeoutMs);\n\treturn signal ? AbortSignal.any([signal, timeout]) : timeout;\n}\n\nasync function probeModel(model: Model<Api>, apiKey: string, options: ProbeOptions): Promise<ProbeResult> {\n\tconst result: ProbeResult = {\n\t\tprovider: model.provider,\n\t\tmodelId: model.id,\n\t\tok: false,\n\t\tlatencyMs: 0,\n\t\tmismatches: [],\n\t\tnotes: [],\n\t};\n\n\tconst reasoningLevel = pickReasoningLevel(model);\n\tconst start = Date.now();\n\n\ttry {\n\t\tconst message = await completeSimple(\n\t\t\tmodel,\n\t\t\t{ messages: [{ role: \"user\", content: \"Reply with the single word: OK\", timestamp: Date.now() }] },\n\t\t\t{\n\t\t\t\tapiKey,\n\t\t\t\tmaxTokens: 64,\n\t\t\t\treasoning: reasoningLevel,\n\t\t\t\tsignal: combineSignals(options.signal, options.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS),\n\t\t\t},\n\t\t);\n\t\tresult.latencyMs = Date.now() - start;\n\n\t\tif (message.stopReason === \"error\" || message.stopReason === \"aborted\") {\n\t\t\tresult.error = message.errorMessage ?? `request ended with stopReason=${message.stopReason}`;\n\t\t} else {\n\t\t\tresult.ok = true;\n\t\t\tif (reasoningLevel !== undefined) {\n\t\t\t\tresult.reasoningProbe = {\n\t\t\t\t\tlevel: reasoningLevel,\n\t\t\t\t\tobservedThinking: message.content.some((c) => c.type === \"thinking\"),\n\t\t\t\t};\n\t\t\t}\n\t\t}\n\t} catch (error) {\n\t\tresult.latencyMs = Date.now() - start;\n\t\tresult.error = error instanceof Error ? error.message : String(error);\n\t}\n\n\tconst claims = evaluateThinkingClaims(model, {\n\t\trequestedLevel: reasoningLevel,\n\t\tobservedThinking: result.reasoningProbe?.observedThinking,\n\t\terror: result.error,\n\t});\n\tresult.mismatches.push(...claims.mismatches);\n\tresult.notes.push(...claims.notes);\n\n\treturn result;\n}\n\n/** Run live probes for all providers with credentials. Never throws. */\nexport async function runModelProbes(options: ProbeOptions = {}): Promise<ProbeReport> {\n\tconst providerFilter = options.provider?.toLowerCase();\n\tconst providers = getProviders().filter((p) => !providerFilter || p.toLowerCase() === providerFilter);\n\n\tconst resolveApiKey = options.resolveApiKey ?? getEnvApiKey;\n\tconst available: { provider: string; apiKey: string }[] = [];\n\tconst skipped: string[] = [];\n\tfor (const provider of providers) {\n\t\tconst apiKey = await resolveApiKey(provider);\n\t\tif (apiKey) {\n\t\t\tavailable.push({ provider, apiKey });\n\t\t} else {\n\t\t\tskipped.push(provider);\n\t\t}\n\t}\n\n\tif (available.length === 0) {\n\t\treturn {\n\t\t\tprobed: [],\n\t\t\tskippedProviders: skipped,\n\t\t\tmessage: options.provider\n\t\t\t\t? `No credentials found for provider \"${options.provider}\". Set the provider's API key environment variable to enable probing.`\n\t\t\t\t: \"No providers configured. Set an API key environment variable (e.g. OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY) to enable model probes.\",\n\t\t};\n\t}\n\n\tconst maxModels = Math.max(1, Math.floor(options.maxModels ?? DEFAULT_MAX_MODELS));\n\tconst groups: { models: Model<Api>[]; apiKey: string }[] = [];\n\tlet budget = maxModels;\n\tfor (const { provider, apiKey } of available) {\n\t\tif (budget <= 0) break;\n\t\tconst all = getModels(provider as KnownProvider) as Model<Api>[];\n\t\tconst models = selectModelsForProvider(all, options.modelPattern).slice(0, budget);\n\t\tif (models.length > 0) {\n\t\t\tgroups.push({ models, apiKey });\n\t\t\tbudget -= models.length;\n\t\t}\n\t}\n\n\tif (groups.length === 0) {\n\t\treturn {\n\t\t\tprobed: [],\n\t\t\tskippedProviders: skipped,\n\t\t\tmessage: options.modelPattern\n\t\t\t\t? `No registry models match pattern \"${options.modelPattern}\" for the configured providers.`\n\t\t\t\t: \"No registry models found for the configured providers.\",\n\t\t};\n\t}\n\n\t// Probe providers in parallel, models within a provider sequentially (rate-limit friendly).\n\tconst results = await Promise.all(\n\t\tgroups.map(async ({ models, apiKey }) => {\n\t\t\tconst groupResults: ProbeResult[] = [];\n\t\t\tfor (const model of models) {\n\t\t\t\tif (options.signal?.aborted) break;\n\t\t\t\tgroupResults.push(await probeModel(model, apiKey, options));\n\t\t\t}\n\t\t\treturn groupResults;\n\t\t}),\n\t);\n\n\treturn { probed: results.flat(), skippedProviders: skipped };\n}\n\n// ── Pure logic: report formatting ──\n\nexport function formatProbeReport(report: ProbeReport): string {\n\tif (report.message) {\n\t\treturn `# Model Probe Report\\n\\n${report.message}`;\n\t}\n\n\tconst ok = report.probed.filter((r) => r.ok);\n\tconst failed = report.probed.filter((r) => !r.ok);\n\tconst mismatchCount = report.probed.reduce((sum, r) => sum + r.mismatches.length, 0);\n\n\tconst lines: string[] = [\n\t\t\"# Model Probe Report\",\n\t\t\"\",\n\t\t`Probed: ${report.probed.length} | OK: ${ok.length} | Failed: ${failed.length} | Mismatches: ${mismatchCount}`,\n\t\t\"\",\n\t\t\"## Results\",\n\t];\n\n\tfor (const r of report.probed) {\n\t\tconst status = r.ok ? `OK (${r.latencyMs}ms)` : `FAILED (${r.latencyMs}ms): ${r.error ?? \"unknown error\"}`;\n\t\tconst thinking = r.reasoningProbe\n\t\t\t? `, thinking(${r.reasoningProbe.level}): ${r.reasoningProbe.observedThinking ? \"observed\" : \"not observed\"}`\n\t\t\t: \"\";\n\t\tlines.push(`- [${r.provider}] ${r.modelId} -- ${status}${thinking}`);\n\t}\n\tlines.push(\"\");\n\n\tif (mismatchCount > 0) {\n\t\tlines.push(\"## Mismatches\");\n\t\tfor (const r of report.probed) {\n\t\t\tfor (const m of r.mismatches) {\n\t\t\t\tlines.push(`- [${r.provider}] ${r.modelId}: ${m}`);\n\t\t\t}\n\t\t}\n\t\tlines.push(\"\");\n\t}\n\n\tconst noteCount = report.probed.reduce((sum, r) => sum + r.notes.length, 0);\n\tif (noteCount > 0) {\n\t\tlines.push(\"## Notes\");\n\t\tfor (const r of report.probed) {\n\t\t\tfor (const n of r.notes) {\n\t\t\t\tlines.push(`- [${r.provider}] ${r.modelId}: ${n}`);\n\t\t\t}\n\t\t}\n\t\tlines.push(\"\");\n\t}\n\n\tif (report.skippedProviders.length > 0) {\n\t\tlines.push(\"## Skipped Providers (no credentials)\");\n\t\tlines.push(report.skippedProviders.join(\", \"));\n\t}\n\n\treturn lines.join(\"\\n\").trimEnd();\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@elyracode/ai",
-	"version": "0.9.7",
+	"version": "0.9.9",
 	"description": "Unified LLM API with automatic model discovery and provider configuration",
 	"type": "module",
 	"main": "./dist/index.js",