@tonyclaw/llm-inspector 1.14.1 → 1.14.2

package/src/proxy/providers.ts

@@ -1,7 +1,7 @@
 import { z } from "zod";
 import Conf from "conf";
 import { randomUUID } from "crypto";
-import { mkdirSync } from "node:fs";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { logger } from "./logger";
 import { getDataDir, hasExplicitDataDir } from "./dataDir";
@@ -119,6 +119,79 @@ function migrateProviders(): void {
 // Run migration on module load
 migrateProviders();
 
+/**
+ * Migrate: merge providers with identical (apiKey, anthropicBaseUrl, openaiBaseUrl)
+ * into multi-model providers. Existing single-model providers get their `model`
+ * value promoted to `models: [model]`.
+ */
+function migrateMultiModel(): void {
+  const providers = store.get("providers", []);
+  if (providers.length === 0) return;
+  let changed = false;
+
+  // Step 1: Promote single model to models array for any provider missing models
+  const promoted = providers.map((p) => {
+    if (p.models !== undefined && p.models.length > 0) return p;
+    const models = p.model !== undefined && p.model !== "" ? [p.model] : [];
+    changed = true;
+    return { ...p, models };
+  });
+
+  // Step 2: Merge providers with same (apiKey, anthropicBaseUrl, openaiBaseUrl)
+  const groups = new Map<string, ProviderConfig[]>();
+  for (const p of promoted) {
+    const key = `${p.apiKey}::${p.anthropicBaseUrl ?? ""}::${p.openaiBaseUrl ?? ""}`;
+    const group = groups.get(key);
+    if (group !== undefined) {
+      group.push(p);
+    } else {
+      groups.set(key, [p]);
+    }
+  }
+
+  const merged: ProviderConfig[] = [];
+  for (const group of groups.values()) {
+    const first = group[0];
+    if (group.length === 1 && first !== undefined) {
+      merged.push(first);
+      continue;
+    }
+    changed = true;
+    // Merge: keep the earliest provider's metadata, combine all models (deduplicated)
+    const sorted = group.toSorted(
+      (a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime(),
+    );
+    const earliest = sorted[0];
+    if (earliest === undefined) continue;
+    const allModels = new Set<string>();
+    for (const p of sorted) {
+      for (const m of p.models ?? []) {
+        if (m !== "") allModels.add(m);
+      }
+    }
+    // Take first non-empty source
+    const mergedSource = sorted.find((p) => p.source !== undefined)?.source;
+    merged.push({
+      ...earliest,
+      models: [...allModels],
+      source: mergedSource,
+      updatedAt: new Date().toISOString(),
+    });
+  }
+
+  if (changed && merged.length > 0) {
+    // Safety backup before writing
+    try {
+      writeFileSync(`${store.path}.bak`, readFileSync(store.path, "utf-8"));
+    } catch {
+      // best-effort, ignore
+    }
+    store.set("providers", merged);
+  }
+}
+
+migrateMultiModel();
+
 // Override with JSON env var if provided (for testing)
 const providersJson = process.env["LLM_INSPECTOR_PROVIDERS_JSON"];
 if (providersJson !== undefined) {
@@ -155,7 +228,7 @@ export function addProvider(
   apiKey: string,
   format?: "anthropic" | "openai",
   baseUrl?: string,
-  model?: string,
+  models?: string[],
   authHeader?: "bearer" | "x-api-key",
   apiDocsUrl?: string,
   anthropicBaseUrl?: string,
@@ -163,11 +236,37 @@ export function addProvider(
   source?: "company" | "personal",
 ): ProviderConfig {
   const providers = getProviders();
+  const normalizedKey = normalizeApiKey(apiKey);
+  const resolvedAnthropicUrl =
+    anthropicBaseUrl ?? (format === "anthropic" && baseUrl !== undefined ? baseUrl : "");
+  const resolvedOpenaiUrl =
+    openaiBaseUrl ?? (format === "openai" && baseUrl !== undefined ? baseUrl : "");
   const now = new Date().toISOString();
+
+  // If a provider with the same (apiKey, anthropicBaseUrl, openaiBaseUrl) already exists,
+  // merge the new models into it instead of creating a duplicate.
+  const existing = providers.find(
+    (p) =>
+      p.apiKey === normalizedKey &&
+      (p.anthropicBaseUrl ?? "") === (resolvedAnthropicUrl ?? "") &&
+      (p.openaiBaseUrl ?? "") === (resolvedOpenaiUrl ?? ""),
+  );
+  if (existing) {
+    const newModels = (models ?? []).filter((m) => m !== "");
+    if (newModels.length > 0) {
+      const mergedModels = new Set(existing.models ?? []);
+      for (const m of newModels) mergedModels.add(m);
+      existing.models = [...mergedModels];
+      existing.updatedAt = now;
+      store.set("providers", providers);
+    }
+    return existing;
+  }
+
   const newProvider: ProviderConfig = {
     id: randomUUID(),
     name,
-    apiKey: normalizeApiKey(apiKey),
+    apiKey: normalizedKey,
     format:
       format ??
       (anthropicBaseUrl !== undefined
@@ -176,14 +275,13 @@ export function addProvider(
           ? "openai"
           : undefined),
     baseUrl,
-    model,
+    models: models ?? [],
     authHeader: authHeader ?? "bearer",
     apiDocsUrl,
     createdAt: now,
     updatedAt: now,
-    anthropicBaseUrl:
-      anthropicBaseUrl ?? (format === "anthropic" && baseUrl !== undefined ? baseUrl : ""),
-    openaiBaseUrl: openaiBaseUrl ?? (format === "openai" && baseUrl !== undefined ? baseUrl : ""),
+    anthropicBaseUrl: resolvedAnthropicUrl,
+    openaiBaseUrl: resolvedOpenaiUrl,
     source,
   };
   providers.push(newProvider);
@@ -203,7 +301,7 @@ export function updateProvider(
     id: existing.id,
     name: updates.name ?? existing.name,
     apiKey: updates.apiKey !== undefined ? normalizeApiKey(updates.apiKey) : existing.apiKey,
-    model: updates.model !== undefined ? updates.model : existing.model,
+    models: updates.models !== undefined ? updates.models : existing.models,
     format: updates.format ?? existing.format,
     baseUrl: updates.baseUrl !== undefined ? updates.baseUrl : existing.baseUrl,
     authHeader: updates.authHeader ?? existing.authHeader,
@@ -371,21 +469,21 @@ export function findProviderByModel(model: string): ProviderConfig | null {
     if (modelLower.startsWith(providerPrefix)) {
       return provider;
     }
-    // Strategy 2: match against provider.model field with normalization
-    if (
-      provider.model !== undefined &&
-      provider.model !== "" &&
-      modelNormalized === normalizeModelName(provider.model)
-    ) {
-      return provider;
+    // Strategy 2: match against provider.models array with normalization
+    const modelList = provider.models ?? (provider.model !== undefined ? [provider.model] : []);
+    for (const m of modelList) {
+      if (m !== "" && modelNormalized === normalizeModelName(m)) {
+        return provider;
+      }
     }
     // Strategy 3: fallback - concatenate provider name with model part and compare
-    // This handles cases like "MiniMax" + "M3" → "MiniMax-M3" when case differs
-    if (provider.model !== undefined && provider.model !== "") {
-      const modelPart = modelNormalized.replace(normalizeModelName(provider.name), "");
-      const concatenated = normalizeModelName(provider.name) + modelPart;
-      if (modelNormalized === concatenated) {
-        return provider;
+    for (const m of modelList) {
+      if (m !== "") {
+        const modelPart = modelNormalized.replace(normalizeModelName(provider.name), "");
+        const concatenated = normalizeModelName(provider.name) + modelPart;
+        if (modelNormalized === concatenated) {
+          return provider;
+        }
       }
     }
   }

package/src/routes/api/providers.$providerId.test.log.ts

@@ -0,0 +1,293 @@
+import { createFileRoute } from "@tanstack/react-router";
+import { getProvider, getModelUsageName } from "../../proxy/providers";
+import { appendLogEntry } from "../../proxy/logger";
+import { addTestLogEntry } from "../../proxy/store";
+import {
+  ProviderTestResultsSchema,
+  type ProviderTestResult as TestResult,
+  type ProviderTestState,
+} from "../../lib/providerTestContract";
+
+function hasSuccessField(result: ProviderTestState): result is TestResult {
+  return Object.prototype.hasOwnProperty.call(result, "success");
+}
+
+function createTestLogEntry(
+  providerName: string,
+  path: string,
+  body: string,
+  upstreamUrl: string,
+  result: TestResult,
+  isTest: boolean,
+): Record<string, unknown> {
+  return {
+    timestamp: new Date().toISOString(),
+    id: `test-${Date.now()}`,
+    method: "POST",
+    path,
+    model: isTest ? result.model : undefined,
+    sessionId: null,
+    rawRequestBody: body,
+    responseStatus: result.success ? 200 : 500,
+    responseText: result.rawResponse ?? JSON.stringify(result),
+    inputTokens: result.inputTokens ?? null,
+    outputTokens: result.outputTokens ?? null,
+    elapsedMs: result.latencyMs ?? 0,
+    streaming: result.streaming ?? false,
+    userAgent: "provider-test",
+    origin: null,
+    upstreamUrl,
+    error: result.success ? null : (result.error?.message ?? String(result.error)),
+    isTest: true,
+    providerName,
+    headers: result.requestHeaders ?? {},
+  };
+}
+
+async function logModelResults(
+  displayName: string,
+  providerName: string,
+  anthropicUrl: string | undefined,
+  openaiUrl: string | undefined,
+  modelResults: {
+    anthropic: { nonStreaming: ProviderTestState; streaming: ProviderTestState };
+    openai: { nonStreaming: ProviderTestState; streaming: ProviderTestState };
+  },
+): Promise<void> {
+  const usageModel = getModelUsageName(displayName, providerName);
+
+  if (anthropicUrl !== undefined) {
+    const nsResult = modelResults.anthropic.nonStreaming;
+    const sResult = modelResults.anthropic.streaming;
+    if (hasSuccessField(nsResult) && hasSuccessField(sResult)) {
+      const nonStreamingResult = nsResult;
+      const streamingResult = sResult;
+
+      const requestBody = JSON.stringify({
+        model: usageModel,
+        messages: [{ role: "user", content: "say hello and briefly introduce yourself" }],
+        max_tokens: 1024,
+      });
+      const upstreamUrl = `${anthropicUrl}/v1/messages`;
+
+      await addTestLogEntry({
+        timestamp: new Date().toISOString(),
+        method: "POST",
+        path: "/v1/messages",
+        model: nonStreamingResult.model ?? displayName,
+        sessionId: null,
+        rawRequestBody: requestBody,
+        responseStatus: nonStreamingResult.success ? 200 : 500,
+        responseText: nonStreamingResult.rawResponse ?? JSON.stringify(nonStreamingResult),
+        inputTokens: nonStreamingResult.inputTokens ?? null,
+        outputTokens: nonStreamingResult.outputTokens ?? null,
+        cacheCreationInputTokens: nonStreamingResult.cacheCreationInputTokens ?? null,
+        cacheReadInputTokens: nonStreamingResult.cacheReadInputTokens ?? null,
+        elapsedMs: nonStreamingResult.latencyMs ?? 0,
+        streaming: false,
+        userAgent: "provider-test",
+        origin: null,
+        apiFormat: "anthropic",
+        isTest: true,
+        providerName,
+        headers: nonStreamingResult.requestHeaders ?? {},
+      });
+
+      appendLogEntry(
+        createTestLogEntry(
+          providerName,
+          "/v1/messages",
+          requestBody,
+          upstreamUrl,
+          nonStreamingResult,
+          true,
+        ),
+      );
+
+      const streamingRequestBody = JSON.stringify({
+        model: usageModel,
+        messages: [{ role: "user", content: "say hello" }],
+        max_tokens: 256,
+        stream: true,
+      });
+
+      await addTestLogEntry({
+        timestamp: new Date().toISOString(),
+        method: "POST",
+        path: "/v1/messages",
+        model: streamingResult.model ?? displayName,
+        sessionId: null,
+        rawRequestBody: streamingRequestBody,
+        responseStatus: streamingResult.success ? 200 : 500,
+        responseText: streamingResult.rawResponse ?? JSON.stringify(streamingResult),
+        inputTokens: streamingResult.inputTokens ?? null,
+        outputTokens: streamingResult.outputTokens ?? null,
+        cacheCreationInputTokens: streamingResult.cacheCreationInputTokens ?? null,
+        cacheReadInputTokens: streamingResult.cacheReadInputTokens ?? null,
+        elapsedMs: streamingResult.latencyMs ?? 0,
+        streaming: true,
+        streamingChunks: streamingResult.streamingChunks,
+        userAgent: "provider-test",
+        origin: null,
+        apiFormat: "anthropic",
+        isTest: true,
+        providerName,
+        headers: streamingResult.requestHeaders ?? {},
+      });
+
+      appendLogEntry(
+        createTestLogEntry(
+          providerName,
+          "/v1/messages",
+          streamingRequestBody,
+          upstreamUrl,
+          streamingResult,
+          true,
+        ),
+      );
+    }
+  }
+
+  if (openaiUrl !== undefined) {
+    const nsResult = modelResults.openai.nonStreaming;
+    const sResult = modelResults.openai.streaming;
+    if (hasSuccessField(nsResult) && hasSuccessField(sResult)) {
+      const nonStreamingResult = nsResult;
+      const streamingResult = sResult;
+
+      const requestBody = JSON.stringify({
+        model: usageModel,
+        messages: [{ role: "user", content: "say hello and briefly introduce yourself" }],
+        max_tokens: 1024,
+      });
+      const upstreamUrl = `${openaiUrl}/v1/chat/completions`;
+
+      await addTestLogEntry({
+        timestamp: new Date().toISOString(),
+        method: "POST",
+        path: "/v1/chat/completions",
+        model: nonStreamingResult.model ?? displayName,
+        sessionId: null,
+        rawRequestBody: requestBody,
+        responseStatus: nonStreamingResult.success ? 200 : 500,
+        responseText: nonStreamingResult.rawResponse ?? JSON.stringify(nonStreamingResult),
+        inputTokens: nonStreamingResult.inputTokens ?? null,
+        outputTokens: nonStreamingResult.outputTokens ?? null,
+        cacheCreationInputTokens: null,
+        cacheReadInputTokens: null,
+        elapsedMs: nonStreamingResult.latencyMs ?? 0,
+        streaming: false,
+        userAgent: "provider-test",
+        origin: null,
+        apiFormat: "openai",
+        isTest: true,
+        providerName,
+        headers: nonStreamingResult.requestHeaders ?? {},
+      });
+
+      appendLogEntry(
+        createTestLogEntry(
+          providerName,
+          "/v1/chat/completions",
+          requestBody,
+          upstreamUrl,
+          nonStreamingResult,
+          true,
+        ),
+      );
+
+      const streamingRequestBody = JSON.stringify({
+        model: usageModel,
+        messages: [{ role: "user", content: "say hello" }],
+        max_tokens: 256,
+        stream: true,
+      });
+
+      await addTestLogEntry({
+        timestamp: new Date().toISOString(),
+        method: "POST",
+        path: "/v1/chat/completions",
+        model: streamingResult.model ?? displayName,
+        sessionId: null,
+        rawRequestBody: streamingRequestBody,
+        responseStatus: streamingResult.success ? 200 : 500,
+        responseText: streamingResult.rawResponse ?? JSON.stringify(streamingResult),
+        inputTokens: streamingResult.inputTokens ?? null,
+        outputTokens: streamingResult.outputTokens ?? null,
+        cacheCreationInputTokens: null,
+        cacheReadInputTokens: null,
+        elapsedMs: streamingResult.latencyMs ?? 0,
+        streaming: true,
+        streamingChunks: streamingResult.streamingChunks,
+        userAgent: "provider-test",
+        origin: null,
+        apiFormat: "openai",
+        isTest: true,
+        providerName,
+        headers: streamingResult.requestHeaders ?? {},
+      });
+
+      appendLogEntry(
+        createTestLogEntry(
+          providerName,
+          "/v1/chat/completions",
+          streamingRequestBody,
+          upstreamUrl,
+          streamingResult,
+          true,
+        ),
+      );
+    }
+  }
+}
+
+export const Route = createFileRoute("/api/providers/$providerId/test/log")({
+  server: {
+    handlers: {
+      POST: async ({ params, request }: { params: { providerId: string }; request: Request }) => {
+        const provider = getProvider(params.providerId);
+        if (!provider) {
+          return Response.json({ error: "Provider not found" }, { status: 404 });
+        }
+
+        let body: unknown;
+        try {
+          body = await request.json();
+        } catch {
+          return Response.json({ error: "Invalid JSON body" }, { status: 400 });
+        }
+
+        const parsed = ProviderTestResultsSchema.safeParse(body);
+        if (!parsed.success) {
+          return Response.json(
+            { error: `Invalid test results: ${parsed.error.message}` },
+            { status: 400 },
+          );
+        }
+
+        const results = parsed.data;
+        const anthropicUrl =
+          provider.anthropicBaseUrl !== undefined && provider.anthropicBaseUrl.length > 0
+            ? provider.anthropicBaseUrl
+            : undefined;
+        const openaiUrl =
+          provider.openaiBaseUrl !== undefined && provider.openaiBaseUrl.length > 0
+            ? provider.openaiBaseUrl
+            : undefined;
+
+        const entries: Promise<void>[] = [];
+        if (results.models !== undefined) {
+          for (const [modelName, modelResult] of Object.entries(results.models)) {
+            entries.push(
+              logModelResults(modelName, provider.name, anthropicUrl, openaiUrl, modelResult),
+            );
+          }
+        }
+
+        await Promise.all(entries);
+
+        return Response.json({ logged: entries.length });
+      },
+    },
+  },
+});

package/src/routes/api/providers.$providerId.ts

@@ -7,7 +7,8 @@ const ProviderUpdateSchema = z.object({
   apiKey: z.string().min(1, "API key is required").optional(),
   format: z.enum(["anthropic", "openai"]).optional(),
   baseUrl: z.string().min(1, "Base URL is required").optional(),
-  model: z.string().min(1, "Model is required").optional(),
+  model: z.string().optional(),
+  models: z.array(z.string()).optional(),
   authHeader: z.enum(["bearer", "x-api-key"]).optional(),
   anthropicBaseUrl: z.string().optional(),
   openaiBaseUrl: z.string().optional(),

package/src/routes/api/providers.ts

@@ -8,7 +8,8 @@ const ProviderInputSchema = z.object({
   format: z.enum(["anthropic", "openai"]).optional(),
   anthropicBaseUrl: z.string().optional(),
   openaiBaseUrl: z.string().optional(),
-  model: z.string().min(1, "Model is required"),
+  models: z.array(z.string()).min(1, "At least one model is required"),
+  model: z.string().optional(),
   authHeader: z.enum(["bearer", "x-api-key"]).optional().default("bearer"),
   apiDocsUrl: z.string().optional(),
   source: z.enum(["company", "personal"]).optional(),
@@ -30,7 +31,7 @@ export const Route = createFileRoute("/api/providers")({
           parsed.data.apiKey,
           parsed.data.format,
           undefined, // baseUrl (legacy) — use format-specific URLs instead
-          parsed.data.model,
+          parsed.data.models,
           parsed.data.authHeader,
           parsed.data.apiDocsUrl,
           parsed.data.anthropicBaseUrl,