@tonyclaw/llm-inspector 1.14.7 → 1.14.9

package/src/proxy/formats/openai/handler.ts

@@ -25,11 +25,41 @@ export const OpenAIFormatHandler: FormatHandler = {
   extractTokens(responseBody: string): TokenUsage {
     const parsed = parseOpenAIResponse(responseBody);
     if (parsed) {
+      // OpenAI puts cached_tokens in usage.prompt_tokens_details (passthrough field)
+      let cacheReadInputTokens: number | null = null;
+      try {
+        const raw: unknown = JSON.parse(responseBody);
+        if (raw !== null && typeof raw === "object" && !Array.isArray(raw)) {
+          const usageDesc = Object.getOwnPropertyDescriptor(raw, "usage");
+          if (
+            usageDesc !== undefined &&
+            typeof usageDesc.value === "object" &&
+            usageDesc.value !== null
+          ) {
+            const detailsDesc = Object.getOwnPropertyDescriptor(
+              usageDesc.value,
+              "prompt_tokens_details",
+            );
+            if (
+              detailsDesc !== undefined &&
+              typeof detailsDesc.value === "object" &&
+              detailsDesc.value !== null
+            ) {
+              const cacheDesc = Object.getOwnPropertyDescriptor(detailsDesc.value, "cached_tokens");
+              if (cacheDesc !== undefined && typeof cacheDesc.value === "number") {
+                cacheReadInputTokens = cacheDesc.value;
+              }
+            }
+          }
+        }
+      } catch {
+        // ignore parse errors
+      }
       return {
         inputTokens: parsed.usage.prompt_tokens ?? null,
         outputTokens: parsed.usage.completion_tokens ?? null,
         cacheCreationInputTokens: null,
-        cacheReadInputTokens: null,
+        cacheReadInputTokens,
       };
     }
     return {
@@ -55,12 +85,8 @@ export const OpenAIFormatHandler: FormatHandler = {
       const json: unknown = JSON.parse(rawBody);
       if (typeof json === "object" && json !== null && !Array.isArray(json)) {
         const keys = Object.keys(json);
-        if (keys.includes("model") && keys.includes("messages")) {
-          // OpenAI doesn't use "system" or "tools" keys at root level
-          if (!keys.includes("system") && !keys.includes("tools")) {
-            return true;
-          }
-        }
+        // OpenAI has `model` and `messages` at the top level, but NOT `system`
+        return keys.includes("model") && keys.includes("messages") && !keys.includes("system");
       }
       return false;
     } catch {

package/src/proxy/formats/openai/schemas.ts

@@ -73,6 +73,7 @@ export const OpenAIRequestSchema = z.object({
   tools: z.array(OpenAIToolDefinition).optional(),
   tool_choice: z
     .union([
+      z.enum(["auto", "none", "required"]),
       z.object({ type: z.literal("auto") }),
       z.object({ type: z.literal("none") }),
       z.object({ type: z.literal("function"), function: z.object({ name: z.string() }) }),

package/src/proxy/formats/openai/stream.ts

@@ -93,6 +93,30 @@ export function extractOpenAIStream(
         promptTokens = chunk.usage.prompt_tokens ?? 0;
         completionTokens = chunk.usage.completion_tokens ?? 0;
         log.inputTokens = promptTokens;
+        // Extract cached_tokens from raw parsed object (passthrough field in usage)
+        if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) {
+          const usageDesc = Object.getOwnPropertyDescriptor(parsed, "usage");
+          if (
+            usageDesc !== undefined &&
+            typeof usageDesc.value === "object" &&
+            usageDesc.value !== null
+          ) {
+            const detailsDesc = Object.getOwnPropertyDescriptor(
+              usageDesc.value,
+              "prompt_tokens_details",
+            );
+            if (
+              detailsDesc !== undefined &&
+              typeof detailsDesc.value === "object" &&
+              detailsDesc.value !== null
+            ) {
+              const cacheDesc = Object.getOwnPropertyDescriptor(detailsDesc.value, "cached_tokens");
+              if (cacheDesc !== undefined && typeof cacheDesc.value === "number") {
+                log.cacheReadInputTokens = cacheDesc.value;
+              }
+            }
+          }
+        }
         usageCaptured = true;
       }
 

package/src/proxy/handler.ts

@@ -5,7 +5,7 @@ import { extractRequestMetadata } from "./schemas";
 import { registry } from "./formats";
 import { findProviderByModel } from "./providers";
 import { getClientInfo } from "./socketTracker";
-import { formatForPath, type FormatHandler } from "./formats";
+import { formatForPath, formatRegistry, type FormatHandler } from "./formats";
 import {
   PROXY_IDENTITY,
   PRESERVE_HEADERS,
@@ -308,6 +308,12 @@ export async function handleProxy(req: Request): Promise<Response> {
   upstreamHeaders.forEach((value, key) => {
     upstreamHeadersObj[key.toLowerCase()] = value;
   });
+  // Detect the true format from the request body for accurate UI display.
+  // The path-based format (formatHandler.format) drives routing, but the body
+  // structure determines whether it's actually Anthropic or OpenAI.
+  const bodyFormat = formatRegistry.detectFormat(requestBody);
+  const displayApiFormat = bodyFormat !== "unknown" ? bodyFormat : formatHandler.format;
+
   const log = await createLog(
     req.method,
     parsed.apiPath,
@@ -316,7 +322,7 @@ export async function handleProxy(req: Request): Promise<Response> {
     clientInfo,
     rawHeaders,
     upstreamHeadersObj,
-    formatHandler.format,
+    displayApiFormat,
     model,
     sessionId,
     preAcquiredId,

package/src/proxy/schemas.ts

@@ -111,9 +111,12 @@ function detectFormat(rawBody: string | null): RequestFormat {
   try {
     const json: unknown = JSON.parse(rawBody);
     if (typeof json === "object" && json !== null && !Array.isArray(json)) {
-      const keys = Object.keys(json);
-      if (keys.includes("model") && keys.includes("messages")) {
-        if (keys.includes("system") || keys.includes("tools")) {
+      const hasModel = safeGetProperty(json, "model") !== undefined;
+      const hasMessages = safeGetProperty(json, "messages") !== undefined;
+      if (hasModel && hasMessages) {
+        // Anthropic requests put `system` as a top-level key; OpenAI does not.
+        // Both formats can have `tools`, so we check `system` as the discriminator.
+        if (safeGetProperty(json, "system") !== undefined) {
           return "anthropic";
         }
         const msgVal = safeGetProperty(json, "messages");