@tonyclaw/llm-inspector 1.6.0

package/src/proxy/formats/openai/handler.ts

@@ -0,0 +1,70 @@
+import type { FormatHandler, ParsedRequest } from "../handler";
+import type { CapturedLog, TokenUsage } from "../../schemas";
+import { OpenAIRequestSchema, OpenAIResponseSchema, parseOpenAIResponse } from "./schemas";
+import { extractOpenAIStream } from "./stream";
+
+export const OpenAIFormatHandler: FormatHandler = {
+  format: "openai",
+
+  parseRequest(rawBody: string, headers?: Headers): ParsedRequest | null {
+    try {
+      const json: unknown = JSON.parse(rawBody);
+      const result = OpenAIRequestSchema.safeParse(json);
+      if (result.success) {
+        return {
+          model: result.data.model,
+          sessionId: headers?.get("x-session-affinity") ?? null,
+        };
+      }
+      return null;
+    } catch {
+      return null;
+    }
+  },
+
+  extractTokens(responseBody: string): TokenUsage {
+    const parsed = parseOpenAIResponse(responseBody);
+    if (parsed) {
+      return {
+        inputTokens: parsed.usage.prompt_tokens,
+        outputTokens: parsed.usage.completion_tokens,
+        cacheCreationInputTokens: null,
+        cacheReadInputTokens: null,
+      };
+    }
+    return {
+      inputTokens: null,
+      outputTokens: null,
+      cacheCreationInputTokens: null,
+      cacheReadInputTokens: null,
+    };
+  },
+
+  extractStream(
+    raw: string,
+    log: CapturedLog,
+    fallbackModel?: string,
+    collectChunks?: boolean,
+  ): string {
+    return extractOpenAIStream(raw, log, fallbackModel, collectChunks);
+  },
+
+  detectFormat(rawBody: string | null): boolean {
+    if (rawBody === null) return false;
+    try {
+      const json: unknown = JSON.parse(rawBody);
+      if (typeof json === "object" && json !== null && !Array.isArray(json)) {
+        const keys = Object.keys(json);
+        if (keys.includes("model") && keys.includes("messages")) {
+          // OpenAI doesn't use "system" or "tools" keys at root level
+          if (!keys.includes("system") && !keys.includes("tools")) {
+            return true;
+          }
+        }
+      }
+      return false;
+    } catch {
+      return false;
+    }
+  },
+};

package/src/proxy/formats/openai/index.ts

@@ -0,0 +1,25 @@
+// Register format handler
+import "./register";
+
+// Schemas
+export {
+  OpenAIRequestSchema,
+  OpenAIResponseSchema,
+  OpenAISSERawChunkSchema,
+  OpenAIMessage,
+  OpenAIMessageContent,
+  OpenAIToolDefinition,
+  OpenAIChoice,
+  OpenAIChoiceDelta,
+  parseOpenAIResponse,
+} from "./schemas";
+
+// Handler
+export { OpenAIFormatHandler } from "./handler";
+
+// Stream extraction
+export { extractOpenAIStream } from "./stream";
+
+// Providers
+export { openaiProvider } from "./provider";
+export { alibabaProvider } from "./alibabaProvider";

package/src/proxy/formats/openai/provider.ts

@@ -0,0 +1,50 @@
+import type { CapturedLog } from "../../schemas";
+import type { ProviderProtocol } from "../protocol";
+import { registry } from "../providers";
+import { extractOpenAIStream } from "./stream";
+
+const DEFAULT_OPENAI_UPSTREAM = "https://api.openai.com/v1";
+
+/**
+ * OpenAI Provider Implementation
+ */
+export const openaiProvider: ProviderProtocol = {
+  name: "openai",
+
+  matches(model: string): boolean {
+    const modelLower = model.toLowerCase();
+    // Match OpenAI models
+    if (modelLower.startsWith("openai-")) {
+      return true;
+    }
+    // Also match common OpenAI model names
+    if (
+      modelLower.includes("gpt-") ||
+      modelLower.includes("o1") ||
+      modelLower.includes("o2") ||
+      modelLower.includes("o3")
+    ) {
+      return true;
+    }
+    return false;
+  },
+
+  getUpstreamBase(_isChatCompletions: boolean, providerConfig?: { baseUrl?: string }): string {
+    if (providerConfig?.baseUrl !== undefined) {
+      return providerConfig.baseUrl;
+    }
+    return DEFAULT_OPENAI_UPSTREAM;
+  },
+
+  extractStream(
+    raw: string,
+    log: CapturedLog,
+    fallbackModel?: string,
+    collectChunks?: boolean,
+  ): string {
+    return extractOpenAIStream(raw, log, fallbackModel, collectChunks);
+  },
+};
+
+// Register the provider
+registry.register(openaiProvider);

package/src/proxy/formats/openai/register.ts

@@ -0,0 +1,4 @@
+import { formatRegistry } from "../registry";
+import { OpenAIFormatHandler } from "./handler";
+
+formatRegistry.register(OpenAIFormatHandler);

package/src/proxy/formats/openai/schemas.ts

@@ -0,0 +1,150 @@
+import { z } from "zod";
+import { JsonValueSchema } from "../jsonSchema";
+
+// OpenAI Chat Completions Schemas
+
+export const OpenAIMessageContent = z.union([
+  z.string(),
+  z.array(
+    z.discriminatedUnion("type", [
+      z.object({ type: z.literal("text"), text: z.string() }),
+      z.object({
+        type: z.literal("image_url"),
+        image_url: z.object({ url: z.string(), detail: z.string().optional() }),
+      }),
+    ]),
+  ),
+]);
+
+export const OpenAIMessage = z.object({
+  role: z.enum(["system", "user", "assistant", "tool"]),
+  content: OpenAIMessageContent,
+  name: z.string().optional(),
+  reasoning_content: z.string().optional(),
+});
+
+export const OpenAIFunctionCall = z.object({
+  name: z.string(),
+  arguments: z.string(),
+});
+
+const OpenAIMessageWithFunctionCall = OpenAIMessage.extend({
+  content: z
+    .union([z.string(), z.array(z.object({ type: z.literal("text"), text: z.string() }))])
+    .optional(),
+  function_call: OpenAIFunctionCall.optional(),
+});
+
+export const OpenAIToolDefinition = z.object({
+  type: z.literal("function"),
+  function: z.object({
+    name: z.string(),
+    description: z.string().optional(),
+    parameters: z.record(z.string(), JsonValueSchema),
+  }),
+});
+
+export const OpenAIRequestSchema = z.object({
+  model: z.string(),
+  messages: z.array(OpenAIMessage),
+  temperature: z.number().optional(),
+  max_tokens: z.number().optional(),
+  stream: z.boolean().optional(),
+  tools: z.array(OpenAIToolDefinition).optional(),
+  tool_choice: z
+    .union([
+      z.object({ type: z.literal("auto") }),
+      z.object({ type: z.literal("none") }),
+      z.object({ type: z.literal("function"), function: z.object({ name: z.string() }) }),
+    ])
+    .optional(),
+  user: z.string().optional(),
+});
+
+export const OpenAIChoiceDelta = z.object({
+  role: z.enum(["assistant"]).optional(),
+  content: z.string().nullable().optional(),
+  reasoning_content: z.string().nullable().optional(),
+  function_call: z
+    .object({ name: z.string().optional(), arguments: z.string().optional() })
+    .optional(),
+  tool_calls: z
+    .array(
+      z.object({
+        index: z.number(),
+        id: z.string().optional(),
+        type: z.literal("function").optional(),
+        function: z.object({
+          name: z.string().optional(),
+          arguments: z.string().optional(),
+        }),
+      }),
+    )
+    .optional(),
+});
+
+export const OpenAIChoice = z.object({
+  index: z.number(),
+  message: z
+    .object({
+      role: z.enum(["assistant"]),
+      content: z.string().nullable(),
+      reasoning_content: z.string().optional(),
+      function_call: z.object({ name: z.string(), arguments: z.string() }).optional(),
+    })
+    .optional(),
+  delta: OpenAIChoiceDelta.optional(),
+  finish_reason: z.string().nullable(),
+});
+
+export const OpenAIResponseSchema = z.object({
+  id: z.string(),
+  object: z.literal("chat.completion"),
+  created: z.number(),
+  model: z.string(),
+  choices: z.array(OpenAIChoice),
+  usage: z.object({
+    prompt_tokens: z.number(),
+    completion_tokens: z.number(),
+    total_tokens: z.number(),
+  }),
+});
+
+// Schema for actual OpenAI SSE chunk format (not wrapped in event/data)
+export const OpenAISSERawChunkSchema = z.object({
+  id: z.string(),
+  object: z.literal("chat.completion.chunk"),
+  created: z.number(),
+  model: z.string(),
+  choices: z.array(
+    z.object({
+      index: z.number(),
+      delta: OpenAIChoiceDelta,
+      finish_reason: z.string().nullable().optional(),
+    }),
+  ),
+  usage: z
+    .object({
+      prompt_tokens: z.number(),
+      completion_tokens: z.number(),
+      total_tokens: z.number(),
+    })
+    .nullable()
+    .optional(),
+});
+
+export type OpenAIResponse = z.infer<typeof OpenAIResponseSchema>;
+
+/**
+ * Parse an OpenAI response body.
+ */
+export function parseOpenAIResponse(rawBody: string): OpenAIResponse | null {
+  try {
+    const json: unknown = JSON.parse(rawBody);
+    const result = OpenAIResponseSchema.safeParse(json);
+    if (result.success) return result.data;
+    return null;
+  } catch {
+    return null;
+  }
+}

package/src/proxy/formats/openai/stream.ts

@@ -0,0 +1,153 @@
+import type { CapturedLog, JsonValue } from "../../schemas";
+import { JsonValueSchema } from "../jsonSchema";
+import { OpenAISSERawChunkSchema } from "./schemas";
+
+/**
+ * Extract OpenAI streaming response and return normalized JSON.
+ * Optionally collects SSE chunks into log.streamingChunks for debugging/display.
+ */
+export function extractOpenAIStream(
+  raw: string,
+  log: CapturedLog,
+  fallbackModel?: string,
+  collectChunks: boolean = true,
+): string {
+  let id = "";
+  let model = "";
+  let completionContent = "";
+  let reasoningContent = "";
+  let finishReason: string | null = null;
+  let promptTokens = 0;
+  let completionTokens = 0;
+  let usageCaptured = false;
+  let started = false;
+  const toolCalls: Array<{
+    index: number;
+    id?: string;
+    type: "function";
+    function: { name: string; arguments: string };
+  }> = [];
+
+  // Chunk collection state
+  const MAX_CHUNKS = 1000;
+  let chunkIndex = 0;
+  let streamStartMs = 0;
+  const chunks: Array<{ index: number; timestamp: number; type: string; data: JsonValue }> = [];
+
+  for (const line of raw.split("\n")) {
+    if (!line.startsWith("data: ")) continue;
+    const dataStr = line.slice(6).trim();
+    if (dataStr === "[DONE]") break;
+
+    try {
+      const parsed: unknown = JSON.parse(dataStr);
+      const chunkResult = OpenAISSERawChunkSchema.safeParse(parsed);
+      if (!chunkResult.success) continue;
+      const chunk = chunkResult.data;
+
+      // Track stream start for timestamps
+      if (chunkIndex === 0) streamStartMs = Date.now();
+
+      // Collect chunks if enabled
+      if (collectChunks === true && chunks.length < MAX_CHUNKS) {
+        const jsonResult = JsonValueSchema.safeParse(chunk);
+        if (jsonResult.success) {
+          chunks.push({
+            index: chunkIndex,
+            timestamp: Date.now() - streamStartMs,
+            type: "chat.completion.chunk",
+            data: jsonResult.data,
+          });
+        }
+      }
+      chunkIndex++;
+
+      if (!started) {
+        id = chunk.id;
+        model = chunk.model;
+        started = true;
+      }
+
+      if (!usageCaptured && chunk.usage !== undefined && chunk.usage !== null) {
+        promptTokens = chunk.usage.prompt_tokens;
+        completionTokens = chunk.usage.completion_tokens;
+        log.inputTokens = promptTokens;
+        usageCaptured = true;
+      }
+
+      for (const choice of chunk.choices) {
+        const delta = choice.delta;
+        if (delta.content !== undefined && delta.content !== null) {
+          completionContent += delta.content;
+        }
+        if (delta.reasoning_content !== undefined && delta.reasoning_content !== null) {
+          reasoningContent += delta.reasoning_content;
+        }
+        if (choice.finish_reason !== undefined && choice.finish_reason !== null) {
+          finishReason = choice.finish_reason;
+        }
+        // Accumulate tool_calls
+        if (delta.tool_calls !== undefined && delta.tool_calls !== null) {
+          for (const tc of delta.tool_calls) {
+            // Find or create tool call entry by index
+            let existing = toolCalls.find((t) => t.index === tc.index);
+            if (!existing) {
+              existing = {
+                index: tc.index,
+                type: "function",
+                function: { name: "", arguments: "" },
+              };
+              toolCalls.push(existing);
+            }
+            if (tc.id !== undefined) existing.id = tc.id;
+            if (tc.function?.name !== undefined) existing.function.name += tc.function.name;
+            if (tc.function?.arguments !== undefined)
+              existing.function.arguments += tc.function.arguments;
+          }
+        }
+      }
+    } catch {
+      // non-JSON SSE line, skip
+    }
+  }
+
+  // Store collected chunks on the log
+  if (collectChunks === true) {
+    log.streamingChunks = {
+      chunks,
+      truncated: chunkIndex > MAX_CHUNKS,
+    };
+  }
+
+  if (!usageCaptured) {
+    completionTokens = Math.ceil(completionContent.length / 4);
+  }
+
+  log.outputTokens = completionTokens;
+
+  const message: Record<string, unknown> = {
+    role: "assistant",
+    content: completionContent,
+  };
+  if (reasoningContent) message.reasoning_content = reasoningContent;
+  if (toolCalls.length > 0) message.tool_calls = toolCalls;
+
+  return JSON.stringify({
+    id,
+    object: "chat.completion",
+    created: Math.floor(Date.now() / 1000),
+    model: model !== "" ? model : (fallbackModel ?? ""),
+    choices: [
+      {
+        index: 0,
+        message,
+        finish_reason: finishReason,
+      },
+    ],
+    usage: {
+      prompt_tokens: promptTokens,
+      completion_tokens: completionTokens,
+      total_tokens: promptTokens + completionTokens,
+    },
+  });
+}

package/src/proxy/formats/protocol.ts

@@ -0,0 +1,50 @@
+import type { CapturedLog, RequestFormat } from "../schemas";
+
+/**
+ * Provider Protocol Interface
+ *
+ * Defines the contract that all providers must implement to support
+ * different LLM APIs (Anthropic, OpenAI, etc.).
+ */
+export type ProviderProtocol = {
+  /** Provider identifier (e.g., "anthropic", "openai") */
+  name: string;
+
+  /**
+   * Check if this provider handles the given model.
+   * @param model - Model name from request
+   * @returns true if this provider should handle the model
+   */
+  matches(model: string): boolean;
+
+  /**
+   * Get the upstream base URL for this provider.
+   * @param isChatCompletions - Whether the request uses OpenAI chat completions format
+   * @param providerConfig - Optional provider configuration for custom base URL
+   * @returns Upstream base URL
+   */
+  getUpstreamBase(isChatCompletions: boolean, providerConfig?: { baseUrl?: string }): string;
+
+  /**
+   * Extract structured response from streaming data.
+   * @param raw - Raw SSE stream data
+   * @param log - Captured log entry to update
+   * @param fallbackModel - Fallback model name if not in stream
+   * @param collectChunks - Whether to collect SSE chunks into log.streamingChunks
+   * @returns JSON string of structured response
+   */
+  extractStream(
+    raw: string,
+    log: CapturedLog,
+    fallbackModel?: string,
+    collectChunks?: boolean,
+  ): string;
+
+  /**
+   * Optional request transformation hook.
+   * @param body - Original request body
+   * @param apiFormat - Request format (anthropic/openai/unknown)
+   * @returns Transformed body or null to use original
+   */
+  transformRequest?(body: string, apiFormat: RequestFormat): string | null;
+};

package/src/proxy/formats/providerRegistry.ts

@@ -0,0 +1,51 @@
+import type { ProviderProtocol } from "./protocol";
+
+/**
+ * Provider Registry
+ *
+ * Singleton registry for provider implementations.
+ * Providers register themselves at module load time.
+ */
+class ProviderRegistryImpl {
+  private providers: ProviderProtocol[] = [];
+
+  /**
+   * Register a provider with the registry.
+   * @param provider - Provider implementation to register
+   */
+  register(provider: ProviderProtocol): void {
+    this.providers.push(provider);
+  }
+
+  /**
+   * Find the first provider that matches the given model.
+   * @param model - Model name to match
+   * @returns Matching provider or null if none found
+   */
+  findProvider(model: string): ProviderProtocol | null {
+    let fallback: ProviderProtocol | null = null;
+
+    for (const provider of this.providers) {
+      if (provider.matches(model)) {
+        // Non-anthropic providers win immediately (explicit match)
+        if (provider.name !== "anthropic") {
+          return provider;
+        }
+        // Anthropic is the fallback — only use if nothing else matches
+        fallback = provider;
+      }
+    }
+    return fallback;
+  }
+
+  /**
+   * Get all registered providers.
+   * @returns Array of all registered providers
+   */
+  getAll(): ProviderProtocol[] {
+    return [...this.providers];
+  }
+}
+
+/** Global registry instance */
+export const registry = new ProviderRegistryImpl();

package/src/proxy/formats/providers/index.ts

@@ -0,0 +1,3 @@
+// Provider types and registry
+export type { ProviderProtocol } from "../protocol";
+export { registry } from "../providerRegistry";

package/src/proxy/formats/registry.ts

@@ -0,0 +1,61 @@
+import type { FormatHandler } from "./handler";
+import type { RequestFormat } from "../schemas";
+import { PATH_V1_CHAT_COMPLETIONS, PATH_CHAT_COMPLETIONS, PATH_V1_MESSAGES } from "../constants";
+
+/**
+ * Format registry — maps request paths and formats to their FormatHandler.
+ *
+ * Registered at module load time by each format's init module.
+ */
+class FormatRegistryImpl {
+  private handlers = new Map<RequestFormat, FormatHandler>();
+  private pathMap = new Map<string, RequestFormat>();
+
+  register(handler: FormatHandler): void {
+    this.handlers.set(handler.format, handler);
+  }
+
+  registerPath(path: string, format: RequestFormat): void {
+    this.pathMap.set(path, format);
+  }
+
+  /** Get handler by format identifier */
+  get(format: RequestFormat): FormatHandler | undefined {
+    return this.handlers.get(format);
+  }
+
+  /** Get handler matching a request path */
+  getByPath(path: string): FormatHandler | undefined {
+    const messagesPath = path.split("?")[0] ?? "";
+    // DeepSeek /v1/messages is chat completions replay
+    if (messagesPath === PATH_V1_MESSAGES) return this.handlers.get("anthropic");
+    if (messagesPath === PATH_V1_CHAT_COMPLETIONS || messagesPath === PATH_CHAT_COMPLETIONS) {
+      return this.handlers.get("openai");
+    }
+    return undefined;
+  }
+
+  /** Detect format from request body content */
+  detectFormat(rawBody: string | null): RequestFormat {
+    if (rawBody === null) return "unknown";
+    for (const handler of this.handlers.values()) {
+      if (handler.detectFormat(rawBody)) return handler.format;
+    }
+    return "unknown";
+  }
+}
+
+export const formatRegistry = new FormatRegistryImpl();
+
+/**
+ * Get the FormatHandler for a request path.
+ * Returns null if no format matches the path.
+ */
+export function formatForPath(path: string): FormatHandler | null {
+  return formatRegistry.getByPath(path) ?? null;
+}
+
+// Register known paths
+formatRegistry.registerPath(PATH_V1_MESSAGES, "anthropic");
+formatRegistry.registerPath(PATH_V1_CHAT_COMPLETIONS, "openai");
+formatRegistry.registerPath(PATH_CHAT_COMPLETIONS, "openai");