@tonyclaw/llm-inspector 1.6.0

package/src/proxy/handler.ts

@@ -0,0 +1,389 @@
+import { createLog, emitLogUpdate, type CapturedLog } from "./store";
+import { appendLogEntry } from "./logger";
+import { writeChunks } from "./chunkStorage";
+import { extractModelFromBody, type RequestFormat } from "./schemas";
+import { registry } from "./formats";
+import { findProviderByModel } from "./providers";
+import { getClientInfo } from "./socketTracker";
+import { formatForPath, formatRegistry, type FormatHandler } from "./formats";
+import {
+  DEFAULT_UPSTREAM,
+  DEFAULT_OPENAI_UPSTREAM,
+  PROXY_IDENTITY,
+  PRESERVE_HEADERS,
+  PATH_V1_CHAT_COMPLETIONS,
+  PATH_CHAT_COMPLETIONS,
+  PATH_V1_MESSAGES,
+  HEADER_CONTENT_TYPE,
+  HEADER_USER_AGENT,
+  HEADER_X_PROXY_IDENTITY,
+  HEADER_AUTHORIZATION,
+  HEADER_X_API_KEY,
+  HEADER_CONTENT_ENCODING,
+  HEADER_CONTENT_LENGTH,
+  HEADER_HOST,
+  AUTH_HEADER_X_API_KEY,
+  CONTENT_TYPE_EVENT_STREAM,
+  STATUS_FORBIDDEN,
+  STATUS_BAD_GATEWAY,
+} from "./constants";
+
+/**
+ * Strips all custom/non-standard headers from the request and replaces with
+ * unified proxy identity. Only preserves standard HTTP headers needed for API calls.
+ */
+function buildProxyHeaders(originalHeaders: Headers): {
+  headers: Headers;
+  rawHeaders: Record<string, string>;
+} {
+  const rawHeaders: Record<string, string> = {};
+  const headers = new Headers();
+
+  originalHeaders.forEach((value, key) => {
+    rawHeaders[key.toLowerCase()] = value;
+  });
+
+  headers.set(HEADER_USER_AGENT, PROXY_IDENTITY);
+  headers.set(HEADER_X_PROXY_IDENTITY, PROXY_IDENTITY);
+
+  for (const name of PRESERVE_HEADERS) {
+    const value = originalHeaders.get(name);
+    if (value !== null) {
+      headers.set(name, value);
+    }
+  }
+
+  return { headers, rawHeaders };
+}
+
+function getHostFromUrl(urlStr: string): string {
+  try {
+    const url = new URL(urlStr);
+    return url.host;
+  } catch {
+    return "api.anthropic.com";
+  }
+}
+
+function buildFileLogEntry(log: CapturedLog, upstreamUrl: string): Record<string, unknown> {
+  return {
+    timestamp: log.timestamp,
+    id: log.id,
+    method: log.method,
+    path: log.path,
+    model: log.model,
+    sessionId: log.sessionId,
+    rawRequestBody: log.rawRequestBody,
+    responseStatus: log.responseStatus,
+    responseText: log.responseText,
+    inputTokens: log.inputTokens,
+    outputTokens: log.outputTokens,
+    elapsedMs: log.elapsedMs,
+    streaming: log.streaming,
+    userAgent: log.userAgent,
+    origin: log.origin,
+    upstreamUrl,
+    clientPort: log.clientPort,
+    clientPid: log.clientPid,
+    clientCwd: log.clientCwd,
+    clientProjectFolder: log.clientProjectFolder,
+    streamingChunks: log.streamingChunks,
+    streamingChunksPath: log.streamingChunksPath,
+  };
+}
+
+type ParsedRequestPath = {
+  apiPath: string;
+  messagesPath: string;
+  isChatCompletionsV1: boolean;
+  isChatCompletions: boolean;
+  isMessages: boolean;
+  normalizedPath: string;
+};
+
+function parseRequestPath(req: Request, url: URL): ParsedRequestPath {
+  const apiPath = url.pathname.replace(/^\/proxy/, "") + url.search;
+  const messagesPath = apiPath.split("?")[0] ?? "";
+
+  const isChatCompletionsV1 = messagesPath === PATH_V1_CHAT_COMPLETIONS;
+  const isChatCompletions = messagesPath === PATH_CHAT_COMPLETIONS || isChatCompletionsV1;
+  const isMessages =
+    req.method === "POST" &&
+    (messagesPath === PATH_V1_MESSAGES ||
+      messagesPath === PATH_V1_CHAT_COMPLETIONS ||
+      messagesPath === PATH_CHAT_COMPLETIONS);
+
+  const normalizedPath =
+    isChatCompletions && !apiPath.startsWith("/v1/") ? "/v1" + apiPath : apiPath;
+
+  return {
+    apiPath,
+    messagesPath,
+    isChatCompletionsV1,
+    isChatCompletions,
+    isMessages,
+    normalizedPath,
+  };
+}
+
+function buildUpstreamUrl(upstreamBase: string, normalizedPath: string): string {
+  return upstreamBase + normalizedPath;
+}
+
+function selectUpstreamBase(
+  isChatCompletions: boolean,
+  matchedProviderConfig: ReturnType<typeof findProviderByModel>,
+): string {
+  let upstreamBase: string;
+
+  if (matchedProviderConfig) {
+    // Choose URL based on request format: ChatCompletions uses OpenAI URL, otherwise Anthropic URL
+    if (
+      isChatCompletions &&
+      matchedProviderConfig.openaiBaseUrl !== undefined &&
+      matchedProviderConfig.openaiBaseUrl !== ""
+    ) {
+      upstreamBase = matchedProviderConfig.openaiBaseUrl;
+    } else if (
+      !isChatCompletions &&
+      matchedProviderConfig.anthropicBaseUrl !== undefined &&
+      matchedProviderConfig.anthropicBaseUrl !== ""
+    ) {
+      upstreamBase = matchedProviderConfig.anthropicBaseUrl;
+    } else if (
+      matchedProviderConfig.baseUrl !== undefined &&
+      matchedProviderConfig.baseUrl !== ""
+    ) {
+      // Fallback to baseUrl for backward compatibility
+      upstreamBase = matchedProviderConfig.baseUrl;
+    } else {
+      // Fall back to defaults based on format
+      upstreamBase =
+        matchedProviderConfig.format === "openai" ? DEFAULT_OPENAI_UPSTREAM : DEFAULT_UPSTREAM;
+    }
+  } else {
+    // No provider config, use defaults
+    upstreamBase = isChatCompletions ? DEFAULT_OPENAI_UPSTREAM : DEFAULT_UPSTREAM;
+  }
+
+  return upstreamBase;
+}
+
+function injectAuthHeaders(
+  upstreamHeaders: Headers,
+  matchedProviderConfig: ReturnType<typeof findProviderByModel>,
+): void {
+  if (!matchedProviderConfig) return;
+
+  const apiKey = matchedProviderConfig.apiKey.replace(/^Bearer\s+/i, "").trim();
+  if (matchedProviderConfig.authHeader === AUTH_HEADER_X_API_KEY) {
+    upstreamHeaders.set(HEADER_X_API_KEY, apiKey);
+    upstreamHeaders.delete(HEADER_AUTHORIZATION);
+  } else {
+    upstreamHeaders.set(HEADER_AUTHORIZATION, `Bearer ${apiKey}`);
+  }
+}
+
+function handleNonStreamingResponse(
+  upstreamRes: Response,
+  responseBody: string,
+  startTime: number,
+  formatHandler: FormatHandler,
+  upstreamUrl: string,
+  log: CapturedLog,
+): Response {
+  const elapsedMs = Date.now() - startTime;
+  const tokens = formatHandler.extractTokens(responseBody);
+
+  log.elapsedMs = elapsedMs;
+  log.responseStatus = upstreamRes.status;
+  log.responseText = responseBody;
+  log.inputTokens = tokens.inputTokens;
+  log.outputTokens = tokens.outputTokens;
+  log.cacheCreationInputTokens = tokens.cacheCreationInputTokens;
+  log.cacheReadInputTokens = tokens.cacheReadInputTokens;
+
+  appendLogEntry({ ...buildFileLogEntry(log, upstreamUrl), error: null });
+
+  const responseHeaders = new Headers(upstreamRes.headers);
+  responseHeaders.delete(HEADER_CONTENT_ENCODING);
+  responseHeaders.delete(HEADER_CONTENT_LENGTH);
+
+  return new Response(responseBody, {
+    status: upstreamRes.status,
+    headers: responseHeaders,
+  });
+}
+
+function handleStreamingResponse(
+  upstreamRes: Response,
+  req: Request,
+  startTime: number,
+  formatHandler: FormatHandler,
+  upstreamUrl: string,
+  log: CapturedLog,
+): Response {
+  log.streaming = true;
+  log.responseStatus = upstreamRes.status;
+
+  const chunks: string[] = [];
+  const decoder = new TextDecoder();
+
+  const transform = new TransformStream<Uint8Array, Uint8Array>({
+    transform(chunk, controller) {
+      controller.enqueue(chunk);
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    },
+    flush() {
+      const full = chunks.join("");
+      log.elapsedMs = Date.now() - startTime;
+      log.responseText = formatHandler.extractStream(full, log, log.model ?? undefined, true);
+      // Persist chunks to disk
+      if (log.streamingChunks && log.streamingChunks.chunks.length > 0) {
+        const chunkPath = writeChunks(
+          log.id,
+          log.streamingChunks.chunks,
+          log.streamingChunks.truncated,
+        );
+        log.streamingChunksPath = chunkPath;
+      }
+      appendLogEntry({ ...buildFileLogEntry(log, upstreamUrl), error: null });
+      emitLogUpdate(log);
+    },
+  });
+
+  if (upstreamRes.body === null) {
+    return new Response("No response body", { status: STATUS_BAD_GATEWAY });
+  }
+
+  const loggedStream = upstreamRes.body.pipeThrough(transform);
+
+  req.signal?.addEventListener("abort", () => {
+    if (log.responseText === null && chunks.length > 0) {
+      const full = chunks.join("");
+      log.elapsedMs = Date.now() - startTime;
+      log.responseText = formatHandler.extractStream(full, log, log.model ?? undefined, true);
+      // Persist chunks to disk on abort
+      if (log.streamingChunks && log.streamingChunks.chunks.length > 0) {
+        const chunkPath = writeChunks(
+          log.id,
+          log.streamingChunks.chunks,
+          log.streamingChunks.truncated,
+        );
+        log.streamingChunksPath = chunkPath;
+      }
+      appendLogEntry({ ...buildFileLogEntry(log, upstreamUrl), error: "Client aborted" });
+      emitLogUpdate(log);
+    }
+  });
+
+  const responseHeaders = new Headers(upstreamRes.headers);
+  responseHeaders.delete(HEADER_CONTENT_ENCODING);
+  responseHeaders.delete(HEADER_CONTENT_LENGTH);
+
+  return new Response(loggedStream, {
+    status: upstreamRes.status,
+    headers: responseHeaders,
+  });
+}
+
+export async function handleProxy(req: Request): Promise<Response> {
+  const url = new URL(req.url);
+  const parsed = parseRequestPath(req, url);
+
+  let requestBody: string | null = null;
+  if (req.body && req.method !== "GET" && req.method !== "HEAD") {
+    requestBody = await req.text();
+  }
+
+  // Extract model once and reuse
+  const model = requestBody !== null ? extractModelFromBody(requestBody) : null;
+
+  const matchedProviderConfig = findProviderByModelFromConfig(requestBody);
+  const upstreamBase = selectUpstreamBase(parsed.isChatCompletions, matchedProviderConfig);
+  const upstreamUrl = buildUpstreamUrl(upstreamBase, parsed.normalizedPath);
+  const upstreamHost = getHostFromUrl(upstreamBase);
+  const startTime = Date.now();
+
+  const { headers: upstreamHeaders, rawHeaders } = buildProxyHeaders(req.headers);
+  upstreamHeaders.set(HEADER_HOST, upstreamHost);
+  injectAuthHeaders(upstreamHeaders, matchedProviderConfig);
+
+  const provider = model !== null ? registry.findProvider(model) : null;
+
+  // Only proxy requests matching a registered provider
+  if (model === null || provider === null) {
+    return new Response("Forbidden: unsupported provider", { status: STATUS_FORBIDDEN });
+  }
+
+  // Get the format handler based on provider format (preferred) or request path fallback
+  let formatHandler: FormatHandler | null;
+  if (matchedProviderConfig?.format) {
+    formatHandler = formatRegistry.get(matchedProviderConfig.format) ?? null;
+  } else {
+    formatHandler = formatForPath(parsed.apiPath);
+  }
+  if (formatHandler === null) {
+    return new Response("Forbidden: unsupported format", { status: STATUS_FORBIDDEN });
+  }
+
+  const clientInfo = await getClientInfo(req);
+  const upstreamHeadersObj: Record<string, string> = {};
+  upstreamHeaders.forEach((value, key) => {
+    upstreamHeadersObj[key.toLowerCase()] = value;
+  });
+  const log = await createLog(
+    req.method,
+    parsed.apiPath,
+    requestBody,
+    req.headers,
+    clientInfo,
+    rawHeaders,
+    upstreamHeadersObj,
+    formatHandler.format,
+  );
+
+  let upstreamRes: Response;
+  try {
+    upstreamRes = await fetch(upstreamUrl, {
+      method: req.method,
+      headers: upstreamHeaders,
+      body: requestBody,
+    });
+  } catch (err) {
+    log.elapsedMs = Date.now() - startTime;
+    log.responseStatus = STATUS_BAD_GATEWAY;
+    log.responseText = String(err);
+    appendLogEntry({ ...buildFileLogEntry(log, upstreamUrl), error: String(err) });
+    return new Response(`Proxy error: ${err}`, { status: STATUS_BAD_GATEWAY });
+  }
+
+  const isStream =
+    upstreamRes.headers.get(HEADER_CONTENT_TYPE)?.includes(CONTENT_TYPE_EVENT_STREAM) ?? false;
+
+  if (!isStream) {
+    const responseBody = await upstreamRes.text();
+    return handleNonStreamingResponse(
+      upstreamRes,
+      responseBody,
+      startTime,
+      formatHandler,
+      upstreamUrl,
+      log,
+    );
+  }
+
+  return handleStreamingResponse(upstreamRes, req, startTime, formatHandler, upstreamUrl, log);
+}
+
+/**
+ * Find provider configuration by model name (from persistent store).
+ */
+function findProviderByModelFromConfig(
+  requestBody: string | null,
+): ReturnType<typeof findProviderByModel> {
+  if (requestBody === null) return null;
+  const model = extractModelFromBody(requestBody);
+  if (model === null) return null;
+  return findProviderByModel(model);
+}

package/src/proxy/logIndex.ts

@@ -0,0 +1,187 @@
+import { readFile, writeFile, stat, readdir, mkdir } from "node:fs/promises";
+import { createReadStream, existsSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { createInterface } from "node:readline";
+import { resolveLogDir } from "./logger";
+
+type LogIndexEntry = {
+  id: number;
+  file: string;
+  lineStart: number;
+  lineEnd: number;
+};
+
+type LogIndex = {
+  version: number;
+  entries: Record<number, LogIndexEntry>;
+  maxId: number;
+};
+
+const INDEX_VERSION = 1;
+const INDEX_FILE = "logs.idx";
+
+export { resolveLogDir as getLogDir };
+
+function getIndexPath(): string {
+  return join(resolveLogDir(), INDEX_FILE);
+}
+
+let cachedIndex: LogIndex | null = null;
+
+function createEmptyIndex(): LogIndex {
+  return {
+    version: INDEX_VERSION,
+    entries: {},
+    maxId: 0,
+  };
+}
+
+function isLogIndex(obj: unknown): obj is LogIndex {
+  if (typeof obj !== "object" || obj === null || Array.isArray(obj)) return false;
+  const versionDesc = Object.getOwnPropertyDescriptor(obj, "version");
+  const entriesDesc = Object.getOwnPropertyDescriptor(obj, "entries");
+  const maxIdDesc = Object.getOwnPropertyDescriptor(obj, "maxId");
+  return (
+    versionDesc !== undefined &&
+    typeof versionDesc.value === "number" &&
+    entriesDesc !== undefined &&
+    typeof entriesDesc.value === "object" &&
+    entriesDesc.value !== null &&
+    maxIdDesc !== undefined &&
+    typeof maxIdDesc.value === "number"
+  );
+}
+
+export async function loadIndex(): Promise<LogIndex> {
+  if (cachedIndex !== null) return cachedIndex;
+
+  const indexPath = getIndexPath();
+  if (!existsSync(indexPath)) {
+    cachedIndex = createEmptyIndex();
+    return cachedIndex;
+  }
+
+  try {
+    const content = await readFile(indexPath, "utf-8");
+    const parsed: unknown = JSON.parse(content);
+    if (isLogIndex(parsed)) {
+      cachedIndex = parsed;
+    } else {
+      cachedIndex = createEmptyIndex();
+    }
+    return cachedIndex;
+  } catch (err) {
+    // eslint-disable-next-line no-console
+    console.error("[logIndex] Failed to load index:", err);
+    cachedIndex = createEmptyIndex();
+    return cachedIndex;
+  }
+}
+
+export async function saveIndex(index: LogIndex): Promise<void> {
+  const indexPath = getIndexPath();
+  const dir = dirname(indexPath);
+
+  try {
+    await mkdir(dir, { recursive: true });
+  } catch {
+    // Ignore
+  }
+
+  try {
+    await writeFile(indexPath, JSON.stringify(index), "utf-8");
+  } catch (err) {
+    // eslint-disable-next-line no-console
+    console.error("[logIndex] Failed to save index:", err);
+  }
+}
+
+export async function addToIndex(
+  id: number,
+  file: string,
+  lineStart: number,
+  lineEnd: number,
+): Promise<void> {
+  const index = await loadIndex();
+  index.entries[id] = { id, file, lineStart, lineEnd };
+  if (id > index.maxId) {
+    index.maxId = id;
+  }
+  await saveIndex(index);
+}
+
+export async function findInIndex(id: number): Promise<LogIndexEntry | null> {
+  const index = await loadIndex();
+  return index.entries[id] ?? null;
+}
+
+export async function rebuildIndex(): Promise<LogIndex> {
+  const logDir = resolveLogDir();
+  const newIndex = createEmptyIndex();
+
+  if (!existsSync(logDir)) {
+    cachedIndex = newIndex;
+    await saveIndex(newIndex);
+    return newIndex;
+  }
+
+  let currentId = 0;
+
+  const files = (await readdir(logDir)).filter((f) => f.endsWith(".jsonl")).sort();
+
+  for (const file of files) {
+    const filePath = join(logDir, file);
+    try {
+      const fileStream = createInterface({
+        input: createReadStream(filePath),
+        crlfDelay: Infinity,
+      });
+
+      let lineNumber = 0;
+
+      for await (const line of fileStream) {
+        lineNumber++;
+        if (line.trim() === "") continue;
+
+        try {
+          const entry: unknown = JSON.parse(line);
+          if (typeof entry === "object" && entry !== null && !Array.isArray(entry)) {
+            const idDesc = Object.getOwnPropertyDescriptor(entry, "id");
+            if (idDesc !== undefined && typeof idDesc.value === "number") {
+              const entryId = idDesc.value;
+              currentId = Math.max(currentId, entryId);
+              newIndex.entries[entryId] = {
+                id: entryId,
+                file,
+                lineStart: lineNumber,
+                lineEnd: lineNumber,
+              };
+            }
+          }
+        } catch {
+          // Skip malformed lines
+        }
+      }
+    } catch {
+      // Skip files that can't be read
+    }
+  }
+
+  newIndex.maxId = currentId;
+  cachedIndex = newIndex;
+  await saveIndex(newIndex);
+  return newIndex;
+}
+
+export async function getNextLogId(): Promise<number> {
+  const index = await loadIndex();
+  return index.maxId + 1;
+}
+
+export function getCurrentLogFile(): string {
+  const now = new Date();
+  const yyyy = now.getUTCFullYear();
+  const mm = String(now.getUTCMonth() + 1).padStart(2, "0");
+  const dd = String(now.getUTCDate()).padStart(2, "0");
+  return `${yyyy}-${mm}-${dd}.jsonl`;
+}

package/src/proxy/logger.ts

@@ -0,0 +1,99 @@
+import { readdir, stat, unlink, appendFile, mkdir } from "node:fs/promises";
+import path from "node:path";
+
+const LOG_DIR_ENV = process.env["LOG_DIR"];
+const RETENTION_DAYS = Number(process.env["LOG_RETENTION_DAYS"] ?? "7");
+
+function getUserDataDir(): string {
+  if (process.platform === "win32") {
+    return (
+      process.env["APPDATA"] ?? path.join(process.env["USERPROFILE"] ?? "C:\\", ".llm-inspector")
+    );
+  }
+  return process.env["HOME"] ?? path.join("/tmp");
+}
+
+let resolvedLogDir: string | null = null;
+
+export function resolveLogDir(): string {
+  if (resolvedLogDir !== null) return resolvedLogDir;
+
+  const base = getUserDataDir();
+  if (LOG_DIR_ENV !== undefined) {
+    resolvedLogDir = path.isAbsolute(LOG_DIR_ENV) ? LOG_DIR_ENV : path.join(base, LOG_DIR_ENV);
+  } else {
+    resolvedLogDir = path.join(base, ".llm-inspector", "logs");
+  }
+  return resolvedLogDir;
+}
+
+export function getLogFilePath(): string {
+  const date = new Date();
+  const yyyy = date.getUTCFullYear();
+  const mm = String(date.getUTCMonth() + 1).padStart(2, "0");
+  const dd = String(date.getUTCDate()).padStart(2, "0");
+  return path.join(resolveLogDir(), `${yyyy}-${mm}-${dd}.jsonl`);
+}
+
+export async function initLogger(): Promise<void> {
+  const dir = resolveLogDir();
+  const retentionMs = RETENTION_DAYS * 24 * 60 * 60 * 1000;
+  const cutoff = Date.now() - retentionMs;
+
+  try {
+    const entries = await readdir(dir);
+    for (const entry of entries) {
+      if (!entry.endsWith(".jsonl")) continue;
+      const fullPath = path.join(dir, entry);
+      try {
+        const s = await stat(fullPath);
+        if (s.mtimeMs < cutoff) {
+          await unlink(fullPath);
+        }
+      } catch (err) {
+        // eslint-disable-next-line no-console
+        console.error("[logger] Failed to stat/remove log file:", err);
+      }
+    }
+  } catch (err) {
+    // eslint-disable-next-line no-console
+    console.error("[logger] Failed to initialize logger:", err);
+  }
+}
+
+// Write buffer for batching async writes
+let writeBuffer: string[] = [];
+let writeScheduled = false;
+
+async function flushWriteBuffer(): Promise<void> {
+  if (writeBuffer.length === 0) return;
+
+  const toWrite = writeBuffer.join("");
+  writeBuffer = [];
+
+  try {
+    const filePath = getLogFilePath();
+    await mkdir(path.dirname(filePath), { recursive: true });
+    await appendFile(filePath, toWrite, "utf-8");
+  } catch (err) {
+    // eslint-disable-next-line no-console
+    console.error("[logger] Failed to write log entries:", err);
+  }
+}
+
+function scheduleFlush(): void {
+  if (writeScheduled) return;
+  writeScheduled = true;
+  // Use Promise.resolve().then to flush after current event loop tick
+  // This batches multiple synchronous appendLogEntry calls
+  void Promise.resolve().then(() => {
+    writeScheduled = false;
+    void flushWriteBuffer();
+  });
+}
+
+export function appendLogEntry(entry: Record<string, unknown>): void {
+  const line = JSON.stringify(entry) + "\n";
+  writeBuffer.push(line);
+  scheduleFlush();
+}