@tonyclaw/llm-inspector 1.6.0

package/src/proxy/store.ts

@@ -0,0 +1,386 @@
+import { z } from "zod";
+import { readFileSync, existsSync } from "node:fs";
+import { createReadStream } from "node:fs";
+import { createInterface } from "node:readline";
+import { join } from "node:path";
+import { appendLogEntry, resolveLogDir } from "./logger";
+import {
+  addToIndex,
+  findInIndex,
+  getNextLogId,
+  getCurrentLogFile,
+  saveIndex,
+  loadIndex,
+} from "./logIndex";
+import { writeChunks } from "./chunkStorage";
+import type { CapturedLog } from "./schemas";
+import { CapturedLogSchema } from "./schemas";
+
+export type { CapturedLog };
+
+const LooseRequestSchema = z.object({
+  model: z.string().optional(),
+  metadata: z.object({ user_id: z.string().optional() }).passthrough().optional(),
+});
+
+const MAX_MEMORY_CACHE = 100;
+
+// Memory cache: id -> CapturedLog (recent logs only)
+const memoryCache: Map<number, CapturedLog> = new Map();
+
+// O(1) LRU ordering using Map's insertion order
+// head -> ... -> tail (head = oldest, tail = newest)
+let headId: number | null = null;
+let tailId: number | null = null;
+const prevMap: Map<number, number | null> = new Map(); // id -> previous id (null if head)
+const nextMap: Map<number, number | null> = new Map(); // id -> next id (null if tail)
+
+function evictOldestIfNeeded(): void {
+  while (memoryCache.size >= MAX_MEMORY_CACHE && headId !== null) {
+    const oldest = headId;
+    const next = nextMap.get(oldest) ?? null;
+
+    memoryCache.delete(oldest);
+    prevMap.delete(oldest);
+    nextMap.delete(oldest);
+
+    if (next !== null) {
+      prevMap.set(next, null);
+      headId = next;
+    } else {
+      headId = null;
+      tailId = null;
+    }
+  }
+}
+
+function addToCache(log: CapturedLog): void {
+  evictOldestIfNeeded();
+
+  // Add at tail (newest position)
+  if (tailId !== null) {
+    nextMap.set(tailId, log.id);
+    prevMap.set(log.id, tailId);
+  } else {
+    // First element
+    headId = log.id;
+    prevMap.set(log.id, null);
+  }
+  nextMap.set(log.id, null);
+  tailId = log.id;
+
+  memoryCache.set(log.id, log);
+}
+
+function removeFromCache(id: number): void {
+  const prev = prevMap.get(id) ?? null;
+  const next = nextMap.get(id) ?? null;
+
+  if (prev !== null) {
+    nextMap.set(prev, next);
+  } else {
+    headId = next;
+  }
+
+  if (next !== null) {
+    prevMap.set(next, prev);
+  } else {
+    tailId = prev;
+  }
+
+  prevMap.delete(id);
+  nextMap.delete(id);
+  memoryCache.delete(id);
+}
+
+/**
+ * Add a test log entry directly to the in-memory store (for dashboard display).
+ * This is used by the provider test endpoint alongside appendLogEntry (file logging).
+ */
+export async function addTestLogEntry(entry: Omit<CapturedLog, "id">): Promise<CapturedLog> {
+  const id = await getNextLogId();
+  // Update the index with the new maxId so subsequent calls get unique IDs
+  const index = await loadIndex();
+  if (id > index.maxId) {
+    index.maxId = id;
+    await saveIndex(index);
+  }
+
+  // Persist streaming chunks to disk if present
+  let streamingChunksPath: string | null = null;
+  if (entry.streamingChunks !== undefined && entry.streamingChunks.chunks.length > 0) {
+    streamingChunksPath = writeChunks(
+      id,
+      entry.streamingChunks.chunks,
+      entry.streamingChunks.truncated,
+    );
+  }
+
+  const log: CapturedLog = {
+    id,
+    ...entry,
+    streamingChunksPath,
+  };
+  addToCache(log);
+  emitLogUpdate(log);
+  return log;
+}
+
+type ClientInfo = {
+  port: number | null;
+  pid: number | null;
+  cwd: string | null;
+  projectFolder: string | null;
+};
+
+export async function createLog(
+  method: string,
+  path: string,
+  requestBody: string | null,
+  headers: Headers,
+  clientInfo?: ClientInfo,
+  rawHeaders?: Record<string, string>,
+  upstreamHeaders?: Record<string, string>,
+  apiFormat: "anthropic" | "openai" | "unknown" = "unknown",
+): Promise<CapturedLog> {
+  let model: string | null = null;
+  let sessionId: string | null = null;
+
+  if (requestBody !== null) {
+    try {
+      const json: unknown = JSON.parse(requestBody);
+      const loose = LooseRequestSchema.safeParse(json);
+      if (loose.success) {
+        model = loose.data.model ?? null;
+        sessionId = loose.data.metadata?.user_id ?? headers.get("x-session-affinity") ?? null;
+      }
+    } catch {
+      // request body not valid JSON, skip parsing
+    }
+  }
+
+  const userAgent = headers.get("user-agent");
+  const origin = headers.get("origin");
+
+  const id = await getNextLogId();
+  const log: CapturedLog = {
+    id,
+    timestamp: new Date().toISOString(),
+    method,
+    path,
+    model,
+    sessionId,
+    rawRequestBody: requestBody,
+    responseStatus: null,
+    responseText: null,
+    inputTokens: null,
+    outputTokens: null,
+    cacheCreationInputTokens: null,
+    cacheReadInputTokens: null,
+    elapsedMs: null,
+    streaming: false,
+    userAgent,
+    origin,
+    rawHeaders: rawHeaders,
+    headers: upstreamHeaders,
+    apiFormat,
+    isTest: false,
+    providerName: null,
+    clientPort: clientInfo?.port ?? null,
+    clientPid: clientInfo?.pid ?? null,
+    clientCwd: clientInfo?.cwd ?? null,
+    clientProjectFolder: clientInfo?.projectFolder ?? null,
+    streamingChunksPath: null,
+  };
+
+  // Write to disk and update index
+  const logFile = getCurrentLogFile();
+  appendLogEntry(log);
+  await addToIndex(id, logFile, -1, -1); // line numbers not tracked precisely
+
+  // Add to memory cache
+  addToCache(log);
+  emitLogUpdate(log);
+
+  return log;
+}
+
+/**
+ * Get a log by ID, checking memory cache first then disk.
+ * Each request has two entries in the log file: initial (incomplete) and complete (with response).
+ * We scan for the complete entry (responseStatus !== null) first, falling back to the last match.
+ */
+export async function getLogById(id: number): Promise<CapturedLog | null> {
+  // Check memory cache first
+  const cached = memoryCache.get(id);
+  if (cached !== undefined) {
+    return cached;
+  }
+
+  // Look up in index and load from disk
+  const entry = await findInIndex(id);
+  if (entry === null) {
+    return null;
+  }
+
+  // Load from disk using index entry
+  try {
+    const filePath = join(resolveLogDir(), entry.file);
+    if (!existsSync(filePath)) {
+      return null;
+    }
+
+    const content = readFileSync(filePath, "utf-8");
+    const lines = content.split("\n");
+
+    let lastMatch: CapturedLog | null = null;
+    for (const line of lines) {
+      if (line.trim() === "") continue;
+      try {
+        const parsed: unknown = JSON.parse(line);
+        if (typeof parsed === "object" && parsed !== null) {
+          const desc = Object.getOwnPropertyDescriptor(parsed, "id");
+          if (desc !== undefined && typeof desc.value === "number" && desc.value === id) {
+            const result = CapturedLogSchema.safeParse(parsed);
+            if (result.success) {
+              lastMatch = result.data;
+              // Return immediately if we found a completed entry (has responseStatus)
+              if (result.data.responseStatus !== null) {
+                return result.data;
+              }
+            }
+          }
+        }
+      } catch {
+        // Skip malformed lines
+      }
+    }
+    // Fallback: return the last matching entry (incomplete is better than nothing)
+    if (lastMatch !== null) return lastMatch;
+  } catch (err) {
+    // eslint-disable-next-line no-console
+    console.error("[store] Failed to read log from disk:", err);
+  }
+
+  return null;
+}
+
+export function getFilteredLogs(sessionId?: string, model?: string): CapturedLog[] {
+  // Get all logs from memory cache sorted by ID
+  const cachedLogs = Array.from(memoryCache.values()).sort((a, b) => a.id - b.id);
+
+  return cachedLogs.filter((l) => {
+    if (sessionId !== undefined && l.sessionId !== sessionId) return false;
+    if (model !== undefined && l.model !== model) return false;
+    return true;
+  });
+}
+
+export function getSessions(): string[] {
+  const set = new Set<string>();
+  for (const l of memoryCache.values()) {
+    if (l.sessionId !== null && l.sessionId !== "") set.add(l.sessionId);
+  }
+  return [...set];
+}
+
+export function getModels(): string[] {
+  const set = new Set<string>();
+  for (const l of memoryCache.values()) {
+    if (l.model !== null && l.model !== "") set.add(l.model);
+  }
+  return [...set];
+}
+
+/**
+ * Load recent completed log entries from today's log file into the memory cache.
+ * Called on server startup so the frontend shows existing logs.
+ */
+export async function loadLogsIntoMemory(): Promise<void> {
+  const logDir = resolveLogDir();
+  if (!existsSync(logDir)) return;
+
+  const today = getCurrentLogFile();
+  const filePath = join(logDir, today);
+  if (!existsSync(filePath)) return;
+
+  // Keep track of IDs seen so we only load the completed entry for each
+  const seenComplete = new Set<number>();
+
+  try {
+    const fileStream = createInterface({
+      input: createReadStream(filePath),
+      crlfDelay: Infinity,
+    });
+
+    const lines: string[] = [];
+    for await (const line of fileStream) {
+      lines.push(line);
+    }
+
+    // Process in reverse so we find completed entries first
+    for (let i = lines.length - 1; i >= 0; i--) {
+      const line = lines[i] ?? "";
+      if (line.trim() === "") continue;
+      try {
+        const parsed: unknown = JSON.parse(line);
+        if (typeof parsed === "object" && parsed !== null) {
+          const desc = Object.getOwnPropertyDescriptor(parsed, "id");
+          if (desc !== undefined && typeof desc.value === "number") {
+            const entryId = desc.value;
+            // Only load completed entries (those with responseStatus)
+            const statusDesc = Object.getOwnPropertyDescriptor(parsed, "responseStatus");
+            if (
+              statusDesc !== undefined &&
+              statusDesc.value !== null &&
+              !seenComplete.has(entryId)
+            ) {
+              const result = CapturedLogSchema.safeParse(parsed);
+              if (result.success) {
+                addToCache(result.data);
+                seenComplete.add(entryId);
+              }
+            }
+          }
+        }
+      } catch {
+        // Skip malformed lines
+      }
+    }
+  } catch (err) {
+    // eslint-disable-next-line no-console
+    console.error("[store] Failed to load logs into memory:", err);
+  }
+}
+
+export function clearAllLogs(): { cleared: number } {
+  const count = memoryCache.size;
+  memoryCache.clear();
+  headId = null;
+  tailId = null;
+  prevMap.clear();
+  nextMap.clear();
+  return { cleared: count };
+}
+
+// SSE event system for real-time log updates
+type LogUpdateHandler = (log: CapturedLog) => void;
+const sseHandlers: Set<LogUpdateHandler> = new Set();
+
+export function onLogUpdate(handler: LogUpdateHandler): () => void {
+  sseHandlers.add(handler);
+  return () => {
+    sseHandlers.delete(handler);
+  };
+}
+
+export function emitLogUpdate(log: CapturedLog): void {
+  for (const handler of sseHandlers) {
+    try {
+      handler(log);
+    } catch {
+      // Remove handler that threw
+      sseHandlers.delete(handler);
+    }
+  }
+}

package/src/router.tsx

@@ -0,0 +1,16 @@
+import { createRouter } from "@tanstack/react-router";
+import { routeTree } from "./routeTree.gen";
+
+export function getRouter() {
+  const router = createRouter({
+    routeTree,
+    scrollRestoration: false,
+  });
+  return router;
+}
+
+declare module "@tanstack/react-router" {
+  interface Register {
+    router: ReturnType<typeof getRouter>;
+  }
+}

package/src/routes/__root.tsx

@@ -0,0 +1,38 @@
+/// <reference types="vite/client" />
+import { Outlet, createRootRoute, HeadContent, Scripts } from "@tanstack/react-router";
+import type { ReactNode } from "react";
+import appCss from "../index.css?url";
+
+export const Route = createRootRoute({
+  head: () => ({
+    meta: [
+      { charSet: "utf-8" },
+      { name: "viewport", content: "width=device-width, initial-scale=1" },
+      { title: "llm-inspector" },
+    ],
+    links: [{ rel: "stylesheet", href: appCss }],
+  }),
+  component: RootComponent,
+});
+
+function RootComponent() {
+  return (
+    <RootDocument>
+      <Outlet />
+    </RootDocument>
+  );
+}
+
+function RootDocument({ children }: Readonly<{ children: ReactNode }>) {
+  return (
+    <html lang="en" className="dark">
+      <head>
+        <HeadContent />
+      </head>
+      <body>
+        {children}
+        <Scripts />
+      </body>
+    </html>
+  );
+}

package/src/routes/api/config.paths.ts

@@ -0,0 +1,14 @@
+import { createFileRoute } from "@tanstack/react-router";
+import { store } from "../../proxy/providers";
+
+export const Route = createFileRoute("/api/config/paths")({
+  server: {
+    handlers: {
+      GET: () => {
+        return Response.json({
+          providerConfig: store.path,
+        });
+      },
+    },
+  },
+});

package/src/routes/api/health.ts

@@ -0,0 +1,11 @@
+import { createFileRoute } from "@tanstack/react-router";
+
+export const Route = createFileRoute("/api/health")({
+  server: {
+    handlers: {
+      GET: () => {
+        return Response.json({ status: "ok" });
+      },
+    },
+  },
+});

package/src/routes/api/logs.$id.chunks.ts

@@ -0,0 +1,36 @@
+import { createFileRoute } from "@tanstack/react-router";
+import { getLogById } from "../../proxy/store";
+import { readChunks } from "../../proxy/chunkStorage";
+
+export const Route = createFileRoute("/api/logs/$id/chunks")({
+  server: {
+    handlers: {
+      GET: async ({ params }: { params: { id: string } }) => {
+        const id = Number(params.id);
+        if (isNaN(id)) {
+          return Response.json({ error: "Invalid log ID" }, { status: 400 });
+        }
+
+        const log = await getLogById(id);
+        if (log === null) {
+          return Response.json({ error: "Log not found" }, { status: 404 });
+        }
+
+        // Try disk file first (persistent chunks from real proxy streaming)
+        if (log.streamingChunksPath !== null && log.streamingChunksPath !== undefined) {
+          const chunksData = readChunks(log.streamingChunksPath);
+          if (chunksData !== null) {
+            return Response.json(chunksData);
+          }
+        }
+
+        // Fall back to in-memory streamingChunks (e.g., from provider test)
+        if (log.streamingChunks !== undefined && log.streamingChunks.chunks.length > 0) {
+          return Response.json(log.streamingChunks);
+        }
+
+        return Response.json({ error: "Chunks not found" }, { status: 404 });
+      },
+    },
+  },
+});