@tonyclaw/llm-inspector 1.17.1 → 1.18.0

package/src/proxy/logFinalizer.ts

@@ -0,0 +1,301 @@
+import { Worker } from "node:worker_threads";
+import { writeChunks } from "./chunkStorage";
+import { formatForPath } from "./formats";
+import { appendLogEntry, logger } from "./logger";
+import type { CapturedLog } from "./schemas";
+import { getSessionProcess } from "./sessionProcess";
+import { finalizeLogUpdate } from "./store";
+
+type BaseFinalizeLogJob = {
+  log: CapturedLog;
+  upstreamUrl: string;
+  elapsedMs: number;
+};
+
+export type FinalizeNonStreamingLogJob = BaseFinalizeLogJob & {
+  type: "non-streaming";
+  responseStatus: number;
+  responseBody: string;
+};
+
+export type FinalizeStreamingLogJob = BaseFinalizeLogJob & {
+  type: "streaming";
+  responseStatus: number;
+  rawStream: string;
+};
+
+export type FinalizeStreamAbortLogJob = BaseFinalizeLogJob & {
+  type: "stream-abort";
+  rawStream: string;
+  hasChunks: boolean;
+};
+
+export type FinalizeLogJob =
+  | FinalizeNonStreamingLogJob
+  | FinalizeStreamingLogJob
+  | FinalizeStreamAbortLogJob;
+
+export type FinalizeLogResult = {
+  log: CapturedLog;
+  upstreamUrl: string;
+  error: string | null;
+};
+
+export function buildFileLogEntry(log: CapturedLog, upstreamUrl: string): Record<string, unknown> {
+  return {
+    timestamp: log.timestamp,
+    id: log.id,
+    method: log.method,
+    path: log.path,
+    model: log.model,
+    sessionId: log.sessionId,
+    rawRequestBody: log.rawRequestBody,
+    responseStatus: log.responseStatus,
+    responseText: log.responseText,
+    inputTokens: log.inputTokens,
+    outputTokens: log.outputTokens,
+    cacheCreationInputTokens: log.cacheCreationInputTokens,
+    cacheReadInputTokens: log.cacheReadInputTokens,
+    elapsedMs: log.elapsedMs,
+    streaming: log.streaming,
+    userAgent: log.userAgent,
+    origin: log.origin,
+    upstreamUrl,
+    clientPort: log.clientPort,
+    clientPid: log.clientPid,
+    clientCwd: log.clientCwd,
+    clientProjectFolder: log.clientProjectFolder,
+    streamingChunks: log.streamingChunks,
+    streamingChunksPath: log.streamingChunksPath,
+    error: log.error,
+  };
+}
+
+function cloneLog(log: CapturedLog): CapturedLog {
+  return { ...log };
+}
+
+function errorMessage(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}
+
+function persistStreamingChunks(log: CapturedLog): void {
+  if (log.streamingChunks !== undefined && log.streamingChunks.chunks.length > 0) {
+    const chunkPath = writeChunks(
+      log.id,
+      log.streamingChunks.chunks,
+      log.streamingChunks.truncated,
+    );
+    log.streamingChunksPath = chunkPath;
+  }
+}
+
+function finalizeWithError(
+  job: FinalizeLogJob,
+  log: CapturedLog,
+  fallbackStatus: number,
+  fallbackResponseText: string,
+  err: unknown,
+): FinalizeLogResult {
+  const message = errorMessage(err);
+  logger.error(`[logFinalizer] Failed to finalize log #${log.id}:`, message);
+  log.responseStatus = log.responseStatus ?? fallbackStatus;
+  log.responseText = log.responseText ?? fallbackResponseText;
+  log.elapsedMs = job.elapsedMs;
+  log.error = message;
+  return { log, upstreamUrl: job.upstreamUrl, error: message };
+}
+
+function finalizeNonStreaming(
+  job: FinalizeNonStreamingLogJob,
+  log: CapturedLog,
+): FinalizeLogResult {
+  const formatHandler = formatForPath(log.path);
+  if (formatHandler === null) {
+    return finalizeWithError(job, log, job.responseStatus, job.responseBody, "Unsupported format");
+  }
+
+  try {
+    const tokens = formatHandler.extractTokens(job.responseBody);
+
+    log.elapsedMs = job.elapsedMs;
+    log.responseStatus = job.responseStatus;
+    log.responseText = job.responseBody;
+    log.inputTokens = tokens.inputTokens;
+    log.outputTokens = tokens.outputTokens;
+    log.cacheCreationInputTokens = tokens.cacheCreationInputTokens;
+    log.cacheReadInputTokens = tokens.cacheReadInputTokens;
+
+    return { log, upstreamUrl: job.upstreamUrl, error: null };
+  } catch (err) {
+    return finalizeWithError(job, log, job.responseStatus, job.responseBody, err);
+  }
+}
+
+function finalizeStreaming(job: FinalizeStreamingLogJob, log: CapturedLog): FinalizeLogResult {
+  const formatHandler = formatForPath(log.path);
+  if (formatHandler === null) {
+    return finalizeWithError(job, log, job.responseStatus, job.rawStream, "Unsupported format");
+  }
+
+  try {
+    log.elapsedMs = job.elapsedMs;
+    log.responseStatus = job.responseStatus;
+    log.responseText = formatHandler.extractStream(
+      job.rawStream,
+      log,
+      log.model ?? undefined,
+      true,
+    );
+    persistStreamingChunks(log);
+
+    return { log, upstreamUrl: job.upstreamUrl, error: null };
+  } catch (err) {
+    return finalizeWithError(job, log, job.responseStatus, job.rawStream, err);
+  }
+}
+
+function finalizeStreamAbort(job: FinalizeStreamAbortLogJob, log: CapturedLog): FinalizeLogResult {
+  const formatHandler = formatForPath(log.path);
+  if (formatHandler === null && job.hasChunks) {
+    return finalizeWithError(job, log, 499, "Client aborted", "Unsupported format");
+  }
+
+  try {
+    log.elapsedMs = job.elapsedMs;
+    if (job.hasChunks && formatHandler !== null) {
+      log.responseText = formatHandler.extractStream(
+        job.rawStream,
+        log,
+        log.model ?? undefined,
+        true,
+      );
+      persistStreamingChunks(log);
+    } else {
+      log.responseText = "Client aborted";
+    }
+
+    return { log, upstreamUrl: job.upstreamUrl, error: "Client aborted" };
+  } catch (err) {
+    return finalizeWithError(job, log, 499, "Client aborted", err);
+  }
+}
+
+export function buildFinalizeLogResult(job: FinalizeLogJob): FinalizeLogResult {
+  const log = cloneLog(job.log);
+
+  switch (job.type) {
+    case "non-streaming":
+      return finalizeNonStreaming(job, log);
+    case "streaming":
+      return finalizeStreaming(job, log);
+    case "stream-abort":
+      return finalizeStreamAbort(job, log);
+  }
+}
+
+export function commitFinalizeLogResult(result: FinalizeLogResult): void {
+  appendLogEntry({ ...buildFileLogEntry(result.log, result.upstreamUrl), error: result.error });
+  finalizeLogUpdate(result.log);
+}
+
+// ── Worker pool ────────────────────────────────────────────────
+// buildFinalizeLogResult is the CPU-heavy step (token extraction,
+// SSE parsing, chunk persistence).  We offload it to a fixed pool
+// of Worker Threads so the main thread stays responsive even under
+// concurrent sessions.  commitFinalizeLogResult stays in-process
+// because it touches the in-memory cache, SSE emitter, and
+// append‑only log — all of which belong to the main thread.
+
+// ── Routing ─────────────────────────────────────────────────────
+// FINALIZER_RUNTIME selects the execution backend:
+//   "process" → per-session child process (default, max isolation)
+//   "worker"  → shared Worker Thread pool
+//   "inline"  → synchronous in-process (debug / fallback)
+// For backward compatibility, FINALIZER_USE_WORKER=0 forces "inline".
+
+const RUNTIME: "process" | "worker" | "inline" = (() => {
+  if (process.env["FINALIZER_USE_WORKER"] === "0") return "inline";
+  const mode = process.env["FINALIZER_RUNTIME"];
+  if (mode === "worker") return "worker";
+  if (mode === "inline") return "inline";
+  return "process";
+})();
+
+function executeBuildInSessionProcess(job: FinalizeLogJob): Promise<FinalizeLogResult> {
+  const sessionId = job.log.sessionId ?? "__unassigned__";
+  return getSessionProcess(sessionId)
+    .enqueue(job)
+    .catch((err: unknown) => {
+      const message = err instanceof Error ? err.message : String(err);
+      logger.error(
+        `[logFinalizer] Session process failed for log #${job.log.id}, ` +
+          `falling back to in-process: ${message}`,
+      );
+      return buildFinalizeLogResult(job);
+    });
+}
+
+function resolveBuildPromise(job: FinalizeLogJob): Promise<FinalizeLogResult> {
+  switch (RUNTIME) {
+    case "process":
+      return executeBuildInSessionProcess(job);
+    case "worker":
+      return executeBuildInWorker(job);
+    case "inline":
+      return Promise.resolve(buildFinalizeLogResult(job));
+  }
+}
+
+const WORKER_COUNT = Math.max(1, Number(process.env["FINALIZER_WORKER_COUNT"]) || 4);
+let _workers: Worker[] | null = null;
+const _pending = new Map<string, (result: FinalizeLogResult) => void>();
+let _nextWorker = 0;
+let _jobSeq = 0;
+
+function getWorkers(): Worker[] {
+  if (_workers !== null) return _workers;
+  _workers = [];
+  for (let i = 0; i < WORKER_COUNT; i++) {
+    const w = new Worker(new URL("./logFinalizer.worker.ts", import.meta.url));
+    w.on("message", (msg: { id: string; result: FinalizeLogResult }) => {
+      const resolve = _pending.get(msg.id);
+      if (resolve !== undefined) {
+        _pending.delete(msg.id);
+        resolve(msg.result);
+      }
+    });
+    w.on("error", (err) => {
+      logger.error(
+        "[logFinalizer] Worker error:",
+        err instanceof Error ? err.message : String(err),
+      );
+    });
+    _workers.push(w);
+  }
+  return _workers;
+}
+
+export function executeBuildInWorker(job: FinalizeLogJob): Promise<FinalizeLogResult> {
+  return new Promise((resolve) => {
+    const list = getWorkers();
+    // getWorkers() always returns at least 1 worker; the array index is safe
+    const idx = _nextWorker % list.length;
+    _nextWorker++;
+    const w = list[idx];
+    if (w === undefined) return; // unreachable, satisfies type-checker
+    const id = String(++_jobSeq);
+    _pending.set(id, resolve);
+    w.postMessage({ id, job });
+  });
+}
+
+export function executeFinalizeLogJob(job: FinalizeLogJob): Promise<void> {
+  return resolveBuildPromise(job).then((result) => {
+    commitFinalizeLogResult(result);
+    if (result.error !== null && result.error !== "Client aborted") {
+      return Promise.reject(new Error(result.error));
+    }
+    return undefined;
+  });
+}

package/src/proxy/logFinalizer.worker.ts

@@ -0,0 +1,24 @@
+import { isMainThread, parentPort } from "node:worker_threads";
+import {
+  buildFinalizeLogResult,
+  type FinalizeLogJob,
+  type FinalizeLogResult,
+} from "./logFinalizer";
+
+if (!isMainThread && parentPort !== null) {
+  const port = parentPort;
+  port.on("message", (msg: { id: string; job: FinalizeLogJob }) => {
+    let result: FinalizeLogResult;
+    try {
+      result = buildFinalizeLogResult(msg.job);
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      result = {
+        log: msg.job.log,
+        upstreamUrl: msg.job.upstreamUrl,
+        error: message,
+      };
+    }
+    port.postMessage({ id: msg.id, result });
+  });
+}

package/src/proxy/schemas.ts

@@ -160,6 +160,10 @@ export type RequestMetadata = {
   sessionId: string | null;
 };
 
+function extractSessionIdFromHeaders(headers: Headers): string | null {
+  return headers.get("x-llm-inspector-session-id") ?? headers.get("x-session-affinity");
+}
+
 /**
  * Parse a request body exactly once and pull out everything the handler needs
  * for routing + log creation: the model (used for provider selection) and the
@@ -171,20 +175,21 @@ export type RequestMetadata = {
  * for the session id.
  */
 export function extractRequestMetadata(body: string | null, headers: Headers): RequestMetadata {
-  if (body === null) return { model: null, sessionId: null };
+  const headerSessionId = extractSessionIdFromHeaders(headers);
+  if (body === null) return { model: null, sessionId: headerSessionId };
   try {
     const json: unknown = JSON.parse(body);
     const loose = LooseRequestSchema.safeParse(json);
     if (loose.success) {
       return {
         model: loose.data.model ?? null,
-        sessionId: loose.data.metadata?.user_id ?? headers.get("x-session-affinity") ?? null,
+        sessionId: loose.data.metadata?.user_id ?? headerSessionId,
       };
     }
   } catch {
     // body not valid JSON
   }
-  return { model: null, sessionId: null };
+  return { model: null, sessionId: headerSessionId };
 }
 
 /**

package/src/proxy/sessionProcess.ts

@@ -0,0 +1,133 @@
+import { fork, type ChildProcess } from "node:child_process";
+import { logger } from "./logger";
+import type { FinalizeLogJob, FinalizeLogResult } from "./logFinalizer";
+
+const IDLE_TIMEOUT_MS = Number(process.env["SESSION_PROCESS_IDLE_MS"]) || 5 * 60 * 1000; // 5 min default
+const MAX_RESTARTS = 3;
+
+const _processes = new Map<string, SessionProcess>();
+
+type PendingJob = {
+  resolve: (result: FinalizeLogResult) => void;
+  reject: (err: Error) => void;
+};
+
+export class SessionProcess {
+  private child: ChildProcess | null = null;
+  private pending = new Map<string, PendingJob>();
+  private nextId = 0;
+  private idleTimer: ReturnType<typeof setTimeout> | null = null;
+  private restartCount = 0;
+  private destroyed = false;
+
+  constructor(private sessionId: string) {}
+
+  /** Number of outstanding jobs sent to this process. */
+  get pendingCount(): number {
+    return this.pending.size;
+  }
+
+  private ensureRunning(): ChildProcess {
+    if (this.child !== null && this.child.connected) return this.child;
+
+    const entryPath = new URL("./sessionWorkerEntry.ts", import.meta.url).pathname;
+    // On Windows, the path from URL.pathname starts with "/" which needs to
+    // be stripped when it's a drive letter path (e.g. "/C:/..." → "C:/...")
+    const resolvedPath =
+      process.platform === "win32" && entryPath.startsWith("/") ? entryPath.slice(1) : entryPath;
+
+    this.child = fork(resolvedPath, [], {
+      stdio: ["pipe", "pipe", "pipe", "ipc"],
+    });
+
+    this.restartCount += 1;
+
+    this.child.on("message", (msg: { id: string; result: FinalizeLogResult }) => {
+      const pending = this.pending.get(msg.id);
+      if (pending !== undefined) {
+        this.pending.delete(msg.id);
+        pending.resolve(msg.result);
+      }
+      this.resetIdleTimer();
+    });
+
+    this.child.on("error", (err) => {
+      logger.error(`[sessionProcess] Session ${this.sessionId} process error:`, err.message);
+    });
+
+    this.child.on("exit", (code, signal) => {
+      const wasConnected = this.child !== null;
+      this.child = null;
+
+      if (this.destroyed) return;
+
+      // Reject all pending jobs on unexpected exit
+      for (const [, pending] of this.pending) {
+        pending.reject(
+          new Error(
+            `Session process exited with code ${code ?? signal}, session=${this.sessionId}`,
+          ),
+        );
+      }
+      this.pending.clear();
+
+      if (wasConnected && this.restartCount <= MAX_RESTARTS) {
+        logger.warn(
+          `[sessionProcess] Session ${this.sessionId} worker exited (code=${code ?? signal}), ` +
+            `restart ${this.restartCount}/${MAX_RESTARTS}`,
+        );
+      }
+    });
+
+    this.resetIdleTimer();
+    return this.child;
+  }
+
+  enqueue(job: FinalizeLogJob): Promise<FinalizeLogResult> {
+    return new Promise((resolve, reject) => {
+      const child = this.ensureRunning();
+      const id = String(++this.nextId);
+      this.pending.set(id, { resolve, reject });
+      child.send({ id, job });
+    });
+  }
+
+  private resetIdleTimer(): void {
+    if (this.idleTimer !== null) clearTimeout(this.idleTimer);
+    this.idleTimer = setTimeout(() => {
+      if (this.pending.size === 0) {
+        this.destroy();
+      }
+    }, IDLE_TIMEOUT_MS);
+  }
+
+  destroy(): void {
+    this.destroyed = true;
+    if (this.idleTimer !== null) {
+      clearTimeout(this.idleTimer);
+      this.idleTimer = null;
+    }
+    if (this.child !== null) {
+      this.child.kill();
+      this.child = null;
+    }
+    _processes.delete(this.sessionId);
+  }
+}
+
+/** Get or create a dedicated child process for a session. */
+export function getSessionProcess(sessionId: string): SessionProcess {
+  const existing = _processes.get(sessionId);
+  if (existing !== undefined) return existing;
+
+  const sp = new SessionProcess(sessionId);
+  _processes.set(sessionId, sp);
+  return sp;
+}
+
+/** Forcefully tear down all session processes (e.g. on server shutdown). */
+export function destroyAllSessionProcesses(): void {
+  for (const [, sp] of _processes) {
+    sp.destroy();
+  }
+}

package/src/proxy/sessionRuntime.ts

@@ -0,0 +1,85 @@
+import { logger } from "./logger";
+import { executeFinalizeLogJob, type FinalizeLogJob } from "./logFinalizer";
+import type { CapturedLog } from "./schemas";
+import {
+  getLogSessionId,
+  markSessionTaskFinished,
+  markSessionTaskQueued,
+  markSessionTaskStarted,
+} from "./sessionSupervisor";
+
+const UNASSIGNED_SESSION_QUEUE = "__unassigned__";
+
+export type SessionTaskName = "finalize-log";
+
+type SessionTask<T> = () => Promise<T> | T;
+
+const queues = new Map<string, Promise<void>>();
+
+function queueKeyForSession(sessionId: string | null): string {
+  return sessionId ?? UNASSIGNED_SESSION_QUEUE;
+}
+
+function errorMessage(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}
+
+async function runSessionTask<T>(
+  sessionId: string | null,
+  taskName: SessionTaskName,
+  task: SessionTask<T>,
+): Promise<T> {
+  markSessionTaskStarted(sessionId);
+  try {
+    const result = await task();
+    markSessionTaskFinished(sessionId, null);
+    return result;
+  } catch (err) {
+    const message = errorMessage(err);
+    logger.error(`[sessionRuntime] ${taskName} failed for session ${sessionId ?? "unassigned"}`);
+    markSessionTaskFinished(sessionId, message);
+    return Promise.reject(err);
+  }
+}
+
+export function enqueueSessionTask<T>(
+  sessionId: string | null,
+  taskName: SessionTaskName,
+  task: SessionTask<T>,
+): Promise<T> {
+  const queueKey = queueKeyForSession(sessionId);
+  const previous = queues.get(queueKey) ?? Promise.resolve();
+
+  markSessionTaskQueued(sessionId);
+
+  const current = previous
+    .catch(() => undefined)
+    .then(() => runSessionTask(sessionId, taskName, task));
+  const settled = current.then(
+    () => undefined,
+    () => undefined,
+  );
+
+  queues.set(queueKey, settled);
+  void settled.then(() => {
+    if (queues.get(queueKey) === settled) {
+      queues.delete(queueKey);
+    }
+  });
+
+  return current;
+}
+
+export function enqueueLogTask<T>(
+  log: CapturedLog,
+  taskName: SessionTaskName,
+  task: SessionTask<T>,
+): Promise<T> {
+  return enqueueSessionTask(getLogSessionId(log), taskName, task);
+}
+
+export function enqueueFinalizeLogJob(job: FinalizeLogJob): Promise<void> {
+  return enqueueSessionTask(getLogSessionId(job.log), "finalize-log", () =>
+    executeFinalizeLogJob(job),
+  );
+}