@tonyclaw/agent-inspector 2.0.18 → 2.0.19

package/src/proxy/store.ts

@@ -1,11 +1,25 @@
-import { readFileSync, existsSync, createReadStream } from "node:fs";
-import { open } from "node:fs/promises";
+import { readFileSync, existsSync, createReadStream, readdirSync, statSync } from "node:fs";
+import { open, readFile, unlink, writeFile } from "node:fs/promises";
 import { readdir as readdirCallback } from "node:fs/promises";
 import { createInterface } from "node:readline";
 import { Buffer } from "node:buffer";
 import { join } from "node:path";
-import { appendLogEntry, resolveLogDir, logger } from "./logger";
-import { addToIndex, findInIndex, getNextLogId, getCurrentLogFile } from "./logIndex";
+import {
+  appendLogEntry,
+  resolveLogDir,
+  logger,
+  runWithFlushedLogWriteLock,
+  runWithLogWriteLock,
+} from "./logger";
+import { getChunksDir } from "./chunkStorage";
+import {
+  addToIndex,
+  findInIndex,
+  flushIndex,
+  getNextLogId,
+  getCurrentLogFile,
+  rebuildIndex,
+} from "./logIndex";
 import { writeChunks } from "./chunkStorage";
 import type { CapturedLog } from "./schemas";
 import { CapturedLogSchema } from "./schemas";
@@ -27,6 +41,42 @@ export { getLogSessionId, getSessionSnapshots } from "./sessionSupervisor";
 
 const MAX_MEMORY_CACHE = 100;
 
+export type LogStorageStats = {
+  memoryCount: number;
+  logDir: string;
+  logFileCount: number;
+  logBytes: number;
+  chunkDir: string;
+  chunkFileCount: number;
+  chunkBytes: number;
+};
+
+export type ClearPersistedLogStorageResult = {
+  cleared: number;
+  logFilesDeleted: number;
+  chunkFilesDeleted: number;
+};
+
+export type ClearPersistedLogsByIdsResult = {
+  cleared: number;
+  logFilesRewritten: number;
+  chunkFilesDeleted: number;
+};
+
+export type ListLogsPageOptions = {
+  sessionId?: string;
+  model?: string;
+  offset: number;
+  limit: number;
+};
+
+export type ListLogsPageResult = {
+  logs: CapturedLog[];
+  total: number;
+  offset: number;
+  limit: number;
+};
+
 // Memory cache: id -> CapturedLog (recent logs only, FIFO eviction)
 const memoryCache: Map<number, CapturedLog> = new Map();
 
@@ -88,9 +138,11 @@ export async function addTestLogEntry(entry: Omit<CapturedLog, "id">): Promise<C
     streamingChunksPath,
   };
 
-  const logFile = getCurrentLogFile();
-  appendLogEntry(log);
-  await addToIndex(id, logFile, -1, -1);
+  await runWithLogWriteLock(async () => {
+    const logFile = getCurrentLogFile();
+    appendLogEntry(log);
+    await addToIndex(id, logFile, -1, -1);
+  });
 
   addToCache(log);
   observeSessionLog(log, session.source);
@@ -154,9 +206,11 @@ export async function createLog(
   };
 
   // Write to disk and update index
-  const logFile = getCurrentLogFile();
-  appendLogEntry(log);
-  await addToIndex(id, logFile, -1, -1); // line numbers not tracked precisely
+  await runWithLogWriteLock(async () => {
+    const logFile = getCurrentLogFile();
+    appendLogEntry(log);
+    await addToIndex(id, logFile, -1, -1); // line numbers not tracked precisely
+  });
 
   // Add to memory cache
   addToCache(log);
@@ -262,12 +316,105 @@ export function getFilteredLogs(sessionId?: string, model?: string): CapturedLog
   // Cache maintains insertion order (sorted by ID since logs are added in ID order)
   // Use spread instead of Array.from for slightly better performance
   return [...memoryCache.values()].filter((l) => {
-    if (sessionId !== undefined && getLogSessionId(l) !== sessionId) return false;
-    if (model !== undefined && l.model !== model) return false;
-    return true;
+    return matchesLogFilters(l, sessionId, model);
   });
 }
 
+function matchesLogFilters(log: CapturedLog, sessionId?: string, model?: string): boolean {
+  if (sessionId !== undefined && getLogSessionId(log) !== sessionId) return false;
+  if (model !== undefined && log.model !== model) return false;
+  return true;
+}
+
+function requestedWindowFitsMemory(
+  filteredMemoryCount: number,
+  offset: number,
+  limit: number,
+): boolean {
+  return offset + limit <= filteredMemoryCount;
+}
+
+function paginateLogs(logs: CapturedLog[], offset: number, limit: number): CapturedLog[] {
+  return logs.slice(offset, offset + limit);
+}
+
+async function readPersistedLogsById(): Promise<Map<number, CapturedLog>> {
+  const byId = new Map<number, CapturedLog>();
+  const logDir = resolveLogDir();
+  if (!existsSync(logDir)) return byId;
+
+  try {
+    const entries = await readdirCallback(logDir);
+    const files = entries.filter((file) => file.endsWith(".jsonl")).sort();
+    for (const file of files) {
+      await readPersistedLogFile(join(logDir, file), byId);
+    }
+  } catch (err) {
+    logger.error("[store] Failed to list persisted logs:", String(err));
+  }
+
+  for (const log of memoryCache.values()) {
+    byId.set(log.id, log);
+  }
+
+  return byId;
+}
+
+async function readPersistedLogFile(
+  filePath: string,
+  byId: Map<number, CapturedLog>,
+): Promise<void> {
+  if (!existsSync(filePath)) return;
+
+  try {
+    const fileStream = createInterface({
+      input: createReadStream(filePath),
+      crlfDelay: Infinity,
+    });
+
+    for await (const line of fileStream) {
+      if (line.trim() === "") continue;
+      try {
+        const parsed: unknown = JSON.parse(line);
+        const result = CapturedLogSchema.safeParse(parsed);
+        if (result.success) {
+          const log = normalizeLogSession(result.data);
+          byId.set(log.id, log);
+          observeSessionLog(log);
+        }
+      } catch {
+        // Skip malformed lines.
+      }
+    }
+  } catch (err) {
+    logger.error("[store] Failed to read persisted log file:", filePath, String(err));
+  }
+}
+
+export async function listLogsPage(options: ListLogsPageOptions): Promise<ListLogsPageResult> {
+  const memoryLogs = getFilteredLogs(options.sessionId, options.model);
+  if (requestedWindowFitsMemory(memoryLogs.length, options.offset, options.limit)) {
+    return {
+      logs: paginateLogs(memoryLogs, options.offset, options.limit),
+      total: memoryLogs.length,
+      offset: options.offset,
+      limit: options.limit,
+    };
+  }
+
+  const persistedById = await readPersistedLogsById();
+  const persistedLogs = [...persistedById.values()]
+    .filter((log) => matchesLogFilters(log, options.sessionId, options.model))
+    .sort((left, right) => left.id - right.id);
+
+  return {
+    logs: paginateLogs(persistedLogs, options.offset, options.limit),
+    total: persistedLogs.length,
+    offset: options.offset,
+    limit: options.limit,
+  };
+}
+
 export function getSessions(): string[] {
   return getSessionIds();
 }
@@ -280,6 +427,60 @@ export function getModels(): string[] {
   return [...set];
 }
 
+export function getReplayLogsForSource(sourceLogId: number): CapturedLog[] {
+  return [...memoryCache.values()].filter((log) => log.replayOfLogId === sourceLogId);
+}
+
+function collectDirectoryStats(
+  dir: string,
+  shouldCount: (fileName: string) => boolean,
+): { fileCount: number; bytes: number } {
+  if (!existsSync(dir)) return { fileCount: 0, bytes: 0 };
+  let fileCount = 0;
+  let bytes = 0;
+  const visit = (currentDir: string, depth: number): void => {
+    if (depth > 8) return;
+    let entries: string[];
+    try {
+      entries = readdirSync(currentDir);
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const path = join(currentDir, entry);
+      try {
+        const stats = statSync(path);
+        if (stats.isDirectory()) {
+          visit(path, depth + 1);
+        } else if (shouldCount(entry)) {
+          fileCount += 1;
+          bytes += stats.size;
+        }
+      } catch {
+        // Ignore files that disappear while stats are being collected.
+      }
+    }
+  };
+  visit(dir, 0);
+  return { fileCount, bytes };
+}
+
+export function getLogStorageStats(): LogStorageStats {
+  const logDir = resolveLogDir();
+  const chunkDir = getChunksDir();
+  const logStats = collectDirectoryStats(logDir, (fileName) => fileName.endsWith(".jsonl"));
+  const chunkStats = collectDirectoryStats(chunkDir, (fileName) => fileName.endsWith(".json"));
+  return {
+    memoryCount: memoryCache.size,
+    logDir,
+    logFileCount: logStats.fileCount,
+    logBytes: logStats.bytes,
+    chunkDir,
+    chunkFileCount: chunkStats.fileCount,
+    chunkBytes: chunkStats.bytes,
+  };
+}
+
 /**
  * Load recent completed log entries from all log files into the memory cache.
  * Called on server startup so the frontend shows existing logs.
@@ -359,6 +560,206 @@ export function clearAllLogs(): { cleared: number } {
   return { cleared: count };
 }
 
+async function deleteMatchingFiles(
+  dir: string,
+  shouldDelete: (fileName: string) => boolean,
+): Promise<number> {
+  if (!existsSync(dir)) return 0;
+
+  let deleted = 0;
+  let entries: string[];
+  try {
+    entries = await readdirCallback(dir);
+  } catch (err) {
+    logger.warn("[store] Failed to list log storage directory:", String(err));
+    return deleted;
+  }
+
+  for (const entry of entries) {
+    if (!shouldDelete(entry)) continue;
+    try {
+      await unlink(join(dir, entry));
+      deleted += 1;
+    } catch (err) {
+      logger.warn("[store] Failed to delete log storage file:", entry, String(err));
+    }
+  }
+  return deleted;
+}
+
+export async function clearPersistedLogStorage(): Promise<ClearPersistedLogStorageResult> {
+  return await runWithFlushedLogWriteLock(async () => {
+    await flushIndex();
+
+    const result = clearAllLogs();
+    const [logFilesDeleted, chunkFilesDeleted] = await Promise.all([
+      deleteMatchingFiles(
+        resolveLogDir(),
+        (fileName) => fileName.endsWith(".jsonl") || fileName === "logs.idx",
+      ),
+      deleteMatchingFiles(
+        getChunksDir(),
+        (fileName) =>
+          fileName.endsWith(".json") || (fileName.startsWith(".") && fileName.endsWith(".tmp")),
+      ),
+    ]);
+
+    await rebuildIndex();
+
+    return {
+      cleared: result.cleared,
+      logFilesDeleted,
+      chunkFilesDeleted,
+    };
+  });
+}
+
+function readLogIdFromLine(line: string): number | null {
+  if (line.trim() === "") return null;
+  try {
+    const parsed: unknown = JSON.parse(line);
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null;
+    const idDesc = Object.getOwnPropertyDescriptor(parsed, "id");
+    if (idDesc === undefined || typeof idDesc.value !== "number") return null;
+    return idDesc.value;
+  } catch {
+    return null;
+  }
+}
+
+type RewriteLogFileResult = {
+  rewritten: boolean;
+  removedIds: Set<number>;
+};
+
+async function rewriteLogFileWithoutIds(
+  filePath: string,
+  ids: ReadonlySet<number>,
+): Promise<RewriteLogFileResult> {
+  let content: string;
+  try {
+    content = await readFile(filePath, "utf-8");
+  } catch (err) {
+    logger.warn("[store] Failed to read log file for deletion:", filePath, String(err));
+    return { rewritten: false, removedIds: new Set() };
+  }
+
+  const lines = content.split("\n");
+  const keptLines: string[] = [];
+  const removedIds = new Set<number>();
+  for (const line of lines) {
+    if (line === "") continue;
+    const id = readLogIdFromLine(line);
+    if (id !== null && ids.has(id)) {
+      removedIds.add(id);
+    } else {
+      keptLines.push(line);
+    }
+  }
+
+  if (removedIds.size === 0) return { rewritten: false, removedIds };
+
+  try {
+    if (keptLines.length === 0) {
+      await unlink(filePath);
+    } else {
+      await writeFile(filePath, `${keptLines.join("\n")}\n`, "utf-8");
+    }
+  } catch (err) {
+    logger.warn("[store] Failed to rewrite log file after deletion:", filePath, String(err));
+    return { rewritten: false, removedIds: new Set() };
+  }
+
+  return { rewritten: true, removedIds };
+}
+
+type RewriteLogFilesResult = {
+  logFilesRewritten: number;
+  removedIds: Set<number>;
+};
+
+async function rewriteLogFilesWithoutIds(ids: ReadonlySet<number>): Promise<RewriteLogFilesResult> {
+  const logDir = resolveLogDir();
+  if (!existsSync(logDir)) return { logFilesRewritten: 0, removedIds: new Set() };
+
+  let entries: string[];
+  try {
+    entries = await readdirCallback(logDir);
+  } catch (err) {
+    logger.warn("[store] Failed to list log directory for deletion:", String(err));
+    return { logFilesRewritten: 0, removedIds: new Set() };
+  }
+
+  let logFilesRewritten = 0;
+  const removedIds = new Set<number>();
+  for (const entry of entries) {
+    if (!entry.endsWith(".jsonl")) continue;
+    const result = await rewriteLogFileWithoutIds(join(logDir, entry), ids);
+    if (result.rewritten) logFilesRewritten += 1;
+    for (const removedId of result.removedIds) {
+      removedIds.add(removedId);
+    }
+  }
+  return { logFilesRewritten, removedIds };
+}
+
+async function deleteChunkFilesByIds(ids: ReadonlySet<number>): Promise<number> {
+  const chunkDir = getChunksDir();
+  if (!existsSync(chunkDir)) return 0;
+
+  let deleted = 0;
+  for (const id of ids) {
+    const candidates = [`${String(id)}.json`, `.${String(id)}.tmp`];
+    for (const candidate of candidates) {
+      try {
+        await unlink(join(chunkDir, candidate));
+        deleted += 1;
+      } catch {
+        // Missing chunk files are expected for non-streaming logs.
+      }
+    }
+  }
+  return deleted;
+}
+
+export async function clearPersistedLogsByIds(
+  ids: readonly number[],
+): Promise<ClearPersistedLogsByIdsResult> {
+  const uniqueIds = new Set(ids);
+  if (uniqueIds.size === 0) {
+    return { cleared: 0, logFilesRewritten: 0, chunkFilesDeleted: 0 };
+  }
+
+  return await runWithFlushedLogWriteLock(async () => {
+    await flushIndex();
+
+    const memoryRemovedIds = new Set<number>();
+    for (const id of uniqueIds) {
+      if (memoryCache.has(id)) memoryRemovedIds.add(id);
+    }
+    const result = clearLogsByIds([...uniqueIds]);
+    const [rewriteResult, chunkFilesDeleted] = await Promise.all([
+      rewriteLogFilesWithoutIds(uniqueIds),
+      deleteChunkFilesByIds(uniqueIds),
+    ]);
+
+    await rebuildIndex();
+
+    const clearedIds = new Set<number>(rewriteResult.removedIds);
+    if (result.cleared > 0) {
+      for (const id of memoryRemovedIds) {
+        clearedIds.add(id);
+      }
+    }
+
+    return {
+      cleared: clearedIds.size,
+      logFilesRewritten: rewriteResult.logFilesRewritten,
+      chunkFilesDeleted,
+    };
+  });
+}
+
 /**
  * Remove a specific set of log IDs from the in-memory cache. Returns the
  * number of logs actually removed. IDs not present in the cache are ignored.

package/src/routes/api/logs.$id.replay.ts

@@ -1,6 +1,6 @@
 import { createFileRoute } from "@tanstack/react-router";
 import { z } from "zod";
-import { getLogById } from "../../proxy/store";
+import { addTestLogEntry, getLogById } from "../../proxy/store";
 import { findProviderByModel } from "../../proxy/providers";
 import { registry } from "../../proxy/formats";
 import { formatForPath } from "../../proxy/formats";
@@ -39,6 +39,7 @@ type ReplayResponse = {
   outputTokens?: number;
   elapsedMs?: number;
   streaming?: boolean;
+  replayLogId?: number;
 };
 
 export const Route = createFileRoute("/api/logs/$id/replay")({
@@ -161,6 +162,37 @@ export const Route = createFileRoute("/api/logs/$id/replay")({
           const mockLog: typeof log = { ...log };
           const responseText = formatHandler.extractStream(fullResponse, mockLog, model, false);
           const tokens = formatHandler.extractTokens(responseText);
+          const savedReplayLog = await addTestLogEntry({
+            timestamp: new Date().toISOString(),
+            method: log.method,
+            path: log.path,
+            model,
+            sessionId: log.sessionId,
+            rawRequestBody: modifiedBody,
+            responseStatus: upstreamRes.status,
+            responseText,
+            inputTokens: tokens.inputTokens ?? null,
+            outputTokens: tokens.outputTokens ?? null,
+            cacheCreationInputTokens: tokens.cacheCreationInputTokens ?? null,
+            cacheReadInputTokens: tokens.cacheReadInputTokens ?? null,
+            elapsedMs,
+            streaming: true,
+            userAgent: log.userAgent,
+            origin: log.origin,
+            rawHeaders: log.rawHeaders,
+            headers: log.headers,
+            apiFormat: log.apiFormat,
+            isTest: true,
+            replayOfLogId: log.id,
+            providerName: log.providerName,
+            clientPort: log.clientPort,
+            clientPid: log.clientPid,
+            clientCwd: log.clientCwd,
+            clientProjectFolder: log.clientProjectFolder,
+            streamingChunks: undefined,
+            streamingChunksPath: null,
+            error: null,
+          });
 
           return Response.json({
             success: true,
@@ -170,10 +202,42 @@ export const Route = createFileRoute("/api/logs/$id/replay")({
             outputTokens: tokens.outputTokens ?? undefined,
             elapsedMs,
             streaming: true,
+            replayLogId: savedReplayLog.id,
           } satisfies ReplayResponse);
         } else {
           const responseText = await upstreamRes.text();
           const tokens = formatHandler.extractTokens(responseText);
+          const savedReplayLog = await addTestLogEntry({
+            timestamp: new Date().toISOString(),
+            method: log.method,
+            path: log.path,
+            model,
+            sessionId: log.sessionId,
+            rawRequestBody: modifiedBody,
+            responseStatus: upstreamRes.status,
+            responseText,
+            inputTokens: tokens.inputTokens ?? null,
+            outputTokens: tokens.outputTokens ?? null,
+            cacheCreationInputTokens: tokens.cacheCreationInputTokens ?? null,
+            cacheReadInputTokens: tokens.cacheReadInputTokens ?? null,
+            elapsedMs,
+            streaming: false,
+            userAgent: log.userAgent,
+            origin: log.origin,
+            rawHeaders: log.rawHeaders,
+            headers: log.headers,
+            apiFormat: log.apiFormat,
+            isTest: true,
+            replayOfLogId: log.id,
+            providerName: log.providerName,
+            clientPort: log.clientPort,
+            clientPid: log.clientPid,
+            clientCwd: log.clientCwd,
+            clientProjectFolder: log.clientProjectFolder,
+            streamingChunks: undefined,
+            streamingChunksPath: null,
+            error: null,
+          });
 
           return Response.json({
             success: true,
@@ -183,6 +247,7 @@ export const Route = createFileRoute("/api/logs/$id/replay")({
             outputTokens: tokens.outputTokens ?? undefined,
             elapsedMs,
             streaming: false,
+            replayLogId: savedReplayLog.id,
           } satisfies ReplayResponse);
         }
       },

package/src/routes/api/logs.stream.ts

@@ -1,7 +1,9 @@
 import { createFileRoute } from "@tanstack/react-router";
-import { onLogUpdate, getFilteredLogs, getLogSessionId } from "../../proxy/store";
+import { onLogUpdate, getLogSessionId, listLogsPage } from "../../proxy/store";
 import { type CapturedLog } from "../../proxy/schemas";
 
+const INITIAL_STREAM_LOG_LIMIT = 500;
+
 export const Route = createFileRoute("/api/logs/stream")({
   server: {
     handlers: {
@@ -49,10 +51,16 @@ export const Route = createFileRoute("/api/logs/stream")({
         request.signal.addEventListener("abort", cleanup, { once: true });
 
         const stream = new ReadableStream<Uint8Array>({
-          start(controller) {
+          async start(controller) {
             controllerRef = controller;
             // Send initial batch of logs immediately
-            const logs = getFilteredLogs(sessionId, model);
+            const result = await listLogsPage({
+              sessionId,
+              model,
+              offset: 0,
+              limit: INITIAL_STREAM_LOG_LIMIT,
+            });
+            const logs = result.logs;
             const initData = `data: ${JSON.stringify({ type: "init", logs })}\n\n`;
             controller.enqueue(new TextEncoder().encode(initData));
           },

package/src/routes/api/logs.ts

@@ -1,6 +1,12 @@
 import { createFileRoute } from "@tanstack/react-router";
 import { z } from "zod";
-import { clearAllLogs, clearLogsByIds, getFilteredLogs } from "../../proxy/store";
+import {
+  clearPersistedLogStorage,
+  clearPersistedLogsByIds,
+  getLogStorageStats,
+  getReplayLogsForSource,
+  listLogsPage,
+} from "../../proxy/store";
 
 const DeleteBodySchema = z
   .object({
@@ -8,25 +14,43 @@ const DeleteBodySchema = z
   })
   .optional();
 
+function parseNonNegativeInt(value: string | null, fallback: number): number {
+  if (value === null) return fallback;
+  const parsed = Number(value);
+  if (!Number.isInteger(parsed) || parsed < 0) return fallback;
+  return parsed;
+}
+
+function parsePositiveInt(value: string | null, fallback: number): number {
+  if (value === null) return fallback;
+  const parsed = Number(value);
+  if (!Number.isInteger(parsed) || parsed < 1) return fallback;
+  return parsed;
+}
+
 export const Route = createFileRoute("/api/logs")({
   server: {
     handlers: {
-      GET: ({ request }: { request: Request }) => {
+      GET: async ({ request }: { request: Request }) => {
         const url = new URL(request.url);
+        if (url.searchParams.get("stats") === "1") {
+          return Response.json(getLogStorageStats());
+        }
+        const replayOf = url.searchParams.get("replayOf");
+        if (replayOf !== null) {
+          const id = Number(replayOf);
+          if (!Number.isInteger(id) || id <= 0) {
+            return Response.json({ error: "Invalid replayOf log ID" }, { status: 400 });
+          }
+          const logs = getReplayLogsForSource(id);
+          return Response.json({ logs, total: logs.length, offset: 0, limit: logs.length });
+        }
         const sessionId = url.searchParams.get("sessionId") ?? undefined;
         const model = url.searchParams.get("model") ?? undefined;
-        const offset = Number(url.searchParams.get("offset") ?? 0);
-        const limit = Number(url.searchParams.get("limit") ?? 50);
+        const offset = parseNonNegativeInt(url.searchParams.get("offset"), 0);
+        const limit = parsePositiveInt(url.searchParams.get("limit"), 50);
 
-        const allLogs = getFilteredLogs(sessionId, model);
-        const paginatedLogs = allLogs.slice(offset, offset + limit);
-
-        return Response.json({
-          logs: paginatedLogs,
-          total: allLogs.length,
-          offset,
-          limit,
-        });
+        return Response.json(await listLogsPage({ sessionId, model, offset, limit }));
       },
       DELETE: async ({ request }: { request: Request }) => {
         let body: z.infer<typeof DeleteBodySchema> = undefined;
@@ -48,11 +72,21 @@ export const Route = createFileRoute("/api/logs")({
         }
 
         if (body?.ids !== undefined && body.ids.length > 0) {
-          const result = clearLogsByIds(body.ids);
-          return Response.json({ success: true, cleared: result.cleared });
+          const result = await clearPersistedLogsByIds(body.ids);
+          return Response.json({
+            success: true,
+            cleared: result.cleared,
+            logFilesRewritten: result.logFilesRewritten,
+            chunkFilesDeleted: result.chunkFilesDeleted,
+          });
         }
-        const result = clearAllLogs();
-        return Response.json({ success: true, cleared: result.cleared });
+        const result = await clearPersistedLogStorage();
+        return Response.json({
+          success: true,
+          cleared: result.cleared,
+          logFilesDeleted: result.logFilesDeleted,
+          chunkFilesDeleted: result.chunkFilesDeleted,
+        });
       },
     },
   },