@tonyclaw/agent-inspector 2.0.22 → 2.0.23

package/src/proxy/store.ts

@@ -1,5 +1,12 @@
-import { readFileSync, existsSync, createReadStream, readdirSync, statSync } from "node:fs";
-import { open, readFile, unlink, writeFile } from "node:fs/promises";
+import {
+  existsSync,
+  createReadStream,
+  createWriteStream,
+  readdirSync,
+  statSync,
+  type WriteStream,
+} from "node:fs";
+import { open, unlink, rename } from "node:fs/promises";
 import { readdir as readdirCallback } from "node:fs/promises";
 import { createInterface } from "node:readline";
 import { Buffer } from "node:buffer";
@@ -273,33 +280,7 @@ export async function getLogById(id: number): Promise<CapturedLog | null> {
       return null;
     }
 
-    // Fallback: scan file for matching ID (for entries written before index rebuild)
-    const content = readFileSync(filePath, "utf-8");
-    const lines = content.split("\n");
-
-    let lastMatch: CapturedLog | null = null;
-    for (const line of lines) {
-      if (line.trim() === "") continue;
-      try {
-        const parsed: unknown = JSON.parse(line);
-        if (typeof parsed === "object" && parsed !== null) {
-          const desc = Object.getOwnPropertyDescriptor(parsed, "id");
-          if (desc !== undefined && typeof desc.value === "number" && desc.value === id) {
-            const result = CapturedLogSchema.safeParse(parsed);
-            if (result.success) {
-              lastMatch = normalizeLogSession(result.data);
-              if (result.data.responseStatus !== null) {
-                addToCache(lastMatch);
-                observeSessionLog(lastMatch);
-                return lastMatch;
-              }
-            }
-          }
-        }
-      } catch {
-        // Skip malformed lines
-      }
-    }
+    const lastMatch = await scanLogFileForId(filePath, id);
     if (lastMatch !== null) {
       addToCache(lastMatch);
       observeSessionLog(lastMatch);
@@ -312,6 +293,37 @@ export async function getLogById(id: number): Promise<CapturedLog | null> {
   return null;
 }
 
+async function scanLogFileForId(filePath: string, id: number): Promise<CapturedLog | null> {
+  let lastMatch: CapturedLog | null = null;
+  const fileStream = createInterface({
+    input: createReadStream(filePath),
+    crlfDelay: Infinity,
+  });
+
+  for await (const line of fileStream) {
+    if (line.trim() === "") continue;
+    try {
+      const parsed: unknown = JSON.parse(line);
+      if (typeof parsed === "object" && parsed !== null) {
+        const desc = Object.getOwnPropertyDescriptor(parsed, "id");
+        if (desc !== undefined && typeof desc.value === "number" && desc.value === id) {
+          const result = CapturedLogSchema.safeParse(parsed);
+          if (result.success) {
+            lastMatch = normalizeLogSession(result.data);
+            if (result.data.responseStatus !== null) {
+              return lastMatch;
+            }
+          }
+        }
+      }
+    } catch {
+      // Skip malformed lines
+    }
+  }
+
+  return lastMatch;
+}
+
 export function getFilteredLogs(sessionId?: string, model?: string): CapturedLog[] {
   // Cache maintains insertion order (sorted by ID since logs are added in ID order)
   // Use spread instead of Array.from for slightly better performance
@@ -326,43 +338,26 @@ function matchesLogFilters(log: CapturedLog, sessionId?: string, model?: string)
   return true;
 }
 
-function requestedWindowFitsMemory(
-  filteredMemoryCount: number,
-  offset: number,
-  limit: number,
-): boolean {
-  return offset + limit <= filteredMemoryCount;
-}
+type PersistedLogVisitor = (log: CapturedLog) => void;
 
-function paginateLogs(logs: CapturedLog[], offset: number, limit: number): CapturedLog[] {
-  return logs.slice(offset, offset + limit);
-}
-
-async function readPersistedLogsById(): Promise<Map<number, CapturedLog>> {
-  const byId = new Map<number, CapturedLog>();
+async function visitPersistedLogs(visitor: PersistedLogVisitor): Promise<void> {
   const logDir = resolveLogDir();
-  if (!existsSync(logDir)) return byId;
+  if (!existsSync(logDir)) return;
 
   try {
     const entries = await readdirCallback(logDir);
     const files = entries.filter((file) => file.endsWith(".jsonl")).sort();
     for (const file of files) {
-      await readPersistedLogFile(join(logDir, file), byId);
+      await visitPersistedLogFile(join(logDir, file), visitor);
     }
   } catch (err) {
     logger.error("[store] Failed to list persisted logs:", String(err));
   }
-
-  for (const log of memoryCache.values()) {
-    byId.set(log.id, log);
-  }
-
-  return byId;
 }
 
-async function readPersistedLogFile(
+async function visitPersistedLogFile(
   filePath: string,
-  byId: Map<number, CapturedLog>,
+  visitor: PersistedLogVisitor,
 ): Promise<void> {
   if (!existsSync(filePath)) return;
 
@@ -379,8 +374,7 @@ async function readPersistedLogFile(
         const result = CapturedLogSchema.safeParse(parsed);
         if (result.success) {
           const log = normalizeLogSession(result.data);
-          byId.set(log.id, log);
-          observeSessionLog(log);
+          visitor(log);
         }
       } catch {
         // Skip malformed lines.
@@ -392,24 +386,44 @@ async function readPersistedLogFile(
 }
 
 export async function listLogsPage(options: ListLogsPageOptions): Promise<ListLogsPageResult> {
-  const memoryLogs = getFilteredLogs(options.sessionId, options.model);
-  if (requestedWindowFitsMemory(memoryLogs.length, options.offset, options.limit)) {
-    return {
-      logs: paginateLogs(memoryLogs, options.offset, options.limit),
-      total: memoryLogs.length,
-      offset: options.offset,
-      limit: options.limit,
-    };
+  const seenIds = new Set<number>();
+  const pageById = new Map<number, CapturedLog>();
+  const pageStart = options.offset;
+  const pageEnd = options.offset + options.limit;
+  let total = 0;
+
+  const visitLog = (log: CapturedLog): void => {
+    if (!matchesLogFilters(log, options.sessionId, options.model)) return;
+
+    if (!seenIds.has(log.id)) {
+      seenIds.add(log.id);
+      const position = total;
+      total += 1;
+      if (position >= pageStart && position < pageEnd) {
+        pageById.set(log.id, log);
+      }
+      return;
+    }
+
+    if (pageById.has(log.id)) {
+      pageById.set(log.id, log);
+    }
+  };
+
+  await visitPersistedLogs((log) => {
+    observeSessionLog(log);
+    visitLog(log);
+  });
+
+  for (const log of memoryCache.values()) {
+    visitLog(log);
   }
 
-  const persistedById = await readPersistedLogsById();
-  const persistedLogs = [...persistedById.values()]
-    .filter((log) => matchesLogFilters(log, options.sessionId, options.model))
-    .sort((left, right) => left.id - right.id);
+  const logs = [...pageById.values()].sort((left, right) => left.id - right.id);
 
   return {
-    logs: paginateLogs(persistedLogs, options.offset, options.limit),
-    total: persistedLogs.length,
+    logs,
+    total,
     offset: options.offset,
     limit: options.limit,
   };
@@ -492,65 +506,41 @@ export async function loadLogsIntoMemory(): Promise<void> {
   const logDir = resolveLogDir();
   if (!existsSync(logDir)) return;
 
-  // lastCompletedById: id -> CapturedLog (overwrite on each occurrence)
-  const lastCompletedById = new Map<number, CapturedLog>();
-
-  try {
-    const entries = await readdirCallback(logDir);
-    const filesWithExt = entries.filter((f: string) => f.endsWith(".jsonl")).sort();
-    for (const file of filesWithExt) {
-      const filePath = join(logDir, file);
-      await loadLogFile(filePath, lastCompletedById);
-    }
-  } catch (err) {
-    logger.error("[store] Failed to read log directory:", String(err));
-  }
+  const recentCompletedById = new Map<number, CapturedLog>();
+  await visitPersistedLogs((log) => {
+    rememberRecentCompletedLog(log, recentCompletedById);
+  });
 
   // Sort by id descending, keep only MAX_MEMORY_CACHE newest completed entries
-  const sorted = [...lastCompletedById.values()].sort((a, b) => b.id - a.id);
-  for (let i = 0; i < Math.min(sorted.length, MAX_MEMORY_CACHE); i++) {
-    const entry = sorted[i];
-    if (entry !== undefined) addToCache(entry);
+  const sorted = [...recentCompletedById.values()].sort((a, b) => b.id - a.id);
+  for (const entry of sorted) {
+    addToCache(entry);
   }
 
   rebuildSessionRegistry(memoryCache.values());
 }
 
-async function loadLogFile(
-  filePath: string,
-  lastCompletedById: Map<number, CapturedLog>,
-): Promise<void> {
-  if (!existsSync(filePath)) return;
-
-  try {
-    const fileStream = createInterface({
-      input: createReadStream(filePath),
-      crlfDelay: Infinity,
-    });
+function rememberRecentCompletedLog(
+  log: CapturedLog,
+  recentCompletedById: Map<number, CapturedLog>,
+): void {
+  if (log.responseStatus === null) return;
+  recentCompletedById.set(log.id, log);
+  while (recentCompletedById.size > MAX_MEMORY_CACHE) {
+    const oldestId = findSmallestLogId(recentCompletedById);
+    if (oldestId === null) return;
+    recentCompletedById.delete(oldestId);
+  }
+}
 
-    for await (const line of fileStream) {
-      if (line.trim() === "") continue;
-      try {
-        const parsed: unknown = JSON.parse(line);
-        if (typeof parsed === "object" && parsed !== null) {
-          const desc = Object.getOwnPropertyDescriptor(parsed, "id");
-          if (desc !== undefined && typeof desc.value === "number") {
-            const statusDesc = Object.getOwnPropertyDescriptor(parsed, "responseStatus");
-            if (statusDesc !== undefined && statusDesc.value !== null) {
-              const result = CapturedLogSchema.safeParse(parsed);
-              if (result.success) {
-                lastCompletedById.set(desc.value, result.data);
-              }
-            }
-          }
-        }
-      } catch {
-        // Skip malformed lines
-      }
+function findSmallestLogId(logsById: ReadonlyMap<number, CapturedLog>): number | null {
+  let smallestId: number | null = null;
+  for (const id of logsById.keys()) {
+    if (smallestId === null || id < smallestId) {
+      smallestId = id;
     }
-  } catch (err) {
-    logger.error("[store] Failed to load logs into memory:", String(err));
   }
+  return smallestId;
 }
 
 export function clearAllLogs(): { cleared: number } {
@@ -632,40 +622,113 @@ type RewriteLogFileResult = {
   removedIds: Set<number>;
 };
 
+function tempRewritePath(filePath: string): string {
+  return `${filePath}.rewrite-${String(process.pid)}-${String(Date.now())}.tmp`;
+}
+
+async function removeTempFile(filePath: string): Promise<void> {
+  try {
+    await unlink(filePath);
+  } catch {
+    // Best-effort cleanup for temp rewrite files.
+  }
+}
+
+function waitForWriterDrain(writer: WriteStream): Promise<boolean> {
+  return new Promise((resolve) => {
+    const cleanup = (): void => {
+      writer.off("drain", onDrain);
+      writer.off("error", onError);
+    };
+    const onDrain = (): void => {
+      cleanup();
+      resolve(true);
+    };
+    const onError = (): void => {
+      cleanup();
+      resolve(false);
+    };
+    writer.once("drain", onDrain);
+    writer.once("error", onError);
+  });
+}
+
+async function writeKeptLogLine(writer: WriteStream, line: string): Promise<boolean> {
+  if (writer.write(`${line}\n`, "utf-8")) return true;
+  return await waitForWriterDrain(writer);
+}
+
+function finishWriter(writer: WriteStream): Promise<boolean> {
+  return new Promise((resolve) => {
+    const cleanup = (): void => {
+      writer.off("finish", onFinish);
+      writer.off("error", onError);
+    };
+    const onFinish = (): void => {
+      cleanup();
+      resolve(true);
+    };
+    const onError = (): void => {
+      cleanup();
+      resolve(false);
+    };
+    writer.once("finish", onFinish);
+    writer.once("error", onError);
+    writer.end();
+  });
+}
+
 async function rewriteLogFileWithoutIds(
   filePath: string,
   ids: ReadonlySet<number>,
 ): Promise<RewriteLogFileResult> {
-  let content: string;
+  const tempPath = tempRewritePath(filePath);
+  const writer = createWriteStream(tempPath, { encoding: "utf-8" });
+  const removedIds = new Set<number>();
+  let keptLineCount = 0;
+  let writeOk = true;
+
   try {
-    content = await readFile(filePath, "utf-8");
+    const fileStream = createInterface({
+      input: createReadStream(filePath),
+      crlfDelay: Infinity,
+    });
+
+    for await (const line of fileStream) {
+      if (line === "") continue;
+      const id = readLogIdFromLine(line);
+      if (id !== null && ids.has(id)) {
+        removedIds.add(id);
+      } else {
+        keptLineCount += 1;
+        writeOk = await writeKeptLogLine(writer, line);
+        if (!writeOk) break;
+      }
+    }
   } catch (err) {
-    logger.warn("[store] Failed to read log file for deletion:", filePath, String(err));
-    return { rewritten: false, removedIds: new Set() };
+    logger.warn("[store] Failed to stream log file for deletion:", filePath, String(err));
+    writeOk = false;
   }
 
-  const lines = content.split("\n");
-  const keptLines: string[] = [];
-  const removedIds = new Set<number>();
-  for (const line of lines) {
-    if (line === "") continue;
-    const id = readLogIdFromLine(line);
-    if (id !== null && ids.has(id)) {
-      removedIds.add(id);
-    } else {
-      keptLines.push(line);
-    }
+  if (!(await finishWriter(writer)) || !writeOk) {
+    await removeTempFile(tempPath);
+    return { rewritten: false, removedIds: new Set() };
   }
 
-  if (removedIds.size === 0) return { rewritten: false, removedIds };
+  if (removedIds.size === 0) {
+    await removeTempFile(tempPath);
+    return { rewritten: false, removedIds };
+  }
 
   try {
-    if (keptLines.length === 0) {
+    if (keptLineCount === 0) {
+      await removeTempFile(tempPath);
       await unlink(filePath);
     } else {
-      await writeFile(filePath, `${keptLines.join("\n")}\n`, "utf-8");
+      await rename(tempPath, filePath);
     }
   } catch (err) {
+    await removeTempFile(tempPath);
     logger.warn("[store] Failed to rewrite log file after deletion:", filePath, String(err));
     return { rewritten: false, removedIds: new Set() };
   }

package/src/routes/api/logs.stream.ts

@@ -2,7 +2,7 @@ import { createFileRoute } from "@tanstack/react-router";
 import { onLogUpdate, getLogSessionId, listLogsPage } from "../../proxy/store";
 import { type CapturedLog } from "../../proxy/schemas";
 
-const INITIAL_STREAM_LOG_LIMIT = 500;
+const INITIAL_STREAM_LOG_LIMIT = 100;
 
 export const Route = createFileRoute("/api/logs/stream")({
   server: {

package/.output/public/assets/_sessionId-b1KDnByv.js

@@ -1 +0,0 @@
-import{R as s,j as e}from"./main-N-sjHJUM.js";import{P as i}from"./ProxyViewerContainer-Dwhi4Q1c.js";function t(){const{sessionId:o}=s.useParams();return e.jsx(i,{initialSessionId:o},o)}export{t as component};

package/.output/public/assets/index-CUyRKgII.js

@@ -1 +0,0 @@
-import{P as o}from"./ProxyViewerContainer-Dwhi4Q1c.js";import"./main-N-sjHJUM.js";const r=o;export{r as component};