@tonyclaw/agent-inspector 2.0.17 → 2.0.19

package/src/proxy/schemas.ts

@@ -8,8 +8,6 @@ export type { JsonValue };
 
 export type RequestFormat = "anthropic" | "openai" | "unknown";
 
-import { safeGetOwnProperty } from "../lib/objectUtils";
-
 /**
  * A single SSE event from a streaming response.
  */
@@ -50,6 +48,7 @@ export const CapturedLogSchema = z.object({
   headers: z.record(z.string(), z.string()).optional(),
   apiFormat: z.enum(["anthropic", "openai", "unknown"]).default("unknown"),
   isTest: z.boolean().optional().default(false),
+  replayOfLogId: z.number().nullable().optional(),
   providerName: z.string().nullable().optional(),
   clientPort: z.number().nullable().optional(),
   clientPid: z.number().nullable().optional(),
@@ -74,9 +73,6 @@ export type TokenUsage = {
 // Import types and schemas from formats
 // ============================================================
 
-// Import types and schemas from formats for internal use
-import { InspectorRequestSchema, type InspectorRequest } from "./formats/anthropic/schemas";
-
 // Re-export types and schemas from formats for backward compatibility
 export {
   InspectorRequestSchema,
@@ -84,7 +80,6 @@ export {
   SseEventSchema,
 } from "./formats/anthropic/schemas";
 export type {
-  InspectorRequest,
   InspectorResponse,
   ResponseContentBlockType,
 } from "./formats/anthropic/schemas";
@@ -98,35 +93,6 @@ export {
   parseOpenAIResponse,
 } from "./formats/openai/schemas";
 
-// ============================================================
-// Utility functions
-// ============================================================
-
-function detectFormat(rawBody: string | null): RequestFormat {
-  if (rawBody === null) return "unknown";
-  try {
-    const json: unknown = JSON.parse(rawBody);
-    if (typeof json === "object" && json !== null && !Array.isArray(json)) {
-      const hasModel = safeGetOwnProperty(json, "model") !== undefined;
-      const hasMessages = safeGetOwnProperty(json, "messages") !== undefined;
-      if (hasModel && hasMessages) {
-        // Anthropic requests put `system` as a top-level key; OpenAI does not.
-        // Both formats can have `tools`, so we check `system` as the discriminator.
-        if (safeGetOwnProperty(json, "system") !== undefined) {
-          return "anthropic";
-        }
-        const msgVal = safeGetOwnProperty(json, "messages");
-        if (Array.isArray(msgVal)) {
-          return "openai";
-        }
-      }
-    }
-    return "unknown";
-  } catch {
-    return "unknown";
-  }
-}
-
 // Schema for model name in request body (used for routing)
 const RequestModelSchema = z.object({
   model: z.string(),
@@ -191,19 +157,3 @@ export function extractRequestMetadata(body: string | null, headers: Headers): R
   }
   return { model: null, sessionId: headerSessionId };
 }
-
-/**
- * Parse a raw request body into an InspectorRequest.
- * @deprecated Use FormatHandler.parseRequest() instead
- */
-export function parseRequest(rawBody: string | null): InspectorRequest | null {
-  if (rawBody === null) return null;
-  try {
-    const json: unknown = JSON.parse(rawBody);
-    const result = InspectorRequestSchema.safeParse(json);
-    if (result.success) return result.data;
-    return null;
-  } catch {
-    return null;
-  }
-}

package/src/proxy/sessionRuntime.ts

@@ -1,6 +1,5 @@
 import { logger } from "./logger";
 import { executeFinalizeLogJob, type FinalizeLogJob } from "./logFinalizer";
-import type { CapturedLog } from "./schemas";
 import {
   getLogSessionId,
   markSessionTaskFinished,
@@ -70,14 +69,6 @@ export function enqueueSessionTask<T>(
   return current;
 }
 
-export function enqueueLogTask<T>(
-  log: CapturedLog,
-  taskName: SessionTaskName,
-  task: SessionTask<T>,
-): Promise<T> {
-  return enqueueSessionTask(getLogSessionId(log), taskName, task);
-}
-
 export function enqueueFinalizeLogJob(job: FinalizeLogJob): Promise<void> {
   return enqueueSessionTask(getLogSessionId(job.log), "finalize-log", () =>
     executeFinalizeLogJob(job),

package/src/proxy/socketTracker.ts

@@ -1,5 +1,6 @@
 import { exec, execFile } from "node:child_process";
 import { promisify } from "node:util";
+import { z } from "zod";
 import { logger } from "./logger";
 
 const execAsync = promisify(exec);
@@ -20,6 +21,12 @@ type CacheEntry = ClientInfo & {
 const cache = new Map<number, CacheEntry>();
 const CACHE_TTL_MS = 5 * 60 * 1000;
 const MAX_CACHE_SIZE = 200;
+const RemotePortSchema = z.number().int().min(1).max(65_535);
+const SocketSchema = z
+  .object({
+    remotePort: RemotePortSchema.nullish(),
+  })
+  .passthrough();
 
 // Deduplicate concurrent lookups for the same port
 const inflight = new Map<number, Promise<ClientInfo>>();
@@ -50,14 +57,14 @@ function setCache(port: number, info: ClientInfo): void {
  * Get the remote port from a Request's underlying socket.
  * Works with Bun, Node.js, and Nitro's event.node.req.
  */
-function extractRemotePort(request: Request): number | null {
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
-  const socket = (request as any).socket;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
-  const remotePort = (socket as any)?.remotePort;
-  // eslint-disable-next-line @typescript-eslint/no-unsafe-return
-  if (remotePort !== undefined && remotePort !== null) return remotePort;
-  return null;
+export function extractRemotePort(request: Request): number | null {
+  const descriptor = Object.getOwnPropertyDescriptor(request, "socket");
+  if (descriptor === undefined || descriptor.get !== undefined) return null;
+  const parsed = SocketSchema.safeParse(descriptor.value);
+  if (!parsed.success) return null;
+  const remotePort = parsed.data.remotePort;
+  if (remotePort === undefined || remotePort === null) return null;
+  return remotePort;
 }
 
 /**

package/src/proxy/store.ts

@@ -1,11 +1,25 @@
-import { readFileSync, existsSync, createReadStream } from "node:fs";
-import { open } from "node:fs/promises";
+import { readFileSync, existsSync, createReadStream, readdirSync, statSync } from "node:fs";
+import { open, readFile, unlink, writeFile } from "node:fs/promises";
 import { readdir as readdirCallback } from "node:fs/promises";
 import { createInterface } from "node:readline";
 import { Buffer } from "node:buffer";
 import { join } from "node:path";
-import { appendLogEntry, resolveLogDir, logger } from "./logger";
-import { addToIndex, findInIndex, getNextLogId, getCurrentLogFile } from "./logIndex";
+import {
+  appendLogEntry,
+  resolveLogDir,
+  logger,
+  runWithFlushedLogWriteLock,
+  runWithLogWriteLock,
+} from "./logger";
+import { getChunksDir } from "./chunkStorage";
+import {
+  addToIndex,
+  findInIndex,
+  flushIndex,
+  getNextLogId,
+  getCurrentLogFile,
+  rebuildIndex,
+} from "./logIndex";
 import { writeChunks } from "./chunkStorage";
 import type { CapturedLog } from "./schemas";
 import { CapturedLogSchema } from "./schemas";
@@ -27,6 +41,42 @@ export { getLogSessionId, getSessionSnapshots } from "./sessionSupervisor";
 
 const MAX_MEMORY_CACHE = 100;
 
+export type LogStorageStats = {
+  memoryCount: number;
+  logDir: string;
+  logFileCount: number;
+  logBytes: number;
+  chunkDir: string;
+  chunkFileCount: number;
+  chunkBytes: number;
+};
+
+export type ClearPersistedLogStorageResult = {
+  cleared: number;
+  logFilesDeleted: number;
+  chunkFilesDeleted: number;
+};
+
+export type ClearPersistedLogsByIdsResult = {
+  cleared: number;
+  logFilesRewritten: number;
+  chunkFilesDeleted: number;
+};
+
+export type ListLogsPageOptions = {
+  sessionId?: string;
+  model?: string;
+  offset: number;
+  limit: number;
+};
+
+export type ListLogsPageResult = {
+  logs: CapturedLog[];
+  total: number;
+  offset: number;
+  limit: number;
+};
+
 // Memory cache: id -> CapturedLog (recent logs only, FIFO eviction)
 const memoryCache: Map<number, CapturedLog> = new Map();
 
@@ -88,9 +138,11 @@ export async function addTestLogEntry(entry: Omit<CapturedLog, "id">): Promise<C
     streamingChunksPath,
   };
 
-  const logFile = getCurrentLogFile();
-  appendLogEntry(log);
-  await addToIndex(id, logFile, -1, -1);
+  await runWithLogWriteLock(async () => {
+    const logFile = getCurrentLogFile();
+    appendLogEntry(log);
+    await addToIndex(id, logFile, -1, -1);
+  });
 
   addToCache(log);
   observeSessionLog(log, session.source);
@@ -154,9 +206,11 @@ export async function createLog(
   };
 
   // Write to disk and update index
-  const logFile = getCurrentLogFile();
-  appendLogEntry(log);
-  await addToIndex(id, logFile, -1, -1); // line numbers not tracked precisely
+  await runWithLogWriteLock(async () => {
+    const logFile = getCurrentLogFile();
+    appendLogEntry(log);
+    await addToIndex(id, logFile, -1, -1); // line numbers not tracked precisely
+  });
 
   // Add to memory cache
   addToCache(log);
@@ -262,12 +316,105 @@ export function getFilteredLogs(sessionId?: string, model?: string): CapturedLog
   // Cache maintains insertion order (sorted by ID since logs are added in ID order)
   // Use spread instead of Array.from for slightly better performance
   return [...memoryCache.values()].filter((l) => {
-    if (sessionId !== undefined && getLogSessionId(l) !== sessionId) return false;
-    if (model !== undefined && l.model !== model) return false;
-    return true;
+    return matchesLogFilters(l, sessionId, model);
   });
 }
 
+function matchesLogFilters(log: CapturedLog, sessionId?: string, model?: string): boolean {
+  if (sessionId !== undefined && getLogSessionId(log) !== sessionId) return false;
+  if (model !== undefined && log.model !== model) return false;
+  return true;
+}
+
+function requestedWindowFitsMemory(
+  filteredMemoryCount: number,
+  offset: number,
+  limit: number,
+): boolean {
+  return offset + limit <= filteredMemoryCount;
+}
+
+function paginateLogs(logs: CapturedLog[], offset: number, limit: number): CapturedLog[] {
+  return logs.slice(offset, offset + limit);
+}
+
+async function readPersistedLogsById(): Promise<Map<number, CapturedLog>> {
+  const byId = new Map<number, CapturedLog>();
+  const logDir = resolveLogDir();
+  if (!existsSync(logDir)) return byId;
+
+  try {
+    const entries = await readdirCallback(logDir);
+    const files = entries.filter((file) => file.endsWith(".jsonl")).sort();
+    for (const file of files) {
+      await readPersistedLogFile(join(logDir, file), byId);
+    }
+  } catch (err) {
+    logger.error("[store] Failed to list persisted logs:", String(err));
+  }
+
+  for (const log of memoryCache.values()) {
+    byId.set(log.id, log);
+  }
+
+  return byId;
+}
+
+async function readPersistedLogFile(
+  filePath: string,
+  byId: Map<number, CapturedLog>,
+): Promise<void> {
+  if (!existsSync(filePath)) return;
+
+  try {
+    const fileStream = createInterface({
+      input: createReadStream(filePath),
+      crlfDelay: Infinity,
+    });
+
+    for await (const line of fileStream) {
+      if (line.trim() === "") continue;
+      try {
+        const parsed: unknown = JSON.parse(line);
+        const result = CapturedLogSchema.safeParse(parsed);
+        if (result.success) {
+          const log = normalizeLogSession(result.data);
+          byId.set(log.id, log);
+          observeSessionLog(log);
+        }
+      } catch {
+        // Skip malformed lines.
+      }
+    }
+  } catch (err) {
+    logger.error("[store] Failed to read persisted log file:", filePath, String(err));
+  }
+}
+
+export async function listLogsPage(options: ListLogsPageOptions): Promise<ListLogsPageResult> {
+  const memoryLogs = getFilteredLogs(options.sessionId, options.model);
+  if (requestedWindowFitsMemory(memoryLogs.length, options.offset, options.limit)) {
+    return {
+      logs: paginateLogs(memoryLogs, options.offset, options.limit),
+      total: memoryLogs.length,
+      offset: options.offset,
+      limit: options.limit,
+    };
+  }
+
+  const persistedById = await readPersistedLogsById();
+  const persistedLogs = [...persistedById.values()]
+    .filter((log) => matchesLogFilters(log, options.sessionId, options.model))
+    .sort((left, right) => left.id - right.id);
+
+  return {
+    logs: paginateLogs(persistedLogs, options.offset, options.limit),
+    total: persistedLogs.length,
+    offset: options.offset,
+    limit: options.limit,
+  };
+}
+
 export function getSessions(): string[] {
   return getSessionIds();
 }
@@ -280,6 +427,60 @@ export function getModels(): string[] {
   return [...set];
 }
 
+export function getReplayLogsForSource(sourceLogId: number): CapturedLog[] {
+  return [...memoryCache.values()].filter((log) => log.replayOfLogId === sourceLogId);
+}
+
+function collectDirectoryStats(
+  dir: string,
+  shouldCount: (fileName: string) => boolean,
+): { fileCount: number; bytes: number } {
+  if (!existsSync(dir)) return { fileCount: 0, bytes: 0 };
+  let fileCount = 0;
+  let bytes = 0;
+  const visit = (currentDir: string, depth: number): void => {
+    if (depth > 8) return;
+    let entries: string[];
+    try {
+      entries = readdirSync(currentDir);
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const path = join(currentDir, entry);
+      try {
+        const stats = statSync(path);
+        if (stats.isDirectory()) {
+          visit(path, depth + 1);
+        } else if (shouldCount(entry)) {
+          fileCount += 1;
+          bytes += stats.size;
+        }
+      } catch {
+        // Ignore files that disappear while stats are being collected.
+      }
+    }
+  };
+  visit(dir, 0);
+  return { fileCount, bytes };
+}
+
+export function getLogStorageStats(): LogStorageStats {
+  const logDir = resolveLogDir();
+  const chunkDir = getChunksDir();
+  const logStats = collectDirectoryStats(logDir, (fileName) => fileName.endsWith(".jsonl"));
+  const chunkStats = collectDirectoryStats(chunkDir, (fileName) => fileName.endsWith(".json"));
+  return {
+    memoryCount: memoryCache.size,
+    logDir,
+    logFileCount: logStats.fileCount,
+    logBytes: logStats.bytes,
+    chunkDir,
+    chunkFileCount: chunkStats.fileCount,
+    chunkBytes: chunkStats.bytes,
+  };
+}
+
 /**
  * Load recent completed log entries from all log files into the memory cache.
  * Called on server startup so the frontend shows existing logs.
@@ -359,6 +560,206 @@ export function clearAllLogs(): { cleared: number } {
   return { cleared: count };
 }
 
+async function deleteMatchingFiles(
+  dir: string,
+  shouldDelete: (fileName: string) => boolean,
+): Promise<number> {
+  if (!existsSync(dir)) return 0;
+
+  let deleted = 0;
+  let entries: string[];
+  try {
+    entries = await readdirCallback(dir);
+  } catch (err) {
+    logger.warn("[store] Failed to list log storage directory:", String(err));
+    return deleted;
+  }
+
+  for (const entry of entries) {
+    if (!shouldDelete(entry)) continue;
+    try {
+      await unlink(join(dir, entry));
+      deleted += 1;
+    } catch (err) {
+      logger.warn("[store] Failed to delete log storage file:", entry, String(err));
+    }
+  }
+  return deleted;
+}
+
+export async function clearPersistedLogStorage(): Promise<ClearPersistedLogStorageResult> {
+  return await runWithFlushedLogWriteLock(async () => {
+    await flushIndex();
+
+    const result = clearAllLogs();
+    const [logFilesDeleted, chunkFilesDeleted] = await Promise.all([
+      deleteMatchingFiles(
+        resolveLogDir(),
+        (fileName) => fileName.endsWith(".jsonl") || fileName === "logs.idx",
+      ),
+      deleteMatchingFiles(
+        getChunksDir(),
+        (fileName) =>
+          fileName.endsWith(".json") || (fileName.startsWith(".") && fileName.endsWith(".tmp")),
+      ),
+    ]);
+
+    await rebuildIndex();
+
+    return {
+      cleared: result.cleared,
+      logFilesDeleted,
+      chunkFilesDeleted,
+    };
+  });
+}
+
+function readLogIdFromLine(line: string): number | null {
+  if (line.trim() === "") return null;
+  try {
+    const parsed: unknown = JSON.parse(line);
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null;
+    const idDesc = Object.getOwnPropertyDescriptor(parsed, "id");
+    if (idDesc === undefined || typeof idDesc.value !== "number") return null;
+    return idDesc.value;
+  } catch {
+    return null;
+  }
+}
+
+type RewriteLogFileResult = {
+  rewritten: boolean;
+  removedIds: Set<number>;
+};
+
+async function rewriteLogFileWithoutIds(
+  filePath: string,
+  ids: ReadonlySet<number>,
+): Promise<RewriteLogFileResult> {
+  let content: string;
+  try {
+    content = await readFile(filePath, "utf-8");
+  } catch (err) {
+    logger.warn("[store] Failed to read log file for deletion:", filePath, String(err));
+    return { rewritten: false, removedIds: new Set() };
+  }
+
+  const lines = content.split("\n");
+  const keptLines: string[] = [];
+  const removedIds = new Set<number>();
+  for (const line of lines) {
+    if (line === "") continue;
+    const id = readLogIdFromLine(line);
+    if (id !== null && ids.has(id)) {
+      removedIds.add(id);
+    } else {
+      keptLines.push(line);
+    }
+  }
+
+  if (removedIds.size === 0) return { rewritten: false, removedIds };
+
+  try {
+    if (keptLines.length === 0) {
+      await unlink(filePath);
+    } else {
+      await writeFile(filePath, `${keptLines.join("\n")}\n`, "utf-8");
+    }
+  } catch (err) {
+    logger.warn("[store] Failed to rewrite log file after deletion:", filePath, String(err));
+    return { rewritten: false, removedIds: new Set() };
+  }
+
+  return { rewritten: true, removedIds };
+}
+
+type RewriteLogFilesResult = {
+  logFilesRewritten: number;
+  removedIds: Set<number>;
+};
+
+async function rewriteLogFilesWithoutIds(ids: ReadonlySet<number>): Promise<RewriteLogFilesResult> {
+  const logDir = resolveLogDir();
+  if (!existsSync(logDir)) return { logFilesRewritten: 0, removedIds: new Set() };
+
+  let entries: string[];
+  try {
+    entries = await readdirCallback(logDir);
+  } catch (err) {
+    logger.warn("[store] Failed to list log directory for deletion:", String(err));
+    return { logFilesRewritten: 0, removedIds: new Set() };
+  }
+
+  let logFilesRewritten = 0;
+  const removedIds = new Set<number>();
+  for (const entry of entries) {
+    if (!entry.endsWith(".jsonl")) continue;
+    const result = await rewriteLogFileWithoutIds(join(logDir, entry), ids);
+    if (result.rewritten) logFilesRewritten += 1;
+    for (const removedId of result.removedIds) {
+      removedIds.add(removedId);
+    }
+  }
+  return { logFilesRewritten, removedIds };
+}
+
+async function deleteChunkFilesByIds(ids: ReadonlySet<number>): Promise<number> {
+  const chunkDir = getChunksDir();
+  if (!existsSync(chunkDir)) return 0;
+
+  let deleted = 0;
+  for (const id of ids) {
+    const candidates = [`${String(id)}.json`, `.${String(id)}.tmp`];
+    for (const candidate of candidates) {
+      try {
+        await unlink(join(chunkDir, candidate));
+        deleted += 1;
+      } catch {
+        // Missing chunk files are expected for non-streaming logs.
+      }
+    }
+  }
+  return deleted;
+}
+
+export async function clearPersistedLogsByIds(
+  ids: readonly number[],
+): Promise<ClearPersistedLogsByIdsResult> {
+  const uniqueIds = new Set(ids);
+  if (uniqueIds.size === 0) {
+    return { cleared: 0, logFilesRewritten: 0, chunkFilesDeleted: 0 };
+  }
+
+  return await runWithFlushedLogWriteLock(async () => {
+    await flushIndex();
+
+    const memoryRemovedIds = new Set<number>();
+    for (const id of uniqueIds) {
+      if (memoryCache.has(id)) memoryRemovedIds.add(id);
+    }
+    const result = clearLogsByIds([...uniqueIds]);
+    const [rewriteResult, chunkFilesDeleted] = await Promise.all([
+      rewriteLogFilesWithoutIds(uniqueIds),
+      deleteChunkFilesByIds(uniqueIds),
+    ]);
+
+    await rebuildIndex();
+
+    const clearedIds = new Set<number>(rewriteResult.removedIds);
+    if (result.cleared > 0) {
+      for (const id of memoryRemovedIds) {
+        clearedIds.add(id);
+      }
+    }
+
+    return {
+      cleared: clearedIds.size,
+      logFilesRewritten: rewriteResult.logFilesRewritten,
+      chunkFilesDeleted,
+    };
+  });
+}
+
 /**
  * Remove a specific set of log IDs from the in-memory cache. Returns the
  * number of logs actually removed. IDs not present in the cache are ignored.