@tonyclaw/agent-inspector 3.1.20 → 3.1.22

package/src/proxy/storageLifecycle.ts

@@ -0,0 +1,432 @@
+import { existsSync, lstatSync, readdirSync, unlinkSync } from "node:fs";
+import { basename, dirname, join, relative, resolve } from "node:path";
+import { getChunksDir } from "./chunkStorage";
+import { getDataDir } from "./dataDir";
+import { resolveApplicationLogPath, resolveLogDir } from "./logger";
+import { getRawStreamDir } from "./rawStreamCapture";
+import { getSessionArchiveRoot } from "./sessionArchive";
+
+export type StorageArtifactKind =
+  | "captured-log-jsonl"
+  | "sqlite-log-index"
+  | "session-archive"
+  | "streaming-chunk"
+  | "raw-stream-temp"
+  | "application-log"
+  | "evidence"
+  | "backup";
+
+export type StorageArtifactSensitivity = "captured-data" | "secret-adjacent" | "operational";
+
+export type StorageArtifactCleanupBehavior =
+  | "retention-gc"
+  | "orphan-reclaim"
+  | "rotation"
+  | "manual-owner-delete";
+
+export type StorageArtifactInventoryRule = {
+  kind: StorageArtifactKind;
+  root: string;
+  sensitivity: StorageArtifactSensitivity;
+  cleanupBehavior: StorageArtifactCleanupBehavior;
+  shouldCount: (fileName: string, path: string) => boolean;
+};
+
+export type StorageArtifactSummary = {
+  kind: StorageArtifactKind;
+  root: string;
+  sensitivity: StorageArtifactSensitivity;
+  cleanupBehavior: StorageArtifactCleanupBehavior;
+  fileCount: number;
+  bytes: number;
+  oldestModifiedAt: string | null;
+  newestModifiedAt: string | null;
+};
+
+export type StorageLifecycleInventory = {
+  checkedAt: string;
+  totalFiles: number;
+  totalBytes: number;
+  artifacts: StorageArtifactSummary[];
+};
+
+export type StorageLifecycleQuotaPolicy = {
+  maxAgeMs: number;
+  maxTotalBytes: number;
+};
+
+export type StorageLifecycleGcError = {
+  kind: StorageArtifactKind;
+  path: string;
+  message: string;
+};
+
+export type StorageLifecycleGcResult = {
+  checkedAt: string;
+  policy: StorageLifecycleQuotaPolicy;
+  before: StorageLifecycleInventory;
+  after: StorageLifecycleInventory;
+  filesScanned: number;
+  bytesScanned: number;
+  filesDeleted: number;
+  bytesDeleted: number;
+  reclaimableBytes: number;
+  errors: StorageLifecycleGcError[];
+};
+
+type MutableDirectoryStats = {
+  fileCount: number;
+  bytes: number;
+  oldestModifiedMs: number | null;
+  newestModifiedMs: number | null;
+};
+
+type StorageArtifactFile = {
+  rule: StorageArtifactInventoryRule;
+  path: string;
+  bytes: number;
+  modifiedMs: number;
+};
+
+const MAX_INVENTORY_DEPTH = 8;
+const DAY_MS = 24 * 60 * 60 * 1000;
+const DEFAULT_RETENTION_DAYS = 7;
+const DEFAULT_MAX_TOTAL_BYTES = 2 * 1024 * 1024 * 1024;
+const DEFAULT_MAINTENANCE_INTERVAL_MS = 60 * 60 * 1000;
+
+let maintenanceTimer: ReturnType<typeof setInterval> | null = null;
+let lastStorageLifecycleGcResult: StorageLifecycleGcResult | null = null;
+
+function emptyStats(): MutableDirectoryStats {
+  return {
+    fileCount: 0,
+    bytes: 0,
+    oldestModifiedMs: null,
+    newestModifiedMs: null,
+  };
+}
+
+function recordFile(stats: MutableDirectoryStats, bytes: number, modifiedMs: number): void {
+  stats.fileCount += 1;
+  stats.bytes += bytes;
+  stats.oldestModifiedMs =
+    stats.oldestModifiedMs === null ? modifiedMs : Math.min(stats.oldestModifiedMs, modifiedMs);
+  stats.newestModifiedMs =
+    stats.newestModifiedMs === null ? modifiedMs : Math.max(stats.newestModifiedMs, modifiedMs);
+}
+
+function isoFromMs(value: number | null): string | null {
+  return value === null ? null : new Date(value).toISOString();
+}
+
+function isPathInsideRoot(root: string, path: string): boolean {
+  const relativePath = relative(resolve(root), resolve(path));
+  return (
+    relativePath.length === 0 ||
+    (!relativePath.startsWith("..") &&
+      !relativePath.startsWith("/") &&
+      !relativePath.startsWith("\\"))
+  );
+}
+
+function collectRuleFiles(rule: StorageArtifactInventoryRule): StorageArtifactFile[] {
+  const files: StorageArtifactFile[] = [];
+  if (!existsSync(rule.root)) return files;
+
+  const visit = (currentDir: string, depth: number): void => {
+    if (depth > MAX_INVENTORY_DEPTH) return;
+    let entries: string[];
+    try {
+      entries = readdirSync(currentDir);
+    } catch {
+      return;
+    }
+
+    for (const entry of entries) {
+      const path = join(currentDir, entry);
+      if (!isPathInsideRoot(rule.root, path)) continue;
+      try {
+        const fileStats = lstatSync(path);
+        if (fileStats.isDirectory()) {
+          visit(path, depth + 1);
+          continue;
+        }
+        if (!fileStats.isFile() || !rule.shouldCount(entry, path)) continue;
+        files.push({ rule, path, bytes: fileStats.size, modifiedMs: fileStats.mtimeMs });
+      } catch {
+        // Files may disappear while inventory is collected; keep inventory best-effort and safe.
+      }
+    }
+  };
+
+  visit(rule.root, 0);
+  return files;
+}
+
+function collectRuleStats(rule: StorageArtifactInventoryRule): MutableDirectoryStats {
+  const stats = emptyStats();
+  for (const file of collectRuleFiles(rule)) {
+    recordFile(stats, file.bytes, file.modifiedMs);
+  }
+  return stats;
+}
+
+function isAutoDeletableCleanupBehavior(behavior: StorageArtifactCleanupBehavior): boolean {
+  switch (behavior) {
+    case "retention-gc":
+    case "orphan-reclaim":
+      return true;
+    case "rotation":
+    case "manual-owner-delete":
+      return false;
+  }
+}
+
+function summarizeRule(rule: StorageArtifactInventoryRule): StorageArtifactSummary {
+  const stats = collectRuleStats(rule);
+  return {
+    kind: rule.kind,
+    root: rule.root,
+    sensitivity: rule.sensitivity,
+    cleanupBehavior: rule.cleanupBehavior,
+    fileCount: stats.fileCount,
+    bytes: stats.bytes,
+    oldestModifiedAt: isoFromMs(stats.oldestModifiedMs),
+    newestModifiedAt: isoFromMs(stats.newestModifiedMs),
+  };
+}
+
+function isSessionArchiveFile(fileName: string): boolean {
+  return fileName === "session.sqlite" || fileName.startsWith("session.sqlite-");
+}
+
+function isSqliteLogIndexFile(fileName: string): boolean {
+  return fileName === "inspector.sqlite" || fileName.startsWith("inspector.sqlite-");
+}
+
+function isBackupFile(fileName: string): boolean {
+  return fileName.endsWith(".bak") || fileName.endsWith(".backup");
+}
+
+function applicationLogMatcher(logPath: string): (fileName: string, path: string) => boolean {
+  const directory = dirname(logPath);
+  const baseName = basename(logPath);
+  return (fileName, path) => {
+    if (dirname(path) !== directory) return false;
+    if (fileName === baseName) return true;
+    return fileName.startsWith(`${baseName}.`) && /^\d+$/.test(fileName.slice(baseName.length + 1));
+  };
+}
+
+function parsePositiveIntegerEnv(name: string, fallback: number): number {
+  const value = process.env[name];
+  if (value === undefined) return fallback;
+  const parsed = Number.parseInt(value, 10);
+  if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
+  return parsed;
+}
+
+export function resolveStorageLifecycleQuotaPolicy(): StorageLifecycleQuotaPolicy {
+  const retentionDays = parsePositiveIntegerEnv(
+    "AGENT_INSPECTOR_STORAGE_RETENTION_DAYS",
+    parsePositiveIntegerEnv("LOG_RETENTION_DAYS", DEFAULT_RETENTION_DAYS),
+  );
+  return {
+    maxAgeMs: retentionDays * DAY_MS,
+    maxTotalBytes: parsePositiveIntegerEnv(
+      "AGENT_INSPECTOR_STORAGE_MAX_BYTES",
+      DEFAULT_MAX_TOTAL_BYTES,
+    ),
+  };
+}
+
+export function getStorageLifecycleInventoryRules(): StorageArtifactInventoryRule[] {
+  const applicationLogPath = resolveApplicationLogPath();
+  return [
+    {
+      kind: "captured-log-jsonl",
+      root: resolveLogDir(),
+      sensitivity: "captured-data",
+      cleanupBehavior: "retention-gc",
+      shouldCount: (fileName) => fileName.endsWith(".jsonl"),
+    },
+    {
+      kind: "sqlite-log-index",
+      root: resolveLogDir(),
+      sensitivity: "captured-data",
+      cleanupBehavior: "orphan-reclaim",
+      shouldCount: isSqliteLogIndexFile,
+    },
+    {
+      kind: "session-archive",
+      root: getSessionArchiveRoot(),
+      sensitivity: "captured-data",
+      cleanupBehavior: "retention-gc",
+      shouldCount: isSessionArchiveFile,
+    },
+    {
+      kind: "streaming-chunk",
+      root: getChunksDir(),
+      sensitivity: "captured-data",
+      cleanupBehavior: "retention-gc",
+      shouldCount: (fileName) => fileName.endsWith(".json"),
+    },
+    {
+      kind: "raw-stream-temp",
+      root: getRawStreamDir(),
+      sensitivity: "captured-data",
+      cleanupBehavior: "orphan-reclaim",
+      shouldCount: () => true,
+    },
+    {
+      kind: "application-log",
+      root: dirname(applicationLogPath),
+      sensitivity: "operational",
+      cleanupBehavior: "rotation",
+      shouldCount: applicationLogMatcher(applicationLogPath),
+    },
+    {
+      kind: "evidence",
+      root: join(getDataDir(), "evidence"),
+      sensitivity: "captured-data",
+      cleanupBehavior: "manual-owner-delete",
+      shouldCount: () => true,
+    },
+    {
+      kind: "backup",
+      root: getDataDir(),
+      sensitivity: "secret-adjacent",
+      cleanupBehavior: "orphan-reclaim",
+      shouldCount: isBackupFile,
+    },
+  ];
+}
+
+export function getStorageLifecycleInventory(
+  rules: readonly StorageArtifactInventoryRule[] = getStorageLifecycleInventoryRules(),
+): StorageLifecycleInventory {
+  const artifacts = rules.map(summarizeRule);
+  return {
+    checkedAt: new Date().toISOString(),
+    totalFiles: artifacts.reduce((sum, artifact) => sum + artifact.fileCount, 0),
+    totalBytes: artifacts.reduce((sum, artifact) => sum + artifact.bytes, 0),
+    artifacts,
+  };
+}
+
+function collectLifecycleFiles(
+  rules: readonly StorageArtifactInventoryRule[] = getStorageLifecycleInventoryRules(),
+): StorageArtifactFile[] {
+  return rules.flatMap((rule) => collectRuleFiles(rule));
+}
+
+function selectGcCandidates(
+  files: readonly StorageArtifactFile[],
+  policy: StorageLifecycleQuotaPolicy,
+  nowMs: number,
+): StorageArtifactFile[] {
+  const autoDeletableFiles = files.filter((file) =>
+    isAutoDeletableCleanupBehavior(file.rule.cleanupBehavior),
+  );
+  const expiredFiles = autoDeletableFiles.filter(
+    (file) => nowMs - file.modifiedMs > policy.maxAgeMs,
+  );
+  const expiredPaths = new Set(expiredFiles.map((file) => file.path));
+  const candidates = [...expiredFiles];
+  let totalBytes =
+    files.reduce((sum, file) => sum + file.bytes, 0) -
+    expiredFiles.reduce((sum, file) => sum + file.bytes, 0);
+
+  if (totalBytes <= policy.maxTotalBytes) return candidates;
+
+  const quotaCandidates = autoDeletableFiles
+    .filter((file) => !expiredPaths.has(file.path))
+    .sort((a, b) => a.modifiedMs - b.modifiedMs);
+
+  for (const file of quotaCandidates) {
+    if (totalBytes <= policy.maxTotalBytes) break;
+    candidates.push(file);
+    totalBytes -= file.bytes;
+  }
+
+  return candidates;
+}
+
+function deleteLifecycleFile(file: StorageArtifactFile): StorageLifecycleGcError | null {
+  if (!isPathInsideRoot(file.rule.root, file.path)) {
+    return { kind: file.rule.kind, path: file.path, message: "path outside lifecycle root" };
+  }
+  try {
+    unlinkSync(file.path);
+    return null;
+  } catch (err) {
+    if (err instanceof Error && err.message.includes("ENOENT")) return null;
+    return { kind: file.rule.kind, path: file.path, message: String(err) };
+  }
+}
+
+export function runStorageLifecycleGc(
+  policy: StorageLifecycleQuotaPolicy = resolveStorageLifecycleQuotaPolicy(),
+  rules: readonly StorageArtifactInventoryRule[] = getStorageLifecycleInventoryRules(),
+  nowMs: number = Date.now(),
+): StorageLifecycleGcResult {
+  const before = getStorageLifecycleInventory(rules);
+  const files = collectLifecycleFiles(rules);
+  const candidates = selectGcCandidates(files, policy, nowMs);
+  const errors: StorageLifecycleGcError[] = [];
+  let filesDeleted = 0;
+  let bytesDeleted = 0;
+
+  for (const file of candidates) {
+    const error = deleteLifecycleFile(file);
+    if (error !== null) {
+      errors.push(error);
+      continue;
+    }
+    filesDeleted += 1;
+    bytesDeleted += file.bytes;
+  }
+
+  const after = getStorageLifecycleInventory(rules);
+  const autoDeletableBytes = after.artifacts
+    .filter((artifact) => isAutoDeletableCleanupBehavior(artifact.cleanupBehavior))
+    .reduce((sum, artifact) => sum + artifact.bytes, 0);
+
+  const result = {
+    checkedAt: new Date(nowMs).toISOString(),
+    policy,
+    before,
+    after,
+    filesScanned: files.length,
+    bytesScanned: files.reduce((sum, file) => sum + file.bytes, 0),
+    filesDeleted,
+    bytesDeleted,
+    reclaimableBytes: autoDeletableBytes,
+    errors,
+  };
+  lastStorageLifecycleGcResult = result;
+  return result;
+}
+
+export function getLastStorageLifecycleGcResult(): StorageLifecycleGcResult | null {
+  return lastStorageLifecycleGcResult;
+}
+
+export function installStorageLifecycleMaintenance(): void {
+  if (maintenanceTimer !== null) return;
+  if (process.env["NODE_ENV"] === "test") return;
+  const intervalMs = parsePositiveIntegerEnv(
+    "AGENT_INSPECTOR_STORAGE_GC_INTERVAL_MS",
+    DEFAULT_MAINTENANCE_INTERVAL_MS,
+  );
+  maintenanceTimer = setInterval(() => {
+    runStorageLifecycleGc();
+  }, intervalMs);
+  maintenanceTimer.unref();
+}
+
+export function stopStorageLifecycleMaintenanceForTests(): void {
+  if (maintenanceTimer === null) return;
+  clearInterval(maintenanceTimer);
+  maintenanceTimer = null;
+}

package/src/proxy/store.ts

@@ -34,7 +34,7 @@ import {
   type LogIndexSummary,
 } from "./logIndex";
 import { writeChunks } from "./chunkStorage";
-import { closeSqliteLogIndex } from "./sqliteLogIndex";
+import { closeSqliteLogIndex, listSqliteLogCursorEntries } from "./sqliteLogIndex";
 import type { SessionInfo, SessionLogSummary } from "../lib/sessionInfoContract";
 import { extractToolTraceEvents } from "../lib/toolTrace";
 import { extractStopReason } from "../lib/stopReason";
@@ -72,6 +72,7 @@ import {
   type SessionClientInfo,
   type SessionSnapshot,
 } from "./sessionSupervisor";
+import { installStorageLifecycleMaintenance, runStorageLifecycleGc } from "./storageLifecycle";
 
 export type { CapturedLog };
 export { getNextLogId };
@@ -869,21 +870,50 @@ export async function listLogsPage(options: ListLogsPageOptions): Promise<ListLo
   };
 }
 
-function matchesCursorWindow(id: number, options: ListLogsCursorPageOptions): boolean {
-  if (options.beforeLogId !== undefined && id >= options.beforeLogId) return false;
-  if (options.afterLogId !== undefined && id <= options.afterLogId) return false;
-  return true;
+function lowerBoundLogId(ids: readonly number[], target: number): number {
+  let low = 0;
+  let high = ids.length;
+  while (low < high) {
+    const middle = Math.floor((low + high) / 2);
+    const id = ids[middle];
+    if (id === undefined || id >= target) {
+      high = middle;
+      continue;
+    }
+    low = middle + 1;
+  }
+  return low;
+}
+
+function upperBoundLogId(ids: readonly number[], target: number): number {
+  let low = 0;
+  let high = ids.length;
+  while (low < high) {
+    const middle = Math.floor((low + high) / 2);
+    const id = ids[middle];
+    if (id === undefined || id > target) {
+      high = middle;
+      continue;
+    }
+    low = middle + 1;
+  }
+  return low;
 }
 
 function selectCursorPageLogs(
   ids: readonly number[],
   options: ListLogsCursorPageOptions,
 ): number[] {
-  const matchingWindow = ids.filter((id) => matchesCursorWindow(id, options));
+  const windowStart =
+    options.afterLogId === undefined ? 0 : upperBoundLogId(ids, options.afterLogId);
+  const windowEnd =
+    options.beforeLogId === undefined ? ids.length : lowerBoundLogId(ids, options.beforeLogId);
+  if (windowStart >= windowEnd) return [];
+
   if (options.afterLogId !== undefined || options.anchor === "oldest") {
-    return matchingWindow.slice(0, options.limit);
+    return ids.slice(windowStart, Math.min(windowEnd, windowStart + options.limit));
   }
-  return matchingWindow.slice(Math.max(0, matchingWindow.length - options.limit));
+  return ids.slice(Math.max(windowStart, windowEnd - options.limit), windowEnd);
 }
 
 function rememberLogCursorPageIndex(index: LogCursorPageIndex): LogCursorPageIndex {
@@ -929,12 +959,18 @@ async function buildLogCursorPageIndexFromLogIndex(
   options: Pick<ListLogsCursorPageOptions, "sessionId" | "model" | "clientPid">,
   allowRebuild: boolean,
 ): Promise<LogCursorPageIndex | null> {
-  if (options.clientPid !== undefined) return null;
-
-  let entries = await listFilteredIndexEntries(options);
+  let entries = await listSqliteLogCursorEntries(options);
+  if (entries === null) {
+    if (options.clientPid !== undefined) return null;
+    entries = await listFilteredIndexEntries(options);
+  }
   if (entries.length === 0 && allowRebuild) {
     await rebuildIndex();
-    entries = await listFilteredIndexEntries(options);
+    entries = await listSqliteLogCursorEntries(options);
+    if (entries === null) {
+      if (options.clientPid !== undefined) return null;
+      entries = await listFilteredIndexEntries(options);
+    }
   }
 
   if (entries.some((entry) => !hasLogIndexFilterMetadata(entry))) {
@@ -1314,6 +1350,20 @@ export function getLogStorageStats(): LogStorageStats {
  * then only retain the newest MAX_MEMORY_CACHE entries in cache.
  */
 export async function loadLogsIntoMemory(): Promise<void> {
+  try {
+    const gcResult = runStorageLifecycleGc();
+    if (gcResult.filesDeleted > 0 || gcResult.errors.length > 0) {
+      logger.info(
+        "[store] Storage lifecycle GC completed:",
+        `${gcResult.filesDeleted} files deleted, ${gcResult.bytesDeleted} bytes reclaimed,`,
+        `${gcResult.errors.length} errors`,
+      );
+    }
+  } catch (err) {
+    logger.warn("[store] Storage lifecycle GC failed:", String(err));
+  }
+  installStorageLifecycleMaintenance();
+
   const logDir = resolveLogDir();
   if (!existsSync(logDir)) return;
 

package/.output/server/_tanstack-start-manifest_v-DnbdNeun.mjs

@@ -1,4 +0,0 @@
-const tsrStartManifest = () => ({ routes: { __root__: { filePath: "C:/Users/claw/workspace/agent-inspector/src/backend/routes/__root.ts", children: ["/$", "/livez", "/readyz", "/api/alerts", "/api/config", "/api/groups", "/api/health", "/api/instances", "/api/logs", "/api/mcp", "/api/models", "/api/pi-agent", "/api/providers", "/api/runs", "/api/runtime", "/api/sessions", "/proxy/$", "/api/ecosystem/packages", "/api/ecosystem/tasks", "/api/knowledge/candidates", "/api/knowledge/project-context", "/api/knowledge/search", "/api/ecosystem/recipes/$recipeId/run", "/api/knowledge/sessions/$sessionId/candidates"], preloads: ["/assets/index-D4LX-sw-.js"], scripts: [{ attrs: { type: "module", async: true, src: "/assets/index-D4LX-sw-.js" } }] } } });
-export {
-  tsrStartManifest
-};