agentwaste-core 0.1.0 → 0.1.1

package/src/scanner.js

@@ -1,14 +1,25 @@
 import { createReadStream, existsSync, readdirSync, readFileSync, statSync } from "node:fs";
-import { homedir } from "node:os";
+import { readFile } from "node:fs/promises";
+import { cpus, homedir } from "node:os";
 import { basename, join, relative } from "node:path";
 import { createInterface } from "node:readline";
 
 const DEFAULT_SCAN_DAYS = 365;
-const DEFAULT_MAX_FILES = Number.MAX_SAFE_INTEGER;
+const DEFAULT_MAX_FILES = 5000;
+const EXHAUSTIVE_MAX_FILES = Number.MAX_SAFE_INTEGER;
+const DEFAULT_SCAN_CONCURRENCY = Math.min(32, Math.max(4, cpus().length * 2 || 4));
+const MAX_SCAN_CONCURRENCY = 64;
 const MAX_FILE_BYTES = 64 * 1024 * 1024;
 const CONTEXT_REVIEW_FRESH_INPUT_PER_TOOL_CALL = 4000;
 const DEFAULT_COST_PER_MILLION_TOKENS = 3.0;
 
+export const SCAN_DEFAULTS = Object.freeze({
+  days: DEFAULT_SCAN_DAYS,
+  maxFiles: DEFAULT_MAX_FILES,
+  exhaustiveMaxFiles: EXHAUSTIVE_MAX_FILES,
+  concurrency: DEFAULT_SCAN_CONCURRENCY,
+});
+
 const IGNORED_DIRS = new Set([
   ".git",
   ".hg",
@@ -66,20 +77,21 @@ export async function scanAgentWaste(input = {}) {
   const homeDir = input.homeDir ?? homedir();
   const projectDir = input.projectDir ?? process.cwd();
   const days = normalizeScanDays(input.days ?? DEFAULT_SCAN_DAYS);
-  const maxFiles = input.maxFiles ?? DEFAULT_MAX_FILES;
+  const maxFiles = normalizeMaxFiles(input.maxFiles ?? DEFAULT_MAX_FILES);
+  const concurrency = normalizeConcurrency(input.concurrency ?? DEFAULT_SCAN_CONCURRENCY);
   const now = input.now ?? Date.now();
   const sinceMs = days === "all" ? 0 : now - days * 24 * 60 * 60 * 1000;
   const totals = createTotals({ homeDir, projectDir, days, sinceMs, now });
 
-  await scanCodex(homeDir, sinceMs, maxFiles, totals);
-  await scanClaude(homeDir, sinceMs, maxFiles, totals);
-  await scanPi(homeDir, sinceMs, maxFiles, totals);
-  await scanGemini(homeDir, sinceMs, maxFiles, totals);
-  await scanOpenCode(homeDir, sinceMs, maxFiles, totals);
-  await scanCursor(homeDir, sinceMs, maxFiles, totals);
-  await scanWindsurf(homeDir, sinceMs, maxFiles, totals);
-  await scanEditorExtensionAgents(homeDir, sinceMs, maxFiles, totals);
-  await scanCopilot(homeDir, sinceMs, maxFiles, totals);
+  await scanCodex(homeDir, sinceMs, maxFiles, totals, concurrency);
+  await scanClaude(homeDir, sinceMs, maxFiles, totals, concurrency);
+  await scanPi(homeDir, sinceMs, maxFiles, totals, concurrency);
+  await scanGemini(homeDir, sinceMs, maxFiles, totals, concurrency);
+  await scanOpenCode(homeDir, sinceMs, maxFiles, totals, concurrency);
+  await scanCursor(homeDir, sinceMs, maxFiles, totals, concurrency);
+  await scanWindsurf(homeDir, sinceMs, maxFiles, totals, concurrency);
+  await scanEditorExtensionAgents(homeDir, sinceMs, maxFiles, totals, concurrency);
+  await scanCopilot(homeDir, sinceMs, maxFiles, totals, concurrency);
   scanAider(projectDir, sinceMs, totals);
   scanConfigFiles(homeDir, sinceMs, totals);
   scanGlueCode(projectDir, totals);
@@ -93,6 +105,17 @@ function normalizeScanDays(value) {
   return Number.isFinite(days) && days > 0 ? days : DEFAULT_SCAN_DAYS;
 }
 
+function normalizeMaxFiles(value) {
+  const maxFiles = Number.parseInt(value, 10);
+  return Number.isFinite(maxFiles) && maxFiles > 0 ? maxFiles : DEFAULT_MAX_FILES;
+}
+
+function normalizeConcurrency(value) {
+  const concurrency = Number.parseInt(value, 10);
+  if (!Number.isFinite(concurrency) || concurrency <= 0) return DEFAULT_SCAN_CONCURRENCY;
+  return Math.min(MAX_SCAN_CONCURRENCY, concurrency);
+}
+
 function createTotals({ homeDir, projectDir, days, sinceMs, now }) {
   return {
     homeDir,
@@ -175,7 +198,7 @@ function createSourceStats(label) {
   };
 }
 
-async function scanCodex(homeDir, sinceMs, maxFiles, totals) {
+async function scanCodex(homeDir, sinceMs, maxFiles, totals, concurrency) {
   const root = join(homeDir, ".codex", "sessions");
   const source = totals.sources.codex;
   if (!existsSync(root)) {
@@ -186,7 +209,7 @@ async function scanCodex(homeDir, sinceMs, maxFiles, totals) {
   const files = recentFiles(root, sinceMs, maxFiles, (path) => path.endsWith(".jsonl"));
   source.files = files.length;
   totals.files += files.length;
-  for (const path of files) await scanCodexFile(path, totals);
+  await scanFiles(files, concurrency, (path) => scanCodexFile(path, totals));
 }
 
 async function scanCodexFile(path, totals) {
@@ -274,7 +297,7 @@ async function scanCodexFile(path, totals) {
   if (!session.id) addSession(totals, source, path);
 }
 
-async function scanClaude(homeDir, sinceMs, maxFiles, totals) {
+async function scanClaude(homeDir, sinceMs, maxFiles, totals, concurrency) {
   const root = join(homeDir, ".claude", "projects");
   const source = totals.sources.claude;
   if (!existsSync(root)) {
@@ -285,7 +308,7 @@ async function scanClaude(homeDir, sinceMs, maxFiles, totals) {
   const files = recentFiles(root, sinceMs, maxFiles, (path) => path.endsWith(".jsonl"));
   source.files = files.length;
   totals.files += files.length;
-  for (const path of files) await scanClaudeFile(path, totals);
+  await scanFiles(files, concurrency, (path) => scanClaudeFile(path, totals));
 
   const facetsRoot = join(homeDir, ".claude", "usage-data", "facets");
   if (existsSync(facetsRoot)) {
@@ -357,12 +380,12 @@ async function scanClaudeFile(path, totals) {
   if (!session.id) addSession(totals, source, path);
 }
 
-async function scanPi(homeDir, sinceMs, maxFiles, totals) {
+async function scanPi(homeDir, sinceMs, maxFiles, totals, concurrency) {
   const source = totals.sources.pi;
   const sessionRoot = join(homeDir, ".pi", "agent", "sessions");
   const files = recentFilesFromRoots([sessionRoot], sinceMs, maxFiles, (path) => path.endsWith(".jsonl"));
   if (files.length === 0 && !existsSync(sessionRoot)) source.unavailable = true;
-  for (const path of files) await scanGenericAgentFile(path, totals, source, { modelFacing: true, sessionPerFile: true });
+  await scanFiles(files, concurrency, (path) => scanGenericAgentFile(path, totals, source, { modelFacing: true, sessionPerFile: true }));
 
   scanKnownFiles([
     join(homeDir, ".pi", "agent", "settings.json"),
@@ -372,7 +395,7 @@ async function scanPi(homeDir, sinceMs, maxFiles, totals) {
   ], sinceMs, totals, source, { modelFacing: false, sessionPerFile: false });
 }
 
-async function scanGemini(homeDir, sinceMs, maxFiles, totals) {
+async function scanGemini(homeDir, sinceMs, maxFiles, totals, concurrency) {
   const source = totals.sources.gemini;
   const root = join(homeDir, ".gemini");
   const sessionRoots = [join(root, "tmp"), join(root, "sessions"), join(root, "exports")];
@@ -384,7 +407,7 @@ async function scanGemini(homeDir, sinceMs, maxFiles, totals) {
       name === "collector.log";
   });
   if (files.length === 0 && !existsSync(root)) source.unavailable = true;
-  for (const path of files) await scanGenericAgentFile(path, totals, source, { modelFacing: true, sessionPerFile: /(?:session|chat|\.jsonl)/i.test(path) });
+  await scanFiles(files, concurrency, (path) => scanGenericAgentFile(path, totals, source, { modelFacing: true, sessionPerFile: /(?:session|chat|\.jsonl)/i.test(path) }));
 
   scanKnownFiles([
     join(root, "settings.json"),
@@ -393,12 +416,12 @@ async function scanGemini(homeDir, sinceMs, maxFiles, totals) {
   ], sinceMs, totals, source, { modelFacing: false, sessionPerFile: false });
 }
 
-async function scanOpenCode(homeDir, sinceMs, maxFiles, totals) {
+async function scanOpenCode(homeDir, sinceMs, maxFiles, totals, concurrency) {
   const source = totals.sources.opencode;
   const root = join(homeDir, ".local", "share", "opencode");
   const files = recentFilesFromRoots([join(root, "log"), join(root, "project")], sinceMs, maxFiles, isLikelyAgentDataFile);
   if (files.length === 0 && !existsSync(root)) source.unavailable = true;
-  for (const path of files) await scanGenericAgentFile(path, totals, source, { modelFacing: true, sessionPerFile: path.includes("/project/") });
+  await scanFiles(files, concurrency, (path) => scanGenericAgentFile(path, totals, source, { modelFacing: true, sessionPerFile: path.includes("/project/") }));
 
   scanKnownFiles([
     join(root, "auth.json"),
@@ -407,7 +430,7 @@ async function scanOpenCode(homeDir, sinceMs, maxFiles, totals) {
   ], sinceMs, totals, source, { modelFacing: false, sessionPerFile: false });
 }
 
-async function scanCursor(homeDir, sinceMs, maxFiles, totals) {
+async function scanCursor(homeDir, sinceMs, maxFiles, totals, concurrency) {
   const source = totals.sources.cursor;
   const roots = editorUserRoots(homeDir, "Cursor");
   const files = recentFilesFromRoots([
@@ -417,10 +440,10 @@ async function scanCursor(homeDir, sinceMs, maxFiles, totals) {
     join(homeDir, ".cursor"),
   ], sinceMs, maxFiles, isLikelyAgentDataFile);
   if (files.length === 0 && roots.every((root) => !existsSync(root)) && !existsSync(join(homeDir, ".cursor"))) source.unavailable = true;
-  for (const path of files) await scanGenericAgentFile(path, totals, source, { modelFacing: true, sessionPerFile: false });
+  await scanFiles(files, concurrency, (path) => scanGenericAgentFile(path, totals, source, { modelFacing: true, sessionPerFile: false }));
 }
 
-async function scanWindsurf(homeDir, sinceMs, maxFiles, totals) {
+async function scanWindsurf(homeDir, sinceMs, maxFiles, totals, concurrency) {
   const source = totals.sources.windsurf;
   const roots = editorUserRoots(homeDir, "Windsurf");
   const files = recentFilesFromRoots([
@@ -431,10 +454,10 @@ async function scanWindsurf(homeDir, sinceMs, maxFiles, totals) {
     join(homeDir, ".windsurf"),
   ], sinceMs, maxFiles, isLikelyAgentDataFile);
   if (files.length === 0 && roots.every((root) => !existsSync(root)) && !existsSync(join(homeDir, ".windsurf"))) source.unavailable = true;
-  for (const path of files) await scanGenericAgentFile(path, totals, source, { modelFacing: true, sessionPerFile: false });
+  await scanFiles(files, concurrency, (path) => scanGenericAgentFile(path, totals, source, { modelFacing: true, sessionPerFile: false }));
 }
 
-async function scanEditorExtensionAgents(homeDir, sinceMs, maxFiles, totals) {
+async function scanEditorExtensionAgents(homeDir, sinceMs, maxFiles, totals, concurrency) {
   const editorRoots = [
     ...editorUserRoots(homeDir, "Code"),
     ...editorUserRoots(homeDir, "Code - Insiders"),
@@ -455,11 +478,11 @@ async function scanEditorExtensionAgents(homeDir, sinceMs, maxFiles, totals) {
     }
     const files = recentFilesFromRoots(roots, sinceMs, Math.min(maxFiles, 1000), isLikelyAgentDataFile);
     if (files.length === 0 && roots.every((root) => !existsSync(root))) item.source.unavailable = true;
-    for (const path of files) await scanGenericAgentFile(path, totals, item.source, { modelFacing: true, sessionPerFile: /(?:task|history|session|conversation)/i.test(path) });
+    await scanFiles(files, concurrency, (path) => scanGenericAgentFile(path, totals, item.source, { modelFacing: true, sessionPerFile: /(?:task|history|session|conversation)/i.test(path) }));
   }
 }
 
-async function scanCopilot(homeDir, sinceMs, maxFiles, totals) {
+async function scanCopilot(homeDir, sinceMs, maxFiles, totals, concurrency) {
   const source = totals.sources.copilot;
   const roots = [
     join(homeDir, ".copilot", "session-state"),
@@ -472,7 +495,7 @@ async function scanCopilot(homeDir, sinceMs, maxFiles, totals) {
     return isLikelyAgentDataFile(path);
   });
   if (files.length === 0 && roots.every((root) => !existsSync(root))) source.unavailable = true;
-  for (const path of files) await scanGenericAgentFile(path, totals, source, { modelFacing: true, sessionPerFile: !path.endsWith(".vscdb") });
+  await scanFiles(files, concurrency, (path) => scanGenericAgentFile(path, totals, source, { modelFacing: true, sessionPerFile: !path.endsWith(".vscdb") }));
 }
 
 function scanAider(projectDir, sinceMs, totals) {
@@ -529,16 +552,27 @@ function scanKnownFiles(paths, sinceMs, totals, source, options) {
 }
 
 async function scanGenericAgentFile(path, totals, source, options = {}) {
-  registerAgentDataFile(totals, source, path);
+  if (!registerAgentDataFile(totals, source, path)) return;
   if (path.endsWith(".jsonl")) {
     await scanGenericJsonLines(path, totals, source, options);
     return;
   }
-  scanGenericAgentFileSync(path, totals, source, options);
+  await scanGenericAgentFileText(path, totals, source, options);
+}
+
+async function scanGenericAgentFileText(path, totals, source, options = {}) {
+  let text;
+  try {
+    text = await readFile(path, "utf8");
+  } catch {
+    totals.diagnostics.push(`Skipped unreadable ${source.label} file ${path}`);
+    return;
+  }
+  scanGenericText(text, totals, source, path, options);
 }
 
 function scanGenericAgentFileSync(path, totals, source, options = {}) {
-  registerAgentDataFile(totals, source, path);
+  if (!registerAgentDataFile(totals, source, path)) return;
   let text;
   try {
     text = readFileSync(path, "utf8");
@@ -546,6 +580,10 @@ function scanGenericAgentFileSync(path, totals, source, options = {}) {
     totals.diagnostics.push(`Skipped unreadable ${source.label} file ${path}`);
     return;
   }
+  scanGenericText(text, totals, source, path, options);
+}
+
+function scanGenericText(text, totals, source, path, options = {}) {
   scanTextForSecrets(text, totals, { modelFacing: options.modelFacing === true, file: path });
   scanTextForStateLoss(text, totals, path);
   scanTextForTrace(text, totals, path);
@@ -553,6 +591,20 @@ function scanGenericAgentFileSync(path, totals, source, options = {}) {
   if (path.endsWith(".json")) scanGenericJsonDocument(text, totals, source, path);
 }
 
+async function scanFiles(files, concurrency, scanFile) {
+  if (files.length === 0) return;
+  const workerCount = Math.min(concurrency, files.length);
+  let next = 0;
+  const workers = Array.from({ length: workerCount }, async () => {
+    while (next < files.length) {
+      const path = files[next];
+      next += 1;
+      await scanFile(path);
+    }
+  });
+  await Promise.all(workers);
+}
+
 async function scanGenericJsonLines(path, totals, source, options = {}) {
   const session = createSessionAccumulator(source.label.toLowerCase(), path);
   let sawRecord = false;
@@ -586,9 +638,11 @@ function noteGenericRecordSignals(record, totals, source, session) {
     addSession(totals, source, id);
   }
 
-  const usage = normalizeGenericUsage(record.usage ?? record.tokenUsage ?? record.token_usage ?? record.metadata?.usage);
+  const usage = normalizeGenericUsage(record.usage ?? record.tokenUsage ?? record.token_usage ?? record.message?.usage ?? record.metadata?.usage);
   if (usage.total > 0) addUsage(totals, source, usage);
 
+  noteNestedMessageToolSignals(record, totals, source, session);
+
   if (isGenericToolStart(record)) {
     const callId = record.id ?? record.call_id ?? record.tool_call_id ?? stableSnippet(record).slice(0, 64);
     addToolCall(totals, source);
@@ -601,6 +655,27 @@ function noteGenericRecordSignals(record, totals, source, session) {
   }
 }
 
+function noteNestedMessageToolSignals(record, totals, source, session) {
+  const message = record.message;
+  if (!message || typeof message !== "object") return;
+
+  for (const block of array(message.content)) {
+    if (!block || typeof block !== "object" || String(block.type).toLowerCase() !== "toolcall") continue;
+    const callId = block.id ?? record.id ?? stableSnippet(block).slice(0, 64);
+    addToolCall(totals, source);
+    noteToolStart(session, callId, toolSignature(block.name, block.arguments ?? block.input));
+  }
+
+  if (String(message.role).toLowerCase() !== "toolresult" && !message.toolCallId && !message.toolName) return;
+  const failed = message.isError === true || isGenericToolFailure({ ...record, ...message });
+  const callId = message.toolCallId ?? record.toolCallId ?? record.tool_call_id;
+  const signature = callId && session.calls.has(callId)
+    ? session.calls.get(callId)
+    : toolSignature(message.toolName ?? record.toolName ?? record.tool_name ?? "tool_result", message.content ?? record.output ?? record.result);
+  if (failed) addFailure(totals, source);
+  noteToolEnd(totals, session, signature, failed, parseTime(message.timestamp ?? record.timestamp), 0);
+}
+
 function visitGenericJson(value, visit, depth = 0) {
   if (depth > 8 || value == null) return;
   if (Array.isArray(value)) {
@@ -640,7 +715,12 @@ function scanGlueCode(projectDir, totals) {
 }
 
 function recentFiles(root, sinceMs, maxFiles, predicate) {
+  return recentFileEntries(root, sinceMs, maxFiles, predicate).map((item) => item.path);
+}
+
+function recentFileEntries(root, sinceMs, maxFiles, predicate) {
   const files = [];
+  const shouldTrim = maxFiles < EXHAUSTIVE_MAX_FILES;
   walk(root, (path, dirent) => {
     if (dirent.isDirectory()) {
       if (IGNORED_DIRS.has(dirent.name)) return false;
@@ -650,27 +730,35 @@ function recentFiles(root, sinceMs, maxFiles, predicate) {
     const stat = safeStat(path);
     if (!stat || stat.size > MAX_FILE_BYTES || stat.mtimeMs < sinceMs) return false;
     files.push({ path, mtimeMs: stat.mtimeMs });
+    trimRecentFileEntries(files, maxFiles, shouldTrim);
     return false;
   });
   files.sort((a, b) => b.mtimeMs - a.mtimeMs);
-  return files.slice(0, maxFiles).map((item) => item.path);
+  return files.slice(0, maxFiles);
 }
 
 function recentFilesFromRoots(roots, sinceMs, maxFiles, predicate) {
   const seen = new Set();
   const files = [];
+  const shouldTrim = maxFiles < EXHAUSTIVE_MAX_FILES;
   for (const root of roots) {
-    for (const path of recentFiles(root, sinceMs, maxFiles, predicate)) {
-      if (seen.has(path)) continue;
-      seen.add(path);
-      const stat = safeStat(path);
-      if (stat) files.push({ path, mtimeMs: stat.mtimeMs });
+    for (const item of recentFileEntries(root, sinceMs, maxFiles, predicate)) {
+      if (seen.has(item.path)) continue;
+      seen.add(item.path);
+      files.push(item);
+      trimRecentFileEntries(files, maxFiles, shouldTrim);
     }
   }
   files.sort((a, b) => b.mtimeMs - a.mtimeMs);
   return files.slice(0, maxFiles).map((item) => item.path);
 }
 
+function trimRecentFileEntries(files, maxFiles, shouldTrim) {
+  if (!shouldTrim || files.length <= maxFiles * 2) return;
+  files.sort((a, b) => b.mtimeMs - a.mtimeMs);
+  files.length = maxFiles;
+}
+
 function isLikelyAgentDataFile(path) {
   const ext = extension(path);
   const name = basename(path);
@@ -906,7 +994,7 @@ function hasToolResult(content) {
 
 function isGenericModelFacing(record) {
   const role = String(record?.role ?? record?.message?.role ?? record?.type ?? "").toLowerCase();
-  return role === "user" || role === "assistant" || role === "model" || role === "tool_use" || role === "function_call";
+  return role === "user" || role === "assistant" || role === "model" || role === "tool_use" || role === "function_call" || role === "toolresult" || role === "tool_result" || role === "tool";
 }
 
 function genericRecordText(record) {
@@ -984,10 +1072,10 @@ function normalizeClaudeUsage(usage) {
 
 function normalizeGenericUsage(usage) {
   if (!usage || typeof usage !== "object") return { input: 0, freshInput: 0, output: 0, cached: 0, reasoning: 0, total: 0, active: 0 };
-  const input = firstNumber(usage.input_tokens, usage.inputTokens, usage.prompt_tokens, usage.promptTokens, usage.promptTokenCount, usage.inputTokenCount);
-  const output = firstNumber(usage.output_tokens, usage.outputTokens, usage.completion_tokens, usage.completionTokens, usage.candidatesTokenCount, usage.outputTokenCount);
-  const cached = firstNumber(usage.cached_input_tokens, usage.cache_read_input_tokens, usage.cachedTokens, usage.cachedContentTokenCount);
-  const cacheCreation = firstNumber(usage.cache_creation_input_tokens, usage.cacheCreationInputTokens);
+  const input = firstNumber(usage.input_tokens, usage.inputTokens, usage.prompt_tokens, usage.promptTokens, usage.promptTokenCount, usage.inputTokenCount, usage.input);
+  const output = firstNumber(usage.output_tokens, usage.outputTokens, usage.completion_tokens, usage.completionTokens, usage.candidatesTokenCount, usage.outputTokenCount, usage.output);
+  const cached = firstNumber(usage.cached_input_tokens, usage.cache_read_input_tokens, usage.cachedTokens, usage.cachedContentTokenCount, usage.cacheRead);
+  const cacheCreation = firstNumber(usage.cache_creation_input_tokens, usage.cacheCreationInputTokens, usage.cacheWrite);
   const reasoning = firstNumber(usage.reasoning_output_tokens, usage.reasoningTokens, usage.thoughtsTokenCount);
   const total = firstNumber(usage.total_tokens, usage.totalTokens, usage.totalTokenCount) || input + output + cached + cacheCreation;
   const freshInput = Math.max(0, input + cacheCreation - cached);