agentaudit 3.9.35 → 3.9.36

package/cli.mjs

@@ -1444,34 +1444,39 @@ async function auditRepo(url) {
   process.stdout.write(`  ${c.dim}[3/4]${c.reset} Preparing audit payload...`);
   const auditPrompt = loadAuditPrompt();
   
-  // Build code block with smart truncation to fit context windows.
-  // Reserve ~16k tokens for system prompt + output → budget ~48k tokens for code (~192k chars).
-  // Smaller models may have 32-65k context; we aim for safe default.
-  const MAX_CODE_CHARS = 180_000; // ~45k tokens
-  let codeBlock = '';
-  let totalChars = 0;
-  let truncatedFiles = 0;
+  // Build code chunks for multi-pass analysis.
+  // Budget ~45k tokens (~180k chars) per chunk for code, leaving room for prompt + output.
+  const MAX_CHUNK_CHARS = 180_000;
+  const chunks = []; // array of code block strings
+  let currentChunk = '';
+  let currentChars = 0;
   for (const file of files) {
     const entry = `\n### FILE: ${file.path}\n\`\`\`\n${file.content}\n\`\`\`\n`;
-    if (totalChars + entry.length > MAX_CODE_CHARS) {
-      // Try to fit a truncated version of this file
-      const remaining = MAX_CODE_CHARS - totalChars;
-      if (remaining > 200) {
-        const truncContent = file.content.substring(0, remaining - 100);
-        codeBlock += `\n### FILE: ${file.path}\n\`\`\`\n${truncContent}\n[... truncated ...]\n\`\`\`\n`;
-      }
-      truncatedFiles = files.length - codeBlock.split('### FILE:').length + 1;
-      break;
+    if (currentChars + entry.length > MAX_CHUNK_CHARS && currentChars > 0) {
+      chunks.push(currentChunk);
+      currentChunk = '';
+      currentChars = 0;
+    }
+    // If a single file exceeds chunk limit, truncate it
+    if (entry.length > MAX_CHUNK_CHARS) {
+      const truncContent = file.content.substring(0, MAX_CHUNK_CHARS - 200);
+      currentChunk += `\n### FILE: ${file.path}\n\`\`\`\n${truncContent}\n[... file truncated, ${file.content.length} chars total ...]\n\`\`\`\n`;
+      currentChars += MAX_CHUNK_CHARS;
+    } else {
+      currentChunk += entry;
+      currentChars += entry.length;
     }
-    codeBlock += entry;
-    totalChars += entry.length;
   }
-  if (truncatedFiles > 0) {
-    codeBlock += `\n[⚠ ${truncatedFiles} file(s) omitted due to context window limits]\n`;
-    console.log(` ${c.green}done${c.reset} ${c.yellow}(${truncatedFiles} files truncated to fit context window)${c.reset}`);
+  if (currentChunk) chunks.push(currentChunk);
+  
+  const needsMultiPass = chunks.length > 1;
+  if (needsMultiPass) {
+    console.log(` ${c.green}done${c.reset} ${c.yellow}(${chunks.length} passes needed — ${files.length} files across ${chunks.length} chunks)${c.reset}`);
   } else {
     console.log(` ${c.green}done${c.reset}`);
   }
+  // For single-pass, use the only chunk as codeBlock
+  const codeBlock = chunks[0] || '';
   
   // Step 4: LLM Analysis
   // Check for API keys to determine which LLM to use
@@ -1567,128 +1572,220 @@ async function auditRepo(url) {
     actualModel = modelOverride || resolvedProvider.model || 'unknown';
   }
   
-  // We have an API key — run LLM audit
-  process.stdout.write(`  ${c.dim}[4/4]${c.reset} Running LLM analysis ${c.dim}(${resolvedProvider.id}: ${actualModel})${c.reset}...`);
-  
-  const systemPrompt = auditPrompt || 'You are a security auditor. Analyze the code and report findings as JSON.';
-  const userMessage = [
-    `Audit this package: **${slug}** (${url})`,
-    ``,
-    `After analysis, respond with ONLY a valid JSON object. No markdown fences, no explanation, no text before or after. Just the raw JSON:`,
-    `{ "skill_slug": "${slug}", "source_url": "${url}", "package_type": "<mcp-server|agent-skill|library|cli-tool>",`,
-    `  "risk_score": <0-100>, "result": "<safe|caution|unsafe>", "max_severity": "<none|low|medium|high|critical>",`,
-    `  "findings_count": <n>, "findings": [{ "id": "...", "title": "...", "severity": "...", "category": "...",`,
-    `  "description": "...", "file": "...", "line": <n>, "remediation": "...", "confidence": "...", "is_by_design": false }] }`,
-    ``,
-    `## Source Code`,
-    codeBlock,
-  ].join('\n');
-  
+  // ── LLM call helper (reused for multi-pass) ──
+  async function callLLM(codeContent, passLabel) {
+    const systemPrompt = auditPrompt || 'You are a security auditor. Analyze the code and report findings as JSON.';
+    const userMessage = [
+      `Audit this package: **${slug}** (${url})`,
+      ``,
+      `After analysis, respond with ONLY a valid JSON object. No markdown fences, no explanation, no text before or after. Just the raw JSON:`,
+      `{ "skill_slug": "${slug}", "source_url": "${url}", "package_type": "<mcp-server|agent-skill|library|cli-tool>",`,
+      `  "risk_score": <0-100>, "result": "<safe|caution|unsafe>", "max_severity": "<none|low|medium|high|critical>",`,
+      `  "findings_count": <n>, "findings": [{ "id": "...", "title": "...", "severity": "...", "category": "...",`,
+      `  "description": "...", "file": "...", "line": <n>, "remediation": "...", "confidence": "...", "is_by_design": false }] }`,
+      ``,
+      `## Source Code`,
+      codeContent,
+    ].join('\n');
+    
+    let _lastLlmText = '';
+    let result = null;
+    let meta = {};
+    
+    try {
+      if (resolvedProvider.id === 'anthropic') {
+        const res = await fetch('https://api.anthropic.com/v1/messages', {
+          method: 'POST',
+          headers: {
+            'x-api-key': resolvedProvider.key,
+            'anthropic-version': '2023-06-01',
+            'content-type': 'application/json',
+          },
+          body: JSON.stringify({
+            model: modelOverride || 'claude-sonnet-4-20250514',
+            max_tokens: 8192,
+            system: systemPrompt,
+            messages: [{ role: 'user', content: userMessage }],
+          }),
+          signal: AbortSignal.timeout(120_000),
+        });
+        const data = await res.json();
+        if (data.error) {
+          return { error: data.error.message || JSON.stringify(data.error) };
+        }
+        const text = data.content?.[0]?.text || '';
+        _lastLlmText = text;
+        result = extractJSON(text);
+        meta = {
+          provider_msg_id: data.id || null,
+          input_tokens: data.usage?.input_tokens || null,
+          output_tokens: data.usage?.output_tokens || null,
+          reported_model: data.model || null,
+        };
+      } else {
+        let apiUrl, modelName, authHeaders;
+        switch (resolvedProvider.id) {
+          case 'openrouter':
+            apiUrl = 'https://openrouter.ai/api/v1/chat/completions';
+            modelName = modelOverride || process.env.OPENROUTER_MODEL || 'anthropic/claude-sonnet-4';
+            authHeaders = { 'Authorization': `Bearer ${resolvedProvider.key}`, 'HTTP-Referer': 'https://agentaudit.dev', 'X-Title': 'AgentAudit' };
+            break;
+          case 'ollama':
+            apiUrl = `${resolvedProvider.host}/v1/chat/completions`;
+            modelName = modelOverride || resolvedProvider.model;
+            authHeaders = {};
+            break;
+          case 'custom':
+            apiUrl = resolvedProvider.url.endsWith('/chat/completions') ? resolvedProvider.url : `${resolvedProvider.url.replace(/\/$/, '')}/chat/completions`;
+            modelName = modelOverride || resolvedProvider.model;
+            authHeaders = resolvedProvider.key ? { 'Authorization': `Bearer ${resolvedProvider.key}` } : {};
+            break;
+          default:
+            apiUrl = 'https://api.openai.com/v1/chat/completions';
+            modelName = modelOverride || 'gpt-4o';
+            authHeaders = { 'Authorization': `Bearer ${resolvedProvider.key}` };
+        }
+
+        const res = await fetch(apiUrl, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json', ...authHeaders },
+          body: JSON.stringify({
+            model: modelName,
+            max_tokens: 8192,
+            messages: [
+              { role: 'system', content: systemPrompt },
+              { role: 'user', content: userMessage },
+            ],
+          }),
+          signal: AbortSignal.timeout(resolvedProvider.id === 'ollama' ? 300_000 : 120_000),
+        });
+        const data = await res.json();
+        if (data.error) {
+          return { error: data.error.message || JSON.stringify(data.error) };
+        }
+        const text = data.choices?.[0]?.message?.content || '';
+        _lastLlmText = text;
+        result = extractJSON(text);
+        meta = {
+          provider_msg_id: data.id || null,
+          provider_fingerprint: data.system_fingerprint || null,
+          input_tokens: data.usage?.prompt_tokens || null,
+          output_tokens: data.usage?.completion_tokens || null,
+          reported_model: data.model || null,
+        };
+      }
+    } catch (err) {
+      return { error: err.message };
+    }
+    
+    return { report: result, meta, rawText: _lastLlmText };
+  }
+
+  // ── Run LLM analysis (single or multi-pass) ──
   let report = null;
+  let providerMeta = {};
   let _lastLlmText = '';
-  let providerMeta = {}; // Collect provider metadata for attestation
   
-  try {
-    if (resolvedProvider.id === 'anthropic') {
-      const res = await fetch('https://api.anthropic.com/v1/messages', {
-        method: 'POST',
-        headers: {
-          'x-api-key': resolvedProvider.key,
-          'anthropic-version': '2023-06-01',
-          'content-type': 'application/json',
-        },
-        body: JSON.stringify({
-          model: modelOverride || 'claude-sonnet-4-20250514',
-          max_tokens: 8192,
-          system: systemPrompt,
-          messages: [{ role: 'user', content: userMessage }],
-        }),
-        signal: AbortSignal.timeout(120_000),
-      });
-      const data = await res.json();
-      if (data.error) {
-        console.log(` ${c.red}failed${c.reset}`);
-        console.log(`  ${c.red}API error: ${data.error.message || JSON.stringify(data.error)}${c.reset}`);
-        try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
-        return null;
-      }
-      const text = data.content?.[0]?.text || '';
-      _lastLlmText = text;
-      report = extractJSON(text);
-      providerMeta = {
-        provider_msg_id: data.id || null,
-        input_tokens: data.usage?.input_tokens || null,
-        output_tokens: data.usage?.output_tokens || null,
-        reported_model: data.model || null,
-      };
-    } else {
-      // OpenAI, OpenRouter, Ollama, or Custom (all use OpenAI-compatible chat completions API)
-      let apiUrl, modelName, authHeaders;
-      switch (resolvedProvider.id) {
-        case 'openrouter':
-          apiUrl = 'https://openrouter.ai/api/v1/chat/completions';
-          modelName = modelOverride || process.env.OPENROUTER_MODEL || 'anthropic/claude-sonnet-4';
-          authHeaders = { 'Authorization': `Bearer ${resolvedProvider.key}`, 'HTTP-Referer': 'https://agentaudit.dev', 'X-Title': 'AgentAudit' };
-          break;
-        case 'ollama':
-          apiUrl = `${resolvedProvider.host}/v1/chat/completions`;
-          modelName = modelOverride || resolvedProvider.model;
-          authHeaders = {};
-          break;
-        case 'custom':
-          apiUrl = resolvedProvider.url.endsWith('/chat/completions') ? resolvedProvider.url : `${resolvedProvider.url.replace(/\/$/, '')}/chat/completions`;
-          modelName = modelOverride || resolvedProvider.model;
-          authHeaders = resolvedProvider.key ? { 'Authorization': `Bearer ${resolvedProvider.key}` } : {};
-          break;
-        default: // openai
-          apiUrl = 'https://api.openai.com/v1/chat/completions';
-          modelName = modelOverride || 'gpt-4o';
-          authHeaders = { 'Authorization': `Bearer ${resolvedProvider.key}` };
-      }
-
-      const res = await fetch(apiUrl, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json', ...authHeaders },
-        body: JSON.stringify({
-          model: modelName,
-          max_tokens: 8192,
-          messages: [
-            { role: 'system', content: systemPrompt },
-            { role: 'user', content: userMessage },
-          ],
-        }),
-        signal: AbortSignal.timeout(resolvedProvider.id === 'ollama' ? 300_000 : 120_000), // Ollama: 5min (local can be slow)
-      });
-      const data = await res.json();
-      if (data.error) {
+  if (needsMultiPass) {
+    // Multi-pass: analyze each chunk, merge findings
+    console.log(`  ${c.dim}[4/4]${c.reset} Running LLM analysis ${c.dim}(${resolvedProvider.id}: ${actualModel})${c.reset} — ${c.yellow}${chunks.length} passes${c.reset}`);
+    const allFindings = [];
+    let totalInput = 0, totalOutput = 0;
+    let lastMeta = {};
+    let baseReport = null;
+    
+    for (let i = 0; i < chunks.length; i++) {
+      process.stdout.write(`  ${c.dim}  Pass ${i + 1}/${chunks.length}...${c.reset}`);
+      const passStart = Date.now();
+      const result = await callLLM(chunks[i], `pass ${i + 1}`);
+      
+      if (result.error) {
         console.log(` ${c.red}failed${c.reset}`);
-        const errMsg = data.error.message || JSON.stringify(data.error);
+        const errMsg = result.error;
         console.log(`  ${c.red}API error: ${errMsg}${c.reset}`);
         if (/context.length|maximum.*tokens|too.many.tokens/i.test(errMsg)) {
-          console.log(`  ${c.dim}This model's context window is too small for this repository.${c.reset}`);
-          console.log(`  ${c.dim}Try a model with a larger context: --model anthropic/claude-sonnet-4 (200k) or --model openai/gpt-4o (128k)${c.reset}`);
+          console.log(`  ${c.dim}This model's context window is too small even for chunked analysis.${c.reset}`);
+          console.log(`  ${c.dim}Try: --model anthropic/claude-sonnet-4 (200k) or --model openai/gpt-4o (128k)${c.reset}`);
         }
         try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
         return null;
       }
-      const text = data.choices?.[0]?.message?.content || '';
-      _lastLlmText = text;
-      report = extractJSON(text);
-      providerMeta = {
-        provider_msg_id: data.id || null,
-        provider_fingerprint: data.system_fingerprint || null,
-        input_tokens: data.usage?.prompt_tokens || null,
-        output_tokens: data.usage?.completion_tokens || null,
-        reported_model: data.model || null,
-      };
+      
+      if (!result.report) {
+        console.log(` ${c.yellow}no findings (empty/unparseable)${c.reset}`);
+        _lastLlmText = result.rawText || '';
+        continue;
+      }
+      
+      const passElapsed = ((Date.now() - passStart) / 1000).toFixed(1);
+      const passFindings = result.report.findings?.length || 0;
+      console.log(` ${c.green}done${c.reset} ${c.dim}(${passElapsed}s, ${passFindings} findings)${c.reset}`);
+      
+      if (!baseReport) baseReport = result.report;
+      if (result.report.findings) allFindings.push(...result.report.findings);
+      lastMeta = result.meta;
+      totalInput += result.meta.input_tokens || 0;
+      totalOutput += result.meta.output_tokens || 0;
+    }
+    
+    if (!baseReport) {
+      console.log(`  ${c.red}✖ All passes failed to produce a report${c.reset}`);
+      try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
+      return null;
     }
     
+    // Merge: deduplicate findings by title+file, recalculate risk score
+    const seen = new Set();
+    const mergedFindings = [];
+    for (const f of allFindings) {
+      const key = `${f.title}::${f.file || ''}`;
+      if (!seen.has(key)) {
+        seen.add(key);
+        mergedFindings.push(f);
+      }
+    }
+    
+    // Recalculate severity-based risk
+    const sevWeights = { critical: 25, high: 15, medium: 5, low: 1 };
+    const mergedRisk = Math.min(100, mergedFindings.reduce((s, f) => s + (sevWeights[f.severity] || 0), 0));
+    const maxSev = mergedFindings.length === 0 ? 'none' : 
+      mergedFindings.some(f => f.severity === 'critical') ? 'critical' :
+      mergedFindings.some(f => f.severity === 'high') ? 'high' :
+      mergedFindings.some(f => f.severity === 'medium') ? 'medium' : 'low';
+    
+    report = {
+      ...baseReport,
+      findings: mergedFindings,
+      findings_count: mergedFindings.length,
+      risk_score: mergedRisk,
+      result: mergedRisk === 0 ? 'safe' : mergedRisk <= 20 ? 'caution' : 'unsafe',
+      max_severity: maxSev,
+    };
+    providerMeta = { ...lastMeta, input_tokens: totalInput || null, output_tokens: totalOutput || null };
+    
+    console.log(`  ${c.dim}  Merged: ${mergedFindings.length} unique findings from ${chunks.length} passes${c.reset}`);
+    console.log(` ${c.green}done${c.reset} ${c.dim}(${elapsed(start)})${c.reset}`);
+  } else {
+    // Single-pass (original flow)
+    process.stdout.write(`  ${c.dim}[4/4]${c.reset} Running LLM analysis ${c.dim}(${resolvedProvider.id}: ${actualModel})${c.reset}...`);
+    const result = await callLLM(codeBlock);
+    
+    if (result.error) {
+      console.log(` ${c.red}failed${c.reset}`);
+      const errMsg = result.error;
+      console.log(`  ${c.red}API error: ${errMsg}${c.reset}`);
+      if (/context.length|maximum.*tokens|too.many.tokens/i.test(errMsg)) {
+        console.log(`  ${c.dim}This model's context window is too small for this repository.${c.reset}`);
+        console.log(`  ${c.dim}Try a model with a larger context: --model anthropic/claude-sonnet-4 (200k) or --model openai/gpt-4o (128k)${c.reset}`);
+      }
+      try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
+      return null;
+    }
+    
+    report = result.report;
+    providerMeta = result.meta;
+    _lastLlmText = result.rawText || '';
     console.log(` ${c.green}done${c.reset} ${c.dim}(${elapsed(start)})${c.reset}`);
-  } catch (err) {
-    console.log(` ${c.red}failed${c.reset}`);
-    console.log(`  ${c.red}${err.message}${c.reset}`);
-    try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
-    return null;
   }
   
   // Cleanup repo

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentaudit",
-  "version": "3.9.35",
+  "version": "3.9.36",
   "description": "Security scanner for AI packages — MCP server + CLI",
   "type": "module",
   "bin": {