agentaudit 3.9.34 → 3.9.36

package/cli.mjs

@@ -1444,11 +1444,39 @@ async function auditRepo(url) {
   process.stdout.write(`  ${c.dim}[3/4]${c.reset} Preparing audit payload...`);
   const auditPrompt = loadAuditPrompt();
   
-  let codeBlock = '';
+  // Build code chunks for multi-pass analysis.
+  // Budget ~45k tokens (~180k chars) per chunk for code, leaving room for prompt + output.
+  const MAX_CHUNK_CHARS = 180_000;
+  const chunks = []; // array of code block strings
+  let currentChunk = '';
+  let currentChars = 0;
   for (const file of files) {
-    codeBlock += `\n### FILE: ${file.path}\n\`\`\`\n${file.content}\n\`\`\`\n`;
+    const entry = `\n### FILE: ${file.path}\n\`\`\`\n${file.content}\n\`\`\`\n`;
+    if (currentChars + entry.length > MAX_CHUNK_CHARS && currentChars > 0) {
+      chunks.push(currentChunk);
+      currentChunk = '';
+      currentChars = 0;
+    }
+    // If a single file exceeds chunk limit, truncate it
+    if (entry.length > MAX_CHUNK_CHARS) {
+      const truncContent = file.content.substring(0, MAX_CHUNK_CHARS - 200);
+      currentChunk += `\n### FILE: ${file.path}\n\`\`\`\n${truncContent}\n[... file truncated, ${file.content.length} chars total ...]\n\`\`\`\n`;
+      currentChars += MAX_CHUNK_CHARS;
+    } else {
+      currentChunk += entry;
+      currentChars += entry.length;
+    }
+  }
+  if (currentChunk) chunks.push(currentChunk);
+  
+  const needsMultiPass = chunks.length > 1;
+  if (needsMultiPass) {
+    console.log(` ${c.green}done${c.reset} ${c.yellow}(${chunks.length} passes needed — ${files.length} files across ${chunks.length} chunks)${c.reset}`);
+  } else {
+    console.log(` ${c.green}done${c.reset}`);
   }
-  console.log(` ${c.green}done${c.reset}`);
+  // For single-pass, use the only chunk as codeBlock
+  const codeBlock = chunks[0] || '';
   
   // Step 4: LLM Analysis
   // Check for API keys to determine which LLM to use
@@ -1544,134 +1572,241 @@ async function auditRepo(url) {
     actualModel = modelOverride || resolvedProvider.model || 'unknown';
   }
   
-  // We have an API key — run LLM audit
-  process.stdout.write(`  ${c.dim}[4/4]${c.reset} Running LLM analysis ${c.dim}(${resolvedProvider.id}: ${actualModel})${c.reset}...`);
-  
-  const systemPrompt = auditPrompt || 'You are a security auditor. Analyze the code and report findings as JSON.';
-  const userMessage = [
-    `Audit this package: **${slug}** (${url})`,
-    ``,
-    `After analysis, respond with ONLY a valid JSON object. No markdown fences, no explanation, no text before or after. Just the raw JSON:`,
-    `{ "skill_slug": "${slug}", "source_url": "${url}", "package_type": "<mcp-server|agent-skill|library|cli-tool>",`,
-    `  "risk_score": <0-100>, "result": "<safe|caution|unsafe>", "max_severity": "<none|low|medium|high|critical>",`,
-    `  "findings_count": <n>, "findings": [{ "id": "...", "title": "...", "severity": "...", "category": "...",`,
-    `  "description": "...", "file": "...", "line": <n>, "remediation": "...", "confidence": "...", "is_by_design": false }] }`,
-    ``,
-    `## Source Code`,
-    codeBlock,
-  ].join('\n');
-  
+  // ── LLM call helper (reused for multi-pass) ──
+  async function callLLM(codeContent, passLabel) {
+    const systemPrompt = auditPrompt || 'You are a security auditor. Analyze the code and report findings as JSON.';
+    const userMessage = [
+      `Audit this package: **${slug}** (${url})`,
+      ``,
+      `After analysis, respond with ONLY a valid JSON object. No markdown fences, no explanation, no text before or after. Just the raw JSON:`,
+      `{ "skill_slug": "${slug}", "source_url": "${url}", "package_type": "<mcp-server|agent-skill|library|cli-tool>",`,
+      `  "risk_score": <0-100>, "result": "<safe|caution|unsafe>", "max_severity": "<none|low|medium|high|critical>",`,
+      `  "findings_count": <n>, "findings": [{ "id": "...", "title": "...", "severity": "...", "category": "...",`,
+      `  "description": "...", "file": "...", "line": <n>, "remediation": "...", "confidence": "...", "is_by_design": false }] }`,
+      ``,
+      `## Source Code`,
+      codeContent,
+    ].join('\n');
+    
+    let _lastLlmText = '';
+    let result = null;
+    let meta = {};
+    
+    try {
+      if (resolvedProvider.id === 'anthropic') {
+        const res = await fetch('https://api.anthropic.com/v1/messages', {
+          method: 'POST',
+          headers: {
+            'x-api-key': resolvedProvider.key,
+            'anthropic-version': '2023-06-01',
+            'content-type': 'application/json',
+          },
+          body: JSON.stringify({
+            model: modelOverride || 'claude-sonnet-4-20250514',
+            max_tokens: 8192,
+            system: systemPrompt,
+            messages: [{ role: 'user', content: userMessage }],
+          }),
+          signal: AbortSignal.timeout(120_000),
+        });
+        const data = await res.json();
+        if (data.error) {
+          return { error: data.error.message || JSON.stringify(data.error) };
+        }
+        const text = data.content?.[0]?.text || '';
+        _lastLlmText = text;
+        result = extractJSON(text);
+        meta = {
+          provider_msg_id: data.id || null,
+          input_tokens: data.usage?.input_tokens || null,
+          output_tokens: data.usage?.output_tokens || null,
+          reported_model: data.model || null,
+        };
+      } else {
+        let apiUrl, modelName, authHeaders;
+        switch (resolvedProvider.id) {
+          case 'openrouter':
+            apiUrl = 'https://openrouter.ai/api/v1/chat/completions';
+            modelName = modelOverride || process.env.OPENROUTER_MODEL || 'anthropic/claude-sonnet-4';
+            authHeaders = { 'Authorization': `Bearer ${resolvedProvider.key}`, 'HTTP-Referer': 'https://agentaudit.dev', 'X-Title': 'AgentAudit' };
+            break;
+          case 'ollama':
+            apiUrl = `${resolvedProvider.host}/v1/chat/completions`;
+            modelName = modelOverride || resolvedProvider.model;
+            authHeaders = {};
+            break;
+          case 'custom':
+            apiUrl = resolvedProvider.url.endsWith('/chat/completions') ? resolvedProvider.url : `${resolvedProvider.url.replace(/\/$/, '')}/chat/completions`;
+            modelName = modelOverride || resolvedProvider.model;
+            authHeaders = resolvedProvider.key ? { 'Authorization': `Bearer ${resolvedProvider.key}` } : {};
+            break;
+          default:
+            apiUrl = 'https://api.openai.com/v1/chat/completions';
+            modelName = modelOverride || 'gpt-4o';
+            authHeaders = { 'Authorization': `Bearer ${resolvedProvider.key}` };
+        }
+
+        const res = await fetch(apiUrl, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json', ...authHeaders },
+          body: JSON.stringify({
+            model: modelName,
+            max_tokens: 8192,
+            messages: [
+              { role: 'system', content: systemPrompt },
+              { role: 'user', content: userMessage },
+            ],
+          }),
+          signal: AbortSignal.timeout(resolvedProvider.id === 'ollama' ? 300_000 : 120_000),
+        });
+        const data = await res.json();
+        if (data.error) {
+          return { error: data.error.message || JSON.stringify(data.error) };
+        }
+        const text = data.choices?.[0]?.message?.content || '';
+        _lastLlmText = text;
+        result = extractJSON(text);
+        meta = {
+          provider_msg_id: data.id || null,
+          provider_fingerprint: data.system_fingerprint || null,
+          input_tokens: data.usage?.prompt_tokens || null,
+          output_tokens: data.usage?.completion_tokens || null,
+          reported_model: data.model || null,
+        };
+      }
+    } catch (err) {
+      return { error: err.message };
+    }
+    
+    return { report: result, meta, rawText: _lastLlmText };
+  }
+
+  // ── Run LLM analysis (single or multi-pass) ──
   let report = null;
+  let providerMeta = {};
   let _lastLlmText = '';
-  let providerMeta = {}; // Collect provider metadata for attestation
   
-  try {
-    if (resolvedProvider.id === 'anthropic') {
-      const res = await fetch('https://api.anthropic.com/v1/messages', {
-        method: 'POST',
-        headers: {
-          'x-api-key': resolvedProvider.key,
-          'anthropic-version': '2023-06-01',
-          'content-type': 'application/json',
-        },
-        body: JSON.stringify({
-          model: modelOverride || 'claude-sonnet-4-20250514',
-          max_tokens: 8192,
-          system: systemPrompt,
-          messages: [{ role: 'user', content: userMessage }],
-        }),
-        signal: AbortSignal.timeout(120_000),
-      });
-      const data = await res.json();
-      if (data.error) {
+  if (needsMultiPass) {
+    // Multi-pass: analyze each chunk, merge findings
+    console.log(`  ${c.dim}[4/4]${c.reset} Running LLM analysis ${c.dim}(${resolvedProvider.id}: ${actualModel})${c.reset} — ${c.yellow}${chunks.length} passes${c.reset}`);
+    const allFindings = [];
+    let totalInput = 0, totalOutput = 0;
+    let lastMeta = {};
+    let baseReport = null;
+    
+    for (let i = 0; i < chunks.length; i++) {
+      process.stdout.write(`  ${c.dim}  Pass ${i + 1}/${chunks.length}...${c.reset}`);
+      const passStart = Date.now();
+      const result = await callLLM(chunks[i], `pass ${i + 1}`);
+      
+      if (result.error) {
         console.log(` ${c.red}failed${c.reset}`);
-        console.log(`  ${c.red}API error: ${data.error.message || JSON.stringify(data.error)}${c.reset}`);
+        const errMsg = result.error;
+        console.log(`  ${c.red}API error: ${errMsg}${c.reset}`);
+        if (/context.length|maximum.*tokens|too.many.tokens/i.test(errMsg)) {
+          console.log(`  ${c.dim}This model's context window is too small even for chunked analysis.${c.reset}`);
+          console.log(`  ${c.dim}Try: --model anthropic/claude-sonnet-4 (200k) or --model openai/gpt-4o (128k)${c.reset}`);
+        }
         try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
         return null;
       }
-      const text = data.content?.[0]?.text || '';
-      _lastLlmText = text;
-      report = extractJSON(text);
-      providerMeta = {
-        provider_msg_id: data.id || null,
-        input_tokens: data.usage?.input_tokens || null,
-        output_tokens: data.usage?.output_tokens || null,
-        reported_model: data.model || null,
-      };
-    } else {
-      // OpenAI, OpenRouter, Ollama, or Custom (all use OpenAI-compatible chat completions API)
-      let apiUrl, modelName, authHeaders;
-      switch (resolvedProvider.id) {
-        case 'openrouter':
-          apiUrl = 'https://openrouter.ai/api/v1/chat/completions';
-          modelName = modelOverride || process.env.OPENROUTER_MODEL || 'anthropic/claude-sonnet-4';
-          authHeaders = { 'Authorization': `Bearer ${resolvedProvider.key}`, 'HTTP-Referer': 'https://agentaudit.dev', 'X-Title': 'AgentAudit' };
-          break;
-        case 'ollama':
-          apiUrl = `${resolvedProvider.host}/v1/chat/completions`;
-          modelName = modelOverride || resolvedProvider.model;
-          authHeaders = {};
-          break;
-        case 'custom':
-          apiUrl = resolvedProvider.url.endsWith('/chat/completions') ? resolvedProvider.url : `${resolvedProvider.url.replace(/\/$/, '')}/chat/completions`;
-          modelName = modelOverride || resolvedProvider.model;
-          authHeaders = resolvedProvider.key ? { 'Authorization': `Bearer ${resolvedProvider.key}` } : {};
-          break;
-        default: // openai
-          apiUrl = 'https://api.openai.com/v1/chat/completions';
-          modelName = modelOverride || 'gpt-4o';
-          authHeaders = { 'Authorization': `Bearer ${resolvedProvider.key}` };
+      
+      if (!result.report) {
+        console.log(` ${c.yellow}no findings (empty/unparseable)${c.reset}`);
+        _lastLlmText = result.rawText || '';
+        continue;
       }
-
-      const res = await fetch(apiUrl, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json', ...authHeaders },
-        body: JSON.stringify({
-          model: modelName,
-          max_tokens: 8192,
-          messages: [
-            { role: 'system', content: systemPrompt },
-            { role: 'user', content: userMessage },
-          ],
-        }),
-        signal: AbortSignal.timeout(resolvedProvider.id === 'ollama' ? 300_000 : 120_000), // Ollama: 5min (local can be slow)
-      });
-      const data = await res.json();
-      if (data.error) {
-        console.log(` ${c.red}failed${c.reset}`);
-        console.log(`  ${c.red}API error: ${data.error.message || JSON.stringify(data.error)}${c.reset}`);
-        try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
-        return null;
+      
+      const passElapsed = ((Date.now() - passStart) / 1000).toFixed(1);
+      const passFindings = result.report.findings?.length || 0;
+      console.log(` ${c.green}done${c.reset} ${c.dim}(${passElapsed}s, ${passFindings} findings)${c.reset}`);
+      
+      if (!baseReport) baseReport = result.report;
+      if (result.report.findings) allFindings.push(...result.report.findings);
+      lastMeta = result.meta;
+      totalInput += result.meta.input_tokens || 0;
+      totalOutput += result.meta.output_tokens || 0;
+    }
+    
+    if (!baseReport) {
+      console.log(`  ${c.red}✖ All passes failed to produce a report${c.reset}`);
+      try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
+      return null;
+    }
+    
+    // Merge: deduplicate findings by title+file, recalculate risk score
+    const seen = new Set();
+    const mergedFindings = [];
+    for (const f of allFindings) {
+      const key = `${f.title}::${f.file || ''}`;
+      if (!seen.has(key)) {
+        seen.add(key);
+        mergedFindings.push(f);
       }
-      const text = data.choices?.[0]?.message?.content || '';
-      _lastLlmText = text;
-      report = extractJSON(text);
-      providerMeta = {
-        provider_msg_id: data.id || null,
-        provider_fingerprint: data.system_fingerprint || null,
-        input_tokens: data.usage?.prompt_tokens || null,
-        output_tokens: data.usage?.completion_tokens || null,
-        reported_model: data.model || null,
-      };
     }
     
+    // Recalculate severity-based risk
+    const sevWeights = { critical: 25, high: 15, medium: 5, low: 1 };
+    const mergedRisk = Math.min(100, mergedFindings.reduce((s, f) => s + (sevWeights[f.severity] || 0), 0));
+    const maxSev = mergedFindings.length === 0 ? 'none' : 
+      mergedFindings.some(f => f.severity === 'critical') ? 'critical' :
+      mergedFindings.some(f => f.severity === 'high') ? 'high' :
+      mergedFindings.some(f => f.severity === 'medium') ? 'medium' : 'low';
+    
+    report = {
+      ...baseReport,
+      findings: mergedFindings,
+      findings_count: mergedFindings.length,
+      risk_score: mergedRisk,
+      result: mergedRisk === 0 ? 'safe' : mergedRisk <= 20 ? 'caution' : 'unsafe',
+      max_severity: maxSev,
+    };
+    providerMeta = { ...lastMeta, input_tokens: totalInput || null, output_tokens: totalOutput || null };
+    
+    console.log(`  ${c.dim}  Merged: ${mergedFindings.length} unique findings from ${chunks.length} passes${c.reset}`);
+    console.log(` ${c.green}done${c.reset} ${c.dim}(${elapsed(start)})${c.reset}`);
+  } else {
+    // Single-pass (original flow)
+    process.stdout.write(`  ${c.dim}[4/4]${c.reset} Running LLM analysis ${c.dim}(${resolvedProvider.id}: ${actualModel})${c.reset}...`);
+    const result = await callLLM(codeBlock);
+    
+    if (result.error) {
+      console.log(` ${c.red}failed${c.reset}`);
+      const errMsg = result.error;
+      console.log(`  ${c.red}API error: ${errMsg}${c.reset}`);
+      if (/context.length|maximum.*tokens|too.many.tokens/i.test(errMsg)) {
+        console.log(`  ${c.dim}This model's context window is too small for this repository.${c.reset}`);
+        console.log(`  ${c.dim}Try a model with a larger context: --model anthropic/claude-sonnet-4 (200k) or --model openai/gpt-4o (128k)${c.reset}`);
+      }
+      try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
+      return null;
+    }
+    
+    report = result.report;
+    providerMeta = result.meta;
+    _lastLlmText = result.rawText || '';
     console.log(` ${c.green}done${c.reset} ${c.dim}(${elapsed(start)})${c.reset}`);
-  } catch (err) {
-    console.log(` ${c.red}failed${c.reset}`);
-    console.log(`  ${c.red}${err.message}${c.reset}`);
-    try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
-    return null;
   }
   
   // Cleanup repo
   try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
   
   if (!report) {
-    console.log(`  ${c.red}Could not parse LLM response as JSON${c.reset}`);
-    console.log(`  ${c.dim}Hint: run with --debug to see the raw LLM response${c.reset}`);
-    if (process.argv.includes('--debug')) {
+    const rawLen = typeof _lastLlmText === 'string' ? _lastLlmText.length : 0;
+    if (rawLen === 0) {
+      console.log(`  ${c.red}✖ Model returned an empty response${c.reset}`);
+      console.log(`  ${c.dim}This model may not support structured JSON output or the prompt was too large.${c.reset}`);
+      console.log(`  ${c.dim}Try a different model: --model anthropic/claude-sonnet-4 or --model openai/gpt-4o${c.reset}`);
+    } else {
+      console.log(`  ${c.red}✖ Could not parse LLM response as JSON${c.reset}`);
+      console.log(`  ${c.dim}The model returned ${rawLen} chars but not valid JSON. Try a stronger model.${c.reset}`);
+      if (!process.argv.includes('--debug')) {
+        console.log(`  ${c.dim}Hint: run with --debug to see the raw LLM response${c.reset}`);
+      }
+    }
+    if (process.argv.includes('--debug') && rawLen > 0) {
       console.log(`  ${c.dim}--- Raw LLM response (first 2000 chars) ---${c.reset}`);
-      console.log((typeof _lastLlmText === 'string' ? _lastLlmText : '(empty)').slice(0, 2000));
+      console.log(_lastLlmText.slice(0, 2000));
       console.log(`  ${c.dim}--- end ---${c.reset}`);
     }
     return null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentaudit",
-  "version": "3.9.34",
+  "version": "3.9.36",
   "description": "Security scanner for AI packages — MCP server + CLI",
   "type": "module",
   "bin": {