@vibecheckai/cli 3.2.2 → 3.2.4

package/bin/runners/lib/agent-firewall/ai/false-positive-analyzer.js

@@ -0,0 +1,474 @@
+/**
+ * AI-Powered False Positive Analyzer
+ * 
+ * Uses AI to analyze code context and determine if a violation is a false positive.
+ * Helps reduce false positives by understanding code intent and patterns.
+ */
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+
+// Cache for AI responses to avoid repeated calls
+const aiCache = new Map();
+const CACHE_TTL = 24 * 60 * 60 * 1000; // 24 hours
+
+/**
+ * Analyze if a violation is likely a false positive using AI
+ * @param {object} params
+ * @param {object} params.violation - Violation object
+ * @param {object} params.claim - Original claim
+ * @param {string} params.filePath - File path where violation occurred
+ * @param {string} params.projectRoot - Project root directory
+ * @param {object} params.policy - Policy configuration
+ * @returns {Promise<object>} Analysis result with isFalsePositive boolean and confidence
+ */
+async function analyzeFalsePositive({ violation, claim, filePath, projectRoot, policy }) {
+  const ruleConfig = policy.rules?.ai_false_positive_detection;
+  
+  // Check if AI analysis is enabled
+  if (!ruleConfig || !ruleConfig.enabled) {
+    return { isFalsePositive: false, confidence: 0, reason: "AI analysis disabled" };
+  }
+  
+  // Check cache first
+  const cacheKey = `${violation.rule}|${claim.value}|${filePath}`;
+  const cached = aiCache.get(cacheKey);
+  if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
+    return cached.result;
+  }
+  
+  try {
+    // Read the file to get context
+    const fullPath = path.join(projectRoot, filePath);
+    if (!fs.existsSync(fullPath)) {
+      return { isFalsePositive: false, confidence: 0, reason: "File not found" };
+    }
+    
+    const fileContent = fs.readFileSync(fullPath, "utf8");
+    const lines = fileContent.split("\n");
+    
+    // Extract context around the violation (10 lines before and after)
+    const pointer = claim.pointer || violation.claim?.pointer;
+    let startLine = 1;
+    let endLine = lines.length;
+    
+    if (pointer) {
+      const match = pointer.match(/:(\d+)-(\d+)/);
+      if (match) {
+        const lineNum = parseInt(match[1], 10);
+        startLine = Math.max(1, lineNum - 10);
+        endLine = Math.min(lines.length, lineNum + 10);
+      }
+    }
+    
+    const context = lines.slice(startLine - 1, endLine).join("\n");
+    const lineNumbers = Array.from({ length: endLine - startLine + 1 }, (_, i) => startLine + i);
+    const numberedContext = lines.slice(startLine - 1, endLine)
+      .map((line, i) => `${lineNumbers[i]}: ${line}`)
+      .join("\n");
+    
+    // Build AI prompt
+    const prompt = buildAnalysisPrompt({
+      violation,
+      claim,
+      filePath,
+      context: numberedContext,
+      ruleType: violation.rule
+    });
+    
+    // Try LLM first if enabled, otherwise use heuristics
+    let analysis = null;
+    if (ruleConfig.useLLM) {
+      analysis = await analyzeWithLLM(prompt, ruleConfig);
+    }
+    
+    // Fall back to heuristics if LLM not available or failed
+    if (!analysis) {
+      analysis = await analyzeWithHeuristics({
+        violation,
+        claim,
+        filePath,
+        context,
+        numberedContext
+      });
+    }
+    
+    // Cache result
+    aiCache.set(cacheKey, {
+      timestamp: Date.now(),
+      result: analysis
+    });
+    
+    return analysis;
+  } catch (error) {
+    // If AI analysis fails, default to not a false positive
+    return {
+      isFalsePositive: false,
+      confidence: 0,
+      reason: `AI analysis failed: ${error.message}`
+    };
+  }
+}
+
+/**
+ * Build prompt for AI analysis
+ */
+function buildAnalysisPrompt({ violation, claim, filePath, context, ruleType }) {
+  return `Analyze this code violation to determine if it's a false positive.
+
+Rule: ${ruleType}
+Claim Type: ${claim.type}
+Claim Value: ${claim.value}
+File: ${filePath}
+
+Code Context:
+\`\`\`
+${context}
+\`\`\`
+
+Question: Is this a false positive? Consider:
+1. Is this an import path being mistaken for a route? (e.g., "from './api/content'")
+2. Is this an external API call that shouldn't be validated? (e.g., "/content/blog" going to backend)
+3. Is this test/fixture code that should be ignored?
+4. Is this a legitimate pattern that the rule doesn't understand?
+
+Respond with JSON:
+{
+  "isFalsePositive": boolean,
+  "confidence": 0.0-1.0,
+  "reason": "explanation",
+  "suggestedFix": "what to do if false positive"
+}`;
+}
+
+/**
+ * Analyze using heuristics (can be replaced with actual LLM call)
+ * This provides immediate value while LLM integration can be added later
+ */
+async function analyzeWithHeuristics({ violation, claim, filePath, context, numberedContext }) {
+  const claimValue = String(claim.value || "").trim();
+  const filePathLower = filePath.toLowerCase();
+  const contextLower = context.toLowerCase();
+  
+  // Heuristic 1: Import paths
+  if (claimValue.includes("./") || claimValue.includes("../")) {
+    // Check if it's in an import/require statement
+    const importPatterns = [
+      /from\s+['"]\./,
+      /require\(['"]\./,
+      /import\s+.*from\s+['"]\./
+    ];
+    
+    for (const pattern of importPatterns) {
+      if (pattern.test(context)) {
+        return {
+          isFalsePositive: true,
+          confidence: 0.95,
+          reason: "This appears to be an import path, not a route",
+          suggestedFix: "Update route detection to exclude import statements"
+        };
+      }
+    }
+  }
+  
+  // Heuristic 2: External API calls (not starting with /api/)
+  if (violation.rule === "ghost_route" && claimValue.startsWith("/")) {
+    if (!claimValue.startsWith("/api/")) {
+      return {
+        isFalsePositive: true,
+        confidence: 0.9,
+        reason: "This is an external API call to the backend, not a Next.js route",
+        suggestedFix: "Only validate Next.js API routes (starting with /api/)"
+      };
+    }
+  }
+  
+  // Heuristic 3: Test files
+  if (filePathLower.includes("test") || 
+      filePathLower.includes("spec") || 
+      filePathLower.includes("fixture") ||
+      filePathLower.includes("mock")) {
+    return {
+      isFalsePositive: true,
+      confidence: 0.85,
+      reason: "This is in a test/fixture file",
+      suggestedFix: "Add test files to .vibecheckignore"
+    };
+  }
+  
+  // Heuristic 4: Comments or strings
+  if (claimValue.includes("//") || 
+      claimValue.includes("/*") ||
+      contextLower.includes(`"${claimValue}"`) ||
+      contextLower.includes(`'${claimValue}'`)) {
+    // Check if it's in a comment
+    const lines = numberedContext.split("\n");
+    for (const line of lines) {
+      if (line.includes(claimValue)) {
+        const beforeValue = line.substring(0, line.indexOf(claimValue));
+        if (beforeValue.includes("//") || beforeValue.includes("/*")) {
+          return {
+            isFalsePositive: true,
+            confidence: 0.8,
+            reason: "This appears to be in a comment",
+            suggestedFix: "Skip route detection in comments"
+          };
+        }
+      }
+    }
+  }
+  
+  // Heuristic 5: Environment variable names (for ghost_env)
+  if (violation.rule === "ghost_env") {
+    // Check if it's a standard/system env var that doesn't need declaration
+    const systemEnvVars = [
+      "NODE_ENV", "PATH", "HOME", "USER", "SHELL", "TMPDIR",
+      "CI", "GITHUB_ACTIONS", "GITLAB_CI", "CIRCLECI", "BUILDKITE",
+      "COLORTERM", "TERM", "LANG", "LC_ALL"
+    ];
+    
+    if (systemEnvVars.includes(claimValue)) {
+      return {
+        isFalsePositive: true,
+        confidence: 0.9,
+        reason: "This is a standard system environment variable",
+        suggestedFix: "Add system env vars to allowlist"
+      };
+    }
+  }
+  
+  // Heuristic 6: Success messages in UI components (for fake_success_ui)
+  if (violation.rule === "fake_success_ui") {
+    // Check if it's just displaying a version number or static text
+    if (claimValue.match(/v?\d+\.\d+\.\d+/) || // Version numbers
+        claimValue.length < 10) { // Very short messages
+      return {
+        isFalsePositive: true,
+        confidence: 0.7,
+        reason: "This appears to be a version number or static text, not a success message",
+        suggestedFix: "Improve success message detection to exclude version numbers"
+      };
+    }
+  }
+  
+  // Default: not a false positive
+  return {
+    isFalsePositive: false,
+    confidence: 0.5,
+    reason: "No clear indicators of false positive",
+    suggestedFix: null
+  };
+}
+
+/**
+ * Call actual LLM for analysis (when AI is available)
+ * Uses OpenAI or Anthropic API directly
+ */
+async function analyzeWithLLM(prompt, config = {}) {
+  const { useLLM = false, provider = "openai" } = config;
+  
+  if (!useLLM) {
+    return null;
+  }
+  
+  const hasOpenAI = !!process.env.OPENAI_API_KEY;
+  const hasAnthropic = !!process.env.ANTHROPIC_API_KEY;
+  
+  if (!hasOpenAI && !hasAnthropic) {
+    return null;
+  }
+  
+  try {
+    // Try OpenAI first (default)
+    if ((provider === "openai" || !hasAnthropic) && hasOpenAI) {
+      return await callOpenAI(prompt);
+    }
+    
+    // Try Anthropic
+    if ((provider === "anthropic" || !hasOpenAI) && hasAnthropic) {
+      return await callAnthropic(prompt);
+    }
+  } catch (error) {
+    // If LLM call fails, fall back to heuristics
+    console.warn(`[AI] LLM analysis failed: ${error.message}`);
+    return null;
+  }
+  
+  return null;
+}
+
+/**
+ * Call OpenAI API
+ */
+async function callOpenAI(prompt) {
+  const https = require("https");
+  const http = require("http");
+  
+  const apiKey = process.env.OPENAI_API_KEY;
+  if (!apiKey) return null;
+  
+  const requestBody = JSON.stringify({
+    model: "gpt-4o-mini", // Fast and cheap model
+    messages: [
+      {
+        role: "system",
+        content: "You are a code analysis assistant. Analyze if a code violation is a false positive. Respond with JSON only."
+      },
+      {
+        role: "user",
+        content: prompt
+      }
+    ],
+    response_format: { type: "json_object" },
+    temperature: 0.3,
+    max_tokens: 500
+  });
+  
+  return new Promise((resolve, reject) => {
+    const options = {
+      hostname: "api.openai.com",
+      path: "/v1/chat/completions",
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Authorization": `Bearer ${apiKey}`,
+        "Content-Length": Buffer.byteLength(requestBody)
+      },
+      timeout: 5000
+    };
+    
+    const req = https.request(options, (res) => {
+      let data = "";
+      
+      res.on("data", (chunk) => {
+        data += chunk;
+      });
+      
+      res.on("end", () => {
+        try {
+          const response = JSON.parse(data);
+          if (response.error) {
+            reject(new Error(response.error.message));
+            return;
+          }
+          
+          const content = response.choices?.[0]?.message?.content;
+          if (!content) {
+            resolve(null);
+            return;
+          }
+          
+          const result = JSON.parse(content);
+          resolve(result);
+        } catch (error) {
+          reject(error);
+        }
+      });
+    });
+    
+    req.on("error", reject);
+    req.on("timeout", () => {
+      req.destroy();
+      reject(new Error("Request timeout"));
+    });
+    
+    req.write(requestBody);
+    req.end();
+  });
+}
+
+/**
+ * Call Anthropic API
+ */
+async function callAnthropic(prompt) {
+  const https = require("https");
+  
+  const apiKey = process.env.ANTHROPIC_API_KEY;
+  if (!apiKey) return null;
+  
+  const requestBody = JSON.stringify({
+    model: "claude-3-haiku-20240307", // Fast and cheap model
+    max_tokens: 500,
+    messages: [
+      {
+        role: "user",
+        content: prompt
+      }
+    ],
+    system: "You are a code analysis assistant. Analyze if a code violation is a false positive. Respond with JSON only."
+  });
+  
+  return new Promise((resolve, reject) => {
+    const options = {
+      hostname: "api.anthropic.com",
+      path: "/v1/messages",
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "x-api-key": apiKey,
+        "anthropic-version": "2023-06-01",
+        "Content-Length": Buffer.byteLength(requestBody)
+      },
+      timeout: 5000
+    };
+    
+    const req = https.request(options, (res) => {
+      let data = "";
+      
+      res.on("data", (chunk) => {
+        data += chunk;
+      });
+      
+      res.on("end", () => {
+        try {
+          const response = JSON.parse(data);
+          if (response.error) {
+            reject(new Error(response.error.message));
+            return;
+          }
+          
+          const content = response.content?.[0]?.text;
+          if (!content) {
+            resolve(null);
+            return;
+          }
+          
+          // Extract JSON from response (might be wrapped in markdown)
+          const jsonMatch = content.match(/\{[\s\S]*\}/);
+          if (!jsonMatch) {
+            resolve(null);
+            return;
+          }
+          
+          const result = JSON.parse(jsonMatch[0]);
+          resolve(result);
+        } catch (error) {
+          reject(error);
+        }
+      });
+    });
+    
+    req.on("error", reject);
+    req.on("timeout", () => {
+      req.destroy();
+      reject(new Error("Request timeout"));
+    });
+    
+    req.write(requestBody);
+    req.end();
+  });
+}
+
+/**
+ * Clear AI cache (useful for testing)
+ */
+function clearCache() {
+  aiCache.clear();
+}
+
+module.exports = {
+  analyzeFalsePositive,
+  clearCache
+};

package/bin/runners/lib/agent-firewall/claims/extractor.js

@@ -12,6 +12,7 @@ const traverse = require("@babel/traverse").default;
 const t = require("@babel/types");
 const { CLAIM_TYPES, CRITICALITY } = require("./claim-types");
 const { ROUTE_LIKE, AUTH_HINTS, SUCCESS_UI } = require("./patterns");
+const { shouldIgnore } = require("../utils/ignore-checker");
 
 function parse(code) {
   return parser.parse(code, {
@@ -36,11 +37,36 @@ function pushClaim(out, claim) {
 
 function extractStringLiterals(code) {
   // Cheap scan for route-ish strings even if AST misses template parts.
+  // Exclude import paths (from './api/...' or from '../api/...')
   const hits = [];
-  for (const rx of ROUTE_LIKE) {
-    const m = code.match(rx);
-    if (m) hits.push(...m);
+  const lines = code.split('\n');
+  
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    
+    // Skip import/require statements to avoid false positives
+    if (/^\s*(import|export|require)\s+.*from\s+['"]/.test(line) ||
+        /^\s*const\s+\w+\s*=\s*require\(/.test(line)) {
+      continue;
+    }
+    
+    // Check for route-like patterns in this line
+    for (const rx of ROUTE_LIKE) {
+      const matches = line.match(rx);
+      if (matches) {
+        // Filter out matches that are part of import paths
+        for (const match of matches) {
+          // Exclude if it's part of an import path (has './' or '../' before it)
+          const matchIndex = line.indexOf(match);
+          const beforeMatch = line.substring(Math.max(0, matchIndex - 10), matchIndex);
+          if (!/['"]\.\.?\/.*$/.test(beforeMatch)) {
+            hits.push(match);
+          }
+        }
+      }
+    }
   }
+  
   return [...new Set(hits)];
 }
 
@@ -56,21 +82,19 @@ function classifyFileDomain(fileRel) {
 
 function extractClaimsFromFile({ repoRoot, fileAbs }) {
   const fileRel = path.relative(repoRoot, fileAbs).replace(/\\/g, "/");
+  
+  // Skip ignored files (test files, fixtures, etc.)
+  if (shouldIgnore(repoRoot, fileRel)) {
+    return { fileRel, domain: "ignored", claims: [], _dedupe: new Set() };
+  }
+  
   const code = fs.readFileSync(fileAbs, "utf8");
   const ast = parse(code);
 
   const out = { fileRel, domain: classifyFileDomain(fileRel), claims: [], _dedupe: new Set() };
 
-  // 1) quick string route-ish scan
-  for (const r of extractStringLiterals(code)) {
-    pushClaim(out, {
-      type: CLAIM_TYPES.ROUTE,
-      value: r,
-      criticality: CRITICALITY.HARD,
-      pointer: `${fileRel}:1-1`,
-      reason: "route-like string literal"
-    });
-  }
+  // 1) quick string route-ish scan (skip - AST-based detection is more accurate)
+  // Removed to avoid false positives from import paths
 
   // 2) AST-based env extraction
   traverse(ast, {
@@ -116,17 +140,58 @@ function extractClaimsFromFile({ repoRoot, fileAbs }) {
     CallExpression(p) {
       const n = p.node;
 
-      // fetch("/api/...")
+      // fetch("/api/..." or fetch(`${url}/api/...`))
       if (t.isIdentifier(n.callee, { name: "fetch" }) && n.arguments[0]) {
         const a0 = n.arguments[0];
         if (t.isStringLiteral(a0)) {
-          pushClaim(out, {
-            type: CLAIM_TYPES.HTTP_CALL,
-            value: a0.value,
-            criticality: CRITICALITY.HARD,
-            pointer: locPtr(fileRel, n),
-            reason: "fetch call"
-          });
+          const routeValue = a0.value;
+          // Skip if it's an external URL (starts with http:// or https://)
+          // Only track Next.js API routes (starting with /api/)
+          // External API calls (like /content/blog) are handled by backend, not validated here
+          if (!routeValue.startsWith('http://') && !routeValue.startsWith('https://') && routeValue.startsWith('/api/')) {
+            pushClaim(out, {
+              type: CLAIM_TYPES.HTTP_CALL,
+              value: routeValue,
+              criticality: CRITICALITY.HARD,
+              pointer: locPtr(fileRel, n),
+              reason: "fetch call"
+            });
+          }
+        } else if (t.isTemplateLiteral(a0)) {
+          // Template literal like fetch(`${apiUrl}/api/...`)
+          // Check if it contains /api/ pattern - this indicates an HTTP call
+          const templateParts = a0.quasis.map(q => q.value.cooked || q.value.raw).join('');
+          // Check if it uses apiUrl or similar variable (backend API call)
+          const hasApiUrlVar = a0.expressions.some(expr => 
+            t.isIdentifier(expr) && (
+              expr.name.includes('apiUrl') || 
+              expr.name.includes('API_URL') ||
+              expr.name.includes('api')
+            )
+          );
+          
+          // Extract the actual route path if possible (e.g., /api/v1/dashboard/summary)
+          const apiMatch = templateParts.match(/\/api\/[a-zA-Z0-9_\/\-]+/);
+          if (apiMatch) {
+            pushClaim(out, {
+              type: CLAIM_TYPES.HTTP_CALL,
+              value: apiMatch[0], // Use the actual matched route
+              criticality: CRITICALITY.HARD,
+              pointer: locPtr(fileRel, n),
+              reason: hasApiUrlVar ? "fetch call (backend API - template literal)" : "fetch call (template literal)",
+              isBackendApi: hasApiUrlVar // Flag for evidence resolver
+            });
+          } else if (templateParts.includes('/api/')) {
+            // Fallback: if we can't extract exact route, still mark as HTTP call
+            pushClaim(out, {
+              type: CLAIM_TYPES.HTTP_CALL,
+              value: '/api/*', // Pattern for template literals
+              criticality: CRITICALITY.HARD,
+              pointer: locPtr(fileRel, n),
+              reason: hasApiUrlVar ? "fetch call (backend API - pattern)" : "fetch call (template literal - pattern)",
+              isBackendApi: hasApiUrlVar
+            });
+          }
         }
       }
 
@@ -135,13 +200,37 @@ function extractClaimsFromFile({ repoRoot, fileAbs }) {
         const method = t.isIdentifier(n.callee.property) ? n.callee.property.name : "call";
         const a0 = n.arguments[0];
         if (a0 && t.isStringLiteral(a0)) {
-          pushClaim(out, {
-            type: CLAIM_TYPES.HTTP_CALL,
-            value: `${method.toUpperCase()} ${a0.value}`,
-            criticality: CRITICALITY.HARD,
-            pointer: locPtr(fileRel, n),
-            reason: "axios call"
-          });
+          const routeValue = a0.value;
+          // Only track Next.js API routes (starting with /api/)
+          // External API calls (like /content/blog) are handled by backend, not validated here
+          if (routeValue.startsWith('/api/')) {
+            pushClaim(out, {
+              type: CLAIM_TYPES.HTTP_CALL,
+              value: `${method.toUpperCase()} ${routeValue}`,
+              criticality: CRITICALITY.HARD,
+              pointer: locPtr(fileRel, n),
+              reason: "axios call"
+            });
+          }
+        }
+      }
+      
+      // apiRequest("/api/...") - check if it's the apiRequest helper
+      if (t.isIdentifier(n.callee, { name: "apiRequest" }) && n.arguments[0]) {
+        const a0 = n.arguments[0];
+        if (t.isStringLiteral(a0)) {
+          const routeValue = a0.value;
+          // Only track Next.js API routes (starting with /api/)
+          // External API calls (like /content/blog) go to backend API, not Next.js routes
+          if (routeValue.startsWith('/api/')) {
+            pushClaim(out, {
+              type: CLAIM_TYPES.HTTP_CALL,
+              value: routeValue,
+              criticality: CRITICALITY.HARD,
+              pointer: locPtr(fileRel, n),
+              reason: "apiRequest call"
+            });
+          }
         }
       }