@vibecheckai/cli 3.2.2 → 3.2.4

package/bin/runners/lib/agent-firewall/evidence/env-evidence.js

@@ -48,24 +48,33 @@ function resolve(projectRoot, claim) {
   }
   
   // Check if env var exists in .env.example or schema files
-  const envExamplePath = path.join(projectRoot, ".env.example");
-  const envSchemaPath = findEnvSchemaFile(projectRoot);
+  // Check multiple possible locations for .env.example
+  const envExamplePaths = [
+    path.join(projectRoot, ".env.example"),
+    path.join(projectRoot, "apps", "web-ui", ".env.example"),
+    path.join(projectRoot, "apps", "api", ".env.example")
+  ];
   
-  if (fs.existsSync(envExamplePath)) {
-    const envExample = fs.readFileSync(envExamplePath, "utf8");
-    if (envExample.includes(envVarName)) {
-      return {
-        result: "PROVEN",
-        sources: [{
-          type: "repo.search",
-          pointer: ".env.example",
-          confidence: 0.7
-        }],
-        reason: `Environment variable ${envVarName} found in .env.example`
-      };
+  for (const envExamplePath of envExamplePaths) {
+    if (fs.existsSync(envExamplePath)) {
+      const envExample = fs.readFileSync(envExamplePath, "utf8");
+      if (envExample.includes(envVarName)) {
+        const relativePath = path.relative(projectRoot, envExamplePath).replace(/\\/g, "/");
+        return {
+          result: "PROVEN",
+          sources: [{
+            type: "repo.search",
+            pointer: relativePath,
+            confidence: 0.7
+          }],
+          reason: `Environment variable ${envVarName} found in ${relativePath}`
+        };
+      }
     }
   }
   
+  const envSchemaPath = findEnvSchemaFile(projectRoot);
+  
   if (envSchemaPath && fs.existsSync(envSchemaPath)) {
     const envSchema = fs.readFileSync(envSchemaPath, "utf8");
     if (envSchema.includes(envVarName)) {

package/bin/runners/lib/agent-firewall/evidence/route-evidence.js

@@ -9,6 +9,7 @@
 
 const { getRoutes } = require("../truthpack");
 const { canonicalizePath } = require("../../route-truth");
+const { shouldIgnore } = require("../utils/ignore-checker");
 
 /**
  * Resolve route claim evidence
@@ -17,11 +18,81 @@ const { canonicalizePath } = require("../../route-truth");
  * @returns {object} Evidence result
  */
 function resolve(projectRoot, claim) {
-  const routes = getRoutes(projectRoot);
+  // Skip checking routes from ignored files (test files, fixtures, etc.)
+  if (claim.pointer && shouldIgnore(projectRoot, claim.pointer.split(":")[0])) {
+    return {
+      result: "PROVEN",
+      sources: [{
+        type: "ignored",
+        pointer: claim.pointer,
+        confidence: 1.0
+      }],
+      reason: `Route from ignored file (test/fixture)`
+    };
+  }
   
   // Normalize route path from claim
   const routePath = canonicalizePath(claim.value);
   
+  // Skip external API calls (routes that don't start with /api/ in Next.js context)
+  // Routes like /content/blog, /content/faq are external backend API calls, not Next.js routes
+  // Only validate Next.js API routes (those in apps/web-ui/src/app/api/)
+  if (!routePath.startsWith('/api/')) {
+    return {
+      result: "PROVEN",
+      sources: [{
+        type: "external_api",
+        pointer: claim.pointer,
+        confidence: 1.0
+      }],
+      reason: `Route ${routePath} is an external API call, not a Next.js route`
+    };
+  }
+  
+  // Skip wildcard patterns from template literals (they're approximations)
+  // These are detected from fetch(`${url}/api/...`) and are not exact routes
+  if (routePath.includes('*') || routePath === '/api/*') {
+    return {
+      result: "PROVEN",
+      sources: [{
+        type: "pattern",
+        pointer: claim.pointer,
+        confidence: 0.8
+      }],
+      reason: `Route pattern ${routePath} is a template literal approximation, not an exact route`
+    };
+  }
+  
+  // Skip backend API routes (routes from template literals using apiUrl)
+  // These are calls to the backend API server, not Next.js API routes
+  if (claim.isBackendApi || (claim.reason && claim.reason.includes('backend API'))) {
+    return {
+      result: "PROVEN",
+      sources: [{
+        type: "backend_api",
+        pointer: claim.pointer,
+        confidence: 0.9
+      }],
+      reason: `Route ${routePath} is a backend API call (using apiUrl variable), not a Next.js route`
+    };
+  }
+  
+  // Skip routes that are clearly backend API routes (v1, v2, etc. are typically backend)
+  // Next.js API routes are usually simpler like /api/checkout, /api/health
+  if (routePath.match(/^\/api\/v\d+\//)) {
+    return {
+      result: "PROVEN",
+      sources: [{
+        type: "backend_api",
+        pointer: claim.pointer,
+        confidence: 0.85
+      }],
+      reason: `Route ${routePath} appears to be a backend API route (versioned), not a Next.js route`
+    };
+  }
+  
+  const routes = getRoutes(projectRoot);
+  
   // Check if route exists in truthpack
   if (routes && routes.length > 0) {
     // First, check for exact match

package/bin/runners/lib/agent-firewall/interceptor/base.js

@@ -77,7 +77,7 @@ async function interceptFileWrite({
   }];
   
   // Evaluate policy
-  const verdict = evaluatePolicy({
+  const verdict = await evaluatePolicy({
     policy,
     claims,
     evidence,
@@ -159,7 +159,7 @@ async function interceptMultiFileWrite({
   const evidence = resolveEvidence(projectRoot, allClaims);
   
   // Evaluate policy
-  const verdict = evaluatePolicy({
+  const verdict = await evaluatePolicy({
     policy,
     claims: allClaims,
     evidence,

package/bin/runners/lib/agent-firewall/policy/default-policy.json

@@ -55,6 +55,12 @@
     "unsafe_side_effect": {
       "severity": "block",
       "enabled": true
+    },
+    "ai_false_positive_detection": {
+      "enabled": true,
+      "confidence_threshold": 0.8,
+      "logSkipped": true,
+      "useLLM": false
     }
   },
   "evidence": {

package/bin/runners/lib/agent-firewall/policy/engine.js

@@ -2,7 +2,7 @@
  * Policy Engine
  * 
  * Main policy engine that evaluates all rules deterministically.
- * No LLM opinions - pure rule-based evaluation.
+ * Uses AI to reduce false positives when enabled.
  */
 
 "use strict";
@@ -15,6 +15,7 @@ const fakeSuccess = require("./rules/fake-success");
 const scope = require("./rules/scope");
 const unsafeSideEffect = require("./rules/unsafe-side-effect");
 const { generateVerdict } = require("./verdict");
+const { analyzeFalsePositive } = require("../ai/false-positive-analyzer");
 
 /**
  * Evaluate policy against change packet
@@ -24,9 +25,9 @@ const { generateVerdict } = require("./verdict");
  * @param {array} params.evidence - Evidence resolution results
  * @param {array} params.files - Changed files
  * @param {string} params.intent - Agent intent message
- * @returns {object} Verdict object
+ * @returns {Promise<object>} Verdict object
  */
-function evaluatePolicy({ policy, claims, evidence, files, intent }) {
+async function evaluatePolicy({ policy, claims, evidence, files, intent }) {
   const violations = [];
   
   // Evaluate all rules
@@ -44,6 +45,36 @@ function evaluatePolicy({ policy, claims, evidence, files, intent }) {
     try {
       const violation = evaluator.evaluate({ claims, evidence, policy });
       if (violation) {
+        // Use AI to check if this is a false positive
+        const claim = claims.find(c => 
+          c.type === violation.claim?.type && 
+          c.value === violation.claim?.value
+        );
+        
+        if (claim && policy.rules?.ai_false_positive_detection?.enabled) {
+          try {
+            const aiAnalysis = await analyzeFalsePositive({
+              violation,
+              claim,
+              filePath: claim.file || violation.claim?.file || "unknown",
+              projectRoot: process.cwd(),
+              policy
+            });
+            
+            // If AI determines it's a false positive with high confidence, skip it
+            if (aiAnalysis.isFalsePositive && aiAnalysis.confidence >= 0.8) {
+              // Log but don't add to violations
+              if (policy.rules?.ai_false_positive_detection?.logSkipped) {
+                console.log(`[AI] Skipping false positive: ${violation.message} (confidence: ${aiAnalysis.confidence}, reason: ${aiAnalysis.reason})`);
+              }
+              continue;
+            }
+          } catch (aiError) {
+            // If AI analysis fails, proceed with the violation
+            console.warn(`[AI] False positive analysis failed: ${aiError.message}`);
+          }
+        }
+        
         violations.push(violation);
       }
     } catch (error) {

package/bin/runners/lib/agent-firewall/policy/rules/fake-success.js

@@ -30,7 +30,32 @@ function evaluate({ claims, evidence, policy }) {
     return null;
   }
   
-  // Check if there's a corresponding HTTP call or side effect
+  // Filter out false positives: if there's an HTTP call in the same file, it's likely a response check
+  const realSuccessClaims = successClaims.filter(claim => {
+    const claimFile = claim.pointer ? claim.pointer.split(":")[0] : "";
+    if (!claimFile) return true;
+    
+    // Check if there's an HTTP call in the same file (within reasonable distance)
+    const claimLine = claim.pointer ? parseInt(claim.pointer.split(":")[1]?.split("-")[0] || "0", 10) : 0;
+    const hasNearbyHttpCall = claims.some(c => {
+      if (c.type !== CLAIM_TYPES.HTTP_CALL && c.type !== CLAIM_TYPES.ROUTE) return false;
+      const cFile = c.pointer ? c.pointer.split(":")[0] : "";
+      if (cFile !== claimFile) return false;
+      const cLine = c.pointer ? parseInt(c.pointer.split(":")[1]?.split("-")[0] || "0", 10) : 0;
+      // Within 50 lines is considered "nearby"
+      return Math.abs(cLine - claimLine) <= 50;
+    });
+    
+    // If there's a nearby HTTP call, this is likely checking the response, not showing UI
+    // Only flag if there's NO HTTP call nearby
+    return !hasNearbyHttpCall;
+  });
+  
+  if (realSuccessClaims.length === 0) {
+    return null; // All were false positives (likely API response checks)
+  }
+  
+  // Check if there's a corresponding HTTP call or side effect globally
   const hasHttpCall = claims.some(c => 
     c.type === CLAIM_TYPES.HTTP_CALL || c.type === CLAIM_TYPES.ROUTE
   );
@@ -40,7 +65,7 @@ function evaluate({ claims, evidence, policy }) {
   if (!hasHttpCall && !hasSideEffect) {
     // Check if domain requires blocking
     const blockDomains = ruleConfig.block_if_domain || [];
-    const fileDomain = successClaims[0].domain || "general";
+    const fileDomain = realSuccessClaims[0].domain || "general";
     
     const shouldBlock = blockDomains.includes(fileDomain);
     
@@ -48,8 +73,8 @@ function evaluate({ claims, evidence, policy }) {
       rule: "fake_success_ui",
       severity: shouldBlock ? "block" : (ruleConfig.severity || "warn"),
       message: `Fake success UI: Success message shown but no HTTP call or side effect detected`,
-      claimId: `claim_${claims.indexOf(successClaims[0])}`,
-      claim: successClaims[0]
+      claimId: `claim_${claims.indexOf(realSuccessClaims[0])}`,
+      claim: realSuccessClaims[0]
     };
   }
   

package/bin/runners/lib/agent-firewall/policy/rules/ghost-route.js

@@ -30,7 +30,19 @@ function evaluate({ claims, evidence, policy }) {
     if (claim.type === CLAIM_TYPES.ROUTE || claim.type === CLAIM_TYPES.HTTP_CALL) {
       const ev = evidence.find(e => e.claimId === `claim_${i}`);
       
+      // Skip if evidence shows it's an external API call or ignored file
+      if (ev && (ev.result === "PROVEN" && (ev.reason?.includes("external API") || ev.reason?.includes("ignored file")))) {
+        continue;
+      }
+      
       if (ev && ev.result === "UNPROVEN") {
+        // Double-check: skip external API calls (routes not starting with /api/)
+        const routePath = String(claim.value || "").trim();
+        if (!routePath.startsWith('/api/')) {
+          // This is an external API call, not a Next.js route - skip
+          continue;
+        }
+        
         return {
           rule: "ghost_route",
           severity: ruleConfig.severity || "block",

package/bin/runners/lib/agent-firewall/truthpack/loader.js

@@ -65,6 +65,27 @@ function loadTruthpackFile(projectRoot, filename) {
   const filePath = path.join(projectRoot, TRUTHPACK_DIR, filename);
   
   if (!fs.existsSync(filePath)) {
+    // Fallback: try loading from truthpack.json if routes.json doesn't exist
+    if (filename === "routes.json") {
+      const truthpackPath = path.join(projectRoot, ".vibecheck", "truthpack.json");
+      if (fs.existsSync(truthpackPath)) {
+        try {
+          const truthpack = JSON.parse(fs.readFileSync(truthpackPath, "utf8"));
+          // Extract server routes from truthpack.json format
+          if (truthpack.routes && truthpack.routes.server) {
+            return {
+              routes: truthpack.routes.server.map(r => ({
+                path: r.path,
+                method: r.method,
+                handler: r.handler || r.source
+              }))
+            };
+          }
+        } catch (error) {
+          // Fall through to return null
+        }
+      }
+    }
     return null;
   }
   

package/bin/runners/lib/agent-firewall/utils/ignore-checker.js

@@ -0,0 +1,118 @@
+/**
+ * Ignore File Checker
+ * 
+ * Checks if a file matches patterns in .vibecheckignore
+ */
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+
+// Try to use minimatch if available, otherwise use simple pattern matching
+let minimatch;
+try {
+  minimatch = require("minimatch").minimatch;
+} catch {
+  minimatch = null;
+}
+
+let ignorePatterns = null;
+let ignoreFile = null;
+
+/**
+ * Load ignore patterns from .vibecheckignore
+ * @param {string} projectRoot - Project root directory
+ * @returns {array} Array of ignore patterns
+ */
+function loadIgnorePatterns(projectRoot) {
+  if (ignorePatterns !== null && ignoreFile === projectRoot) {
+    return ignorePatterns;
+  }
+
+  const ignorePath = path.join(projectRoot, ".vibecheckignore");
+  ignorePatterns = [];
+  
+  if (fs.existsSync(ignorePath)) {
+    const content = fs.readFileSync(ignorePath, "utf8");
+    ignorePatterns = content
+      .split("\n")
+      .map(line => line.trim())
+      .filter(line => line && !line.startsWith("#"));
+  }
+  
+  ignoreFile = projectRoot;
+  return ignorePatterns;
+}
+
+/**
+ * Check if a file should be ignored
+ * @param {string} projectRoot - Project root directory
+ * @param {string} filePath - File path (relative to project root)
+ * @returns {boolean} True if file should be ignored
+ */
+function shouldIgnore(projectRoot, filePath) {
+  const patterns = loadIgnorePatterns(projectRoot);
+  const normalizedPath = filePath.replace(/\\/g, "/");
+  
+  // Check against all patterns
+  for (const pattern of patterns) {
+    // Handle both glob patterns and simple path matches
+    if (minimatch) {
+      if (minimatch(normalizedPath, pattern, { matchBase: true }) ||
+          minimatch(normalizedPath, `**/${pattern}`, { matchBase: true })) {
+        return true;
+      }
+    } else {
+      // Fallback: simple pattern matching
+      const regexPattern = pattern
+        .replace(/\*\*/g, ".*")
+        .replace(/\*/g, "[^/]*")
+        .replace(/\//g, "\\/");
+      const regex = new RegExp(`^${regexPattern}$`);
+      if (regex.test(normalizedPath)) {
+        return true;
+      }
+    }
+    
+    // Also check if path contains the pattern as a substring (for directory patterns)
+    const cleanPattern = pattern.replace(/\*\*/g, "").replace(/\*/g, "");
+    if (normalizedPath.includes(cleanPattern)) {
+      return true;
+    }
+  }
+  
+  // Also check common test file patterns
+  const testPatterns = [
+    /\/__tests__\//,
+    /\.test\.(ts|tsx|js|jsx)$/,
+    /\.spec\.(ts|tsx|js|jsx)$/,
+    /\/tests\//,
+    /\/test\//,
+    /\/fixtures\//,
+    /\/examples\//,
+    /\/templates\//
+  ];
+  
+  for (const pattern of testPatterns) {
+    if (pattern.test(normalizedPath)) {
+      return true;
+    }
+  }
+  
+  return false;
+}
+
+/**
+ * Clear cached ignore patterns (for testing)
+ */
+function clearCache() {
+  ignorePatterns = null;
+  ignoreFile = null;
+}
+
+module.exports = {
+  shouldIgnore,
+  loadIgnorePatterns,
+  clearCache
+};