@vibecheckai/cli 3.0.4 → 3.0.7

package/bin/runners/lib/detectors-v2.js

@@ -0,0 +1,703 @@
+/**
+ * Canonical Detectors v2
+ * 
+ * Implementation of all spec-defined detectors that produce v2-compliant findings.
+ * Each detector has a unique ID and produces evidence-backed findings.
+ * 
+ * Detector Categories:
+ * - Routes (D_ROUTE_*)
+ * - Auth Coverage (D_AUTH_*)
+ * - Env (D_ENV_*)
+ * - Fake Success / Truth (D_FAKE_SUCCESS_*, D_SILENT_CATCH)
+ * - Dead UI (D_DEAD_*, D_UI_*)
+ * - Billing/Stripe (D_STRIPE_*)
+ * - Entitlements (D_LOCAL_BYPASS_*)
+ * - Drift (D_CONTRACTS_*)
+ */
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+const { createFindingV2, createEvidence, generateFingerprint } = require("./schema-validator");
+
+// =============================================================================
+// B1) Routes Detectors
+// =============================================================================
+
+/**
+ * D_ROUTE_MISSING (BLOCK)
+ * Trigger: clientCalls contains /api/x but truthpack routes has no matching endpoint
+ */
+function detectRouteMissing(truthpack) {
+  const findings = [];
+  const serverRoutes = truthpack.routes || [];
+  const clientCalls = truthpack.clientCalls || [];
+
+  for (const call of clientCalls) {
+    const resolved = call.resolvedPath || call.urlTemplate;
+    const method = call.method || "UNKNOWN";
+
+    const match = serverRoutes.find(r => 
+      routeMatches(r.path, resolved) && 
+      (r.methods.includes(method) || r.methods.includes("*"))
+    );
+
+    if (!match) {
+      findings.push(createFindingV2({
+        detectorId: "ROUTE_MISSING",
+        severity: "BLOCK",
+        category: "Routes",
+        scope: "client",
+        title: `Client calls ${method} ${resolved} but no server route exists`,
+        why: "AI frequently invents endpoints. This will cause 404 errors or silent failures in production.",
+        confidence: call.confidence || "medium",
+        evidence: call.evidence || [
+          createEvidence({
+            kind: "file",
+            reason: "Client call site",
+            file: call.evidence?.[0]?.file || "unknown",
+            lines: call.evidence?.[0]?.lines || "1-1",
+          })
+        ],
+        fixHints: [
+          "Create the missing server route handler",
+          "Or update the client to call an existing route",
+          "Check truthpack.routes for available endpoints"
+        ],
+      }));
+    }
+  }
+
+  return findings;
+}
+
+/**
+ * D_ROUTE_METHOD_MISMATCH (BLOCK/WARN)
+ * Trigger: client uses POST but server exposes GET (or vice versa)
+ */
+function detectRouteMethodMismatch(truthpack) {
+  const findings = [];
+  const serverRoutes = truthpack.routes || [];
+  const clientCalls = truthpack.clientCalls || [];
+
+  for (const call of clientCalls) {
+    const resolved = call.resolvedPath || call.urlTemplate;
+    const clientMethod = call.method || "UNKNOWN";
+
+    const pathMatch = serverRoutes.find(r => routeMatches(r.path, resolved));
+    
+    if (pathMatch && !pathMatch.methods.includes(clientMethod) && !pathMatch.methods.includes("*")) {
+      const isCritical = /checkout|login|save|pay|submit|register/i.test(resolved);
+      
+      findings.push(createFindingV2({
+        detectorId: "ROUTE_METHOD_MISMATCH",
+        severity: isCritical ? "BLOCK" : "WARN",
+        category: "Routes",
+        scope: "client",
+        title: `Method mismatch: client uses ${clientMethod} but server only handles ${pathMatch.methods.join("/")} for ${resolved}`,
+        why: "Method mismatch will cause 405 errors. Critical paths (checkout/login) must match exactly.",
+        confidence: "high",
+        evidence: [
+          createEvidence({
+            kind: "file",
+            reason: "Client call site",
+            file: call.evidence?.[0]?.file || "unknown",
+            lines: call.evidence?.[0]?.lines || "1-1",
+          }),
+          createEvidence({
+            kind: "file",
+            reason: "Server route handler",
+            file: pathMatch.handler?.file || "unknown",
+            lines: "1-1",
+          })
+        ],
+        fixHints: [
+          `Update client to use ${pathMatch.methods[0]} method`,
+          `Or add ${clientMethod} handler to server route`
+        ],
+      }));
+    }
+  }
+
+  return findings;
+}
+
+/**
+ * D_ROUTE_PREFIX_DRIFT (WARN→BLOCK)
+ * Trigger: Fastify registered under /api/v1 but client hits /api/
+ */
+function detectRoutePrefixDrift(truthpack) {
+  const findings = [];
+  const fastifyPrefixes = truthpack.stack?.fastify?.prefixes || [];
+  const clientCalls = truthpack.clientCalls || [];
+
+  if (fastifyPrefixes.length === 0) return findings;
+
+  const prefixSet = new Set(fastifyPrefixes);
+  let mismatchCount = 0;
+
+  for (const call of clientCalls) {
+    const resolved = call.resolvedPath || call.urlTemplate;
+    
+    // Check if client path uses a known prefix
+    const usesKnownPrefix = fastifyPrefixes.some(prefix => resolved.startsWith(prefix));
+    
+    if (!usesKnownPrefix && resolved.startsWith("/api/")) {
+      mismatchCount++;
+    }
+  }
+
+  if (mismatchCount > 0) {
+    findings.push(createFindingV2({
+      detectorId: "ROUTE_PREFIX_DRIFT",
+      severity: mismatchCount > 5 ? "BLOCK" : "WARN",
+      category: "Routes",
+      scope: "client",
+      title: `${mismatchCount} client calls don't match Fastify prefixes: ${fastifyPrefixes.join(", ")}`,
+      why: "Prefix drift causes silent failures. Clients must use the correct API prefix.",
+      confidence: "medium",
+      evidence: [
+        createEvidence({
+          kind: "file",
+          reason: "Fastify entry file with prefix registration",
+          file: truthpack.stack?.fastify?.entryFile || "unknown",
+          lines: "1-1",
+        })
+      ],
+      fixHints: [
+        `Update client calls to use correct prefix (${fastifyPrefixes[0] || "/api"})`,
+        "Or update Fastify prefix registration to match client expectations"
+      ],
+    }));
+  }
+
+  return findings;
+}
+
+// =============================================================================
+// B2) Auth Coverage Detectors
+// =============================================================================
+
+/**
+ * D_AUTH_PROTECTED_ROUTE_ACCESSIBLE_ANON (BLOCK) - runtime
+ * Trigger: --verify-auth ANON pass can access protected route
+ */
+function detectAuthProtectedAccessibleAnon(realityReport, authContract) {
+  const findings = [];
+  if (!realityReport?.run?.pass === "anon") return findings;
+
+  const protectedPatterns = authContract?.protectedRoutes || [];
+  const anonPages = realityReport.pages || [];
+  const anonNetwork = realityReport.network || [];
+
+  for (const pattern of protectedPatterns) {
+    // Check if anon pass successfully accessed a protected route
+    const accessed = anonNetwork.filter(req => 
+      matchPattern(pattern.pattern, req.url) && 
+      req.status >= 200 && req.status < 300
+    );
+
+    for (const req of accessed) {
+      if (pattern.expect?.anon === "deny" || pattern.expect?.anon === "redirect") {
+        findings.push(createFindingV2({
+          detectorId: "AUTH_PROTECTED_ROUTE_ACCESSIBLE_ANON",
+          severity: "BLOCK",
+          category: "AuthCoverage",
+          scope: "runtime",
+          title: `Protected route ${req.url} accessible to anonymous users`,
+          why: "Auth contract expects denial/redirect but got 2xx. This is a security bypass.",
+          confidence: "high",
+          evidence: [
+            createEvidence({
+              kind: "request",
+              reason: "Successful anonymous request to protected route",
+              url: req.url,
+              httpStatus: req.status,
+              requestId: req.id,
+            })
+          ],
+          fixHints: [
+            "Add server-side auth middleware to this route",
+            "Ensure Next.js middleware matcher covers this path",
+            "Verify auth contract pattern is correct"
+          ],
+          repro: {
+            steps: [
+              `Navigate to ${req.url} without authentication`,
+              "Observe that the page loads successfully (should be denied)"
+            ],
+            url: req.url,
+          },
+        }));
+      }
+    }
+  }
+
+  return findings;
+}
+
+/**
+ * D_AUTH_PROTECTED_ROUTE_BLOCKED_WHEN_AUTHED (BLOCK) - runtime
+ * Trigger: AUTH pass still denied/redirected repeatedly
+ */
+function detectAuthProtectedBlockedWhenAuthed(realityReport, authContract) {
+  const findings = [];
+  if (!realityReport?.run?.pass === "authed") return findings;
+
+  const protectedPatterns = authContract?.protectedRoutes || [];
+  const authedNetwork = realityReport.network || [];
+
+  for (const pattern of protectedPatterns) {
+    const blocked = authedNetwork.filter(req =>
+      matchPattern(pattern.pattern, req.url) &&
+      (req.status === 401 || req.status === 403 || req.status >= 300 && req.status < 400)
+    );
+
+    if (blocked.length > 2 && pattern.expect?.authed === "allow") {
+      findings.push(createFindingV2({
+        detectorId: "AUTH_PROTECTED_ROUTE_BLOCKED_WHEN_AUTHED",
+        severity: "BLOCK",
+        category: "AuthCoverage",
+        scope: "runtime",
+        title: `Protected route ${pattern.pattern} blocks authenticated users`,
+        why: "Auth contract expects allow but authenticated user is denied/redirected repeatedly.",
+        confidence: "high",
+        evidence: blocked.slice(0, 3).map(req => createEvidence({
+          kind: "request",
+          reason: "Request denied despite authentication",
+          url: req.url,
+          httpStatus: req.status,
+          requestId: req.id,
+        })),
+        fixHints: [
+          "Check if session/token is being passed correctly",
+          "Verify middleware is not over-blocking",
+          "Check for redirect loops"
+        ],
+      }));
+    }
+  }
+
+  return findings;
+}
+
+/**
+ * D_AUTH_CONTRACT_DRIFT (WARN/BLOCK)
+ * Trigger: contracts/auth.json patterns don't match middleware matcher
+ */
+function detectAuthContractDrift(truthpack, authContract) {
+  const findings = [];
+  const middlewareMatchers = new Set(truthpack.auth?.middlewareMatchers || []);
+  const contractPatterns = new Set(authContract?.protectedRoutes?.map(r => r.pattern) || []);
+
+  // Patterns in contract but not in middleware
+  for (const pattern of contractPatterns) {
+    if (!middlewareMatchers.has(pattern)) {
+      findings.push(createFindingV2({
+        detectorId: "AUTH_CONTRACT_DRIFT",
+        severity: "BLOCK",
+        category: "Drift",
+        scope: "contracts",
+        title: `Auth pattern "${pattern}" in contract but not in middleware`,
+        why: "Contract expects protection but middleware doesn't enforce it. Security boundary may be exposed.",
+        confidence: "high",
+        evidence: [
+          createEvidence({
+            kind: "file",
+            reason: "Auth contract file",
+            file: ".vibecheck/contracts/auth.json",
+            lines: "1-1",
+          })
+        ],
+        fixHints: [
+          "Add pattern to middleware matcher",
+          "Or run 'vibecheck ctx sync' to update contract"
+        ],
+      }));
+    }
+  }
+
+  // Patterns in middleware but not in contract (WARN - might be intentional)
+  for (const pattern of middlewareMatchers) {
+    if (!contractPatterns.has(pattern)) {
+      findings.push(createFindingV2({
+        detectorId: "AUTH_CONTRACT_DRIFT",
+        severity: "WARN",
+        category: "Drift",
+        scope: "contracts",
+        title: `Middleware pattern "${pattern}" not declared in auth contract`,
+        why: "Middleware protects a pattern not in contract. AI agents won't know about this protection.",
+        confidence: "medium",
+        evidence: [
+          createEvidence({
+            kind: "file",
+            reason: "Middleware file",
+            file: truthpack.stack?.next?.middlewareFile || "middleware.ts",
+            lines: "1-1",
+          })
+        ],
+        fixHints: [
+          "Run 'vibecheck ctx sync' to update contract"
+        ],
+      }));
+    }
+  }
+
+  return findings;
+}
+
+// =============================================================================
+// B3) Env Detectors
+// =============================================================================
+
+/**
+ * D_ENV_USED_BUT_UNDECLARED (WARN→BLOCK if required)
+ * Trigger: truthpack envUsage includes FOO but contracts/env.json does not
+ */
+function detectEnvUsedButUndeclared(truthpack, envContract) {
+  const findings = [];
+  const envUsage = truthpack.envUsage || [];
+  const declaredVars = new Set(envContract?.vars?.map(v => v.name) || []);
+
+  for (const usage of envUsage) {
+    if (!declaredVars.has(usage.name)) {
+      const isRequired = usage.inferredRequiredness === "required" || isLikelyRequired(usage.name);
+      
+      findings.push(createFindingV2({
+        detectorId: "ENV_USED_BUT_UNDECLARED",
+        severity: isRequired ? "BLOCK" : "WARN",
+        category: "Env",
+        scope: "server",
+        title: `Env var ${usage.name} used but not declared in contract`,
+        why: isRequired
+          ? "Required env var missing from contract. Deployment will fail if not set."
+          : "Env var used but not documented. AI won't know about this dependency.",
+        confidence: isRequired ? "high" : "medium",
+        evidence: usage.locations?.slice(0, 3).map(loc => createEvidence({
+          kind: "file",
+          reason: `Usage of ${usage.name}`,
+          file: loc.file,
+          lines: loc.lines,
+          snippet: loc.snippetHash,
+        })) || [],
+        fixHints: [
+          "Add to .env.example with appropriate default/placeholder",
+          "Run 'vibecheck ctx sync' to update env contract",
+          isRequired ? "Ensure this var is set in all environments" : null
+        ].filter(Boolean),
+      }));
+    }
+  }
+
+  return findings;
+}
+
+// =============================================================================
+// B4) Fake Success / Truth Detectors
+// =============================================================================
+
+/**
+ * D_FAKE_SUCCESS_TOAST_BEFORE_AWAIT (BLOCK/WARN)
+ * Trigger: toast.success before awaited network call
+ */
+function detectFakeSuccessToastBeforeAwait(projectPath) {
+  const findings = [];
+  // This requires AST analysis - see existing findFakeSuccess in analyzers.js
+  // Placeholder for integration
+  return findings;
+}
+
+/**
+ * D_FAKE_SUCCESS_RESPONSE_OK_IGNORED (BLOCK)
+ * Trigger: fetch result not checked before success UI
+ */
+function detectFakeSuccessResponseIgnored(projectPath) {
+  const findings = [];
+  // This requires AST analysis
+  return findings;
+}
+
+/**
+ * D_SILENT_CATCH (WARN→BLOCK)
+ * Trigger: catch (e) {} OR catch { return null } + UI success continues
+ */
+function detectSilentCatch(projectPath) {
+  const findings = [];
+  // This requires AST analysis
+  return findings;
+}
+
+// =============================================================================
+// B5) Dead UI Detectors (runtime-first)
+// =============================================================================
+
+/**
+ * D_DEAD_CLICK_NO_EFFECT (BLOCK/WARN)
+ * Trigger: action click yields no navigation, no DOM change, no network, no console
+ */
+function detectDeadClickNoEffect(realityReport) {
+  const findings = [];
+  const actions = realityReport?.actions || [];
+
+  for (const action of actions) {
+    if (action.type !== "click") continue;
+
+    const noNavigation = action.pageUrl === action.afterPageUrl;
+    const noDomChange = action.beforeDomHash === action.afterDomHash;
+    const noNetwork = !action.networkRequestIds || action.networkRequestIds.length === 0;
+
+    if (noNavigation && noDomChange && noNetwork) {
+      const isCritical = /save|submit|pay|login|continue|checkout/i.test(action.label || "");
+      
+      findings.push(createFindingV2({
+        detectorId: "DEAD_CLICK_NO_EFFECT",
+        severity: isCritical ? "BLOCK" : "WARN",
+        category: "DeadUI",
+        scope: "runtime",
+        title: `Click on "${action.label || action.selector}" has no effect`,
+        why: "Button/link does nothing - no navigation, no DOM change, no network call. User will think app is broken.",
+        confidence: "high",
+        evidence: [
+          createEvidence({
+            kind: "screenshot",
+            reason: "Screenshot before click",
+            artifactPath: action.screenshotBefore,
+          }),
+          createEvidence({
+            kind: "screenshot",
+            reason: "Screenshot after click (unchanged)",
+            artifactPath: action.screenshotAfter,
+          })
+        ].filter(e => e.artifactPath),
+        fixHints: [
+          "Wire click handler to actual functionality",
+          "If disabled, add aria-disabled and disabled styling",
+          "If feature not ready, remove or hide the element"
+        ],
+        repro: {
+          steps: [
+            `Navigate to ${action.pageUrl}`,
+            `Click on element: ${action.selector || action.label}`,
+            "Observe: nothing happens"
+          ],
+          url: action.pageUrl,
+        },
+      }));
+    }
+  }
+
+  return findings;
+}
+
+/**
+ * D_UI_ACTION_CAUSES_4XX_5XX (BLOCK)
+ * Trigger: click leads to request with 4xx/5xx
+ */
+function detectUIActionCauses4xx5xx(realityReport) {
+  const findings = [];
+  const actions = realityReport?.actions || [];
+  const network = realityReport?.network || [];
+
+  for (const action of actions) {
+    if (!action.networkRequestIds) continue;
+
+    for (const reqId of action.networkRequestIds) {
+      const req = network.find(r => r.id === reqId);
+      if (req && (req.status >= 400 || req.failed)) {
+        findings.push(createFindingV2({
+          detectorId: "UI_ACTION_CAUSES_4XX_5XX",
+          severity: "BLOCK",
+          category: "DeadUI",
+          scope: "runtime",
+          title: `Click on "${action.label || action.selector}" causes ${req.status} error`,
+          why: `UI action triggers failed API call (${req.status}). User will see broken functionality.`,
+          confidence: "high",
+          evidence: [
+            createEvidence({
+              kind: "request",
+              reason: `Failed request triggered by UI action`,
+              url: req.url,
+              httpStatus: req.status,
+              requestId: req.id,
+            })
+          ],
+          fixHints: [
+            "Fix the API endpoint to return success",
+            "Add proper error handling in UI",
+            "If endpoint doesn't exist, create it"
+          ],
+          repro: {
+            steps: [
+              `Navigate to ${action.pageUrl}`,
+              `Click on element: ${action.selector || action.label}`,
+              `Observe: API returns ${req.status}`
+            ],
+            url: action.pageUrl,
+          },
+        }));
+      }
+    }
+  }
+
+  return findings;
+}
+
+// =============================================================================
+// B6) Billing/Stripe Detectors
+// =============================================================================
+
+/**
+ * D_STRIPE_WEBHOOK_NO_SIGNATURE_VERIFY (BLOCK)
+ * Trigger: webhook route handler lacks signature verification
+ */
+function detectStripeWebhookNoSigVerify(truthpack) {
+  const findings = [];
+  const webhookRoutes = truthpack.externals?.stripe?.webhookRoutes || [];
+
+  // This requires checking if routes have proper verification
+  // Placeholder - actual implementation would scan the handler files
+  
+  return findings;
+}
+
+// =============================================================================
+// B7) Entitlements Detectors
+// =============================================================================
+
+/**
+ * D_LOCAL_BYPASS_PAID_FEATURE (BLOCK)
+ * Trigger: code checks env var like OWNER_MODE=true to unlock features
+ */
+function detectLocalBypassPaidFeature(projectPath) {
+  const findings = [];
+  // This requires scanning for patterns like OWNER_MODE, SKIP_AUTH, etc.
+  return findings;
+}
+
+// =============================================================================
+// B8) Drift Detectors
+// =============================================================================
+
+/**
+ * D_CONTRACTS_OUT_OF_DATE (WARN/BLOCK)
+ * Trigger: truthpack fingerprint changed but contracts still reflect old fingerprint
+ */
+function detectContractsOutOfDate(truthpack, contracts) {
+  const findings = [];
+  const truthpackFingerprint = truthpack.fingerprint;
+
+  for (const [type, contract] of Object.entries(contracts)) {
+    if (contract.projectFingerprint && contract.projectFingerprint !== truthpackFingerprint) {
+      findings.push(createFindingV2({
+        detectorId: "CONTRACTS_OUT_OF_DATE",
+        severity: type === "routes" || type === "auth" ? "BLOCK" : "WARN",
+        category: "Drift",
+        scope: "contracts",
+        title: `${type} contract out of date (fingerprint mismatch)`,
+        why: `Contract was generated from old codebase. AI agents will use stale information.`,
+        confidence: "high",
+        evidence: [
+          createEvidence({
+            kind: "hash",
+            reason: "Contract fingerprint",
+            file: `.vibecheck/contracts/${type}.json`,
+            lines: "1-1",
+          })
+        ],
+        fixHints: [
+          "Run 'vibecheck ctx sync' to regenerate contracts"
+        ],
+      }));
+    }
+  }
+
+  return findings;
+}
+
+// =============================================================================
+// Helpers
+// =============================================================================
+
+function routeMatches(pattern, actual) {
+  const patternParts = pattern.split("/").filter(Boolean);
+  const actualParts = actual.split("/").filter(Boolean);
+
+  if (patternParts.length !== actualParts.length) return false;
+
+  for (let i = 0; i < patternParts.length; i++) {
+    const p = patternParts[i];
+    if (p.startsWith(":") || p.startsWith("[") || p === "*") continue;
+    if (p !== actualParts[i]) return false;
+  }
+  return true;
+}
+
+function matchPattern(pattern, url) {
+  // Convert glob-like pattern to regex
+  const regex = new RegExp(
+    "^" + pattern
+      .replace(/\*/g, ".*")
+      .replace(/\//g, "\\/")
+    + "$"
+  );
+  return regex.test(url) || regex.test(new URL(url, "http://localhost").pathname);
+}
+
+function isLikelyRequired(name) {
+  const requiredPatterns = [
+    /^DATABASE_URL$/i,
+    /^NEXTAUTH_SECRET$/i,
+    /^NEXTAUTH_URL$/i,
+    /^JWT_SECRET$/i,
+    /^STRIPE_SECRET_KEY$/i,
+    /^STRIPE_WEBHOOK_SECRET$/i,
+    /SECRET/i,
+    /TOKEN$/i,
+    /API_KEY$/i,
+  ];
+  return requiredPatterns.some(p => p.test(name));
+}
+
+// =============================================================================
+// Exports
+// =============================================================================
+
+module.exports = {
+  // Routes
+  detectRouteMissing,
+  detectRouteMethodMismatch,
+  detectRoutePrefixDrift,
+  
+  // Auth
+  detectAuthProtectedAccessibleAnon,
+  detectAuthProtectedBlockedWhenAuthed,
+  detectAuthContractDrift,
+  
+  // Env
+  detectEnvUsedButUndeclared,
+  
+  // Fake Success
+  detectFakeSuccessToastBeforeAwait,
+  detectFakeSuccessResponseIgnored,
+  detectSilentCatch,
+  
+  // Dead UI
+  detectDeadClickNoEffect,
+  detectUIActionCauses4xx5xx,
+  
+  // Billing
+  detectStripeWebhookNoSigVerify,
+  
+  // Entitlements
+  detectLocalBypassPaidFeature,
+  
+  // Drift
+  detectContractsOutOfDate,
+  
+  // Helpers
+  routeMatches,
+  matchPattern,
+  isLikelyRequired,
+};