@vibecheckai/cli 3.5.1 → 3.5.2

package/bin/runners/lib/engine/index.js

@@ -0,0 +1,207 @@
+// bin/runners/lib/engine/index.js
+// Main entry point for the vNext scanner engine
+
+const { RepoIndex, DEFAULT_IGNORE, PREFILTER_TOKENS } = require("./repo-index");
+const { ASTCache, globalASTCache, DEFAULT_PARSER_OPTIONS } = require("./ast-cache");
+const {
+  extractNextAppRoutes,
+  extractNextPagesRoutes,
+  extractClientRefs,
+  extractFastifyRoutes,
+  detectFastifyEntries,
+  canonicalizeMethod,
+  canonicalizePath,
+  joinPaths,
+} = require("./extractors");
+const {
+  extractEnvUsage,
+  extractEnvDeclared,
+  buildEnvTruthV2,
+} = require("./env-extractor");
+const { buildBillingTruthV2 } = require("./billing-extractor");
+const { buildAuthTruthV2 } = require("./auth-extractor");
+const { buildEnforcementTruthV2 } = require("./enforcement-extractor");
+const { extractExpressRoutes } = require("./express-extractor");
+
+/**
+ * Create a RepoIndex for a repository
+ * 
+ * @param {string} repoRoot - Absolute path to repo root
+ * @param {Object} [options]
+ * @param {string[]} [options.ignorePatterns] - Additional ignore patterns
+ * @param {boolean} [options.buildTokenIndex] - Whether to build token index (default: true)
+ * @returns {Promise<RepoIndex>}
+ * 
+ * @example
+ * const index = await createIndex('/path/to/repo');
+ * console.log(`Indexed ${index.stats.totalFiles} files in ${index.stats.indexTimeMs}ms`);
+ * console.log('Detected frameworks:', Array.from(index.signals.detectedFrameworks));
+ */
+async function createIndex(repoRoot, options = {}) {
+  const index = new RepoIndex(repoRoot, options);
+  await index.build();
+  return index;
+}
+
+/**
+ * Log index summary for debugging
+ * @param {RepoIndex} index
+ */
+function logIndexSummary(index) {
+  const frameworks = Array.from(index.signals.detectedFrameworks).join(", ") || "none";
+  
+  console.log(`📁 RepoIndex Summary:`);
+  console.log(`   Files: ${index.stats.totalFiles}`);
+  console.log(`   Size: ${(index.stats.totalSize / 1024 / 1024).toFixed(2)} MB`);
+  console.log(`   Time: ${index.stats.indexTimeMs}ms`);
+  console.log(`   Frameworks: ${frameworks}`);
+  
+  if (index.signals.hasNextApp || index.signals.hasNextPages) {
+    console.log(`   Next.js: app=${index.signals.hasNextApp}, pages=${index.signals.hasNextPages}`);
+  }
+  if (index.signals.hasOpenAPI) {
+    console.log(`   OpenAPI: detected`);
+  }
+  if (index.signals.hasGraphQL) {
+    console.log(`   GraphQL: detected`);
+  }
+}
+
+/**
+ * Get files that need Fastify scanning (prefiltered)
+ * @param {RepoIndex} index
+ * @returns {string[]} - Absolute paths
+ */
+function getFastifyFiles(index) {
+  if (!index.hasFramework("fastify")) return [];
+  
+  return index.getByAnyToken([
+    "fastify",
+    "Fastify",
+    "register",
+  ]).filter(abs => {
+    const ext = abs.split(".").pop();
+    return ["ts", "tsx", "js", "jsx", "mjs", "cjs"].includes(ext);
+  });
+}
+
+/**
+ * Get files that need Express scanning (prefiltered)
+ * @param {RepoIndex} index
+ * @returns {string[]} - Absolute paths
+ */
+function getExpressFiles(index) {
+  if (!index.hasFramework("express")) return [];
+  
+  return index.getByAnyToken([
+    "express",
+    "Express",
+    "app.get",
+    "app.post",
+    "router.get",
+    "router.post",
+  ]).filter(abs => {
+    const ext = abs.split(".").pop();
+    return ["ts", "tsx", "js", "jsx", "mjs", "cjs"].includes(ext);
+  });
+}
+
+/**
+ * Get files that need client ref scanning (prefiltered)
+ * @param {RepoIndex} index
+ * @returns {string[]} - Absolute paths
+ */
+function getClientRefFiles(index) {
+  return index.getByAnyToken([
+    "fetch(",
+    "axios",
+    "useSWR",
+    "useQuery",
+  ]).filter(abs => {
+    const ext = abs.split(".").pop();
+    return ["ts", "tsx", "js", "jsx", "mjs", "cjs"].includes(ext);
+  });
+}
+
+/**
+ * Get files that need env var scanning (prefiltered)
+ * @param {RepoIndex} index
+ * @returns {string[]} - Absolute paths
+ */
+function getEnvFiles(index) {
+  return index.getByAnyToken([
+    "process.env",
+    "import.meta.env",
+    "os.environ",
+    "getenv",
+  ]);
+}
+
+/**
+ * Get Next.js app router route files
+ * @param {RepoIndex} index
+ * @returns {import('./types').FileRecord[]}
+ */
+function getNextAppRoutes(index) {
+  return index.files.filter(f => 
+    f.rel.includes("/app/") && 
+    f.rel.includes("/api/") && 
+    /route\.(ts|tsx|js|jsx)$/.test(f.rel)
+  );
+}
+
+/**
+ * Get Next.js pages router API files
+ * @param {RepoIndex} index
+ * @returns {import('./types').FileRecord[]}
+ */
+function getNextPagesRoutes(index) {
+  return index.files.filter(f => 
+    f.rel.includes("/pages/api/") && 
+    /\.(ts|tsx|js|jsx)$/.test(f.rel) &&
+    !f.rel.includes("/_")
+  );
+}
+
+module.exports = {
+  // Core classes
+  RepoIndex,
+  ASTCache,
+  
+  // Factory functions
+  createIndex,
+  
+  // Optimized extractors (use RepoIndex)
+  extractNextAppRoutes,
+  extractNextPagesRoutes,
+  extractClientRefs,
+  extractFastifyRoutes,
+  detectFastifyEntries,
+  extractEnvUsage,
+  extractEnvDeclared,
+  buildEnvTruthV2,
+  buildBillingTruthV2,
+  buildAuthTruthV2,
+  buildEnforcementTruthV2,
+  extractExpressRoutes,
+  
+  // Utilities
+  logIndexSummary,
+  getFastifyFiles,
+  getExpressFiles,
+  getClientRefFiles,
+  getEnvFiles,
+  getNextAppRoutes,
+  getNextPagesRoutes,
+  canonicalizeMethod,
+  canonicalizePath,
+  joinPaths,
+  
+  // Shared instances
+  globalASTCache,
+  
+  // Constants
+  DEFAULT_IGNORE,
+  PREFILTER_TOKENS,
+  DEFAULT_PARSER_OPTIONS,
+};

package/bin/runners/lib/engine/repo-index.js

@@ -0,0 +1,514 @@
+// bin/runners/lib/engine/repo-index.js
+// Single-pass file indexer with content cache, token prefilter, and framework detection
+
+const fg = require("fast-glob");
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+
+// Default ignore patterns (performance-critical)
+const DEFAULT_IGNORE = [
+  "**/node_modules/**",
+  "**/.next/**",
+  "**/dist/**",
+  "**/build/**",
+  "**/.turbo/**",
+  "**/.git/**",
+  "**/.vibecheck/**",
+  "**/.guardrail/**",
+  "**/coverage/**",
+  "**/.nyc_output/**",
+  "**/vendor/**",
+  "**/__pycache__/**",
+  "**/.venv/**",
+  "**/venv/**",
+  "**/env/**",
+  "**/site-packages/**",
+  "**/*.min.js",
+  "**/*.min.css",
+  "**/*.map",
+  "**/*.d.ts",
+];
+
+// Tokens we search for to build a prefilter index
+const PREFILTER_TOKENS = [
+  // Route definitions
+  "fastify", "Fastify", "register", "express", "Express", "app.get", "app.post",
+  "router.get", "router.post", "hono", "Hono", "koa", "Koa",
+  // Client calls
+  "fetch(", "axios", "useSWR", "useQuery",
+  // Python
+  "@app.route", "@router", "FastAPI", "Flask", "Django",
+  // Go
+  "gin.Default", "echo.New", "fiber.New",
+  // Auth
+  "requireAuth", "isAuthenticated", "checkPermission", "authorize",
+  // Env
+  "process.env", "import.meta.env", "os.environ", "getenv",
+];
+
+/**
+ * Compute SHA256 hash of content
+ * @param {string} content
+ * @returns {string}
+ */
+function sha256(content) {
+  return crypto.createHash("sha256").update(content).digest("hex");
+}
+
+/**
+ * Check if a file exists
+ * @param {string} p - Path to check
+ * @returns {boolean}
+ */
+function fileExists(p) {
+  try {
+    return fs.statSync(p).isFile();
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Check if a directory exists
+ * @param {string} p - Path to check
+ * @returns {boolean}
+ */
+function dirExists(p) {
+  try {
+    return fs.statSync(p).isDirectory();
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * RepoIndex - Single-pass file indexer
+ * 
+ * Usage:
+ *   const index = new RepoIndex(repoRoot, { ignorePatterns: [...] });
+ *   await index.build();
+ *   
+ *   // Get all files
+ *   const allFiles = index.files;
+ *   
+ *   // Get files by extension
+ *   const tsFiles = index.getByExtension('.ts');
+ *   
+ *   // Get files containing a token
+ *   const fastifyFiles = index.getByToken('fastify');
+ *   
+ *   // Get file content (cached)
+ *   const content = index.getContent(fileAbs);
+ *   
+ *   // Check signals
+ *   if (index.signals.detectedFrameworks.has('fastify')) { ... }
+ */
+class RepoIndex {
+  /**
+   * @param {string} repoRoot - Absolute path to repo root
+   * @param {Object} [options]
+   * @param {string[]} [options.ignorePatterns] - Additional ignore patterns
+   * @param {string[]} [options.includePatterns] - File patterns to include (default: all code files)
+   * @param {boolean} [options.buildTokenIndex] - Whether to build token index (default: true)
+   * @param {number} [options.maxFileSize] - Max file size to index in bytes (default: 1MB)
+   */
+  constructor(repoRoot, options = {}) {
+    this.repoRoot = repoRoot;
+    this.ignorePatterns = [...DEFAULT_IGNORE, ...(options.ignorePatterns || [])];
+    this.includePatterns = options.includePatterns || [
+      "**/*.{ts,tsx,js,jsx,mjs,cjs}",
+      "**/*.py",
+      "**/*.go",
+      "**/*.rb",
+      "**/*.json",
+      "**/*.yaml",
+      "**/*.yml",
+    ];
+    this.buildTokenIndex = options.buildTokenIndex !== false;
+    this.maxFileSize = options.maxFileSize || 1024 * 1024; // 1MB
+
+    /** @type {import('./types').FileRecord[]} */
+    this.files = [];
+
+    /** @type {Map<string, string>} abs -> content */
+    this._contentCache = new Map();
+
+    /** @type {Map<string, Set<string>>} token -> Set<abs> */
+    this._tokenIndex = new Map();
+
+    /** @type {Map<string, import('./types').FileRecord[]>} ext -> FileRecord[] */
+    this._byExtension = new Map();
+
+    /** @type {import('./types').RepoSignals} */
+    this.signals = {
+      hasPackageJson: false,
+      hasRequirementsTxt: false,
+      hasPyprojectToml: false,
+      hasGoMod: false,
+      hasGemfile: false,
+      hasNextApp: false,
+      hasNextPages: false,
+      hasOpenAPI: false,
+      hasGraphQL: false,
+      detectedFrameworks: new Set(),
+    };
+
+    this.stats = {
+      totalFiles: 0,
+      totalSize: 0,
+      indexTimeMs: 0,
+      contentCacheHits: 0,
+      contentCacheMisses: 0,
+    };
+
+    this._built = false;
+  }
+
+  /**
+   * Build the index (single glob pass)
+   * @returns {Promise<this>}
+   */
+  async build() {
+    const startTime = Date.now();
+
+    // Single glob pass
+    const filePaths = await fg(this.includePatterns, {
+      cwd: this.repoRoot,
+      absolute: true,
+      ignore: this.ignorePatterns,
+      stats: true,
+      onlyFiles: true,
+    });
+
+    // Process files
+    for (const entry of filePaths) {
+      const abs = typeof entry === "string" ? entry : entry.path;
+      const rel = path.relative(this.repoRoot, abs).replace(/\\/g, "/");
+      const ext = path.extname(abs).toLowerCase();
+
+      let stat;
+      if (typeof entry === "object" && entry.stats) {
+        stat = entry.stats;
+      } else {
+        try {
+          stat = fs.statSync(abs);
+        } catch {
+          continue;
+        }
+      }
+
+      // Skip files that are too large
+      if (stat.size > this.maxFileSize) {
+        continue;
+      }
+
+      const content = this._readFile(abs);
+      if (content === null) continue;
+
+      const hash = sha256(content);
+
+      /** @type {import('./types').FileRecord} */
+      const record = {
+        abs,
+        rel,
+        size: stat.size,
+        mtime: stat.mtimeMs,
+        hash,
+        ext,
+      };
+
+      this.files.push(record);
+      this._contentCache.set(abs, content);
+      this.stats.totalSize += stat.size;
+
+      // Index by extension
+      if (!this._byExtension.has(ext)) {
+        this._byExtension.set(ext, []);
+      }
+      this._byExtension.get(ext).push(record);
+
+      // Build token index
+      if (this.buildTokenIndex) {
+        this._indexTokens(abs, content);
+      }
+    }
+
+    this.stats.totalFiles = this.files.length;
+
+    // Detect signals
+    this._detectSignals();
+
+    this.stats.indexTimeMs = Date.now() - startTime;
+    this._built = true;
+
+    return this;
+  }
+
+  /**
+   * Read file content (with caching)
+   * @param {string} abs - Absolute path
+   * @returns {string|null}
+   */
+  _readFile(abs) {
+    if (this._contentCache.has(abs)) {
+      this.stats.contentCacheHits++;
+      return this._contentCache.get(abs);
+    }
+
+    this.stats.contentCacheMisses++;
+    try {
+      const content = fs.readFileSync(abs, "utf8");
+      return content;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Index tokens in a file
+   * @param {string} abs
+   * @param {string} content
+   */
+  _indexTokens(abs, content) {
+    for (const token of PREFILTER_TOKENS) {
+      if (content.includes(token)) {
+        if (!this._tokenIndex.has(token)) {
+          this._tokenIndex.set(token, new Set());
+        }
+        this._tokenIndex.get(token).add(abs);
+      }
+    }
+  }
+
+  /**
+   * Detect framework signals from indexed files
+   */
+  _detectSignals() {
+    // Check for manifest files
+    this.signals.hasPackageJson = fileExists(path.join(this.repoRoot, "package.json"));
+    this.signals.hasRequirementsTxt = fileExists(path.join(this.repoRoot, "requirements.txt"));
+    this.signals.hasPyprojectToml = fileExists(path.join(this.repoRoot, "pyproject.toml"));
+    this.signals.hasGoMod = fileExists(path.join(this.repoRoot, "go.mod"));
+    this.signals.hasGemfile = fileExists(path.join(this.repoRoot, "Gemfile"));
+
+    // Check for Next.js directories
+    this.signals.hasNextApp = dirExists(path.join(this.repoRoot, "app")) ||
+      this.files.some(f => f.rel.includes("/app/") && f.rel.includes("route."));
+    this.signals.hasNextPages = dirExists(path.join(this.repoRoot, "pages")) ||
+      this.files.some(f => f.rel.includes("/pages/api/"));
+
+    // Check for OpenAPI/GraphQL
+    this.signals.hasOpenAPI = this.files.some(f =>
+      /openapi\.(json|ya?ml)$/i.test(f.rel) || /swagger\.(json|ya?ml)$/i.test(f.rel)
+    );
+    this.signals.hasGraphQL = this.files.some(f =>
+      /\.graphql$/i.test(f.rel) || /schema\.gql$/i.test(f.rel)
+    );
+
+    // Detect frameworks from package.json
+    if (this.signals.hasPackageJson) {
+      try {
+        const pkg = JSON.parse(fs.readFileSync(path.join(this.repoRoot, "package.json"), "utf8"));
+        const allDeps = { ...pkg.dependencies, ...pkg.devDependencies };
+
+        if (allDeps.fastify) this.signals.detectedFrameworks.add("fastify");
+        if (allDeps.express) this.signals.detectedFrameworks.add("express");
+        if (allDeps.koa) this.signals.detectedFrameworks.add("koa");
+        if (allDeps.hono) this.signals.detectedFrameworks.add("hono");
+        if (allDeps.next) this.signals.detectedFrameworks.add("next");
+        if (allDeps["@nestjs/core"]) this.signals.detectedFrameworks.add("nestjs");
+      } catch {}
+    }
+
+    // Detect Python frameworks
+    if (this.signals.hasRequirementsTxt) {
+      try {
+        const req = fs.readFileSync(path.join(this.repoRoot, "requirements.txt"), "utf8").toLowerCase();
+        if (req.includes("flask")) this.signals.detectedFrameworks.add("flask");
+        if (req.includes("fastapi")) this.signals.detectedFrameworks.add("fastapi");
+        if (req.includes("django")) this.signals.detectedFrameworks.add("django");
+      } catch {}
+    }
+
+    // Detect Go frameworks
+    if (this.signals.hasGoMod) {
+      try {
+        const gomod = fs.readFileSync(path.join(this.repoRoot, "go.mod"), "utf8").toLowerCase();
+        if (gomod.includes("gin-gonic")) this.signals.detectedFrameworks.add("gin");
+        if (gomod.includes("labstack/echo")) this.signals.detectedFrameworks.add("echo");
+        if (gomod.includes("gofiber/fiber")) this.signals.detectedFrameworks.add("fiber");
+        this.signals.detectedFrameworks.add("go");
+      } catch {}
+    }
+
+    // Detect Ruby frameworks
+    if (this.signals.hasGemfile) {
+      try {
+        const gem = fs.readFileSync(path.join(this.repoRoot, "Gemfile"), "utf8").toLowerCase();
+        if (gem.includes("rails")) this.signals.detectedFrameworks.add("rails");
+        if (gem.includes("sinatra")) this.signals.detectedFrameworks.add("sinatra");
+      } catch {}
+    }
+
+    // Add next if we have app/pages directories
+    if (this.signals.hasNextApp || this.signals.hasNextPages) {
+      this.signals.detectedFrameworks.add("next");
+    }
+  }
+
+  /**
+   * Get file content (cached)
+   * @param {string} abs - Absolute path
+   * @returns {string|null}
+   */
+  getContent(abs) {
+    if (this._contentCache.has(abs)) {
+      this.stats.contentCacheHits++;
+      return this._contentCache.get(abs);
+    }
+
+    // Lazy load if not in cache
+    this.stats.contentCacheMisses++;
+    try {
+      const content = fs.readFileSync(abs, "utf8");
+      this._contentCache.set(abs, content);
+      return content;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Get files by extension
+   * @param {string} ext - Extension with dot (e.g., ".ts")
+   * @returns {import('./types').FileRecord[]}
+   */
+  getByExtension(ext) {
+    return this._byExtension.get(ext.toLowerCase()) || [];
+  }
+
+  /**
+   * Get files containing a token (prefilter)
+   * @param {string} token
+   * @returns {string[]} - Array of absolute paths
+   */
+  getByToken(token) {
+    const set = this._tokenIndex.get(token);
+    return set ? Array.from(set) : [];
+  }
+
+  /**
+   * Get files matching multiple tokens (OR)
+   * @param {string[]} tokens
+   * @returns {string[]} - Array of absolute paths
+   */
+  getByAnyToken(tokens) {
+    const result = new Set();
+    for (const token of tokens) {
+      const set = this._tokenIndex.get(token);
+      if (set) {
+        for (const abs of set) {
+          result.add(abs);
+        }
+      }
+    }
+    return Array.from(result);
+  }
+
+  /**
+   * Get files matching all tokens (AND)
+   * @param {string[]} tokens
+   * @returns {string[]} - Array of absolute paths
+   */
+  getByAllTokens(tokens) {
+    if (tokens.length === 0) return [];
+
+    const sets = tokens.map(t => this._tokenIndex.get(t)).filter(Boolean);
+    if (sets.length === 0) return [];
+    if (sets.length !== tokens.length) return []; // Some tokens not found
+
+    // Intersect all sets
+    let result = new Set(sets[0]);
+    for (let i = 1; i < sets.length; i++) {
+      const next = new Set();
+      for (const abs of result) {
+        if (sets[i].has(abs)) {
+          next.add(abs);
+        }
+      }
+      result = next;
+    }
+
+    return Array.from(result);
+  }
+
+  /**
+   * Get files matching a glob pattern (from already-indexed files)
+   * @param {string} pattern - Glob pattern
+   * @returns {import('./types').FileRecord[]}
+   */
+  getByPattern(pattern) {
+    const picomatch = require("picomatch");
+    const isMatch = picomatch(pattern);
+    return this.files.filter(f => isMatch(f.rel));
+  }
+
+  /**
+   * Check if a framework is detected
+   * @param {string} framework
+   * @returns {boolean}
+   */
+  hasFramework(framework) {
+    return this.signals.detectedFrameworks.has(framework.toLowerCase());
+  }
+
+  /**
+   * Get JS/TS files only
+   * @returns {import('./types').FileRecord[]}
+   */
+  getJsFiles() {
+    return [
+      ...this.getByExtension(".ts"),
+      ...this.getByExtension(".tsx"),
+      ...this.getByExtension(".js"),
+      ...this.getByExtension(".jsx"),
+      ...this.getByExtension(".mjs"),
+      ...this.getByExtension(".cjs"),
+    ];
+  }
+
+  /**
+   * Get Python files only
+   * @returns {import('./types').FileRecord[]}
+   */
+  getPyFiles() {
+    return this.getByExtension(".py");
+  }
+
+  /**
+   * Get Go files only
+   * @returns {import('./types').FileRecord[]}
+   */
+  getGoFiles() {
+    return this.getByExtension(".go");
+  }
+
+  /**
+   * Clear content cache to free memory
+   */
+  clearContentCache() {
+    this._contentCache.clear();
+  }
+
+  /**
+   * Get relative path from absolute
+   * @param {string} abs
+   * @returns {string}
+   */
+  relPath(abs) {
+    return path.relative(this.repoRoot, abs).replace(/\\/g, "/");
+  }
+}
+
+module.exports = { RepoIndex, DEFAULT_IGNORE, PREFILTER_TOKENS };