getdoorman 1.0.0

package/src/scanner.js

@@ -0,0 +1,710 @@
+import { readFileSync, writeFileSync, existsSync, mkdirSync, statSync, lstatSync, realpathSync } from 'fs';
+import { readFile } from 'fs/promises';
+import { execSync } from 'child_process';
+import { createHash } from 'crypto';
+import { glob } from 'glob';
+import { cpus } from 'os';
+import { Worker } from 'worker_threads';
+import { fileURLToPath } from 'url';
+import { join, relative, dirname } from 'path';
+import { loadIgnorePatterns } from './ignore.js';
+
+const DOORMAN_VERSION = '1.0.0';
+
+const SOURCE_PATTERNS = [
+  '**/*.js',
+  '**/*.jsx',
+  '**/*.ts',
+  '**/*.tsx',
+  '**/*.mjs',
+  '**/*.cjs',
+  '**/*.py',
+  '**/*.rb',
+  '**/*.go',
+  '**/*.env*',
+  '**/*.tf',
+  '**/*.tfvars',
+  '**/Dockerfile*',
+  '**/docker-compose*.yml',
+  '**/docker-compose*.yaml',
+  '**/.github/workflows/*.yml',
+  '**/.github/workflows/*.yaml',
+  '**/.gitlab-ci.yml',
+  '**/.gitlab-ci.yaml',
+  '**/k8s/**/*.yml',
+  '**/k8s/**/*.yaml',
+  '**/kubernetes/**/*.yml',
+  '**/kubernetes/**/*.yaml',
+  '**/helm/**/*.yaml',
+  '**/manifests/**/*.yaml',
+  '**/deploy/**/*.yaml',
+  '**/serverless.yml',
+  '**/serverless.yaml',
+  '**/package.json',
+  '**/requirements.txt',
+  '**/Gemfile',
+  '**/go.mod',
+  '**/*.sql',
+  '**/*.prisma',
+  '**/nginx.conf',
+  '**/next.config.*',
+  '**/tsconfig.json',
+  '**/.eslintrc*',
+  '**/.gitignore',
+  '**/.npmrc',
+  '**/jest.config.*',
+  '**/webpack.config.*',
+  '**/vite.config.*',
+  '**/rollup.config.*',
+  '**/CODEOWNERS',
+  '**/Makefile',
+  '**/*.sh',
+];
+
+const MAX_FILE_SIZE = 1_000_000; // 1MB
+const MAX_FILE_COUNT = 50_000;
+const PARALLEL_BATCH_SIZE = 50;
+const MEMORY_WARNING_BYTES = 500 * 1024 * 1024; // 500MB
+const RULE_TIMEOUT_MS = 5_000;
+const TOTAL_SCAN_TIMEOUT_MS = 60_000;
+const RULE_CHUNK_SIZE = 20;
+const CACHE_FILE = '.doorman-cache.json';
+
+/**
+ * Check if a file appears to be binary by looking for null bytes in the first 512 bytes.
+ */
+function isBinaryFile(fullPath) {
+  try {
+    const fd = readFileSync(fullPath, { encoding: null, flag: 'r' });
+    const sample = fd.subarray(0, 512);
+    for (let i = 0; i < sample.length; i++) {
+      if (sample[i] === 0) return true;
+    }
+    return false;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Check if a path is a symlink and resolve it, detecting loops.
+ * Returns the real path if safe, or null if it's a symlink loop or points outside target.
+ */
+function resolveSymlink(fullPath, visitedPaths) {
+  try {
+    const stat = lstatSync(fullPath);
+    if (stat.isSymbolicLink()) {
+      const realPath = realpathSync(fullPath);
+      if (visitedPaths.has(realPath)) {
+        return null; // symlink loop detected
+      }
+      visitedPaths.add(realPath);
+      return realPath;
+    }
+    visitedPaths.add(fullPath);
+    return fullPath;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Safely read a file with all edge case checks.
+ * Returns { content, skipped, reason } where content is null if skipped.
+ */
+export function safeReadFile(fullPath, match, visitedPaths, silent = false) {
+  // Check symlink safety
+  const resolved = resolveSymlink(fullPath, visitedPaths);
+  if (resolved === null) {
+    if (!silent) {
+      console.warn(`[scanner] Skipping ${match}: symlink loop or unresolvable symlink`);
+    }
+    return { content: null, skipped: true, reason: 'symlink' };
+  }
+
+  // Check file size before reading full content
+  try {
+    const stat = statSync(resolved);
+    if (stat.size > MAX_FILE_SIZE) {
+      if (!silent) {
+        console.warn(`[scanner] Skipping ${match}: file size ${(stat.size / 1024 / 1024).toFixed(1)}MB exceeds 1MB limit`);
+      }
+      return { content: null, skipped: true, reason: 'too-large' };
+    }
+  } catch (err) {
+    if (err.code === 'EACCES' || err.code === 'EPERM') {
+      if (!silent) {
+        console.warn(`[scanner] Skipping ${match}: permission denied`);
+      }
+      return { content: null, skipped: true, reason: 'permission' };
+    }
+    return { content: null, skipped: true, reason: 'stat-error' };
+  }
+
+  // Check for binary content
+  if (isBinaryFile(resolved)) {
+    return { content: null, skipped: true, reason: 'binary' };
+  }
+
+  // Read the file
+  try {
+    const content = readFileSync(resolved, 'utf-8');
+    return { content, skipped: false, reason: null };
+  } catch (err) {
+    if (err.code === 'EACCES' || err.code === 'EPERM') {
+      if (!silent) {
+        console.warn(`[scanner] Skipping ${match}: permission denied`);
+      }
+      return { content: null, skipped: true, reason: 'permission' };
+    }
+    return { content: null, skipped: true, reason: 'read-error' };
+  }
+}
+
+/**
+ * Async version of safeReadFile — reads the file content asynchronously
+ * while keeping synchronous safety checks (stat, binary detection).
+ * Returns { content, skipped, reason }.
+ */
+async function safeReadFileAsync(fullPath, match, visitedPaths, silent = false) {
+  // Check symlink safety (sync — fast, no I/O wait)
+  const resolved = resolveSymlink(fullPath, visitedPaths);
+  if (resolved === null) {
+    if (!silent) {
+      console.warn(`[scanner] Skipping ${match}: symlink loop or unresolvable symlink`);
+    }
+    return { content: null, skipped: true, reason: 'symlink' };
+  }
+
+  // Check file size before reading full content
+  try {
+    const stat = statSync(resolved);
+    if (stat.size > MAX_FILE_SIZE) {
+      if (!silent) {
+        console.warn(`[scanner] Skipping ${match}: file size ${(stat.size / 1024 / 1024).toFixed(1)}MB exceeds 1MB limit`);
+      }
+      return { content: null, skipped: true, reason: 'too-large' };
+    }
+  } catch (err) {
+    if (err.code === 'EACCES' || err.code === 'EPERM') {
+      if (!silent) {
+        console.warn(`[scanner] Skipping ${match}: permission denied`);
+      }
+      return { content: null, skipped: true, reason: 'permission' };
+    }
+    return { content: null, skipped: true, reason: 'stat-error' };
+  }
+
+  // Check for binary content
+  if (isBinaryFile(resolved)) {
+    return { content: null, skipped: true, reason: 'binary' };
+  }
+
+  // Read the file asynchronously
+  try {
+    const content = await readFile(resolved, 'utf-8');
+    return { content, skipped: false, reason: null };
+  } catch (err) {
+    if (err.code === 'EACCES' || err.code === 'EPERM') {
+      if (!silent) {
+        console.warn(`[scanner] Skipping ${match}: permission denied`);
+      }
+      return { content: null, skipped: true, reason: 'permission' };
+    }
+    return { content: null, skipped: true, reason: 'read-error' };
+  }
+}
+
+/**
+ * Hash file content with SHA-256.
+ */
+function hashContent(content) {
+  return createHash('sha256').update(content).digest('hex');
+}
+
+/**
+ * Check current memory usage and warn if it exceeds the threshold.
+ */
+function checkMemoryUsage() {
+  const used = process.memoryUsage().heapUsed;
+  if (used > MEMORY_WARNING_BYTES) {
+    console.warn(
+      `[scanner] Warning: memory usage is ${Math.round(used / 1024 / 1024)}MB, exceeding 500MB threshold`
+    );
+  }
+  return used;
+}
+
+// ---------------------------------------------------------------------------
+// Cache — keyed by file content SHA-256, invalidated on version change
+// ---------------------------------------------------------------------------
+
+/**
+ * Read the scan cache from disk.
+ * Returns null if cache is missing, corrupted, or from a different Doorman version.
+ */
+function readCache(targetPath) {
+  const cachePath = join(targetPath, CACHE_FILE);
+  try {
+    if (existsSync(cachePath)) {
+      const data = JSON.parse(readFileSync(cachePath, 'utf-8'));
+      // Invalidate if Doorman version has changed
+      if (data.version !== DOORMAN_VERSION) {
+        return null;
+      }
+      return data;
+    }
+  } catch {
+    // Corrupted cache — treat as first run
+  }
+  return null;
+}
+
+/**
+ * Write the scan cache to disk.
+ */
+function writeCache(targetPath, fileHashes, fileResults) {
+  const cachePath = join(targetPath, CACHE_FILE);
+  const cacheDir = dirname(cachePath);
+  if (!existsSync(cacheDir)) {
+    mkdirSync(cacheDir, { recursive: true });
+  }
+  const data = {
+    version: DOORMAN_VERSION,
+    timestamp: new Date().toISOString(),
+    hashes: fileHashes,
+    results: fileResults,
+  };
+  writeFileSync(cachePath, JSON.stringify(data, null, 2));
+}
+
+/**
+ * Get list of changed files from git.
+ * Returns null if not a git repo or git is unavailable.
+ */
+function getGitChangedFiles(targetPath) {
+  try {
+    const output = execSync('git diff --name-only HEAD', {
+      cwd: targetPath,
+      encoding: 'utf-8',
+      timeout: 10_000,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    return output
+      .split('\n')
+      .map((f) => f.trim())
+      .filter(Boolean);
+  } catch {
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Progress reporting
+// ---------------------------------------------------------------------------
+
+/**
+ * Create a progress reporter.
+ * When silent, all callbacks are no-ops.
+ */
+function createProgress(total, silent) {
+  if (silent) {
+    return {
+      update() {},
+      done() {},
+    };
+  }
+
+  let scanned = 0;
+  const startTime = performance.now();
+  const isTTY = process.stderr?.isTTY;
+
+  function render(file) {
+    if (!isTTY) return;
+    const pct = Math.round((scanned / total) * 100);
+    const barLen = 20;
+    const filled = Math.round(barLen * scanned / total);
+    const bar = '\u2588'.repeat(filled) + '\u2591'.repeat(barLen - filled);
+    const short = file.length > 40 ? '...' + file.slice(-37) : file;
+    process.stderr.write(`\r  [${bar}] ${pct}% (${scanned}/${total}) ${short}   `);
+  }
+
+  return {
+    update(file) {
+      scanned++;
+      render(file);
+    },
+    done() {
+      if (isTTY) {
+        process.stderr.write('\r' + ' '.repeat(80) + '\r');
+      }
+      const elapsed = ((performance.now() - startTime) / 1000).toFixed(2);
+      if (!silent) {
+        // Printed by caller via return value
+      }
+      return { scanned, elapsed: parseFloat(elapsed) };
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// File collection
+// ---------------------------------------------------------------------------
+
+/**
+ * Collect all scannable files from the target path.
+ * Reads files in parallel batches for speed. Supports:
+ *   - .doormanignore patterns
+ *   - File hash caching (skip unchanged files)
+ *   - Progress reporting
+ *
+ * Options:
+ *   silent       — suppress warnings and progress
+ *   noCache      — skip cache, force full scan
+ *   batchSize    — parallel read batch size (default 50)
+ *   onProgress   — callback({ scanned, total, file })
+ */
+export async function collectFiles(targetPath, optionsOrSilent = false) {
+  // Backward compat: if a boolean is passed, treat it as `silent`
+  const options = typeof optionsOrSilent === 'object' ? optionsOrSilent : { silent: optionsOrSilent };
+  const silent = options.silent ?? false;
+  const noCache = options.noCache ?? false;
+  const batchSize = options.batchSize ?? PARALLEL_BATCH_SIZE;
+
+  // Load ignore patterns (defaults + .doormanignore + RC config)
+  const { patterns: ignorePatterns } = loadIgnorePatterns(targetPath);
+  const extraIgnores = options.extraIgnores || [];
+  const allIgnorePatterns = [...ignorePatterns, ...extraIgnores];
+
+  const matches = await glob(SOURCE_PATTERNS, {
+    cwd: targetPath,
+    ignore: allIgnorePatterns,
+    nodir: true,
+    dot: true,
+  });
+
+  if (matches.length > MAX_FILE_COUNT) {
+    if (!silent) {
+      console.warn(`[scanner] Warning: found ${matches.length} files, limiting to ${MAX_FILE_COUNT}`);
+    }
+  }
+
+  const capped = matches.slice(0, MAX_FILE_COUNT);
+
+  // Read cache
+  const cache = noCache ? null : readCache(targetPath);
+  const cachedHashes = cache?.hashes ?? {};
+
+  const files = new Map();
+  const visitedPaths = new Set();
+  const newHashes = {};
+  const progress = createProgress(capped.length, silent);
+  let cacheHits = 0;
+
+  // Process in parallel batches
+  for (let i = 0; i < capped.length; i += batchSize) {
+    const batch = capped.slice(i, i + batchSize);
+
+    const results = await Promise.all(
+      batch.map(async (match) => {
+        const fullPath = join(targetPath, match);
+        const result = await safeReadFileAsync(fullPath, match, visitedPaths, silent);
+        return { match, ...result };
+      })
+    );
+
+    for (const { match, content, skipped } of results) {
+      progress.update(match);
+
+      if (skipped || content === null) continue;
+
+      const contentHash = hashContent(content);
+      newHashes[match] = contentHash;
+
+      // If hash matches cache, we can skip re-scanning this file
+      // but we still need its content for rule context
+      if (!noCache && cachedHashes[match] === contentHash) {
+        cacheHits++;
+      }
+
+      files.set(match, content);
+    }
+
+    checkMemoryUsage();
+  }
+
+  // Persist updated cache
+  if (!noCache) {
+    writeCache(targetPath, newHashes, {});
+  }
+
+  const stats = progress.done();
+
+  return files;
+}
+
+/**
+ * Collect files incrementally — only re-scan files that changed since last run.
+ * Falls back to full scan if not a git repo or on first run.
+ */
+export async function collectFilesIncremental(targetPath, optionsOrSilent = false) {
+  const options = typeof optionsOrSilent === 'object' ? optionsOrSilent : { silent: optionsOrSilent };
+  const silent = options.silent ?? false;
+  const noCache = options.noCache ?? false;
+  const batchSize = options.batchSize ?? PARALLEL_BATCH_SIZE;
+
+  const cache = noCache ? null : readCache(targetPath);
+  const gitChanged = getGitChangedFiles(targetPath);
+
+  // If no cache exists, do a full scan and build the cache
+  if (!cache) {
+    return collectFiles(targetPath, options);
+  }
+
+  const cachedHashes = cache.hashes ?? {};
+
+  // Load ignore patterns (defaults + .doormanignore + RC config)
+  const { patterns: ignorePatterns } = loadIgnorePatterns(targetPath);
+  const extraIgnores = options.extraIgnores || [];
+  const allIgnorePatterns = [...ignorePatterns, ...extraIgnores];
+
+  // Determine which files to re-scan
+  const allMatches = await glob(SOURCE_PATTERNS, {
+    cwd: targetPath,
+    ignore: allIgnorePatterns,
+    nodir: true,
+    dot: true,
+  });
+
+  const capped = allMatches.slice(0, MAX_FILE_COUNT);
+  const gitChangedSet = gitChanged ? new Set(gitChanged) : null;
+
+  const files = new Map();
+  const newHashes = {};
+  const visitedPaths = new Set();
+  const progress = createProgress(capped.length, silent);
+
+  for (let i = 0; i < capped.length; i += batchSize) {
+    const batch = capped.slice(i, i + batchSize);
+
+    const results = await Promise.all(
+      batch.map(async (match) => {
+        const fullPath = join(targetPath, match);
+        const result = await safeReadFileAsync(fullPath, match, visitedPaths, silent);
+        return { match, ...result };
+      })
+    );
+
+    for (const { match, content, skipped } of results) {
+      progress.update(match);
+
+      if (skipped || content === null) continue;
+
+      const contentHash = hashContent(content);
+      newHashes[match] = contentHash;
+
+      // Include file if: git says it changed, hash differs from cache, or it is new
+      const prevHash = cachedHashes[match];
+      const changedInGit = gitChangedSet ? gitChangedSet.has(match) : true;
+      const hashChanged = prevHash !== contentHash;
+
+      if (changedInGit || hashChanged) {
+        files.set(match, content);
+      }
+    }
+
+    checkMemoryUsage();
+  }
+
+  if (!noCache) {
+    writeCache(targetPath, newHashes, {});
+  }
+
+  progress.done();
+  return files;
+}
+
+// ---------------------------------------------------------------------------
+// Rule execution
+// ---------------------------------------------------------------------------
+
+/**
+ * Run a single rule with a timeout.
+ * Returns the rule results or an empty array if the rule times out.
+ */
+/**
+ * Run a single rule synchronously (most rules are sync).
+ * Falls back to async with timeout for async rules.
+ */
+const _failedRules = new Set();
+function runRuleSync(rule, context) {
+  const id = rule.id || '';
+  // Skip previously failed rules and irrelevant language rules
+  if (_failedRules.has(id)) return [];
+  if (rule.lang && context._detectedLangs && !context._detectedLangs.has(rule.lang)) return [];
+  try {
+    const result = rule.check(context);
+    if (result && typeof result.then === 'function') return result;
+    return Array.isArray(result) ? result : [];
+  } catch (e) {
+    _failedRules.add(id);
+    return [];
+  }
+}
+
+async function runRuleWithTimeout(rule, context, silent) {
+  let timer;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => {
+      reject(new Error(`Rule "${rule.id || rule.name || 'unknown'}" timed out after ${RULE_TIMEOUT_MS}ms`));
+    }, RULE_TIMEOUT_MS);
+  });
+
+  try {
+    const result = await Promise.race([rule.check(context), timeout]);
+    clearTimeout(timer);
+    return Array.isArray(result) ? result : [];
+  } catch (e) {
+    clearTimeout(timer);
+    if (!silent) {
+      if (e.message.includes('timed out')) {
+        console.warn(`[scanner] ${e.message}`);
+      } else {
+        console.warn(`[scanner] Rule "${rule.id || rule.name || 'unknown'}" failed: ${e.message}`);
+      }
+    }
+    return [];
+  }
+}
+
+/**
+ * Run all applicable rules against the collected files.
+ * Optimized: runs sync rules without Promise overhead, batches async rules.
+ * Pre-filters files to skip empty/irrelevant content.
+ * Supports --profile mode for performance profiling.
+ */
+export async function runRules(rules, context) {
+  const findings = [];
+  const scanStart = Date.now();
+  const silent = context?.silent ?? false;
+  const profile = context?.profile ?? false;
+  const profileData = profile ? [] : null;
+
+  // Pre-compute: skip files with no meaningful content (< 10 chars)
+  if (context.files) {
+    const toDelete = [];
+    for (const [fp, content] of context.files) {
+      if (!content || content.length < 10) toDelete.push(fp);
+    }
+    for (const fp of toDelete) context.files.delete(fp);
+  }
+
+  // Separate sync vs async rules for optimal execution
+  const asyncRules = [];
+
+  // Run rules in chunks to allow timeout checks less frequently
+  const CHUNK = 100;
+  for (let i = 0; i < rules.length; i += CHUNK) {
+    // Check total scan timeout per chunk (not per rule — reduces overhead)
+    if (Date.now() - scanStart > TOTAL_SCAN_TIMEOUT_MS) {
+      if (!silent) console.warn(`[scanner] Scan timeout exceeded, returning partial results`);
+      break;
+    }
+
+    const chunk = rules.slice(i, i + CHUNK);
+    for (const rule of chunk) {
+      const ruleStart = profile ? performance.now() : 0;
+      const result = runRuleSync(rule, context);
+
+      if (result && typeof result.then === 'function') {
+        asyncRules.push({ rule, promise: result });
+      } else if (Array.isArray(result) && result.length > 0) {
+        findings.push(...result);
+      }
+
+      if (profile) {
+        profileData.push({ id: rule.id, ms: performance.now() - ruleStart, findings: Array.isArray(result) ? result.length : 0 });
+      }
+    }
+  }
+
+  // Process async rules with timeout
+  if (asyncRules.length > 0) {
+    const ASYNC_CHUNK = 50;
+    for (let i = 0; i < asyncRules.length; i += ASYNC_CHUNK) {
+      const chunk = asyncRules.slice(i, i + ASYNC_CHUNK);
+      const results = await Promise.all(
+        chunk.map(({ rule }) => runRuleWithTimeout(rule, context, silent))
+      );
+      for (const result of results) {
+        if (Array.isArray(result)) findings.push(...result);
+      }
+    }
+  }
+
+  // Sort by severity
+  const severityOrder = { critical: 0, high: 1, medium: 2, low: 3, info: 4 };
+  findings.sort((a, b) => (severityOrder[a.severity] || 4) - (severityOrder[b.severity] || 4));
+
+  // Attach profile data if requested
+  if (profile && profileData) {
+    findings._profile = profileData.sort((a, b) => b.ms - a.ms);
+  }
+
+  return findings;
+}
+
+/**
+ * Run rules in parallel using worker threads.
+ * Splits rules across CPU cores for ~4-8x speedup on multi-core machines.
+ */
+export async function runRulesParallel(rules, context) {
+  const numCores = Math.min(cpus().length, 8); // Cap at 8 workers
+  if (numCores <= 1 || !context.files || context.files.size === 0) {
+    return runRules(rules, context); // Fallback to serial
+  }
+
+  const workerPath = join(dirname(fileURLToPath(import.meta.url)), 'rule-worker.js');
+
+  // Serialize files Map to array for transfer
+  const filesData = [...context.files.entries()];
+  const stackData = context.stack || {};
+  const category = context._categoryFilter || null;
+  const _detectedLangs = context._detectedLangs ? [...context._detectedLangs] : null;
+
+  // Spawn workers
+  const workers = [];
+  for (let i = 0; i < numCores; i++) {
+    workers.push(new Promise((resolve, reject) => {
+      const worker = new Worker(workerPath, {
+        workerData: {
+          filesData,
+          stackData,
+          category,
+          _detectedLangs,
+          workerIndex: i,
+          totalWorkers: numCores,
+        },
+      });
+      worker.on('message', resolve);
+      worker.on('error', reject);
+      worker.on('exit', (code) => {
+        if (code !== 0) resolve([]); // Don't crash on worker failure
+      });
+    }));
+  }
+
+  try {
+    const results = await Promise.all(workers);
+    const findings = results.flat();
+
+    // Sort by severity
+    const severityOrder = { critical: 0, high: 1, medium: 2, low: 3, info: 4 };
+    findings.sort((a, b) => (severityOrder[a.severity] || 4) - (severityOrder[b.severity] || 4));
+
+    return findings;
+  } catch {
+    // Fallback to serial on any error
+    return runRules(rules, context);
+  }
+}