circle-ir-ai 2.7.17 → 2.8.0

package/dist/secret-scan/scanner.js

@@ -1,42 +1,92 @@
 /**
- * Secret Scanner Module
+ * Secret Scanner Module (Refactored)
  *
- * Scans code and Git history for secrets and credentials.
+ * Architecture:
+ * - SAST detection: Delegates to circle-ir's ScanSecretsPass (no regex duplication)
+ * - Git history: Scans commits for secrets introduced historically (circle-ir-ai domain)
+ * - LLM verification: Reduces false positives via context-aware analysis
+ *
+ * This module consumes CircleIR findings rather than reimplementing SAST logic.
  */
 import { execFileSync } from 'child_process';
 import * as fs from 'fs';
 import * as path from 'path';
 import { minimatch } from 'minimatch';
-import { SECRET_PATTERNS, } from './patterns.js';
+import { initAnalyzer, analyze, isAnalyzerInitialized, detectLanguage } from 'circle-ir';
+// Minimal patterns for git history scanning only.
+// Working-tree scanning uses circle-ir's ScanSecretsPass.
+import { HISTORY_SCAN_PATTERNS } from './history-patterns.js';
+/**
+ * Map circle-ir severity to our severity type
+ */
+function mapSeverity(severity) {
+    const map = {
+        critical: 'critical',
+        high: 'high',
+        medium: 'medium',
+        low: 'low',
+        info: 'low',
+    };
+    return map[severity.toLowerCase()] ?? 'medium';
+}
+/**
+ * Map circle-ir finding to DetectedSecret
+ */
+function sastFindingToSecret(finding, fileContent) {
+    const lines = fileContent.split('\n');
+    const lineContent = lines[finding.line - 1] || '';
+    // Extract pattern info from evidence or rule_id
+    const evidence = finding.evidence || {};
+    const patternName = evidence.provider || finding.rule_id.replace('hardcoded-credential', 'Secret').replace(/-/g, ' ');
+    const match = evidence.match || '[redacted]';
+    // Determine category from evidence or pattern name
+    let category = 'generic';
+    if (evidence.provider) {
+        const provider = evidence.provider.toLowerCase();
+        if (provider.includes('aws'))
+            category = 'aws';
+        else if (provider.includes('github'))
+            category = 'github';
+        else if (provider.includes('stripe'))
+            category = 'stripe';
+        else if (provider.includes('openai'))
+            category = 'openai';
+        else if (provider.includes('anthropic'))
+            category = 'anthropic';
+        else if (provider.includes('slack'))
+            category = 'slack';
+        else if (provider.includes('google'))
+            category = 'gcp';
+        else if (provider.includes('jwt'))
+            category = 'jwt';
+        else if (provider.includes('pem') || provider.includes('private key'))
+            category = 'private-key';
+        else if (provider.includes('npm'))
+            category = 'npm';
+    }
+    else if (evidence.kind === 'entropy') {
+        category = 'high-entropy';
+    }
+    return {
+        patternId: finding.rule_id,
+        patternName,
+        file: finding.file,
+        line: finding.line,
+        column: 1,
+        match: redactSecret(match),
+        lineContent: truncateLine(lineContent),
+        severity: mapSeverity(finding.severity),
+        category,
+        presentInHead: true,
+    };
+}
 /**
  * Secret Scanner class
  */
 export class SecretScanner {
-    patterns;
     options;
     constructor(options = {}) {
         this.options = options;
-        this.patterns = this.selectPatterns(options);
-    }
-    /**
-     * Select patterns based on options
-     */
-    selectPatterns(options) {
-        let patterns = options.patterns || SECRET_PATTERNS;
-        // Filter by category
-        if (options.includeCategories?.length) {
-            patterns = patterns.filter((p) => options.includeCategories.includes(p.category));
-        }
-        if (options.excludeCategories?.length) {
-            patterns = patterns.filter((p) => !options.excludeCategories.includes(p.category));
-        }
-        // Filter by severity
-        if (options.minSeverity) {
-            const severityOrder = ['low', 'medium', 'high', 'critical'];
-            const minIndex = severityOrder.indexOf(options.minSeverity);
-            patterns = patterns.filter((p) => severityOrder.indexOf(p.severity) >= minIndex);
-        }
-        return patterns;
     }
     /**
      * Scan a directory for secrets
@@ -44,6 +94,10 @@ export class SecretScanner {
     async scan(directory) {
         const startTime = Date.now();
         const secrets = [];
+        // Ensure analyzer is initialized
+        if (!isAnalyzerInitialized()) {
+            await initAnalyzer();
+        }
         const progress = {
             phase: 'indexing',
             filesScanned: 0,
@@ -56,11 +110,11 @@ export class SecretScanner {
         progress.totalFiles = files.length;
         progress.phase = 'scanning-files';
         this.options.onProgress?.(progress);
-        // Scan current files
+        // Scan current files using circle-ir SAST
         for (const file of files) {
             progress.currentFile = file;
             this.options.onProgress?.(progress);
-            const fileSecrets = await this.scanFile(file, directory);
+            const fileSecrets = await this.scanFileWithCircleIR(file);
             for (const secret of fileSecrets) {
                 secret.presentInHead = true;
                 secrets.push(secret);
@@ -86,6 +140,26 @@ export class SecretScanner {
             }
             commitsScanned = progress.commitsScanned || 0;
         }
+        // LLM verification (optional)
+        if (this.options.llmVerify && secrets.length > 0) {
+            progress.phase = 'verifying';
+            this.options.onProgress?.(progress);
+            await this.llmVerifySecrets(secrets, directory);
+        }
+        // Apply severity filter
+        let filteredSecrets = secrets;
+        if (this.options.minSeverity) {
+            const severityOrder = ['low', 'medium', 'high', 'critical'];
+            const minIndex = severityOrder.indexOf(this.options.minSeverity);
+            filteredSecrets = secrets.filter((s) => severityOrder.indexOf(s.severity) >= minIndex);
+        }
+        // Apply category filters
+        if (this.options.includeCategories?.length) {
+            filteredSecrets = filteredSecrets.filter((s) => this.options.includeCategories.includes(s.category));
+        }
+        if (this.options.excludeCategories?.length) {
+            filteredSecrets = filteredSecrets.filter((s) => !this.options.excludeCategories.includes(s.category));
+        }
         progress.phase = 'complete';
         this.options.onProgress?.(progress);
         // Calculate statistics
@@ -96,19 +170,19 @@ export class SecretScanner {
             low: 0,
         };
         const byCategory = {};
-        for (const secret of secrets) {
+        for (const secret of filteredSecrets) {
             bySeverity[secret.severity]++;
             byCategory[secret.category] = (byCategory[secret.category] || 0) + 1;
         }
-        const activeSecrets = secrets.filter((s) => s.presentInHead).length;
-        const historicalSecrets = secrets.filter((s) => !s.presentInHead).length;
+        const activeSecrets = filteredSecrets.filter((s) => s.presentInHead).length;
+        const historicalSecrets = filteredSecrets.filter((s) => !s.presentInHead).length;
         // Generate .gitignore recommendations
-        const gitignoreRecommendations = this.generateGitignoreRecommendations(secrets);
+        const gitignoreRecommendations = this.generateGitignoreRecommendations(filteredSecrets);
         return {
             directory,
             filesScanned: files.length,
             commitsScanned,
-            secrets,
+            secrets: filteredSecrets,
             bySeverity,
             byCategory,
             activeSecrets,
@@ -118,112 +192,33 @@ export class SecretScanner {
         };
     }
     /**
-     * Scan a single file for secrets
+     * Scan a single file using circle-ir's ScanSecretsPass
      */
-    async scanFile(filePath, _baseDir) {
+    async scanFileWithCircleIR(filePath) {
         const secrets = [];
         try {
             const content = fs.readFileSync(filePath, 'utf-8');
-            // #13: emit absolute paths in DetectedSecret.file. baseDir is no
-            // longer needed for path formatting (kept in signature for callers).
-            const absolutePath = path.resolve(filePath);
-            const lines = content.split('\n');
-            for (let lineNum = 0; lineNum < lines.length; lineNum++) {
-                const line = lines[lineNum];
-                const lineSecrets = this.scanLine(line, absolutePath, lineNum + 1);
-                secrets.push(...lineSecrets);
+            const language = detectLanguage(filePath);
+            if (!language) {
+                return secrets; // Unsupported language
+            }
+            const result = await analyze(content, path.basename(filePath), language);
+            // Extract hardcoded-credential findings from circle-ir
+            const credentialFindings = (result.findings ?? []).filter((f) => f.rule_id?.includes('hardcoded-credential'));
+            for (const finding of credentialFindings) {
+                const secret = sastFindingToSecret(finding, content);
+                secret.file = path.resolve(filePath); // Absolute path
+                secrets.push(secret);
             }
         }
         catch {
-            // Skip files that can't be read
-        }
-        return secrets;
-    }
-    /**
-     * Scan a single line for secrets
-     */
-    scanLine(line, file, lineNum, commit, author, commitDate) {
-        const secrets = [];
-        // Quick keyword pre-filter
-        const lineLower = line.toLowerCase();
-        for (const pattern of this.patterns) {
-            // Skip if no keywords match (optimization)
-            if (pattern.keywords?.length) {
-                const hasKeyword = pattern.keywords.some((k) => lineLower.includes(k.toLowerCase()));
-                if (!hasKeyword)
-                    continue;
-            }
-            // Reset regex lastIndex for global patterns
-            pattern.pattern.lastIndex = 0;
-            let match;
-            while ((match = pattern.pattern.exec(line)) !== null) {
-                const matchedText = match[0];
-                // Check false positive patterns
-                if (pattern.falsePositivePatterns?.length) {
-                    const isFalsePositive = pattern.falsePositivePatterns.some((fp) => fp.test(line));
-                    if (isFalsePositive)
-                        continue;
-                }
-                // Run validator if present
-                if (pattern.validator && !pattern.validator(matchedText)) {
-                    continue;
-                }
-                // Context-aware false positive detection:
-                // 1. Match inside a regex literal (pattern definition, not real secret)
-                if (this.isInsideRegexLiteral(line, match.index, match.index + matchedText.length)) {
-                    continue;
-                }
-                // 2. Match is a truncated example (e.g. "Bearer eyJ...", "sk-...")
-                if (/\.{2,}["'`]?\s*[,;)}\]]?\s*$/.test(line.slice(match.index)) ||
-                    /\.{2,}["'`]/.test(matchedText) ||
-                    /\.{3}/.test(line.slice(match.index, match.index + matchedText.length + 5))) {
-                    continue;
-                }
-                // 3. Line is in a documentation/example context (comment with "Examples:", "e.g.")
-                if (/^\s*(?:\*|\/\/|#)\s*(?:examples?:|e\.g\.|i\.e\.|sample|usage)/i.test(line)) {
-                    continue;
-                }
-                secrets.push({
-                    patternId: pattern.id,
-                    patternName: pattern.name,
-                    file,
-                    line: lineNum,
-                    column: match.index + 1,
-                    match: this.redactSecret(matchedText),
-                    lineContent: this.truncateLine(line),
-                    severity: pattern.severity,
-                    category: pattern.category,
-                    commit,
-                    author,
-                    commitDate,
-                    presentInHead: false, // Will be updated later
-                });
-            }
+            // Skip files that can't be read or analyzed
         }
         return secrets;
     }
-    /**
-     * Check if a match position falls inside a regex literal (/.../).
-     * Looks for unescaped `/` delimiters surrounding the match range.
-     */
-    isInsideRegexLiteral(line, matchStart, matchEnd) {
-        // Find regex literals in the line: look for /pattern/flags
-        // A regex literal starts with / (not preceded by a word char or /) and
-        // ends with / followed by optional flags [gimsuy]
-        const regexLiteralPattern = /(?:^|[=(:,;!&|?+\-~^%<>[\s])(\/.+?\/[gimsuy]*)/g;
-        let m;
-        while ((m = regexLiteralPattern.exec(line)) !== null) {
-            const regexStart = m.index + m[0].indexOf(m[1]);
-            const regexEnd = regexStart + m[1].length;
-            // If the secret match falls inside this regex literal, it's a pattern definition
-            if (matchStart >= regexStart && matchEnd <= regexEnd) {
-                return true;
-            }
-        }
-        return false;
-    }
     /**
      * Scan git history for secrets
+     * Uses minimal patterns since we can't run circle-ir on diffs
      */
     async scanGitHistory(directory, progress) {
         const secrets = [];
@@ -240,16 +235,9 @@ export class SecretScanner {
                 // Get commit info
                 const commitInfo = execFileSync('git', ['-C', directory, 'log', '-1', '--format=%an|%aI', commit], { encoding: 'utf-8' }).trim();
                 const [author, commitDate] = commitInfo.split('|');
-                // Get diff for this commit.
-                // `--root` is required so the diff against the empty tree is
-                // emitted for the repo's root commit — otherwise `git diff-tree`
-                // returns empty for the initial commit and any secret added in
-                // commit #1 is silently invisible to the scanner. Discovered
-                // while writing the #60 history-exclude test.
                 try {
                     const diff = execFileSync('git', ['-C', directory, 'diff-tree', '--root', '--no-commit-id', '-r', '-p', commit], { encoding: 'utf-8', maxBuffer: 50 * 1024 * 1024 });
-                    // Parse diff and scan for secrets
-                    const diffSecrets = this.scanDiff(diff, commit, author, commitDate);
+                    const diffSecrets = this.scanDiff(diff, commit, author, commitDate, directory);
                     secrets.push(...diffSecrets);
                 }
                 catch {
@@ -266,32 +254,23 @@ export class SecretScanner {
         return secrets;
     }
     /**
-     * Scan a git diff for secrets
+     * Scan a git diff for secrets using minimal history patterns
      */
-    scanDiff(diff, commit, author, commitDate) {
+    scanDiff(diff, commit, author, commitDate, repoDir) {
         const secrets = [];
         let currentFile = '';
         let lineNum = 0;
-        // #60: when the current file is excluded by user globs or built-in
-        // regex skips, drop every line of that diff section until the next
-        // `+++ b/<path>` marker. Working-tree walk applied excludes; history
-        // walk did not, so e.g. Cargo.lock secrets would show up as
-        // "Status: Historical" even with `--exclude Cargo.lock`.
         let currentFileExcluded = false;
         const lines = diff.split('\n');
         for (const line of lines) {
-            // Track current file
             if (line.startsWith('+++ b/')) {
                 currentFile = line.slice(6);
                 lineNum = 0;
                 currentFileExcluded = this.isPathExcluded(currentFile);
                 continue;
             }
-            // #60: while the active file is excluded, skip every line —
-            // including hunk headers — until the next `+++ b/` marker.
             if (currentFileExcluded)
                 continue;
-            // Track line numbers from hunk headers
             const hunkMatch = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)/);
             if (hunkMatch) {
                 lineNum = parseInt(hunkMatch[1], 10) - 1;
@@ -300,8 +279,9 @@ export class SecretScanner {
             // Only scan added lines
             if (line.startsWith('+') && !line.startsWith('+++')) {
                 lineNum++;
-                const content = line.slice(1); // Remove the '+' prefix
-                const lineSecrets = this.scanLine(content, currentFile, lineNum, commit, author, commitDate);
+                const content = line.slice(1);
+                const absoluteFile = path.resolve(repoDir, currentFile);
+                const lineSecrets = this.scanLineWithPatterns(content, absoluteFile, lineNum, commit, author, commitDate);
                 secrets.push(...lineSecrets);
             }
             else if (!line.startsWith('-')) {
@@ -311,12 +291,88 @@ export class SecretScanner {
         return secrets;
     }
     /**
-     * Built-in path-skip regexes shared between the working-tree walk
-     * (`getFiles`) and the git-history diff path (`scanDiff`). #60: previously
-     * only the working-tree walk applied these — git-history scans walked
-     * every diff including `Cargo.lock`, `*.lock`, binary blobs, etc.,
-     * producing noisy "Status: Historical" findings on paths the user had
-     * already excluded from their working-tree scan.
+     * Scan a single line using minimal history patterns
+     * This is only used for git history scanning where we can't use circle-ir
+     */
+    scanLineWithPatterns(line, file, lineNum, commit, author, commitDate) {
+        const secrets = [];
+        for (const pattern of HISTORY_SCAN_PATTERNS) {
+            pattern.pattern.lastIndex = 0;
+            let match;
+            while ((match = pattern.pattern.exec(line)) !== null) {
+                const matchedText = match[0];
+                // Skip false positives
+                if (pattern.falsePositivePatterns?.length) {
+                    const isFalsePositive = pattern.falsePositivePatterns.some((fp) => fp.test(line));
+                    if (isFalsePositive)
+                        continue;
+                }
+                if (pattern.validator && !pattern.validator(matchedText)) {
+                    continue;
+                }
+                secrets.push({
+                    patternId: pattern.id,
+                    patternName: pattern.name,
+                    file,
+                    line: lineNum,
+                    column: match.index + 1,
+                    match: redactSecret(matchedText),
+                    lineContent: truncateLine(line),
+                    severity: pattern.severity,
+                    category: pattern.category,
+                    commit,
+                    author,
+                    commitDate,
+                    presentInHead: false,
+                });
+            }
+        }
+        return secrets;
+    }
+    /**
+     * LLM verification to reduce false positives
+     */
+    async llmVerifySecrets(secrets, _directory) {
+        // Import LLM client dynamically to avoid circular deps
+        try {
+            const { AxLLMClient } = await import('../llm/ax-client.js');
+            const { getDefaultLLMConfig } = await import('../llm/config.js');
+            const config = getDefaultLLMConfig();
+            const client = new AxLLMClient(config);
+            // Batch verify entropy-based findings (most likely FPs)
+            const entropySecrets = secrets.filter(s => s.category === 'high-entropy');
+            const systemPrompt = 'You are a security expert. Analyze code for hardcoded secrets. Respond with JSON only.';
+            for (const secret of entropySecrets) {
+                try {
+                    const userPrompt = `Analyze this code line and determine if it contains a real hardcoded secret or credential.
+
+Line: ${secret.lineContent}
+Context: File ${path.basename(secret.file)}, line ${secret.line}
+
+Respond with JSON: {"isSecret": true/false, "confidence": 0.0-1.0, "reason": "brief explanation"}
+
+Consider:
+- Is this a placeholder, example, or test value?
+- Is this a hash, UUID, or non-sensitive identifier?
+- Is this an actual API key, token, or credential?`;
+                    const response = await client.chatJSON(systemPrompt, userPrompt, 'verification');
+                    if (response) {
+                        secret.llmVerified = response.isSecret;
+                        secret.llmConfidence = response.confidence;
+                    }
+                }
+                catch {
+                    // LLM verification failed, keep the finding
+                    secret.llmVerified = undefined;
+                }
+            }
+        }
+        catch {
+            // LLM not available, skip verification
+        }
+    }
+    /**
+     * Built-in path-skip patterns
      */
     static BUILTIN_EXCLUDE_PATTERNS = [
         /node_modules/,
@@ -348,9 +404,7 @@ export class SecretScanner {
         /\.wasm$/i,
     ];
     /**
-     * #60: returns true when a path should be filtered out per built-in
-     * regex excludes + user-supplied include/exclude minimatch globs.
-     * Used by both the working-tree walk and the git-history diff parser.
+     * Check if path should be excluded
      */
     isPathExcluded(relativePath) {
         if (SecretScanner.BUILTIN_EXCLUDE_PATTERNS.some((p) => p.test(relativePath))) {
@@ -372,10 +426,6 @@ export class SecretScanner {
     getFiles(directory) {
         const files = [];
         const excludePatterns = SecretScanner.BUILTIN_EXCLUDE_PATTERNS;
-        // #18: user-provided include/exclude patterns are globs (matching the
-        // shape used by `cognium.config.json` and the CLI's `--include` /
-        // `--exclude` flags). Match via minimatch — passing them to RegExp
-        // crashed on `**` ("nothing to repeat").
         const userExcludeGlobs = this.options.excludeFiles ?? [];
         const userIncludeGlobs = this.options.includeFiles ?? [];
         const walk = (dir) => {
@@ -384,11 +434,9 @@ export class SecretScanner {
                 for (const entry of entries) {
                     const fullPath = path.join(dir, entry.name);
                     const relativePath = path.relative(directory, fullPath);
-                    // Skip excluded patterns
                     if (excludePatterns.some((p) => p.test(relativePath))) {
                         continue;
                     }
-                    // Apply user exclude globs
                     if (userExcludeGlobs.length) {
                         if (userExcludeGlobs.some((g) => minimatch(relativePath, g, { dot: true })))
                             continue;
@@ -397,7 +445,6 @@ export class SecretScanner {
                         walk(fullPath);
                     }
                     else if (entry.isFile()) {
-                        // Apply user include globs
                         if (userIncludeGlobs.length) {
                             if (!userIncludeGlobs.some((g) => minimatch(relativePath, g, { dot: true })))
                                 continue;
@@ -428,26 +475,6 @@ export class SecretScanner {
             return false;
         }
     }
-    /**
-     * Redact a secret for safe display
-     */
-    redactSecret(secret) {
-        if (secret.length <= 8) {
-            return '*'.repeat(secret.length);
-        }
-        const visibleChars = Math.min(4, Math.floor(secret.length / 4));
-        return (secret.slice(0, visibleChars) +
-            '*'.repeat(secret.length - visibleChars * 2) +
-            secret.slice(-visibleChars));
-    }
-    /**
-     * Truncate long lines
-     */
-    truncateLine(line, maxLength = 200) {
-        if (line.length <= maxLength)
-            return line;
-        return line.slice(0, maxLength) + '...';
-    }
     /**
      * Generate .gitignore recommendations
      */
@@ -455,7 +482,6 @@ export class SecretScanner {
         const recommendations = new Set();
         for (const secret of secrets) {
             const file = secret.file;
-            // Common sensitive file patterns
             if (/\.env($|\.)/.test(file)) {
                 recommendations.add('.env*');
                 recommendations.add('!.env.example');
@@ -491,7 +517,6 @@ export class SecretScanner {
                 recommendations.add('*.pfx');
             }
         }
-        // Always recommend common patterns
         recommendations.add('.env');
         recommendations.add('.env.local');
         recommendations.add('*.pem');
@@ -499,6 +524,26 @@ export class SecretScanner {
         return [...recommendations].sort();
     }
 }
+/**
+ * Redact a secret for safe display
+ */
+function redactSecret(secret) {
+    if (secret.length <= 8) {
+        return '*'.repeat(secret.length);
+    }
+    const visibleChars = Math.min(4, Math.floor(secret.length / 4));
+    return (secret.slice(0, visibleChars) +
+        '*'.repeat(secret.length - visibleChars * 2) +
+        secret.slice(-visibleChars));
+}
+/**
+ * Truncate long lines
+ */
+function truncateLine(line, maxLength = 200) {
+    if (line.length <= maxLength)
+        return line;
+    return line.slice(0, maxLength) + '...';
+}
 /**
  * Scan a directory for secrets (convenience function)
  */
@@ -529,7 +574,6 @@ export function formatSecretReport(result) {
     lines.push(`Commits Scanned: ${result.commitsScanned}`);
     lines.push(`Duration: ${result.durationMs}ms`);
     lines.push('');
-    // Summary
     lines.push('-'.repeat(40));
     lines.push('SUMMARY');
     lines.push('-'.repeat(40));
@@ -548,12 +592,10 @@ export function formatSecretReport(result) {
         lines.push(`  ${category}: ${count}`);
     }
     lines.push('');
-    // Detailed findings
     if (result.secrets.length > 0) {
         lines.push('-'.repeat(40));
         lines.push('FINDINGS');
         lines.push('-'.repeat(40));
-        // Group by severity
         const grouped = new Map();
         for (const secret of result.secrets) {
             const list = grouped.get(secret.severity) || [];
@@ -577,10 +619,12 @@ export function formatSecretReport(result) {
                     lines.push(`    Date: ${secret.commitDate}`);
                 }
                 lines.push(`    Status: ${secret.presentInHead ? 'ACTIVE' : 'Historical'}`);
+                if (secret.llmVerified !== undefined) {
+                    lines.push(`    LLM Verified: ${secret.llmVerified} (confidence: ${secret.llmConfidence?.toFixed(2)})`);
+                }
             }
         }
     }
-    // .gitignore recommendations
     if (result.gitignoreRecommendations.length > 0) {
         lines.push('');
         lines.push('-'.repeat(40));