@oculum/scanner 1.0.0

package/src/layer1/file-flags.ts

@@ -0,0 +1,127 @@
+/**
+ * Layer 1: File Extension Red Flags
+ * Detects dangerous files that should not be in a repository
+ */
+
+import type { Vulnerability } from '../types'
+
+// Dangerous file extensions and names that should not be committed
+const DANGEROUS_FILE_PATTERNS = [
+  // Private keys
+  { pattern: /\.pem$/i, name: 'PEM Private Key', severity: 'critical' as const },
+  { pattern: /\.key$/i, name: 'Private Key File', severity: 'critical' as const },
+  { pattern: /\.p12$/i, name: 'PKCS#12 Certificate', severity: 'critical' as const },
+  { pattern: /\.pfx$/i, name: 'PFX Certificate', severity: 'critical' as const },
+  { pattern: /\.p8$/i, name: 'PKCS#8 Private Key', severity: 'critical' as const },
+  
+  // SSH keys
+  { pattern: /id_rsa$/i, name: 'RSA SSH Private Key', severity: 'critical' as const },
+  { pattern: /id_dsa$/i, name: 'DSA SSH Private Key', severity: 'critical' as const },
+  { pattern: /id_ecdsa$/i, name: 'ECDSA SSH Private Key', severity: 'critical' as const },
+  { pattern: /id_ed25519$/i, name: 'Ed25519 SSH Private Key', severity: 'critical' as const },
+  
+  // Environment files (should be gitignored)
+  { pattern: /^\.env$/i, name: 'Environment File', severity: 'critical' as const },
+  { pattern: /^\.env\.local$/i, name: 'Local Environment File', severity: 'critical' as const },
+  { pattern: /^\.env\.production$/i, name: 'Production Environment File', severity: 'critical' as const },
+  { pattern: /^\.env\.development$/i, name: 'Development Environment File', severity: 'high' as const },
+  { pattern: /^\.env\.staging$/i, name: 'Staging Environment File', severity: 'high' as const },
+  
+  // Credential files
+  { pattern: /\.pgpass$/i, name: 'PostgreSQL Password File', severity: 'critical' as const },
+  { pattern: /\.npmrc$/i, name: 'NPM Configuration (may contain tokens)', severity: 'high' as const },
+  { pattern: /\.pypirc$/i, name: 'PyPI Configuration (may contain tokens)', severity: 'high' as const },
+  { pattern: /\.netrc$/i, name: 'Netrc File (contains credentials)', severity: 'critical' as const },
+  { pattern: /\.htpasswd$/i, name: 'Apache Password File', severity: 'critical' as const },
+  
+  // Database files
+  { pattern: /\.sqlite$/i, name: 'SQLite Database', severity: 'medium' as const },
+  { pattern: /\.sqlite3$/i, name: 'SQLite Database', severity: 'medium' as const },
+  { pattern: /\.db$/i, name: 'Database File', severity: 'medium' as const },
+  
+  // Backup files that may contain secrets
+  { pattern: /\.bak$/i, name: 'Backup File', severity: 'low' as const },
+  { pattern: /\.backup$/i, name: 'Backup File', severity: 'low' as const },
+  { pattern: /\.old$/i, name: 'Old File Backup', severity: 'low' as const },
+  
+  // Keystore files
+  { pattern: /\.jks$/i, name: 'Java Keystore', severity: 'high' as const },
+  { pattern: /\.keystore$/i, name: 'Keystore File', severity: 'high' as const },
+]
+
+// Files that are okay in certain contexts but should be reviewed
+const CONTEXT_SENSITIVE_PATTERNS = [
+  { pattern: /\.env\.example$/i, name: 'Environment Example File', checkContent: true },
+  { pattern: /\.env\.sample$/i, name: 'Environment Sample File', checkContent: true },
+  { pattern: /\.env\.template$/i, name: 'Environment Template File', checkContent: true },
+]
+
+export function detectDangerousFiles(
+  content: string,
+  filePath: string
+): Vulnerability[] {
+  const vulnerabilities: Vulnerability[] = []
+  const fileName = filePath.split('/').pop() || ''
+
+  // Check for dangerous file patterns
+  for (const { pattern, name, severity } of DANGEROUS_FILE_PATTERNS) {
+    if (pattern.test(fileName) || pattern.test(filePath)) {
+      vulnerabilities.push({
+        id: `file-flag-${filePath}`,
+        filePath,
+        lineNumber: 1,
+        lineContent: `File: ${fileName}`,
+        severity,
+        category: 'dangerous_file',
+        title: `Dangerous file detected: ${name}`,
+        description: `This file type (${name}) should not be committed to version control as it may contain sensitive information.`,
+        suggestedFix: 'Add this file to .gitignore and remove it from the repository. If it contains secrets, rotate them immediately.',
+        confidence: 'high',
+        layer: 1,
+      })
+    }
+  }
+
+  // Check context-sensitive files for actual secrets
+  for (const { pattern, name, checkContent } of CONTEXT_SENSITIVE_PATTERNS) {
+    if ((pattern.test(fileName) || pattern.test(filePath)) && checkContent) {
+      // Check if the example/sample file contains what looks like real values
+      const suspiciousPatterns = [
+        /[a-zA-Z0-9_]+=sk-[a-zA-Z0-9]{20,}/,  // API keys
+        /[a-zA-Z0-9_]+=ghp_[a-zA-Z0-9]{36,}/, // GitHub tokens
+        /[a-zA-Z0-9_]+=eyJ[a-zA-Z0-9_-]+\./,  // JWT tokens
+        /password\s*=\s*[^\s$]{8,}/i,          // Passwords (not env vars)
+      ]
+
+      const lines = content.split('\n')
+      for (let i = 0; i < lines.length; i++) {
+        const line = lines[i]
+        // Skip lines that reference environment variables
+        if (line.includes('${') || line.includes('$env:') || line.includes('process.env')) {
+          continue
+        }
+        
+        for (const suspicious of suspiciousPatterns) {
+          if (suspicious.test(line)) {
+            vulnerabilities.push({
+              id: `file-flag-content-${filePath}-${i + 1}`,
+              filePath,
+              lineNumber: i + 1,
+              lineContent: line.trim(),
+              severity: 'high',
+              category: 'dangerous_file',
+              title: `${name} may contain real secrets`,
+              description: 'This example/sample file appears to contain real secret values instead of placeholders.',
+              suggestedFix: 'Replace real values with placeholders like YOUR_API_KEY_HERE or use environment variable references.',
+              confidence: 'medium',
+              layer: 1,
+            })
+            break
+          }
+        }
+      }
+    }
+  }
+
+  return vulnerabilities
+}

package/src/layer1/index.ts

@@ -0,0 +1,181 @@
+/**
+ * Layer 1: Surface Scan
+ * Fast, deterministic scanning using entropy, patterns, config auditing,
+ * file flags, comment analysis, URL detection, and weak crypto detection
+ */
+
+import type { Vulnerability, ScanFile } from '../types'
+import { detectHighEntropyStrings } from './entropy'
+import { detectKnownPatterns } from './patterns'
+import { auditConfiguration } from './config-audit'
+import { detectDangerousFiles } from './file-flags'
+import { detectAICommentPatterns } from './comments'
+import { detectSensitiveURLs } from './urls'
+import { detectWeakCrypto } from './weak-crypto'
+import {
+  type TierStats,
+  computeTierStats,
+  formatTierStats,
+  getLayer1DetectorTier,
+  type Layer1DetectorName,
+} from '../tiers'
+import {
+  filterFindingsByPath,
+  type ExclusionConfig,
+} from '../utils/path-exclusions'
+
+/**
+ * Layer 1 detector stats for raw finding counts before deduplication
+ */
+export interface Layer1Stats {
+  /** Raw finding counts per detector (before dedupe) */
+  raw: Record<string, number>
+  /** Deduped finding counts per category */
+  deduped: Record<string, number>
+  /** Tier breakdown of deduped findings */
+  tiers: TierStats
+  /** Number of findings suppressed by path exclusions */
+  suppressedByPath: number
+}
+
+export interface Layer1Result {
+  vulnerabilities: Vulnerability[]
+  filesScanned: number
+  duration: number
+  /** Heuristic breakdown stats for noise analysis */
+  stats: Layer1Stats
+}
+
+export async function runLayer1Scan(files: ScanFile[]): Promise<Layer1Result> {
+  const startTime = Date.now()
+  const vulnerabilities: Vulnerability[] = []
+
+  // Track raw counts per detector (before dedupe)
+  const rawStats: Record<Layer1DetectorName, number> = {
+    known_secrets: 0,
+    weak_crypto: 0,
+    sensitive_urls: 0,
+    entropy: 0,
+    config_audit: 0,
+    file_flags: 0,
+    ai_comments: 0,
+  }
+
+  for (const file of files) {
+    // Run all Layer 1 detectors and track raw counts
+    const entropyFindings = detectHighEntropyStrings(file.content, file.path)
+    const patternFindings = detectKnownPatterns(file.content, file.path)
+    const configFindings = auditConfiguration(file.content, file.path)
+    const fileFlags = detectDangerousFiles(file.content, file.path)
+    const commentFindings = detectAICommentPatterns(file.content, file.path)
+    const urlFindings = detectSensitiveURLs(file.content, file.path)
+    const cryptoFindings = detectWeakCrypto(file.content, file.path)
+
+    rawStats.entropy += entropyFindings.length
+    rawStats.known_secrets += patternFindings.length
+    rawStats.config_audit += configFindings.length
+    rawStats.file_flags += fileFlags.length
+    rawStats.ai_comments += commentFindings.length
+    rawStats.sensitive_urls += urlFindings.length
+    rawStats.weak_crypto += cryptoFindings.length
+
+    vulnerabilities.push(
+      ...entropyFindings,
+      ...patternFindings,
+      ...configFindings,
+      ...fileFlags,
+      ...commentFindings,
+      ...urlFindings,
+      ...cryptoFindings
+    )
+  }
+
+  // Deduplicate findings (same line might be caught by multiple detectors)
+  const dedupedVulnerabilities = deduplicateFindings(vulnerabilities)
+
+  // Apply path exclusions to filter out findings in test/seed/example files
+  const { kept: uniqueVulnerabilities, suppressed } = filterFindingsByPath(dedupedVulnerabilities)
+
+  // Log suppressed findings
+  if (suppressed.length > 0) {
+    const byReason: Record<string, number> = {}
+    for (const s of suppressed) {
+      const reason = s.reason || 'unknown'
+      byReason[reason] = (byReason[reason] || 0) + 1
+    }
+    console.log(`[Layer 1] Suppressed ${suppressed.length} findings in test/seed/example files:`)
+    for (const [reason, count] of Object.entries(byReason)) {
+      console.log(`  - ${reason}: ${count}`)
+    }
+  }
+
+  // Compute deduped counts per category
+  const dedupedStats: Record<string, number> = {}
+  for (const vuln of uniqueVulnerabilities) {
+    const cat = vuln.category
+    dedupedStats[cat] = (dedupedStats[cat] || 0) + 1
+  }
+
+  // Compute tier breakdown (all Layer 1 findings have layer: 1)
+  const tierStats = computeTierStats(
+    uniqueVulnerabilities.map(v => ({ category: v.category, layer: 1 as const }))
+  )
+
+  // Log heuristic breakdown with tier info
+  console.log('[Layer 1] Heuristic breakdown (raw findings before dedupe):')
+  for (const [name, count] of Object.entries(rawStats)) {
+    if (count > 0) {
+      const tier = getLayer1DetectorTier(name as Layer1DetectorName)
+      console.log(`  - ${name}: ${count} (${tier})`)
+    }
+  }
+  console.log(`[Layer 1] Tier breakdown (after dedupe): ${formatTierStats(tierStats)}`)
+
+  return {
+    vulnerabilities: uniqueVulnerabilities,
+    filesScanned: files.length,
+    duration: Date.now() - startTime,
+    stats: {
+      raw: rawStats,
+      deduped: dedupedStats,
+      tiers: tierStats,
+      suppressedByPath: suppressed.length,
+    },
+  }
+}
+
+// Remove duplicate findings on the same line
+function deduplicateFindings(vulnerabilities: Vulnerability[]): Vulnerability[] {
+  const seen = new Map<string, Vulnerability>()
+
+  for (const vuln of vulnerabilities) {
+    const key = `${vuln.filePath}:${vuln.lineNumber}:${vuln.category}`
+    const existing = seen.get(key)
+
+    // Keep the higher severity finding
+    if (!existing || severityRank(vuln.severity) > severityRank(existing.severity)) {
+      seen.set(key, vuln)
+    }
+  }
+
+  return Array.from(seen.values())
+}
+
+function severityRank(severity: string): number {
+  const ranks: Record<string, number> = {
+    critical: 5,
+    high: 4,
+    medium: 3,
+    low: 2,
+    info: 1,
+  }
+  return ranks[severity] || 0
+}
+
+export { detectHighEntropyStrings } from './entropy'
+export { detectKnownPatterns } from './patterns'
+export { auditConfiguration } from './config-audit'
+export { detectDangerousFiles } from './file-flags'
+export { detectAICommentPatterns } from './comments'
+export { detectSensitiveURLs } from './urls'
+export { detectWeakCrypto } from './weak-crypto'