@oculum/scanner 1.0.0

package/src/modes/incremental.ts

@@ -0,0 +1,293 @@
+/**
+ * Incremental Scan Mode
+ * Optimized scanning for PR workflows - only scan changed files and surface relevant findings
+ */
+
+import type { ScanFile, Vulnerability, ScanResult, ScanModeConfig } from '../types'
+import { runLayer1Scan } from '../layer1'
+import { runLayer2Scan } from '../layer2'
+import {
+  parseDiff,
+  parseChangedFileList,
+  filterToChangedLines,
+  getChangedFilePaths,
+  type FileDiff,
+} from '../utils/diff-parser'
+import { detectGlobalAuthMiddleware } from '../utils/middleware-detector'
+
+/**
+ * Options for incremental scanning
+ */
+export interface IncrementalScanOptions {
+  /** Git diff output (if available) */
+  diffContent?: string
+  /** List of changed file paths (alternative to diff) */
+  changedFiles?: string[]
+  /** Whether to only show findings on exactly changed lines (strict) or nearby (default) */
+  strictLineMatching?: boolean
+  /** Context window for "near changed line" (default: 5) */
+  contextWindow?: number
+  /** Whether to mark findings as "introduced in this PR" */
+  markAsIntroduced?: boolean
+  /** Previous findings to compare against (for suppressing pre-existing issues) */
+  previousFindings?: Vulnerability[]
+}
+
+/**
+ * Result of incremental scan with additional PR-specific metadata
+ */
+export interface IncrementalScanResult {
+  /** All findings (filtered to changed lines) */
+  findings: Vulnerability[]
+  /** Findings that are new in this PR */
+  introduced: Vulnerability[]
+  /** Findings that existed before (if previousFindings provided) */
+  preExisting: Vulnerability[]
+  /** Number of files scanned */
+  filesScanned: number
+  /** Number of files that were changed */
+  filesChanged: number
+  /** Parsed diff information */
+  diffs: Map<string, FileDiff>
+  /** Scan duration in ms */
+  duration: number
+}
+
+/**
+ * Run an incremental scan optimized for PR workflows
+ *
+ * This scans:
+ * 1. All changed files (added + modified)
+ * 2. Files that import changed files (for context)
+ * 3. Middleware files (for auth context)
+ *
+ * And only surfaces findings on/near changed lines.
+ */
+export async function runIncrementalScan(
+  allFiles: ScanFile[],
+  options: IncrementalScanOptions
+): Promise<IncrementalScanResult> {
+  const startTime = Date.now()
+
+  const {
+    diffContent,
+    changedFiles,
+    strictLineMatching = false,
+    contextWindow = 5,
+    markAsIntroduced = true,
+    previousFindings = [],
+  } = options
+
+  // Parse diff or file list to get changed files
+  let diffs: Map<string, FileDiff>
+
+  if (diffContent) {
+    diffs = parseDiff(diffContent, contextWindow)
+  } else if (changedFiles && changedFiles.length > 0) {
+    diffs = parseChangedFileList(changedFiles)
+  } else {
+    // No diff info - scan everything but don't filter
+    console.log('[Incremental] No diff info provided, scanning all files')
+    diffs = new Map()
+  }
+
+  const changedPaths = getChangedFilePaths(diffs)
+  console.log(`[Incremental] Changed files: ${changedPaths.length}`)
+
+  // Build file index for import resolution
+  const fileIndex = new Map<string, ScanFile>()
+  for (const file of allFiles) {
+    fileIndex.set(file.path, file)
+  }
+
+  // Determine which files to scan
+  const filesToScan: ScanFile[] = []
+  const scannedPaths = new Set<string>()
+
+  // 1. Add all changed files
+  for (const path of changedPaths) {
+    const file = fileIndex.get(path)
+    if (file && !scannedPaths.has(path)) {
+      filesToScan.push(file)
+      scannedPaths.add(path)
+    }
+  }
+
+  // 2. Add files that import changed files (for context)
+  // This helps detect issues where changes break dependencies
+  const importers = findImporters(allFiles, changedPaths)
+  for (const path of importers) {
+    if (!scannedPaths.has(path)) {
+      const file = fileIndex.get(path)
+      if (file) {
+        filesToScan.push(file)
+        scannedPaths.add(path)
+      }
+    }
+  }
+
+  // 3. Always include middleware files for auth context
+  const middlewareFile = allFiles.find(f =>
+    f.path.includes('middleware.ts') ||
+    f.path.includes('middleware.js')
+  )
+  if (middlewareFile && !scannedPaths.has(middlewareFile.path)) {
+    filesToScan.push(middlewareFile)
+    scannedPaths.add(middlewareFile.path)
+  }
+
+  console.log(`[Incremental] Scanning ${filesToScan.length} files (${changedPaths.length} changed + ${importers.size} importers)`)
+
+  // Detect auth middleware from ALL files (for context)
+  const middlewareConfig = detectGlobalAuthMiddleware(allFiles)
+
+  // Run Layer 1 + Layer 2 on selected files
+  const layer1Result = await runLayer1Scan(filesToScan)
+  const layer2Result = await runLayer2Scan(filesToScan, { middlewareConfig })
+
+  let allFindings = [...layer1Result.vulnerabilities, ...layer2Result.vulnerabilities]
+
+  // Filter to only findings on/near changed lines
+  if (diffs.size > 0) {
+    const beforeFilter = allFindings.length
+    allFindings = filterToChangedLines(allFindings, diffs, { strictMode: strictLineMatching })
+    console.log(`[Incremental] Filtered findings: ${beforeFilter} → ${allFindings.length} (on/near changed lines)`)
+  }
+
+  // Mark findings as introduced and separate pre-existing
+  const introduced: Vulnerability[] = []
+  const preExisting: Vulnerability[] = []
+
+  if (previousFindings.length > 0) {
+    // Create fingerprints for previous findings
+    const previousFingerprints = new Set(
+      previousFindings.map(f => `${f.filePath}:${f.lineNumber}:${f.category}`)
+    )
+
+    for (const finding of allFindings) {
+      const fingerprint = `${finding.filePath}:${finding.lineNumber}:${finding.category}`
+
+      if (previousFingerprints.has(fingerprint)) {
+        preExisting.push(finding)
+      } else {
+        if (markAsIntroduced) {
+          finding.validationNotes = (finding.validationNotes || '') + ' [Introduced in this PR]'
+        }
+        introduced.push(finding)
+      }
+    }
+  } else {
+    // No previous findings - all are "introduced"
+    introduced.push(...allFindings)
+  }
+
+  const duration = Date.now() - startTime
+  console.log(`[Incremental] Scan completed in ${duration}ms: ${introduced.length} new, ${preExisting.length} pre-existing`)
+
+  return {
+    findings: allFindings,
+    introduced,
+    preExisting,
+    filesScanned: filesToScan.length,
+    filesChanged: changedPaths.length,
+    diffs,
+    duration,
+  }
+}
+
+/**
+ * Find files that import any of the changed files
+ */
+function findImporters(allFiles: ScanFile[], changedPaths: string[]): Set<string> {
+  const importers = new Set<string>()
+
+  // Create patterns to match imports
+  const importPatterns = changedPaths.map(path => {
+    // Remove extension for import matching
+    const withoutExt = path.replace(/\.[^/.]+$/, '')
+    // Get just the filename without path for relative imports
+    const filename = withoutExt.split('/').pop() || ''
+    return { fullPath: withoutExt, filename }
+  })
+
+  for (const file of allFiles) {
+    // Skip if this file is already in changed paths
+    if (changedPaths.includes(file.path)) continue
+
+    // Check if this file imports any changed file
+    for (const { fullPath, filename } of importPatterns) {
+      // Match various import patterns
+      const importRegex = new RegExp(
+        `(?:import|require)\\s*(?:\\([^)]*|[^;]*from\\s*)['"]` +
+        `(?:\\.{0,2}/)?(?:${escapeRegex(fullPath)}|[^'"]*/${escapeRegex(filename)})['"]`,
+        'i'
+      )
+
+      if (importRegex.test(file.content)) {
+        importers.add(file.path)
+        break
+      }
+    }
+  }
+
+  return importers
+}
+
+/**
+ * Escape special regex characters
+ */
+function escapeRegex(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+/**
+ * Create a PR-optimized scan config
+ */
+export function createPRScanConfig(
+  changedFiles: string[],
+  options: Partial<ScanModeConfig> = {}
+): ScanModeConfig {
+  return {
+    mode: 'incremental',
+    changedFiles,
+    skipAIValidation: false,  // Use AI for validation
+    skipLayer3: true,          // Skip deep analysis for speed
+    maxAIValidationFiles: 20,
+    maxLayer3Files: 0,
+    scanDepth: 'cheap',        // Fast feedback for PRs
+    ...options,
+  }
+}
+
+/**
+ * Format incremental scan result for PR comment
+ */
+export function formatIncrementalForPR(result: IncrementalScanResult): {
+  summary: string
+  hasNewIssues: boolean
+  blockingIssues: number
+} {
+  const blocking = result.introduced.filter(
+    f => f.severity === 'critical' || f.severity === 'high'
+  )
+
+  let summary: string
+
+  if (result.introduced.length === 0) {
+    summary = `✅ No new security issues introduced in this PR`
+  } else if (blocking.length > 0) {
+    summary = `🚨 ${blocking.length} blocking issue${blocking.length === 1 ? '' : 's'} introduced`
+  } else {
+    summary = `⚠️ ${result.introduced.length} new issue${result.introduced.length === 1 ? '' : 's'} to review`
+  }
+
+  if (result.preExisting.length > 0) {
+    summary += ` (${result.preExisting.length} pre-existing)`
+  }
+
+  return {
+    summary,
+    hasNewIssues: result.introduced.length > 0,
+    blockingIssues: blocking.length,
+  }
+}

package/src/tiers.ts

@@ -0,0 +1,318 @@
+/**
+ * Detector Tier System
+ *
+ * Provides a shared language for "how much we trust this detector" so we can:
+ * - Filter findings in runScan by tier + depth
+ * - Log tier breakdowns for tuning
+ * - Route AI validation budget toward Tier B
+ *
+ * Security reasoning:
+ * - Makes it explicit which detectors are safe to expose in cheap scans
+ * - Avoids "accidental promotion" of an experimental heuristic to production output
+ */
+
+import type { VulnerabilityCategory } from './types'
+
+/**
+ * Detector tiers control visibility and trust level:
+ *
+ * - core: High-precision SAST + core AI-safety detectors. Visible in all scan depths.
+ * - ai_assisted: Context-heavy heuristics that need AI validation. Shown in validated/deep.
+ * - experimental: High-noise signals used only for internal scoring/AI hints. Hidden from users.
+ */
+export type DetectorTier = 'core' | 'ai_assisted' | 'experimental'
+
+/**
+ * Tier statistics for logging and analysis
+ */
+export interface TierStats {
+  core: number
+  ai_assisted: number
+  experimental: number
+}
+
+// ============================================================================
+// Layer 1 Detector Tier Mappings
+// ============================================================================
+
+/**
+ * Layer 1 detector names (internal identifiers matching detector function names)
+ */
+export type Layer1DetectorName =
+  | 'known_secrets'      // patterns.ts - Known secret patterns (API keys, tokens)
+  | 'weak_crypto'        // weak-crypto.ts - Weak hash/cipher usage
+  | 'sensitive_urls'     // urls.ts - Webhook URLs, URLs with tokens, internal endpoints
+  | 'entropy'            // entropy.ts - High-entropy string detection
+  | 'config_audit'       // config-audit.ts - Risky .env/config and debug flags
+  | 'file_flags'         // file-flags.ts - Dangerous file patterns
+  | 'ai_comments'        // comments.ts - AI comment patterns ("Generated by...")
+
+/**
+ * Layer 1 tier assignments
+ *
+ * Tier A (core):
+ *   - known_secrets: Hardcoded secrets are objectively bad and high-impact
+ *   - weak_crypto: Weak crypto is a classic SAST finding with clear remediation
+ *   - sensitive_urls: Hardcoded webhook URLs + tokens are real data exfil vectors
+ *
+ * Tier B (ai_assisted):
+ *   - entropy: Great at finding candidates, needs AI to separate real secrets from noise
+ *   - config_audit: Depends on project norms; better reviewed with AI + project context
+ *   - file_flags: Subjective items should be AI-triaged (except committed .env = Tier A)
+ *
+ * Tier C (experimental):
+ *   - ai_comments: Not directly a vuln; belongs in separate "AI hygiene" report
+ */
+export const LAYER1_DETECTOR_TIERS: Record<Layer1DetectorName, DetectorTier> = {
+  known_secrets: 'core',
+  weak_crypto: 'core',
+  sensitive_urls: 'core',
+  entropy: 'ai_assisted',
+  config_audit: 'ai_assisted',
+  file_flags: 'ai_assisted',  // Mixed: committed .env is effectively core
+  ai_comments: 'experimental',
+}
+
+/**
+ * Mapping from vulnerability category to Layer 1 detector name
+ * Used for tier lookups when we only have the category
+ */
+export const LAYER1_CATEGORY_TO_DETECTOR: Partial<Record<VulnerabilityCategory, Layer1DetectorName>> = {
+  hardcoded_secret: 'known_secrets',
+  weak_crypto: 'weak_crypto',
+  sensitive_url: 'sensitive_urls',
+  high_entropy_string: 'entropy',
+  insecure_config: 'config_audit',
+  root_container: 'config_audit',
+  dangerous_file: 'file_flags',
+  ai_pattern: 'ai_comments',  // AI comment patterns detected in Layer 1
+}
+
+// ============================================================================
+// Layer 2 Detector Tier Mappings
+// ============================================================================
+
+/**
+ * Layer 2 detector names (internal identifiers matching detector function names)
+ */
+export type Layer2DetectorName =
+  | 'dangerous_functions'  // dangerous-functions.ts
+  | 'byok_patterns'        // byok-patterns.ts
+  | 'ai_execution_sinks'   // ai-execution-sinks.ts
+  | 'ai_agent_tools'       // ai-agent-tools.ts
+  | 'auth_antipatterns'    // auth-antipatterns.ts
+  | 'data_exposure'        // data-exposure.ts
+  | 'ai_fingerprinting'    // ai-fingerprinting.ts
+  | 'ai_prompt_hygiene'    // ai-prompt-hygiene.ts
+  | 'logic_gates'          // logic-gates.ts
+  | 'variables'            // variables.ts
+  | 'risky_imports'        // risky-imports.ts
+  | 'framework_checks'     // framework-checks.ts
+  // M5: New AI-era detectors
+  | 'ai_rag_safety'        // ai-rag-safety.ts - RAG data exfiltration
+  | 'ai_endpoint_protection' // ai-endpoint-protection.ts - Unprotected AI endpoints
+  | 'ai_schema_validation' // ai-schema-validation.ts - Schema mismatch
+
+/**
+ * Layer 2 tier assignments
+ *
+ * Tier A (core) - High-precision, high-risk, clear remediation:
+ *   - dangerous_functions: Classic injection footholds (eval, exec, unsafe SQL)
+ *   - byok_patterns: Storing/logging BYOK is critical AI-era customer trust risk
+ *   - ai_execution_sinks: Core to Oculum's AI story; LLM output as code/commands
+ *   - ai_agent_tools: Over-permissive tools are central AI alignment problem
+ *
+ * Tier B (ai_assisted) - Context-heavy, need AI validation:
+ *   - auth_antipatterns: Very context-dependent; needs middleware awareness
+ *   - data_exposure: Logging queries/user IDs is often acceptable but context-specific
+ *   - ai_fingerprinting: TypeScript 'any' at trust boundaries is risk indicator, not vuln
+ *   - ai_prompt_hygiene: Needs semantic understanding of prompt design
+ *
+ * Tier C (experimental) - High-noise, internal use only:
+ *   - logic_gates: Many hits with limited actionable signal
+ *   - variables: Generic token/password variable names - too generic
+ *   - risky_imports: Most imports are fine; keep tiny whitelist if proved valuable
+ *   - framework_checks: Many findings are style/low-risk; prone to noisy lint
+ */
+export const LAYER2_DETECTOR_TIERS: Record<Layer2DetectorName, DetectorTier> = {
+  // Tier A - Core SAST / AI-safety
+  dangerous_functions: 'core',
+  byok_patterns: 'core',
+  ai_execution_sinks: 'core',
+  ai_agent_tools: 'core',
+
+  // Tier B - AI-assisted heuristics
+  auth_antipatterns: 'ai_assisted',
+  data_exposure: 'ai_assisted',
+  ai_fingerprinting: 'ai_assisted',
+  ai_prompt_hygiene: 'ai_assisted',
+
+  // Tier C - Experimental / high-noise
+  logic_gates: 'experimental',
+  variables: 'experimental',
+  risky_imports: 'experimental',
+  framework_checks: 'experimental',
+
+  // M5: New AI-era detectors
+  ai_rag_safety: 'core',           // Tier A - Cross-tenant data access is critical
+  ai_endpoint_protection: 'core',  // Tier A - Cost abuse / API exposure has clear signals
+  ai_schema_validation: 'ai_assisted', // Tier B - Context-dependent, benefits from AI validation
+}
+
+/**
+ * Mapping from vulnerability category to Layer 2 detector name
+ * Used for tier lookups when we only have the category
+ *
+ * NOTE: Some categories are used by multiple detectors:
+ * - ai_pattern: used by ai-fingerprinting (Tier B), byok-patterns (Tier A), dangerous-functions (Tier A)
+ * - insecure_config: used by config-audit (L1 Tier B), framework-checks (L2 Tier C)
+ *
+ * For ambiguous categories, we use the most conservative (highest trust) tier mapping.
+ * When category alone isn't sufficient, the orchestrator can use detector-specific tracking.
+ */
+export const LAYER2_CATEGORY_TO_DETECTOR: Partial<Record<VulnerabilityCategory, Layer2DetectorName>> = {
+  // Tier A categories (unambiguous)
+  dangerous_function: 'dangerous_functions',
+  sql_injection: 'dangerous_functions',
+  command_injection: 'dangerous_functions',
+  ai_unsafe_execution: 'ai_execution_sinks',
+  ai_overpermissive_tool: 'ai_agent_tools',
+
+  // Tier B categories
+  missing_auth: 'auth_antipatterns',
+  data_exposure: 'data_exposure',
+  ai_prompt_injection: 'ai_prompt_hygiene',
+
+  // ai_pattern is ambiguous - used by multiple detectors with different tiers:
+  // - ai-fingerprinting (Tier B): TypeScript 'any' at boundaries
+  // - byok-patterns (Tier A): BYOK key handling
+  // - dangerous-functions (Tier A): JSON.parse related AI patterns
+  // Default to ai_fingerprinting (Tier B) since it's most common; byok/dangerous_functions
+  // findings are usually categorized differently or handled explicitly
+  ai_pattern: 'ai_fingerprinting',
+
+  // Tier C categories
+  security_bypass: 'logic_gates',
+  sensitive_variable: 'variables',
+  suspicious_package: 'risky_imports',
+  cors_misconfiguration: 'framework_checks',
+  // insecure_config from framework-checks is Tier C in Layer 2
+  // (but same category from config-audit is Tier B in Layer 1 - handled by layer check)
+  insecure_config: 'framework_checks',
+
+  // M5: New AI-era categories
+  ai_rag_exfiltration: 'ai_rag_safety',
+  ai_endpoint_unprotected: 'ai_endpoint_protection',
+  ai_schema_mismatch: 'ai_schema_validation',
+}
+
+// ============================================================================
+// Tier Lookup Helpers
+// ============================================================================
+
+/**
+ * Get the tier for a vulnerability based on its category and layer
+ */
+export function getTierForCategory(
+  category: VulnerabilityCategory,
+  layer: 1 | 2 | 3
+): DetectorTier {
+  if (layer === 1) {
+    const detector = LAYER1_CATEGORY_TO_DETECTOR[category]
+    if (detector) {
+      return LAYER1_DETECTOR_TIERS[detector]
+    }
+  } else if (layer === 2) {
+    const detector = LAYER2_CATEGORY_TO_DETECTOR[category]
+    if (detector) {
+      return LAYER2_DETECTOR_TIERS[detector]
+    }
+  }
+
+  // Layer 3 findings are always core (AI semantic analysis)
+  if (layer === 3) {
+    return 'core'
+  }
+
+  // Default to ai_assisted if unmapped (safe default - will go through AI validation)
+  return 'ai_assisted'
+}
+
+/**
+ * Get tier for a Layer 1 detector by name
+ */
+export function getLayer1DetectorTier(detector: Layer1DetectorName): DetectorTier {
+  return LAYER1_DETECTOR_TIERS[detector]
+}
+
+/**
+ * Get tier for a Layer 2 detector by name
+ */
+export function getLayer2DetectorTier(detector: Layer2DetectorName): DetectorTier {
+  return LAYER2_DETECTOR_TIERS[detector]
+}
+
+/**
+ * Check if a tier should be visible at a given scan depth
+ */
+export function isTierVisibleAtDepth(
+  tier: DetectorTier,
+  depth: 'cheap' | 'validated' | 'deep'
+): boolean {
+  switch (depth) {
+    case 'cheap':
+      // Only Tier A (core) findings are visible in cheap scans
+      return tier === 'core'
+    case 'validated':
+      // Tier A always visible, Tier B visible after AI validation
+      return tier === 'core' || tier === 'ai_assisted'
+    case 'deep':
+      // Same as validated for visibility (deep adds Layer 3, not more tiers)
+      return tier === 'core' || tier === 'ai_assisted'
+  }
+}
+
+/**
+ * Check if a tier should go through AI validation at a given scan depth
+ */
+export function shouldValidateWithAI(
+  tier: DetectorTier,
+  depth: 'cheap' | 'validated' | 'deep'
+): boolean {
+  // Cheap scans skip AI validation entirely
+  if (depth === 'cheap') {
+    return false
+  }
+
+  // In validated/deep, Tier B findings should go through AI validation
+  // Tier A is high-precision and doesn't need AI validation
+  // Tier C is hidden anyway
+  return tier === 'ai_assisted'
+}
+
+/**
+ * Compute tier statistics from an array of vulnerabilities
+ */
+export function computeTierStats(
+  vulnerabilities: Array<{ category: VulnerabilityCategory; layer: 1 | 2 | 3 }>
+): TierStats {
+  const stats: TierStats = {
+    core: 0,
+    ai_assisted: 0,
+    experimental: 0,
+  }
+
+  for (const vuln of vulnerabilities) {
+    const tier = getTierForCategory(vuln.category, vuln.layer)
+    stats[tier]++
+  }
+
+  return stats
+}
+
+/**
+ * Format tier stats for logging
+ */
+export function formatTierStats(stats: TierStats): string {
+  return `tiers={core:${stats.core},ai_assisted:${stats.ai_assisted},experimental:${stats.experimental}}`
+}