@oculum/scanner 1.0.11 → 1.0.13

package/src/postprocess/dedup.ts

@@ -0,0 +1,62 @@
+/**
+ * Deduplication — Remove duplicate vulnerabilities.
+ *
+ * Handles standard dedup (same file, line, category) and cross-layer URL dedup
+ * (e.g., Layer 1 sensitive_url + Layer 2 ai_pattern on same line).
+ */
+
+import type { Vulnerability } from '../shared/types'
+import { severityRank, confidenceRank } from '../shared/parsed-file'
+
+/**
+ * Remove duplicate vulnerabilities (same file, line, category)
+ * Also handles cross-layer URL duplicates (sensitive_url + ai_pattern)
+ */
+export function deduplicateVulnerabilities(vulnerabilities: Vulnerability[]): Vulnerability[] {
+  const seen = new Map<string, Vulnerability>()
+  const urlDedupMap = new Map<string, Vulnerability>()
+
+  for (const vuln of vulnerabilities) {
+    // Special handling for URL duplicates across layers
+    // (e.g., Layer 1 detects as sensitive_url, Layer 2 detects as ai_pattern on same line)
+    // Route based on category, not lineContent regex — a sensitive_url finding is a URL
+    // finding regardless of what scheme appears in the line content.
+    if (vuln.category === 'sensitive_url' || vuln.category === 'ai_pattern') {
+
+      // Create compound key that ignores category differences for URLs
+      const urlKey = `${vuln.filePath}:${vuln.lineNumber}:url_finding`
+      const existing = urlDedupMap.get(urlKey)
+
+      if (!existing) {
+        urlDedupMap.set(urlKey, vuln)
+      } else {
+        // Keep Layer 1 (more specific) over Layer 2 AI pattern
+        // Or keep higher severity
+        if (vuln.layer < existing.layer ||
+            severityRank(vuln.severity) > severityRank(existing.severity)) {
+          urlDedupMap.set(urlKey, vuln)
+        }
+      }
+      continue
+    }
+
+    // Standard deduplication for non-URL findings
+    const key = `${vuln.filePath}:${vuln.lineNumber}:${vuln.category}`
+    const existing = seen.get(key)
+
+    // Keep the higher severity or higher confidence finding
+    if (!existing) {
+      seen.set(key, vuln)
+    } else if (severityRank(vuln.severity) > severityRank(existing.severity)) {
+      seen.set(key, vuln)
+    } else if (
+      severityRank(vuln.severity) === severityRank(existing.severity) &&
+      confidenceRank(vuln.confidence) > confidenceRank(existing.confidence)
+    ) {
+      seen.set(key, vuln)
+    }
+  }
+
+  // Combine URL and non-URL findings
+  return [...Array.from(seen.values()), ...Array.from(urlDedupMap.values())]
+}

package/src/postprocess/filtering/__tests__/pipeline.test.ts

@@ -0,0 +1,134 @@
+import { FilterPipeline } from '../pipeline'
+import type { FilterDecision } from '../pipeline'
+
+describe('FilterPipeline', () => {
+  describe('when disabled', () => {
+    it('does not record decisions', () => {
+      const pipeline = new FilterPipeline(false)
+      pipeline.record('finding-1', {
+        stage: 'auto_dismiss',
+        action: 'dismissed',
+        reason: 'test file',
+      })
+      expect(pipeline.getAuditTrail()).toEqual([])
+    })
+
+    it('reports isEnabled as false', () => {
+      expect(new FilterPipeline(false).isEnabled).toBe(false)
+      expect(new FilterPipeline().isEnabled).toBe(false)
+    })
+  })
+
+  describe('when enabled', () => {
+    let pipeline: FilterPipeline
+
+    beforeEach(() => {
+      pipeline = new FilterPipeline(true)
+    })
+
+    it('reports isEnabled as true', () => {
+      expect(pipeline.isEnabled).toBe(true)
+    })
+
+    it('records and retrieves decisions', () => {
+      const decision: FilterDecision = {
+        stage: 'auto_dismiss',
+        action: 'dismissed',
+        reason: 'test file detected',
+        ruleName: 'test_file_dismiss',
+      }
+      pipeline.record('finding-1', decision)
+
+      const trail = pipeline.getAuditTrail()
+      expect(trail).toHaveLength(1)
+      expect(trail[0].findingId).toBe('finding-1')
+      expect(trail[0].decisions).toEqual([decision])
+      expect(trail[0].finalStatus).toBe('dismissed')
+    })
+
+    it('accumulates multiple decisions for the same finding', () => {
+      pipeline.record('finding-1', {
+        stage: 'file_context_adjustment',
+        action: 'downgraded',
+        reason: 'tooling directory',
+        originalSeverity: 'medium',
+        newSeverity: 'info',
+      })
+      pipeline.record('finding-1', {
+        stage: 'ai_validation',
+        action: 'kept',
+        reason: 'AI confirmed the finding',
+      })
+
+      const decisions = pipeline.explainDismissal('finding-1')
+      expect(decisions).toHaveLength(2)
+      expect(decisions[0].stage).toBe('file_context_adjustment')
+      expect(decisions[1].stage).toBe('ai_validation')
+    })
+
+    it('marks final status as surfaced when no dismissal', () => {
+      pipeline.record('finding-1', {
+        stage: 'ai_validation',
+        action: 'kept',
+        reason: 'confirmed',
+      })
+
+      const trail = pipeline.getAuditTrail()
+      expect(trail[0].finalStatus).toBe('surfaced')
+    })
+
+    it('marks final status as dismissed for deduplicated findings', () => {
+      pipeline.record('finding-1', {
+        stage: 'deduplication',
+        action: 'deduplicated',
+        reason: 'duplicate of finding-2',
+      })
+
+      const trail = pipeline.getAuditTrail()
+      expect(trail[0].finalStatus).toBe('dismissed')
+    })
+
+    it('marks final status as dismissed for suppressed findings', () => {
+      pipeline.record('finding-1', {
+        stage: 'user_suppression',
+        action: 'suppressed',
+        reason: 'inline oculum-ignore comment',
+      })
+
+      const trail = pipeline.getAuditTrail()
+      expect(trail[0].finalStatus).toBe('dismissed')
+    })
+
+    it('returns empty array for unknown finding in explainDismissal', () => {
+      expect(pipeline.explainDismissal('nonexistent')).toEqual([])
+    })
+  })
+
+  describe('getSummaryStats', () => {
+    it('aggregates counts per stage and action', () => {
+      const pipeline = new FilterPipeline(true)
+
+      pipeline.record('f1', { stage: 'auto_dismiss', action: 'dismissed', reason: 'test' })
+      pipeline.record('f2', { stage: 'auto_dismiss', action: 'dismissed', reason: 'css' })
+      pipeline.record('f3', { stage: 'auto_dismiss', action: 'kept', reason: 'real finding' })
+      pipeline.record('f4', { stage: 'ai_validation', action: 'downgraded', reason: 'lower severity' })
+      pipeline.record('f5', { stage: 'deduplication', action: 'deduplicated', reason: 'dup' })
+
+      const stats = pipeline.getSummaryStats()
+
+      expect(stats.auto_dismiss.dismissed).toBe(2)
+      expect(stats.auto_dismiss.kept).toBe(1)
+      expect(stats.ai_validation.downgraded).toBe(1)
+      expect(stats.deduplication.deduplicated).toBe(1)
+      expect(stats.localhost_aggregation.dismissed).toBe(0)
+    })
+
+    it('returns zero counts when no decisions recorded', () => {
+      const pipeline = new FilterPipeline(true)
+      const stats = pipeline.getSummaryStats()
+
+      expect(stats.auto_dismiss.dismissed).toBe(0)
+      expect(stats.confidence_scoring.kept).toBe(0)
+    })
+  })
+})

package/src/postprocess/filtering/context-adjustments.ts

@@ -0,0 +1,111 @@
+/**
+ * Centralized Context-Based Severity Adjustments
+ *
+ * Consolidates the duplicate filter logic from:
+ * - layer2/index.ts applyFileContextAdjustments() — per-file with full FileContext
+ * - index.ts applyGlobalContextAdjustments() — post-layers with tooling-dir only
+ *
+ * Both functions are replaced by a single applyContextAdjustments() that handles
+ * all context-based severity downgrades in one place.
+ */
+
+import type { Vulnerability } from '../../shared/types'
+import type { FileContext } from '../../parse/file-classifier'
+
+/** Categories that are expected and not risky in tooling directories */
+const TOOLING_DOWNGRADABLE_CATEGORIES = [
+  'dangerous_function',   // exec, spawn, etc. are expected in scripts
+  'command_injection',    // child_process usage is expected in build scripts
+  'weak_crypto',          // Math.random() for seeding is fine in scripts
+  'data_exposure',        // Logging is expected in tooling
+  'ai_pattern',           // AI patterns in scripts are typically fine
+]
+
+/** Categories that are expected and not risky in test files */
+const TEST_DOWNGRADABLE_CATEGORIES = [
+  'ai_pattern',
+  'dangerous_function',
+  'sensitive_variable',
+  'data_exposure',
+]
+
+/**
+ * Apply context-based severity adjustments to findings.
+ *
+ * When called with a FileContext (from Layer 2 per-file processing), applies
+ * full context-aware rules: test files, tooling dirs, server-only files.
+ *
+ * When called without a FileContext (from the orchestrator for Layer 1 findings),
+ * applies only tooling-directory downgrades based on file path.
+ */
+export function applyContextAdjustments(
+  findings: Vulnerability[],
+  fileContext?: FileContext
+): Vulnerability[] {
+  return findings.map(finding => {
+    // Never downgrade critical findings
+    if (finding.severity === 'critical') {
+      return finding
+    }
+
+    if (fileContext) {
+      // Full per-file context available (Layer 2 path)
+
+      // Test files: Downgrade specific categories
+      if (fileContext.isTestFile) {
+        if (TEST_DOWNGRADABLE_CATEGORIES.includes(finding.category)) {
+          return {
+            ...finding,
+            severity: 'info' as const,
+            validationNotes: 'Downgraded: test/fixture file',
+          }
+        }
+      }
+
+      // Tooling directories: Downgrade expected patterns
+      if (fileContext.isToolingDir) {
+        if (TOOLING_DOWNGRADABLE_CATEGORIES.includes(finding.category)) {
+          return {
+            ...finding,
+            severity: 'info' as const,
+            validationNotes: 'Downgraded: tooling/scripts directory',
+          }
+        }
+      }
+
+      // Server-only files: Client-exposure concerns don't apply
+      if (fileContext.isServerOnly) {
+        if (finding.category === 'hardcoded_secret' &&
+            finding.title.toLowerCase().includes('service role')) {
+          return {
+            ...finding,
+            severity: 'info' as const,
+            validationNotes: 'Downgraded: server-only file (service role key expected)',
+          }
+        }
+      }
+    } else {
+      // No file context — apply path-based adjustments only (orchestrator path)
+      // This catches Layer 1 findings that didn't go through Layer 2's per-file adjustments
+      const inToolingDir = isToolingPath(finding.filePath)
+
+      if (inToolingDir && TOOLING_DOWNGRADABLE_CATEGORIES.includes(finding.category)) {
+        return {
+          ...finding,
+          severity: 'info' as const,
+          validationNotes: `${finding.validationNotes || ''} Downgraded: tooling/scripts directory`.trim(),
+        }
+      }
+    }
+
+    return finding
+  })
+}
+
+/**
+ * Tooling-path check for findings without FileContext.
+ * Uses the same regex as utils/context-helpers.ts isToolingDirectory().
+ */
+function isToolingPath(filePath: string): boolean {
+  return /\/(scripts?|cli|tools?|bin|devtools|build|tasks)\//i.test(filePath)
+}

package/src/postprocess/filtering/index.ts

@@ -0,0 +1,10 @@
+export {
+  FilterPipeline,
+  type FilterStage,
+  type FilterAction,
+  type FilterDecision,
+  type AnnotatedFinding,
+  type FilterStageSummary,
+} from './pipeline'
+
+export { applyContextAdjustments } from './context-adjustments'

package/src/postprocess/filtering/pipeline.ts

@@ -0,0 +1,130 @@
+/**
+ * FilterPipeline — Records every filter decision for auditability.
+ *
+ * Tracks which findings were dismissed, downgraded, aggregated, or deduplicated
+ * at each stage of the scanning pipeline. This enables answering
+ * "why was finding X suppressed?" without rerunning the scan.
+ *
+ * Zero overhead by default — only records when explicitly enabled.
+ */
+
+export type FilterStage =
+  | 'file_context_adjustment'
+  | 'localhost_aggregation'
+  | 'noisy_aggregation'
+  | 'auto_dismiss'
+  | 'confidence_scoring'
+  | 'ai_validation'
+  | 'global_context'
+  | 'contradiction_resolution'
+  | 'deduplication'
+  | 'user_suppression'
+
+export type FilterAction =
+  | 'kept'
+  | 'dismissed'
+  | 'downgraded'
+  | 'aggregated'
+  | 'deduplicated'
+  | 'suppressed'
+
+export interface FilterDecision {
+  stage: FilterStage
+  action: FilterAction
+  reason: string
+  originalSeverity?: string
+  newSeverity?: string
+  ruleName?: string
+}
+
+export interface AnnotatedFinding {
+  findingId: string
+  decisions: FilterDecision[]
+  finalStatus: 'surfaced' | 'dismissed'
+}
+
+export interface FilterStageSummary {
+  dismissed: number
+  downgraded: number
+  kept: number
+  aggregated: number
+  deduplicated: number
+  suppressed: number
+}
+
+export class FilterPipeline {
+  private decisions = new Map<string, FilterDecision[]>()
+  private readonly enabled: boolean
+
+  constructor(enabled: boolean = false) {
+    this.enabled = enabled
+  }
+
+  /** Record a filter decision for a finding. No-op if pipeline is disabled. */
+  record(findingId: string, decision: FilterDecision): void {
+    if (!this.enabled) return
+
+    const existing = this.decisions.get(findingId)
+    if (existing) {
+      existing.push(decision)
+    } else {
+      this.decisions.set(findingId, [decision])
+    }
+  }
+
+  /** Get the full audit trail for all findings. */
+  getAuditTrail(): AnnotatedFinding[] {
+    const trail: AnnotatedFinding[] = []
+
+    for (const [findingId, decisions] of this.decisions) {
+      const finalStatus = decisions.some(d =>
+        d.action === 'dismissed' || d.action === 'deduplicated' || d.action === 'suppressed'
+      ) ? 'dismissed' as const : 'surfaced' as const
+
+      trail.push({ findingId, decisions, finalStatus })
+    }
+
+    return trail
+  }
+
+  /** Get all decisions that led to a specific finding being filtered. */
+  explainDismissal(findingId: string): FilterDecision[] {
+    return this.decisions.get(findingId) ?? []
+  }
+
+  /** Get summary statistics per filter stage. */
+  getSummaryStats(): Record<FilterStage, FilterStageSummary> {
+    const stages: FilterStage[] = [
+      'file_context_adjustment',
+      'localhost_aggregation',
+      'noisy_aggregation',
+      'auto_dismiss',
+      'confidence_scoring',
+      'ai_validation',
+      'global_context',
+      'contradiction_resolution',
+      'deduplication',
+      'user_suppression',
+    ]
+
+    const stats = {} as Record<FilterStage, FilterStageSummary>
+    for (const stage of stages) {
+      stats[stage] = { dismissed: 0, downgraded: 0, kept: 0, aggregated: 0, deduplicated: 0, suppressed: 0 }
+    }
+
+    for (const decisions of this.decisions.values()) {
+      for (const d of decisions) {
+        if (stats[d.stage]) {
+          stats[d.stage][d.action]++
+        }
+      }
+    }
+
+    return stats
+  }
+
+  /** Whether this pipeline is actively recording. */
+  get isEnabled(): boolean {
+    return this.enabled
+  }
+}

package/src/postprocess/index.ts

@@ -0,0 +1,118 @@
+/**
+ * Postprocessing Pipeline — Runs the full postprocessing chain on findings.
+ *
+ * Dedup → context adjustments → contradiction resolution → suppression.
+ * Records audit decisions for each sub-step via FilterPipeline.
+ */
+
+import type { Vulnerability, ScanFile } from '../shared/types'
+import type { MiddlewareAuthConfig } from '../model/middleware-detector'
+import type { FilterPipeline } from './filtering/pipeline'
+import type { SuppressionResult } from './suppression/types'
+import { deduplicateVulnerabilities } from './dedup'
+import { applyContextAdjustments } from './filtering/context-adjustments'
+import { resolveContradictions } from './contradictions'
+import { SuppressionManager } from './suppression'
+
+export interface PostprocessInput {
+  /** Raw findings to postprocess */
+  findings: Vulnerability[]
+  /** Source files (for inline suppression parsing) */
+  files: ScanFile[]
+  /** Middleware config for contradiction resolution */
+  middlewareConfig?: MiddlewareAuthConfig
+  /** Project path for suppression config loading */
+  projectPath?: string
+  /** FilterPipeline for audit recording */
+  filterPipeline: FilterPipeline
+  /** Whether to show suppressed findings */
+  showSuppressed?: boolean
+}
+
+export interface PostprocessResult {
+  /** Findings after all postprocessing */
+  findings: Vulnerability[]
+  /** Suppression result (for stats and suppressed findings) */
+  suppressionResult: SuppressionResult
+}
+
+const fid = (v: Pick<Vulnerability, 'filePath' | 'lineNumber' | 'category'>) =>
+  `${v.filePath}:${v.lineNumber}:${v.category}`
+
+/**
+ * Run the full postprocessing chain on findings.
+ *
+ * Pipeline: dedup → context adjustments → contradiction resolution → suppression
+ */
+export function postprocessFindings(input: PostprocessInput): PostprocessResult {
+  const {
+    findings,
+    files,
+    middlewareConfig,
+    projectPath,
+    filterPipeline,
+  } = input
+
+  // 1. Deduplicate vulnerabilities
+  const uniqueVulnerabilities = deduplicateVulnerabilities(findings)
+  if (filterPipeline.isEnabled) {
+    const uniqueIds = new Set(uniqueVulnerabilities.map(fid))
+    for (const v of findings) {
+      if (!uniqueIds.has(fid(v))) {
+        filterPipeline.record(fid(v), { stage: 'deduplication', action: 'deduplicated', reason: 'Duplicate finding removed' })
+      }
+    }
+  }
+
+  // 2. Apply file context-based severity adjustments (Phase 2b)
+  const contextAdjustedVulnerabilities = applyContextAdjustments(uniqueVulnerabilities)
+  if (filterPipeline.isEnabled) {
+    for (let i = 0; i < uniqueVulnerabilities.length; i++) {
+      const before = uniqueVulnerabilities[i]
+      const after = contextAdjustedVulnerabilities[i]
+      if (before.severity !== after.severity) {
+        filterPipeline.record(fid(after), { stage: 'global_context', action: 'downgraded', reason: after.validationNotes || 'Context-based severity downgrade', originalSeverity: before.severity, newSeverity: after.severity })
+      }
+    }
+  }
+
+  // 3. Resolve contradictions
+  const resolvedVulnerabilities = resolveContradictions(contextAdjustedVulnerabilities, middlewareConfig)
+  if (filterPipeline.isEnabled) {
+    const resolvedIds = new Set(resolvedVulnerabilities.map(fid))
+    for (const v of contextAdjustedVulnerabilities) {
+      if (!resolvedIds.has(fid(v))) {
+        filterPipeline.record(fid(v), { stage: 'contradiction_resolution', action: 'dismissed', reason: 'Removed by contradiction resolution' })
+      }
+    }
+    for (let i = 0; i < contextAdjustedVulnerabilities.length; i++) {
+      const before = contextAdjustedVulnerabilities[i]
+      const after = resolvedVulnerabilities.find(v => fid(v) === fid(before))
+      if (after && before.severity !== after.severity) {
+        filterPipeline.record(fid(after), { stage: 'contradiction_resolution', action: 'downgraded', reason: 'Downgraded by contradiction resolution', originalSeverity: before.severity, newSeverity: after.severity })
+      }
+    }
+  }
+
+  // 4. Apply suppressions (inline comments + config file)
+  const resolvedProjectPath = projectPath || process.cwd()
+  const suppressionManager = new SuppressionManager({ projectPath: resolvedProjectPath })
+  const suppressionResult = suppressionManager.applySuppressions(resolvedVulnerabilities, files)
+
+  if (filterPipeline.isEnabled) {
+    for (const s of suppressionResult.suppressed) {
+      filterPipeline.record(fid(s.vulnerability), { stage: 'user_suppression', action: 'suppressed', reason: s.suppression.reason || `Suppressed by ${s.suppression.type}` })
+    }
+  }
+
+  return {
+    findings: suppressionResult.findings,
+    suppressionResult,
+  }
+}
+
+// Re-export sub-modules for direct access
+export { deduplicateVulnerabilities } from './dedup'
+export { aggregateNoisyFindings } from './aggregation'
+export { resolveContradictions } from './contradictions'
+export { capValidationCandidatesPerFile, MAX_VALIDATION_CANDIDATES_PER_FILE } from './validation-cap'

package/src/{suppression → postprocess/suppression}/config-loader.ts

@@ -15,7 +15,7 @@ import {
   SUPPRESSION_CONFIG_FILES,
   DEFAULT_SUPPRESSION_CONFIG,
 } from './types'
-import type { VulnerabilityCategory } from '../types'
+import type { VulnerabilityCategory } from '../../shared/types'
 
 /**
  * Result of loading suppression config

package/src/{suppression → postprocess/suppression}/hash.ts

@@ -4,7 +4,7 @@
  */
 
 import { createHash } from 'crypto'
-import type { Vulnerability } from '../types'
+import type { Vulnerability } from '../../shared/types'
 
 /**
  * Normalize a file path for hashing

package/src/{suppression → postprocess/suppression}/inline-parser.ts

@@ -4,7 +4,7 @@
  */
 
 import type { InlineSuppression } from './types'
-import type { VulnerabilityCategory } from '../types'
+import type { VulnerabilityCategory } from '../../shared/types'
 
 /**
  * Valid suppression comment patterns:

package/src/{suppression → postprocess/suppression}/manager.ts

@@ -4,7 +4,7 @@
  */
 
 import { minimatch } from 'minimatch'
-import type { Vulnerability, ScanFile } from '../types'
+import type { Vulnerability, ScanFile } from '../../shared/types'
 import type {
   SuppressionConfig,
   SuppressionMatch,

package/src/{suppression → postprocess/suppression}/types.ts

@@ -3,7 +3,7 @@
  * Defines types for the finding suppression/ignore system
  */
 
-import type { VulnerabilityCategory, VulnerabilitySeverity } from '../types'
+import type { VulnerabilityCategory, VulnerabilitySeverity } from '../../shared/types'
 
 /**
  * Suppression configuration file structure
@@ -131,7 +131,7 @@ export interface SuppressedVulnerability {
  */
 export interface SuppressionResult {
   /** Findings that passed through (not suppressed) */
-  findings: import('../types').Vulnerability[]
+  findings: import('../../shared/types').Vulnerability[]
   /** Findings that were suppressed */
   suppressed: SuppressedVulnerability[]
   /** Count of expired suppressions (findings reappear) */