@oculum/scanner 1.0.0

package/src/__tests__/benchmark/utils/test-runner.ts

@@ -0,0 +1,475 @@
+/**
+ * Test runner utilities for the security benchmark suite
+ */
+
+import { runLayer1Scan } from '../../../layer1'
+import { runLayer2Scan } from '../../../layer2'
+import { computeTierStats, formatTierStats, getTierForCategory } from '../../../tiers'
+import type {
+  TestFixture,
+  TestResult,
+  TestGroup,
+  BenchmarkSummary,
+  BenchmarkMetrics,
+  DetectorMetrics,
+  SeverityMetrics
+} from '../types'
+import type { Vulnerability, VulnerabilityCategory } from '../../../types'
+
+/**
+ * Run a single test fixture
+ */
+export async function runTestFixture(fixture: TestFixture): Promise<TestResult> {
+  const layer1Result = await runLayer1Scan([fixture.file])
+  const layer2Result = await runLayer2Scan([fixture.file])
+
+  const allFindings = [...layer1Result.vulnerabilities, ...layer2Result.vulnerabilities]
+  const foundCategories = new Set(allFindings.map(f => f.category))
+
+  const unexpectedCategories: VulnerabilityCategory[] = []
+  const acceptableInfoFindings: Vulnerability[] = []
+  const problematicFindings: Vulnerability[] = []
+  let passed = true
+  let failureReason: string | undefined
+
+  if (fixture.expectFindings) {
+    // True positive test - should find vulnerabilities
+    if (allFindings.length === 0) {
+      passed = false
+      failureReason = 'Expected vulnerabilities but found none'
+    } else {
+      // Check we found expected categories
+      for (const cat of fixture.expectedCategories || []) {
+        if (!foundCategories.has(cat)) {
+          passed = false
+          failureReason = `Missing expected category: ${cat}`
+        }
+      }
+    }
+  } else {
+    // False negative test - should NOT flag actual vulnerabilities
+    // Default: Fail on medium+ severity (critical, high, medium)
+    // Optional: Allow specific info/low findings via allowedInfoFindings
+
+    const mediumOrHigher = allFindings.filter(f =>
+      f.severity === 'critical' || f.severity === 'high' || f.severity === 'medium'
+    )
+
+    // Check if medium+ findings exist
+    if (mediumOrHigher.length > 0) {
+      passed = false
+      failureReason = `Found ${mediumOrHigher.length} false positive(s) with medium+ severity`
+      problematicFindings.push(...mediumOrHigher)
+      for (const f of mediumOrHigher) {
+        unexpectedCategories.push(f.category)
+      }
+    }
+
+    // Check low-severity findings (low, info)
+    const lowSeverity = allFindings.filter(f =>
+      f.severity === 'low' || f.severity === 'info'
+    )
+
+    if (lowSeverity.length > 0) {
+      // Check if these are allowed
+      const allowedCategories = new Map(
+        (fixture.allowedInfoFindings || []).map(a => [a.category, a.maxCount])
+      )
+
+      for (const finding of lowSeverity) {
+        const allowedCount = allowedCategories.get(finding.category)
+
+        if (allowedCount === undefined) {
+          // Not allowed - this is a false positive
+          passed = false
+          if (!failureReason) {
+            failureReason = `Found unexpected ${finding.severity} finding: ${finding.category}`
+          }
+          problematicFindings.push(finding)
+          unexpectedCategories.push(finding.category)
+        } else {
+          // Allowed but track it
+          acceptableInfoFindings.push(finding)
+
+          // Check if we exceeded the allowed count
+          const countForCategory = lowSeverity.filter(f => f.category === finding.category).length
+          if (countForCategory > allowedCount) {
+            passed = false
+            failureReason = `Exceeded allowed count for ${finding.category}: ${countForCategory} > ${allowedCount}`
+            problematicFindings.push(finding)
+          }
+        }
+      }
+    }
+  }
+
+  return {
+    name: fixture.name,
+    file: fixture.file,
+    layer1Findings: layer1Result.vulnerabilities,
+    layer2Findings: layer2Result.vulnerabilities,
+    expectedCategories: fixture.expectedCategories || [],
+    unexpectedCategories,
+    passed,
+    failureReason,
+    acceptableInfoFindings,
+    problematicFindings,
+  }
+}
+
+/**
+ * Run all fixtures in a test group
+ */
+export async function runTestGroup(group: TestGroup): Promise<TestResult[]> {
+  const results: TestResult[] = []
+
+  // Run true positive tests
+  for (const fixture of group.truePositives) {
+    results.push(await runTestFixture(fixture))
+  }
+
+  // Run false negative tests
+  for (const fixture of group.falseNegatives) {
+    results.push(await runTestFixture(fixture))
+  }
+
+  return results
+}
+
+/**
+ * Print results for a single test
+ */
+export function printTestResult(result: TestResult): void {
+  const totalFindings = result.layer1Findings.length + result.layer2Findings.length
+
+  console.log(`\n📋 ${result.name}`)
+  console.log(`   File: ${result.file.path}`)
+  console.log(`   Layer 1 findings: ${result.layer1Findings.length}`)
+  console.log(`   Layer 2 findings: ${result.layer2Findings.length}`)
+
+  if (totalFindings > 0) {
+    const allFindings = [...result.layer1Findings, ...result.layer2Findings]
+    const byCategory: Record<string, number> = {}
+    const bySeverity: Record<string, number> = {}
+
+    for (const f of allFindings) {
+      byCategory[f.category] = (byCategory[f.category] || 0) + 1
+      bySeverity[f.severity] = (bySeverity[f.severity] || 0) + 1
+    }
+
+    console.log('   By category:', Object.entries(byCategory).map(([k, v]) => `${k}:${v}`).join(', '))
+    console.log('   By severity:', Object.entries(bySeverity).map(([k, v]) => `${k}:${v}`).join(', '))
+
+    // Tier breakdown
+    const tierStats = computeTierStats(allFindings.map(f => ({ category: f.category, layer: f.layer })))
+    console.log(`   ${formatTierStats(tierStats)}`)
+  }
+
+  if (result.passed) {
+    const isTPTest = result.expectedCategories.length > 0 || result.name.includes('True Positive')
+    if (isTPTest) {
+      console.log('   ✅ PASS: Detected expected vulnerabilities')
+    } else {
+      console.log('   ✅ PASS: No false positives')
+      if (result.acceptableInfoFindings && result.acceptableInfoFindings.length > 0) {
+        console.log(`   ℹ️  Note: ${result.acceptableInfoFindings.length} acceptable low-severity finding(s)`)
+      }
+    }
+  } else {
+    console.log(`   ❌ FAIL: ${result.failureReason}`)
+    if (result.problematicFindings && result.problematicFindings.length > 0) {
+      console.log('   Problematic findings:')
+      for (const f of result.problematicFindings) {
+        console.log(`      - ${f.category} (${f.severity}): ${f.title}`)
+      }
+    }
+  }
+}
+
+/**
+ * Compute detailed benchmark metrics
+ */
+export function computeMetrics(results: TestResult[]): BenchmarkMetrics {
+  // Collect all categories and detectors
+  const allCategories = new Set<VulnerabilityCategory>()
+  const detectorStats = new Map<string, { tp: number; fp: number; fn: number }>()
+
+  // Initialize severity distributions
+  const severityDistribution: SeverityMetrics = { critical: 0, high: 0, medium: 0, low: 0, info: 0 }
+  const falsePositiveSeverity: SeverityMetrics = { critical: 0, high: 0, medium: 0, low: 0, info: 0 }
+
+  let totalVulnerabilitiesDetected = 0
+  let truePositiveDetections = 0
+  let falsePositiveDetections = 0
+  let missedVulnerabilities = 0
+
+  // Tier tracking
+  const tierStats = {
+    tierA: { tested: 0, passed: 0 },
+    tierB: { tested: 0, passed: 0 },
+    tierC: { tested: 0, passed: 0 },
+  }
+
+  for (const result of results) {
+    const allFindings = [...result.layer1Findings, ...result.layer2Findings]
+    const isTPTest = result.expectedCategories.length > 0 || result.name.includes('True Positive')
+    const isFNTest = result.name.includes('False Negative') || result.name.includes('Should Not Flag')
+
+    // Track tier stats
+    for (const finding of allFindings) {
+      const tier = getTierForCategory(finding.category, finding.layer)
+      if (tier === 'core') {
+        tierStats.tierA.tested++
+        if (result.passed) tierStats.tierA.passed++
+      } else if (tier === 'ai_assisted') {
+        tierStats.tierB.tested++
+        if (result.passed) tierStats.tierB.passed++
+      } else if (tier === 'experimental') {
+        tierStats.tierC.tested++
+        if (result.passed) tierStats.tierC.passed++
+      }
+    }
+
+    // Process findings
+    for (const finding of allFindings) {
+      allCategories.add(finding.category)
+      totalVulnerabilitiesDetected++
+
+      // Update severity distribution
+      severityDistribution[finding.severity]++
+
+      // Initialize detector stats if not exists
+      if (!detectorStats.has(finding.category)) {
+        detectorStats.set(finding.category, { tp: 0, fp: 0, fn: 0 })
+      }
+      const stats = detectorStats.get(finding.category)!
+
+      if (isTPTest) {
+        // True positive test - findings are expected
+        if (result.expectedCategories?.includes(finding.category)) {
+          stats.tp++
+          truePositiveDetections++
+        }
+      } else if (isFNTest) {
+        // False negative test - findings are false positives
+        const isAcceptable = result.acceptableInfoFindings?.some(f => f.id === finding.id)
+        if (!isAcceptable) {
+          stats.fp++
+          falsePositiveDetections++
+          falsePositiveSeverity[finding.severity]++
+        }
+      }
+    }
+
+    // Track missed vulnerabilities (false negatives)
+    if (isTPTest && !result.passed) {
+      for (const expectedCat of result.expectedCategories || []) {
+        if (!detectorStats.has(expectedCat)) {
+          detectorStats.set(expectedCat, { tp: 0, fp: 0, fn: 0 })
+        }
+        const foundIt = allFindings.some(f => f.category === expectedCat)
+        if (!foundIt) {
+          detectorStats.get(expectedCat)!.fn++
+          missedVulnerabilities++
+        }
+      }
+    }
+  }
+
+  // Compute detector metrics
+  const byDetector: DetectorMetrics[] = Array.from(detectorStats.entries())
+    .map(([name, stats]) => {
+      const precision = stats.tp + stats.fp > 0 ? stats.tp / (stats.tp + stats.fp) : 0
+      const recall = stats.tp + stats.fn > 0 ? stats.tp / (stats.tp + stats.fn) : 0
+      return {
+        name,
+        truePositives: stats.tp,
+        falsePositives: stats.fp,
+        falseNegatives: stats.fn,
+        precision,
+        recall,
+      }
+    })
+    .sort((a, b) => b.falsePositives - a.falsePositives) // Sort by FP count
+
+  // Compute coverage
+  const knownCategories: VulnerabilityCategory[] = [
+    'hardcoded_secret',
+    'high_entropy_string',
+    'weak_crypto',
+    'sensitive_url',
+    'dangerous_function',
+    'missing_auth',
+    'data_exposure',
+    'ai_pattern',
+    'ai_prompt_injection',
+    'ai_unsafe_execution',
+    'ai_overpermissive_tool',
+    'insecure_config',
+    'suspicious_package',
+    'sensitive_variable',
+  ]
+  const categoriesTested = Array.from(allCategories).filter(c => knownCategories.includes(c)).length
+  const untestedCategories = knownCategories.filter(c => !allCategories.has(c))
+
+  return {
+    detection: {
+      totalVulnerabilitiesDetected,
+      truePositiveDetections,
+      falsePositiveDetections,
+      missedVulnerabilities,
+    },
+    byDetector,
+    severityDistribution,
+    falsePositiveSeverity,
+    coverage: {
+      totalCategories: knownCategories.length,
+      categoriesTested,
+      coveragePercent: (categoriesTested / knownCategories.length) * 100,
+      untestedCategories,
+    },
+    byTier: tierStats,
+  }
+}
+
+/**
+ * Compute benchmark summary from results
+ */
+export function computeSummary(results: TestResult[]): BenchmarkSummary {
+  let truePositivePassed = 0
+  let truePositiveFailed = 0
+  let falseNegativePassed = 0
+  let falseNegativeFailed = 0
+
+  for (const result of results) {
+    const isTPTest = result.expectedCategories.length > 0 ||
+                     result.name.includes('True Positive')
+    const isFNTest = result.name.includes('False Negative') ||
+                     result.name.includes('Should Not Flag')
+
+    if (isTPTest) {
+      if (result.passed) {
+        truePositivePassed++
+      } else {
+        truePositiveFailed++
+      }
+    } else if (isFNTest) {
+      if (result.passed) {
+        falseNegativePassed++
+      } else {
+        falseNegativeFailed++
+      }
+    }
+  }
+
+  const totalTests = truePositivePassed + truePositiveFailed + falseNegativePassed + falseNegativeFailed
+  const passedTests = truePositivePassed + falseNegativePassed
+  const passRate = totalTests > 0 ? (passedTests / totalTests) * 100 : 0
+
+  const metrics = computeMetrics(results)
+
+  return {
+    truePositivePassed,
+    truePositiveFailed,
+    falseNegativePassed,
+    falseNegativeFailed,
+    totalTests,
+    passedTests,
+    passRate,
+    results,
+    metrics,
+  }
+}
+
+/**
+ * Print detailed metrics
+ */
+export function printMetrics(metrics: BenchmarkMetrics): void {
+  console.log('\n' + '='.repeat(80))
+  console.log('DETAILED PERFORMANCE METRICS')
+  console.log('='.repeat(80))
+
+  // Overall detection stats
+  console.log('\n📊 Detection Statistics:')
+  console.log(`  Total findings:        ${metrics.detection.totalVulnerabilitiesDetected}`)
+  console.log(`  True positives:        ${metrics.detection.truePositiveDetections}`)
+  console.log(`  False positives:       ${metrics.detection.falsePositiveDetections}`)
+  console.log(`  Missed vulnerabilities: ${metrics.detection.missedVulnerabilities}`)
+
+  // Severity distribution
+  console.log('\n🎯 Severity Distribution (All Findings):')
+  console.log(`  Critical: ${metrics.severityDistribution.critical}`)
+  console.log(`  High:     ${metrics.severityDistribution.high}`)
+  console.log(`  Medium:   ${metrics.severityDistribution.medium}`)
+  console.log(`  Low:      ${metrics.severityDistribution.low}`)
+  console.log(`  Info:     ${metrics.severityDistribution.info}`)
+
+  if (metrics.detection.falsePositiveDetections > 0) {
+    console.log('\n⚠️  False Positive Severity Distribution:')
+    console.log(`  Critical: ${metrics.falsePositiveSeverity.critical}`)
+    console.log(`  High:     ${metrics.falsePositiveSeverity.high}`)
+    console.log(`  Medium:   ${metrics.falsePositiveSeverity.medium}`)
+    console.log(`  Low:      ${metrics.falsePositiveSeverity.low}`)
+    console.log(`  Info:     ${metrics.falsePositiveSeverity.info}`)
+  }
+
+  // Detector performance
+  if (metrics.byDetector.length > 0) {
+    console.log('\n🔍 Detector Performance (sorted by false positives):')
+    console.log('  Detector                    TP   FP   FN   Precision   Recall')
+    console.log('  ' + '─'.repeat(70))
+    for (const detector of metrics.byDetector) {
+      const name = detector.name.padEnd(25)
+      const tp = detector.truePositives.toString().padStart(4)
+      const fp = detector.falsePositives.toString().padStart(4)
+      const fn = detector.falseNegatives.toString().padStart(4)
+      const precision = (detector.precision * 100).toFixed(1).padStart(10) + '%'
+      const recall = (detector.recall * 100).toFixed(1).padStart(8) + '%'
+      console.log(`  ${name} ${tp}  ${fp}  ${fn}  ${precision}  ${recall}`)
+    }
+  }
+
+  // Coverage
+  console.log('\n📋 Category Coverage:')
+  console.log(`  Categories tested: ${metrics.coverage.categoriesTested}/${metrics.coverage.totalCategories} (${metrics.coverage.coveragePercent.toFixed(1)}%)`)
+  if (metrics.coverage.untestedCategories.length > 0) {
+    console.log('  Untested categories:', metrics.coverage.untestedCategories.join(', '))
+  }
+
+  // Tier stats
+  console.log('\n🎚️  Performance by Tier:')
+  console.log(`  Tier A (Core):        ${metrics.byTier.tierA.passed}/${metrics.byTier.tierA.tested} findings validated`)
+  console.log(`  Tier B (AI-Assisted): ${metrics.byTier.tierB.passed}/${metrics.byTier.tierB.tested} findings validated`)
+  console.log(`  Tier C (Experimental): ${metrics.byTier.tierC.passed}/${metrics.byTier.tierC.tested} findings validated`)
+}
+
+/**
+ * Print final benchmark summary
+ */
+export function printSummary(summary: BenchmarkSummary): void {
+  console.log('\n' + '='.repeat(80))
+  console.log('FINAL SCORE')
+  console.log('='.repeat(80))
+
+  console.log(`\n  True Positive Tests:  ${summary.truePositivePassed}/${summary.truePositivePassed + summary.truePositiveFailed} passed`)
+  console.log(`  False Negative Tests: ${summary.falseNegativePassed}/${summary.falseNegativePassed + summary.falseNegativeFailed} passed`)
+  console.log(`  ─────────────────────────────`)
+  console.log(`  Total:                ${summary.passedTests}/${summary.totalTests} (${summary.passRate.toFixed(1)}%)`)
+
+  if (summary.passRate === 100) {
+    console.log('\n  🎉 ALL TESTS PASSED!')
+  } else {
+    console.log('\n  ⚠️  Some tests need attention')
+
+    // Show failed tests
+    const failedTests = summary.results.filter(r => !r.passed)
+    if (failedTests.length > 0) {
+      console.log('\n  Failed tests:')
+      for (const test of failedTests) {
+        console.log(`    - ${test.name}: ${test.failureReason}`)
+      }
+    }
+  }
+
+  console.log('\n' + '='.repeat(80))
+}