@oculum/scanner 1.0.0

package/src/__tests__/benchmark/tier-integration-script.ts

@@ -0,0 +1,177 @@
+/**
+ * Tier System Integration Test
+ *
+ * This test validates that the detector tier system correctly filters
+ * findings based on scan depth (cheap/validated/deep).
+ *
+ * Run with: npx tsx src/lib/scanner/__tests__/tier-integration.test.ts
+ */
+
+import { runLayer1Scan } from '../layer1'
+import { runLayer2Scan } from '../layer2'
+import type { ScanFile, Vulnerability } from '../types'
+import {
+  getTierForCategory,
+  computeTierStats,
+  formatTierStats,
+  LAYER1_DETECTOR_TIERS,
+  LAYER2_DETECTOR_TIERS,
+} from '../tiers'
+
+// Mock files with various vulnerability patterns
+const mockFiles: ScanFile[] = [
+  {
+    path: 'src/api/auth/route.ts',
+    content: `
+// Tier A: Known secret (patterns.ts)
+const API_KEY = "sk-ant-api03-abc123xyz456"
+
+// Tier A: Dangerous function (dangerous-functions.ts)
+const result = eval(userInput)
+
+// Tier B: High entropy (entropy.ts)
+const token = "a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6q7r8s9"
+
+// Tier C: Sensitive variable (variables.ts)
+const password = getPassword()
+
+// Tier C: Logic gate (logic-gates.ts)
+if (isAdmin || bypassAuth) {
+  allowAccess()
+}
+`,
+    language: 'typescript',
+    size: 500,
+  },
+  {
+    path: 'src/lib/ai/client.ts',
+    content: `
+// Tier A: AI execution sink (ai-execution-sinks.ts)
+const code = await openai.chat.completions.create({
+  model: "gpt-4",
+  messages: [{ role: "user", content: prompt }]
+})
+eval(code.choices[0].message.content)
+
+// Tier B: AI fingerprint (ai-fingerprinting.ts)
+function processData(data: any): any {
+  return data
+}
+
+// Tier A: BYOK pattern (byok-patterns.ts)
+const apiKey = request.headers.get('x-api-key')
+await client.chat({
+  apiKey: apiKey,
+  messages: []
+})
+`,
+    language: 'typescript',
+    size: 400,
+  },
+]
+
+async function runTest() {
+  console.log('='.repeat(60))
+  console.log('Tier System Integration Test')
+  console.log('='.repeat(60))
+
+  // Display tier mappings
+  console.log('\n📋 Layer 1 Detector Tiers:')
+  for (const [detector, tier] of Object.entries(LAYER1_DETECTOR_TIERS)) {
+    console.log(`  - ${detector}: ${tier}`)
+  }
+
+  console.log('\n📋 Layer 2 Detector Tiers:')
+  for (const [detector, tier] of Object.entries(LAYER2_DETECTOR_TIERS)) {
+    console.log(`  - ${detector}: ${tier}`)
+  }
+
+  // Run Layer 1
+  console.log('\n' + '='.repeat(60))
+  console.log('Running Layer 1 Scan...')
+  console.log('='.repeat(60))
+
+  const layer1Result = await runLayer1Scan(mockFiles)
+  console.log(`\nLayer 1 found ${layer1Result.vulnerabilities.length} vulnerabilities`)
+  console.log(`Tier stats: ${formatTierStats(layer1Result.stats.tiers)}`)
+
+  // Group Layer 1 findings by tier
+  const layer1ByTier = groupByTier(layer1Result.vulnerabilities)
+  console.log('\nLayer 1 by tier:')
+  console.log(`  - Core (A): ${layer1ByTier.core.length} findings`)
+  console.log(`  - AI-Assisted (B): ${layer1ByTier.ai_assisted.length} findings`)
+  console.log(`  - Experimental (C): ${layer1ByTier.experimental.length} findings`)
+
+  // Run Layer 2
+  console.log('\n' + '='.repeat(60))
+  console.log('Running Layer 2 Scan...')
+  console.log('='.repeat(60))
+
+  const layer2Result = await runLayer2Scan(mockFiles)
+  console.log(`\nLayer 2 found ${layer2Result.vulnerabilities.length} vulnerabilities`)
+  console.log(`Tier stats: ${formatTierStats(layer2Result.stats.tiers)}`)
+
+  // Group Layer 2 findings by tier
+  const layer2ByTier = groupByTier(layer2Result.vulnerabilities)
+  console.log('\nLayer 2 by tier:')
+  console.log(`  - Core (A): ${layer2ByTier.core.length} findings`)
+  console.log(`  - AI-Assisted (B): ${layer2ByTier.ai_assisted.length} findings`)
+  console.log(`  - Experimental (C): ${layer2ByTier.experimental.length} findings`)
+
+  // Print all findings with their tiers
+  console.log('\n' + '='.repeat(60))
+  console.log('All Findings with Tier Classification')
+  console.log('='.repeat(60))
+
+  const allFindings = [...layer1Result.vulnerabilities, ...layer2Result.vulnerabilities]
+  for (const finding of allFindings) {
+    const tier = getTierForCategory(finding.category, finding.layer)
+    console.log(`\n[${tier.toUpperCase()}] ${finding.title}`)
+    console.log(`  Category: ${finding.category}`)
+    console.log(`  Layer: ${finding.layer}`)
+    console.log(`  Severity: ${finding.severity}`)
+    console.log(`  File: ${finding.filePath}:${finding.lineNumber}`)
+  }
+
+  // Summary
+  console.log('\n' + '='.repeat(60))
+  console.log('Scan Depth Behavior Summary')
+  console.log('='.repeat(60))
+
+  const totalStats = computeTierStats(
+    allFindings.map(v => ({ category: v.category, layer: v.layer }))
+  )
+
+  console.log(`\nTotal findings: ${allFindings.length}`)
+  console.log(`  ${formatTierStats(totalStats)}`)
+
+  console.log('\nBehavior by scan depth:')
+  console.log(`  • cheap:     Would surface ${totalStats.core} findings (Tier A only)`)
+  console.log(`  • validated: Would surface ${totalStats.core} + up to ${totalStats.ai_assisted} (after AI validation)`)
+  console.log(`  • deep:      Same as validated + Layer 3 semantic analysis`)
+  console.log(`  • hidden:    ${totalStats.experimental} findings never surfaced (Tier C)`)
+
+  console.log('\n✅ Test completed successfully!')
+}
+
+function groupByTier(vulnerabilities: Vulnerability[]): {
+  core: Vulnerability[]
+  ai_assisted: Vulnerability[]
+  experimental: Vulnerability[]
+} {
+  const result = { core: [], ai_assisted: [], experimental: [] } as {
+    core: Vulnerability[]
+    ai_assisted: Vulnerability[]
+    experimental: Vulnerability[]
+  }
+
+  for (const vuln of vulnerabilities) {
+    const tier = getTierForCategory(vuln.category, vuln.layer)
+    result[tier].push(vuln)
+  }
+
+  return result
+}
+
+// Run the test
+runTest().catch(console.error)

package/src/__tests__/benchmark/types.ts

@@ -0,0 +1,144 @@
+/**
+ * Shared types for the security benchmark test suite
+ */
+
+import type { ScanFile, Vulnerability, VulnerabilityCategory, VulnerabilitySeverity } from '../../types'
+
+/**
+ * A test fixture containing a file to scan and expected results
+ */
+export interface TestFixture {
+  /** Descriptive name for the test */
+  name: string
+  /** The file to scan */
+  file: ScanFile
+  /** Whether findings are expected (true positive test) or not (false negative test) */
+  expectFindings: boolean
+  /** Categories expected to be found (for true positive tests) */
+  expectedCategories?: VulnerabilityCategory[]
+  /** Description of what this test validates */
+  description?: string
+  /**
+   * For false negative tests: Allow specific low-severity findings
+   * Some edge cases may generate info findings (e.g., test files with eval())
+   * By default, false negative tests fail on medium+ severity
+   */
+  allowedInfoFindings?: {
+    category: VulnerabilityCategory
+    maxCount: number
+    reason: string
+  }[]
+}
+
+/**
+ * Result from running a single test
+ */
+export interface TestResult {
+  name: string
+  file: ScanFile
+  layer1Findings: Vulnerability[]
+  layer2Findings: Vulnerability[]
+  expectedCategories: VulnerabilityCategory[]
+  unexpectedCategories: VulnerabilityCategory[]
+  passed: boolean
+  failureReason?: string
+  /** Findings that were unexpected but acceptable (info severity) */
+  acceptableInfoFindings?: Vulnerability[]
+  /** Findings that caused the test to fail (medium+) */
+  problematicFindings?: Vulnerability[]
+}
+
+/**
+ * A group of related test fixtures
+ */
+export interface TestGroup {
+  /** Name of the test group (e.g., "Hardcoded Secrets") */
+  name: string
+  /** Tier classification */
+  tier: 'A' | 'B' | 'C'
+  /** Layer this test group primarily targets */
+  layer: 1 | 2
+  /** Description of the test group */
+  description: string
+  /** True positive test fixtures */
+  truePositives: TestFixture[]
+  /** False negative test fixtures (should NOT flag) */
+  falseNegatives: TestFixture[]
+}
+
+/**
+ * Detector performance metrics
+ */
+export interface DetectorMetrics {
+  /** Detector/category name */
+  name: string
+  /** True positives detected */
+  truePositives: number
+  /** False positives (unexpected findings on safe code) */
+  falsePositives: number
+  /** False negatives (missed detections) */
+  falseNegatives: number
+  /** Precision: TP / (TP + FP) */
+  precision: number
+  /** Recall: TP / (TP + FN) */
+  recall: number
+}
+
+/**
+ * Severity distribution metrics
+ */
+export interface SeverityMetrics {
+  critical: number
+  high: number
+  medium: number
+  low: number
+  info: number
+}
+
+/**
+ * Detailed benchmark metrics
+ */
+export interface BenchmarkMetrics {
+  /** Overall detection statistics */
+  detection: {
+    totalVulnerabilitiesDetected: number
+    truePositiveDetections: number
+    falsePositiveDetections: number
+    missedVulnerabilities: number
+  }
+  /** Performance by detector/category */
+  byDetector: DetectorMetrics[]
+  /** Severity distribution of all findings */
+  severityDistribution: SeverityMetrics
+  /** Severity distribution of false positives */
+  falsePositiveSeverity: SeverityMetrics
+  /** Coverage statistics */
+  coverage: {
+    totalCategories: number
+    categoriesTested: number
+    coveragePercent: number
+    untestedCategories: string[]
+  }
+  /** Tier statistics */
+  byTier: {
+    tierA: { tested: number; passed: number }
+    tierB: { tested: number; passed: number }
+    tierC: { tested: number; passed: number }
+  }
+}
+
+/**
+ * Summary of benchmark results
+ */
+export interface BenchmarkSummary {
+  truePositivePassed: number
+  truePositiveFailed: number
+  falseNegativePassed: number
+  falseNegativeFailed: number
+  totalTests: number
+  passedTests: number
+  passRate: number
+  results: TestResult[]
+  /** Detailed performance metrics */
+  metrics: BenchmarkMetrics
+}