@oculum/scanner 1.0.0

package/src/__tests__/benchmark/run-benchmark.ts

@@ -0,0 +1,144 @@
+/**
+ * Security Benchmark Test Runner
+ * 
+ * Comprehensive security benchmark suite for the Oculum scanner, covering:
+ * - Traditional OWASP-style vulnerabilities
+ * - AI-specific security issues (prompt injection, unsafe execution, BYOK, etc.)
+ * - False positive scenarios (patterns that should NOT be flagged)
+ *
+ * Run with: npx tsx src/lib/scanner/__tests__/benchmark/run-benchmark.ts
+ */
+
+import {
+  allTestGroups,
+  falsePositiveFixtures,
+  highEntropyTests,
+} from './fixtures'
+import {
+  runTestFixture,
+  printTestResult,
+  computeSummary,
+  printSummary,
+  printMetrics,
+} from './utils/test-runner'
+import type { TestResult } from './types'
+
+async function main() {
+  console.log('='.repeat(80))
+  console.log('OCULUM SECURITY BENCHMARK TEST SUITE')
+  console.log('='.repeat(80))
+  console.log('')
+
+  const results: TestResult[] = []
+
+  // ========================================
+  // LAYER 1 TESTS
+  // ========================================
+
+  console.log('\n' + '─'.repeat(80))
+  console.log('LAYER 1: SURFACE SCAN TESTS')
+  console.log('─'.repeat(80))
+
+  const layer1Groups = allTestGroups.filter(g => g.layer === 1)
+  
+  for (const group of layer1Groups) {
+    console.log(`\n📌 TIER ${group.tier} (${group.tier === 'A' ? 'Core' : 'AI-Assisted'}): ${group.name}`)
+    
+    if (group.truePositives.length > 0) {
+      console.log('  Testing TRUE POSITIVES...')
+      for (const fixture of group.truePositives) {
+        const result = await runTestFixture(fixture)
+        results.push(result)
+        printTestResult(result)
+      }
+    }
+
+    if (group.falseNegatives.length > 0) {
+      console.log('  Testing FALSE NEGATIVES (should NOT flag)...')
+      for (const fixture of group.falseNegatives) {
+        const result = await runTestFixture(fixture)
+        results.push(result)
+        printTestResult(result)
+      }
+    }
+  }
+
+  // High Entropy (Tier B)
+  console.log(`\n📌 TIER B (AI-Assisted): ${highEntropyTests.name}`)
+  console.log('  Testing TRUE POSITIVES...')
+  for (const fixture of highEntropyTests.truePositives) {
+    const result = await runTestFixture(fixture)
+    results.push(result)
+    printTestResult(result)
+  }
+
+  // ========================================
+  // LAYER 2 TESTS
+  // ========================================
+
+  console.log('\n' + '─'.repeat(80))
+  console.log('LAYER 2: STRUCTURAL SCAN TESTS')
+  console.log('─'.repeat(80))
+
+  const layer2Groups = allTestGroups.filter(g => g.layer === 2)
+  
+  for (const group of layer2Groups) {
+    console.log(`\n📌 TIER ${group.tier} (${group.tier === 'A' ? 'Core' : 'AI-Assisted'}): ${group.name}`)
+    
+    if (group.truePositives.length > 0) {
+      console.log('  Testing TRUE POSITIVES...')
+      for (const fixture of group.truePositives) {
+        const result = await runTestFixture(fixture)
+        results.push(result)
+        printTestResult(result)
+      }
+    }
+
+    if (group.falseNegatives.length > 0) {
+      console.log('  Testing FALSE NEGATIVES (should NOT flag)...')
+      for (const fixture of group.falseNegatives) {
+        const result = await runTestFixture(fixture)
+        results.push(result)
+        printTestResult(result)
+      }
+    }
+  }
+
+  // ========================================
+  // FALSE POSITIVE TESTS
+  // ========================================
+
+  console.log('\n' + '─'.repeat(80))
+  console.log('FALSE POSITIVE TESTS')
+  console.log('─'.repeat(80))
+
+  for (const fixture of falsePositiveFixtures) {
+    console.log(`\n📌 ${fixture.name}`)
+    const result = await runTestFixture(fixture)
+    results.push(result)
+    printTestResult(result)
+  }
+
+  // ========================================
+  // RESULTS SUMMARY
+  // ========================================
+
+  console.log('\n' + '='.repeat(80))
+  console.log('BENCHMARK RESULTS SUMMARY')
+  console.log('='.repeat(80))
+
+  // Compute summary with metrics
+  const summary = computeSummary(results)
+
+  // Print detailed metrics
+  printMetrics(summary.metrics)
+
+  // Print final summary
+  printSummary(summary)
+
+  // Exit with appropriate code
+  process.exit(summary.passRate === 100 ? 0 : 1)
+}
+
+// Run the test suite
+main().catch(console.error)

package/src/__tests__/benchmark/run-depth-validation.ts

@@ -0,0 +1,206 @@
+/**
+ * Scan Depth Profile Validation
+ * Validates that cheap/validated/deep modes work correctly
+ */
+
+import { runLayer1Scan } from '../../layer1'
+import { runLayer2Scan } from '../../layer2'
+import { getTierForCategory, isTierVisibleAtDepth, formatTierStats, computeTierStats } from '../../tiers'
+import type { ScanFile, ScanDepth, Vulnerability } from '../../types'
+
+// Test fixture with various vulnerability types
+const testFile: ScanFile = {
+  path: 'src/api/test-route.ts',
+  content: `
+// This file contains patterns from different tiers for testing
+
+// Tier A (core): Hardcoded secret - always visible
+const API_KEY = "sk_live_abc123def456ghi789jkl012mno345pqr678"
+
+// Tier A (core): Dangerous function - always visible
+export function executeQuery(input: string) {
+  return db.query(\`SELECT * FROM users WHERE id = '\${input}'\`)
+}
+
+// Tier B (ai_assisted): AI pattern - needs validation in validated/deep modes
+// TODO: This was AI generated and needs review
+const placeholder = "implementation goes here"
+
+// Tier C (experimental): Should always be hidden
+const suspiciousVar = "possibly_risky"
+
+export async function GET() {
+  // Missing auth pattern (Tier B)
+  const data = await db.query('SELECT * FROM data')
+  return NextResponse.json(data)
+}
+`,
+  language: 'typescript',
+  size: 700,
+}
+
+// Helper to classify findings by tier
+function classifyFindings(findings: Vulnerability[]): {
+  core: Vulnerability[]
+  ai_assisted: Vulnerability[]
+  experimental: Vulnerability[]
+} {
+  const result: {
+    core: Vulnerability[]
+    ai_assisted: Vulnerability[]
+    experimental: Vulnerability[]
+  } = { core: [], ai_assisted: [], experimental: [] }
+
+  for (const f of findings) {
+    const tier = getTierForCategory(f.category, f.layer)
+    result[tier].push(f)
+  }
+
+  return result
+}
+
+async function runValidation() {
+  console.log('=' .repeat(70))
+  console.log('SCAN DEPTH PROFILE VALIDATION')
+  console.log('=' .repeat(70))
+
+  // Run scans
+  console.log('\n📋 Running Layer 1 + Layer 2 scans...')
+  const startScan = Date.now()
+  const l1 = await runLayer1Scan([testFile])
+  const l2 = await runLayer2Scan([testFile])
+  const scanDuration = Date.now() - startScan
+
+  const allFindings = [...l1.vulnerabilities, ...l2.vulnerabilities]
+  const classified = classifyFindings(allFindings)
+
+  console.log(`\n⏱️  Scan duration: ${scanDuration}ms`)
+  console.log(`\n📊 Total findings: ${allFindings.length}`)
+  console.log(`   Tier A (core): ${classified.core.length}`)
+  console.log(`   Tier B (ai_assisted): ${classified.ai_assisted.length}`)
+  console.log(`   Tier C (experimental): ${classified.experimental.length}`)
+
+  // Show tier stats
+  const tierStats = computeTierStats(
+    allFindings.map(v => ({ category: v.category, layer: v.layer }))
+  )
+  console.log(`\n📈 ${formatTierStats(tierStats)}`)
+
+  // Test each depth mode
+  const depths: ScanDepth[] = ['cheap', 'validated', 'deep']
+
+  console.log('\n' + '-'.repeat(70))
+  console.log('DEPTH MODE BEHAVIOR')
+  console.log('-'.repeat(70))
+
+  for (const depth of depths) {
+    console.log(`\n🎚️  ${depth.toUpperCase()} mode:`)
+
+    const coreVisible = isTierVisibleAtDepth('core', depth)
+    const aiAssistedVisible = isTierVisibleAtDepth('ai_assisted', depth)
+    const experimentalVisible = isTierVisibleAtDepth('experimental', depth)
+
+    console.log(`   Tier A (core): ${coreVisible ? '✅ VISIBLE' : '❌ hidden'}`)
+    console.log(`   Tier B (ai_assisted): ${aiAssistedVisible ? '✅ VISIBLE (via AI validation)' : '❌ hidden'}`)
+    console.log(`   Tier C (experimental): ${experimentalVisible ? '✅ VISIBLE' : '❌ hidden'}`)
+
+    // Calculate what would be surfaced
+    let surfaced = 0
+    let validated = 0
+    let hidden = 0
+
+    if (coreVisible) surfaced += classified.core.length
+    else hidden += classified.core.length
+
+    if (aiAssistedVisible) validated += classified.ai_assisted.length
+    else hidden += classified.ai_assisted.length
+
+    hidden += classified.experimental.length // Always hidden
+
+    console.log(`   → ${surfaced} surfaced directly, ${validated} to AI validation, ${hidden} hidden`)
+
+    // Show expected behavior
+    switch (depth) {
+      case 'cheap':
+        console.log(`   → AI validation: SKIPPED`)
+        console.log(`   → Layer 3: SKIPPED`)
+        console.log(`   → Target time: <5s for typical PRs`)
+        break
+      case 'validated':
+        console.log(`   → AI validation: ENABLED`)
+        console.log(`   → Layer 3: SKIPPED`)
+        console.log(`   → Target time: <15s with <10 AI calls`)
+        break
+      case 'deep':
+        console.log(`   → AI validation: ENABLED`)
+        console.log(`   → Layer 3: ENABLED`)
+        console.log(`   → Target time: Thorough analysis`)
+        break
+    }
+  }
+
+  // Performance test
+  console.log('\n' + '-'.repeat(70))
+  console.log('PERFORMANCE VALIDATION')
+  console.log('-'.repeat(70))
+
+  // Test with 10 files
+  const files10 = Array.from({ length: 10 }, (_, i) => ({
+    ...testFile,
+    path: `src/api/test-route-${i}.ts`,
+  }))
+
+  console.log('\n⏱️  Testing with 10 files (cheap mode, no AI)...')
+  const start10 = Date.now()
+  await runLayer1Scan(files10)
+  await runLayer2Scan(files10)
+  const duration10 = Date.now() - start10
+  console.log(`   Duration: ${duration10}ms`)
+  console.log(`   ${duration10 < 5000 ? '✅' : '❌'} Target: <5000ms`)
+
+  // Test with 50 files
+  const files50 = Array.from({ length: 50 }, (_, i) => ({
+    ...testFile,
+    path: `src/api/test-route-${i}.ts`,
+  }))
+
+  console.log('\n⏱️  Testing with 50 files (cheap mode, no AI)...')
+  const start50 = Date.now()
+  await runLayer1Scan(files50)
+  await runLayer2Scan(files50)
+  const duration50 = Date.now() - start50
+  console.log(`   Duration: ${duration50}ms`)
+  console.log(`   ${duration50 < 10000 ? '✅' : '❌'} Target: <10000ms`)
+  console.log(`   Rate: ${Math.round(50000 / duration50)} files/sec`)
+
+  // Summary
+  console.log('\n' + '=' .repeat(70))
+  console.log('VALIDATION SUMMARY')
+  console.log('=' .repeat(70))
+
+  const allPassed =
+    classified.core.length > 0 &&  // Should have some Tier A findings
+    duration10 < 5000 &&           // 10 files under 5s
+    duration50 < 10000             // 50 files under 10s
+
+  if (allPassed) {
+    console.log('\n✅ ALL DEPTH PROFILE VALIDATIONS PASSED')
+  } else {
+    console.log('\n❌ SOME VALIDATIONS FAILED')
+  }
+
+  console.log('\nWorkflow Profile Recommendations:')
+  console.log('┌─────────────────┬──────────────┬────────────────────────────────────┐')
+  console.log('│ Workflow        │ Default Depth│ Rationale                          │')
+  console.log('├─────────────────┼──────────────┼────────────────────────────────────┤')
+  console.log('│ GitHub PR       │ cheap        │ Fast feedback, high-signal findings│')
+  console.log('│ VS Code         │ validated    │ Interactive, balance depth + speed │')
+  console.log('│ CLI (default)   │ cheap        │ Fast local scans                   │')
+  console.log('│ CLI --deep      │ deep         │ Thorough analysis when requested   │')
+  console.log('│ Onboarding      │ deep         │ Full picture on first scan         │')
+  console.log('└─────────────────┴──────────────┴────────────────────────────────────┘')
+
+  console.log('\n')
+}
+
+runValidation().catch(console.error)

package/src/__tests__/benchmark/run-real-world-test.ts

@@ -0,0 +1,243 @@
+/**
+ * Real-World Testing Utility
+ * Tests the scanner on actual code patterns and measures performance
+ */
+
+import * as fs from 'fs'
+import * as path from 'path'
+import { runLayer1Scan } from '../../layer1'
+import { runLayer2Scan } from '../../layer2'
+import { formatTierStats, computeTierStats } from '../../tiers'
+import { formatTerminalOutput } from '../../formatters/cli-terminal'
+import type { ScanFile, Vulnerability, ScanResult } from '../../types'
+
+/**
+ * Test result for a directory scan
+ */
+interface RealWorldTestResult {
+  directory: string
+  filesScanned: number
+  totalFindings: number
+  bySeverity: Record<string, number>
+  byCategory: Record<string, number>
+  scanDuration: number
+  filesPerSecond: number
+  topFindings: Vulnerability[]
+  potentialFalsePositives: Vulnerability[]
+}
+
+/**
+ * Scan a local directory and analyze results
+ */
+async function scanDirectory(dirPath: string, maxFiles: number = 100): Promise<RealWorldTestResult> {
+  const startTime = Date.now()
+
+  // Collect files
+  const files: ScanFile[] = []
+  const extensions = new Set(['.ts', '.tsx', '.js', '.jsx', '.py', '.go', '.java', '.rb', '.php'])
+
+  function collectFiles(dir: string, depth: number = 0) {
+    if (depth > 10 || files.length >= maxFiles) return
+
+    try {
+      const entries = fs.readdirSync(dir, { withFileTypes: true })
+
+      for (const entry of entries) {
+        if (files.length >= maxFiles) break
+
+        const fullPath = path.join(dir, entry.name)
+
+        // Skip common non-source directories
+        if (entry.isDirectory()) {
+          if (['node_modules', '.git', 'dist', 'build', '.next', '__pycache__', 'venv'].includes(entry.name)) {
+            continue
+          }
+          collectFiles(fullPath, depth + 1)
+        } else if (entry.isFile()) {
+          const ext = path.extname(entry.name).toLowerCase()
+          if (extensions.has(ext)) {
+            try {
+              const content = fs.readFileSync(fullPath, 'utf-8')
+              if (content.length < 50000) { // Skip very large files
+                files.push({
+                  path: fullPath.replace(dirPath + '/', ''),
+                  content,
+                  language: ext.slice(1),
+                  size: content.length,
+                })
+              }
+            } catch {
+              // Skip unreadable files
+            }
+          }
+        }
+      }
+    } catch {
+      // Skip unreadable directories
+    }
+  }
+
+  collectFiles(dirPath)
+
+  if (files.length === 0) {
+    console.log(`No scannable files found in ${dirPath}`)
+    return {
+      directory: dirPath,
+      filesScanned: 0,
+      totalFindings: 0,
+      bySeverity: {},
+      byCategory: {},
+      scanDuration: 0,
+      filesPerSecond: 0,
+      topFindings: [],
+      potentialFalsePositives: [],
+    }
+  }
+
+  console.log(`Scanning ${files.length} files from ${dirPath}...`)
+
+  // Run scans
+  const layer1Result = await runLayer1Scan(files)
+  const layer2Result = await runLayer2Scan(files)
+
+  const scanDuration = Date.now() - startTime
+  const allFindings = [...layer1Result.vulnerabilities, ...layer2Result.vulnerabilities]
+
+  // Analyze results
+  const bySeverity: Record<string, number> = {
+    critical: 0,
+    high: 0,
+    medium: 0,
+    low: 0,
+    info: 0,
+  }
+
+  const byCategory: Record<string, number> = {}
+
+  for (const finding of allFindings) {
+    bySeverity[finding.severity] = (bySeverity[finding.severity] || 0) + 1
+    byCategory[finding.category] = (byCategory[finding.category] || 0) + 1
+  }
+
+  // Identify potential false positives (info/low in test files, or common patterns)
+  const potentialFalsePositives = allFindings.filter(f => {
+    // Test files with any findings
+    if (f.filePath.includes('test') || f.filePath.includes('spec')) return true
+    // Example/fixture files
+    if (f.filePath.includes('example') || f.filePath.includes('fixture')) return true
+    // Mock files
+    if (f.filePath.includes('mock') || f.filePath.includes('__mocks__')) return true
+    return false
+  })
+
+  // Sort findings by severity for top findings
+  const severityOrder = { critical: 5, high: 4, medium: 3, low: 2, info: 1 }
+  const sortedFindings = [...allFindings].sort(
+    (a, b) => severityOrder[b.severity] - severityOrder[a.severity]
+  )
+
+  return {
+    directory: dirPath,
+    filesScanned: files.length,
+    totalFindings: allFindings.length,
+    bySeverity,
+    byCategory,
+    scanDuration,
+    filesPerSecond: Math.round((files.length * 1000) / scanDuration),
+    topFindings: sortedFindings.slice(0, 10),
+    potentialFalsePositives,
+  }
+}
+
+/**
+ * Print test result summary
+ */
+function printResult(result: RealWorldTestResult) {
+  console.log('\n' + '='.repeat(70))
+  console.log(`REAL-WORLD TEST: ${result.directory}`)
+  console.log('='.repeat(70))
+
+  console.log(`\n📁 Files scanned: ${result.filesScanned}`)
+  console.log(`⏱️  Scan duration: ${result.scanDuration}ms`)
+  console.log(`🚀 Performance: ${result.filesPerSecond} files/sec`)
+
+  console.log(`\n📊 Total findings: ${result.totalFindings}`)
+  console.log('By severity:')
+  for (const [severity, count] of Object.entries(result.bySeverity)) {
+    if (count > 0) console.log(`  ${severity}: ${count}`)
+  }
+
+  console.log('\nBy category:')
+  const sortedCategories = Object.entries(result.byCategory)
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 10)
+  for (const [category, count] of sortedCategories) {
+    console.log(`  ${category}: ${count}`)
+  }
+
+  if (result.potentialFalsePositives.length > 0) {
+    console.log(`\n⚠️  Potential false positives: ${result.potentialFalsePositives.length}`)
+    console.log('(Findings in test/example/mock files)')
+  }
+
+  if (result.topFindings.length > 0) {
+    console.log('\n🔝 Top findings:')
+    for (const finding of result.topFindings.slice(0, 5)) {
+      console.log(`  [${finding.severity.toUpperCase()}] ${finding.category}: ${finding.title}`)
+      console.log(`    ${finding.filePath}:${finding.lineNumber}`)
+    }
+  }
+
+  // Assessment
+  console.log('\n' + '-'.repeat(70))
+  console.log('ASSESSMENT')
+  console.log('-'.repeat(70))
+
+  const fpRate = result.totalFindings > 0
+    ? (result.potentialFalsePositives.length / result.totalFindings * 100).toFixed(1)
+    : '0'
+
+  console.log(`False positive rate (test/mock files): ${fpRate}%`)
+  console.log(`Performance target (<5s for 100 files): ${result.scanDuration < 5000 ? '✅ PASS' : '❌ FAIL'}`)
+
+  const blockingCount = result.bySeverity.critical + result.bySeverity.high
+  if (blockingCount > 0) {
+    console.log(`\n⚠️  ${blockingCount} blocking issues found - review recommended`)
+  } else {
+    console.log('\n✅ No blocking issues found')
+  }
+}
+
+/**
+ * Run real-world tests
+ */
+async function main() {
+  console.log('=' .repeat(70))
+  console.log('OCULUM REAL-WORLD TESTING')
+  console.log('=' .repeat(70))
+
+  // Test on the scanner's own codebase
+  const scannerPath = path.resolve(__dirname, '../..')
+
+  console.log('\n📂 Testing on scanner codebase itself...')
+  const result = await scanDirectory(scannerPath, 50)
+  printResult(result)
+
+  // Summary
+  console.log('\n' + '='.repeat(70))
+  console.log('TESTING COMPLETE')
+  console.log('='.repeat(70))
+
+  console.log('\nTo test on other directories:')
+  console.log('  npx tsx src/lib/scanner/__tests__/benchmark/run-real-world-test.ts /path/to/repo')
+}
+
+// Allow testing specific directories via command line
+const targetDir = process.argv[2]
+if (targetDir) {
+  scanDirectory(path.resolve(targetDir), 100)
+    .then(printResult)
+    .catch(console.error)
+} else {
+  main().catch(console.error)
+}