@oculum/scanner 1.0.0

package/src/__tests__/validation/analyze-results.ts

@@ -0,0 +1,542 @@
+#!/usr/bin/env npx tsx
+/**
+ * M7: Results Analysis Script
+ *
+ * Analyzes scan results from real-repo validation and generates
+ * summary metrics for documentation.
+ *
+ * Usage:
+ *   npx tsx packages/scanner/src/__tests__/validation/analyze-results.ts
+ *   npx tsx packages/scanner/src/__tests__/validation/analyze-results.ts --output docs/RESULTSCOMPARISON.md
+ */
+
+import * as fs from 'fs'
+import * as path from 'path'
+import type { ScanResult, Vulnerability, VulnerabilityCategory, VulnerabilitySeverity } from '../../types'
+
+const RESULTS_DIR = path.join(__dirname, '../../../validation-results')
+
+// M5 AI-era categories we're specifically validating
+const M5_CATEGORIES: VulnerabilityCategory[] = [
+  'ai_rag_exfiltration',
+  'ai_endpoint_unprotected',
+  'ai_schema_mismatch',
+  'ai_prompt_injection',
+  'ai_unsafe_execution',
+  'ai_overpermissive_tool',
+]
+
+interface FindingSummary {
+  category: VulnerabilityCategory
+  count: number
+  bySeverity: Record<VulnerabilitySeverity, number>
+  examples: Array<{
+    file: string
+    line: number
+    title: string
+    severity: VulnerabilitySeverity
+  }>
+}
+
+interface RepoAnalysis {
+  repoName: string
+  scanDepth: string
+  totalFindings: number
+  mediumPlusFindings: number
+  filesScanned: number
+  scanDuration: number
+  bySeverity: Record<VulnerabilitySeverity, number>
+  byCategory: Record<string, number>
+  m5Findings: FindingSummary[]
+  topCategories: Array<{ category: string; count: number }>
+  libraryCodeFindings: number
+  exampleCodeFindings: number
+  testCodeFindings: number
+}
+
+interface ValidationReport {
+  generatedAt: string
+  totalRepos: number
+  totalScans: number
+  analyses: RepoAnalysis[]
+  aggregateMetrics: {
+    totalFindings: number
+    totalMediumPlus: number
+    avgFindingsPerRepo: number
+    m5DetectorPerformance: Record<string, { total: number; mediumPlus: number }>
+  }
+}
+
+/**
+ * Classify file path as library internal, example, or test code
+ */
+function classifyFilePath(filePath: string): 'library' | 'example' | 'test' | 'other' {
+  const lowerPath = filePath.toLowerCase()
+
+  // Test files
+  if (
+    lowerPath.includes('__tests__') ||
+    lowerPath.includes('/test/') ||
+    lowerPath.includes('/tests/') ||
+    lowerPath.includes('.test.') ||
+    lowerPath.includes('.spec.') ||
+    lowerPath.includes('_test.') ||
+    lowerPath.includes('/fixtures/')
+  ) {
+    return 'test'
+  }
+
+  // Example/demo files
+  if (
+    lowerPath.includes('/examples/') ||
+    lowerPath.includes('/example/') ||
+    lowerPath.includes('/demos/') ||
+    lowerPath.includes('/demo/') ||
+    lowerPath.includes('/templates/') ||
+    lowerPath.includes('/cookbook/')
+  ) {
+    return 'example'
+  }
+
+  // Library internal code
+  if (
+    lowerPath.includes('/src/') ||
+    lowerPath.includes('/lib/') ||
+    lowerPath.includes('/libs/') ||
+    lowerPath.includes('/packages/')
+  ) {
+    return 'library'
+  }
+
+  return 'other'
+}
+
+/**
+ * Load scan results from JSON files
+ */
+function loadResults(): Map<string, ScanResult> {
+  const results = new Map<string, ScanResult>()
+
+  if (!fs.existsSync(RESULTS_DIR)) {
+    console.error(`Results directory not found: ${RESULTS_DIR}`)
+    console.error('Run run-validation.ts first to generate results.')
+    process.exit(1)
+  }
+
+  const files = fs.readdirSync(RESULTS_DIR).filter(f =>
+    f.endsWith('.json') && !f.includes('medium-plus')
+  )
+
+  if (files.length === 0) {
+    console.error('No result files found in', RESULTS_DIR)
+    console.error('Run run-validation.ts first to generate results.')
+    process.exit(1)
+  }
+
+  for (const file of files) {
+    const filePath = path.join(RESULTS_DIR, file)
+    const content = fs.readFileSync(filePath, 'utf-8')
+    const result = JSON.parse(content) as ScanResult
+    const key = file.replace('.json', '')
+    results.set(key, result)
+  }
+
+  return results
+}
+
+/**
+ * Analyze a single scan result
+ */
+function analyzeResult(key: string, result: ScanResult): RepoAnalysis {
+  const [repoName, scanDepth] = key.split('-')
+
+  // Severity distribution
+  const bySeverity: Record<VulnerabilitySeverity, number> = {
+    critical: 0,
+    high: 0,
+    medium: 0,
+    low: 0,
+    info: 0,
+  }
+
+  // Category distribution
+  const byCategory: Record<string, number> = {}
+
+  // Code type distribution
+  let libraryCodeFindings = 0
+  let exampleCodeFindings = 0
+  let testCodeFindings = 0
+
+  // M5 detector findings
+  const m5FindingsMap = new Map<VulnerabilityCategory, Vulnerability[]>()
+  for (const cat of M5_CATEGORIES) {
+    m5FindingsMap.set(cat, [])
+  }
+
+  // Process each finding
+  for (const vuln of result.vulnerabilities) {
+    // Severity
+    bySeverity[vuln.severity]++
+
+    // Category
+    byCategory[vuln.category] = (byCategory[vuln.category] || 0) + 1
+
+    // Code type
+    const codeType = classifyFilePath(vuln.filePath)
+    if (codeType === 'library') libraryCodeFindings++
+    else if (codeType === 'example') exampleCodeFindings++
+    else if (codeType === 'test') testCodeFindings++
+
+    // M5 categories
+    if (M5_CATEGORIES.includes(vuln.category as VulnerabilityCategory)) {
+      m5FindingsMap.get(vuln.category as VulnerabilityCategory)!.push(vuln)
+    }
+  }
+
+  // Build M5 findings summary
+  const m5Findings: FindingSummary[] = []
+  for (const [category, findings] of m5FindingsMap) {
+    if (findings.length === 0) continue
+
+    const severityBreakdown: Record<VulnerabilitySeverity, number> = {
+      critical: 0,
+      high: 0,
+      medium: 0,
+      low: 0,
+      info: 0,
+    }
+
+    for (const f of findings) {
+      severityBreakdown[f.severity]++
+    }
+
+    // Get top 5 examples
+    const examples = findings.slice(0, 5).map(f => ({
+      file: f.filePath,
+      line: f.lineNumber,
+      title: f.title,
+      severity: f.severity,
+    }))
+
+    m5Findings.push({
+      category,
+      count: findings.length,
+      bySeverity: severityBreakdown,
+      examples,
+    })
+  }
+
+  // Sort m5Findings by count
+  m5Findings.sort((a, b) => b.count - a.count)
+
+  // Top categories
+  const topCategories = Object.entries(byCategory)
+    .sort(([, a], [, b]) => b - a)
+    .slice(0, 10)
+    .map(([category, count]) => ({ category, count }))
+
+  // Medium+ count
+  const mediumPlusFindings = bySeverity.critical + bySeverity.high + bySeverity.medium
+
+  return {
+    repoName,
+    scanDepth,
+    totalFindings: result.vulnerabilities.length,
+    mediumPlusFindings,
+    filesScanned: result.filesScanned,
+    scanDuration: result.scanDuration,
+    bySeverity,
+    byCategory,
+    m5Findings,
+    topCategories,
+    libraryCodeFindings,
+    exampleCodeFindings,
+    testCodeFindings,
+  }
+}
+
+/**
+ * Generate aggregate metrics across all analyses
+ */
+function computeAggregateMetrics(analyses: RepoAnalysis[]): ValidationReport['aggregateMetrics'] {
+  let totalFindings = 0
+  let totalMediumPlus = 0
+
+  const m5Performance: Record<string, { total: number; mediumPlus: number }> = {}
+  for (const cat of M5_CATEGORIES) {
+    m5Performance[cat] = { total: 0, mediumPlus: 0 }
+  }
+
+  for (const analysis of analyses) {
+    totalFindings += analysis.totalFindings
+    totalMediumPlus += analysis.mediumPlusFindings
+
+    for (const m5 of analysis.m5Findings) {
+      m5Performance[m5.category].total += m5.count
+      m5Performance[m5.category].mediumPlus +=
+        m5.bySeverity.critical + m5.bySeverity.high + m5.bySeverity.medium
+    }
+  }
+
+  return {
+    totalFindings,
+    totalMediumPlus,
+    avgFindingsPerRepo: analyses.length > 0 ? Math.round(totalFindings / analyses.length) : 0,
+    m5DetectorPerformance: m5Performance,
+  }
+}
+
+/**
+ * Generate markdown report
+ */
+function generateMarkdownReport(report: ValidationReport): string {
+  const lines: string[] = []
+
+  lines.push('# M7: Real-Repo Validation Results')
+  lines.push('')
+  lines.push(`> Generated: ${report.generatedAt}`)
+  lines.push('')
+
+  // Executive Summary
+  lines.push('## Executive Summary')
+  lines.push('')
+  lines.push(`- **Repositories scanned:** ${report.totalRepos}`)
+  lines.push(`- **Total scans:** ${report.totalScans}`)
+  lines.push(`- **Total findings:** ${report.aggregateMetrics.totalFindings}`)
+  lines.push(`- **Medium+ findings (to triage):** ${report.aggregateMetrics.totalMediumPlus}`)
+  lines.push('')
+
+  // M5 Detector Performance
+  lines.push('## M5 AI-Era Detector Performance')
+  lines.push('')
+  lines.push('| Detector | Total Findings | Medium+ |')
+  lines.push('|----------|----------------|---------|')
+
+  for (const [category, stats] of Object.entries(report.aggregateMetrics.m5DetectorPerformance)) {
+    if (stats.total > 0) {
+      lines.push(`| ${category} | ${stats.total} | ${stats.mediumPlus} |`)
+    }
+  }
+
+  const emptyM5 = Object.entries(report.aggregateMetrics.m5DetectorPerformance)
+    .filter(([, stats]) => stats.total === 0)
+    .map(([cat]) => cat)
+
+  if (emptyM5.length > 0) {
+    lines.push('')
+    lines.push(`**No findings for:** ${emptyM5.join(', ')}`)
+  }
+
+  lines.push('')
+
+  // Per-Repo Results
+  lines.push('## Per-Repository Results')
+  lines.push('')
+
+  // Group analyses by repo
+  const byRepo = new Map<string, RepoAnalysis[]>()
+  for (const analysis of report.analyses) {
+    const existing = byRepo.get(analysis.repoName) || []
+    existing.push(analysis)
+    byRepo.set(analysis.repoName, existing)
+  }
+
+  for (const [repoName, analyses] of byRepo) {
+    lines.push(`### ${repoName}`)
+    lines.push('')
+
+    for (const analysis of analyses) {
+      lines.push(`#### Depth: ${analysis.scanDepth}`)
+      lines.push('')
+      lines.push(`- Files scanned: ${analysis.filesScanned}`)
+      lines.push(`- Total findings: ${analysis.totalFindings}`)
+      lines.push(`- Duration: ${analysis.scanDuration}ms`)
+      lines.push('')
+      lines.push('**Severity Breakdown:**')
+      lines.push(`| Critical | High | Medium | Low | Info |`)
+      lines.push(`|----------|------|--------|-----|------|`)
+      lines.push(`| ${analysis.bySeverity.critical} | ${analysis.bySeverity.high} | ${analysis.bySeverity.medium} | ${analysis.bySeverity.low} | ${analysis.bySeverity.info} |`)
+      lines.push('')
+
+      // Code type breakdown
+      lines.push('**Finding Distribution by Code Type:**')
+      lines.push(`- Library internals: ${analysis.libraryCodeFindings}`)
+      lines.push(`- Examples/demos: ${analysis.exampleCodeFindings}`)
+      lines.push(`- Test files: ${analysis.testCodeFindings}`)
+      lines.push('')
+
+      // Top categories
+      if (analysis.topCategories.length > 0) {
+        lines.push('**Top Categories:**')
+        for (const { category, count } of analysis.topCategories.slice(0, 5)) {
+          lines.push(`- ${category}: ${count}`)
+        }
+        lines.push('')
+      }
+
+      // M5 findings with examples
+      if (analysis.m5Findings.length > 0) {
+        lines.push('**M5 Detector Findings:**')
+        for (const m5 of analysis.m5Findings) {
+          lines.push(`- **${m5.category}**: ${m5.count} findings`)
+          if (m5.examples.length > 0) {
+            for (const ex of m5.examples.slice(0, 3)) {
+              lines.push(`  - \`${ex.file}:${ex.line}\` (${ex.severity}): ${ex.title}`)
+            }
+          }
+        }
+        lines.push('')
+      }
+    }
+  }
+
+  // Triage Checklist
+  lines.push('## FP Triage Checklist')
+  lines.push('')
+  lines.push('For each medium+ finding, classify as:')
+  lines.push('- [ ] **True Positive** - Real security issue')
+  lines.push('- [ ] **False Positive** - Safe code incorrectly flagged')
+  lines.push('- [ ] **Borderline** - Context-dependent, may need tuning')
+  lines.push('')
+  lines.push('### Triage Template')
+  lines.push('')
+  lines.push('```markdown')
+  lines.push('### Finding: [category]')
+  lines.push('**File:** [path]:[line]')
+  lines.push('**Severity:** [severity]')
+  lines.push('**Classification:** [ ] TP [ ] FP [ ] Borderline')
+  lines.push('**Context:** Library internal / Example code / Test file')
+  lines.push('**Reasoning:** [Why this is/isn\'t a real issue]')
+  lines.push('**Action:** [ ] Keep [ ] Tune detector [ ] Add to known-FP fixtures')
+  lines.push('```')
+  lines.push('')
+
+  // Next Steps
+  lines.push('## Next Steps')
+  lines.push('')
+  lines.push('1. Review all medium+ findings in this report')
+  lines.push('2. Classify each as TP/FP/Borderline')
+  lines.push('3. Calculate FP rate: `FP / (TP + FP)` - target: <20%')
+  lines.push('4. Add FP patterns to `known-false-positives.test.ts`')
+  lines.push('5. Tune detectors if FP rate exceeds 20%')
+  lines.push('')
+
+  return lines.join('\n')
+}
+
+/**
+ * Print console summary
+ */
+function printConsoleSummary(report: ValidationReport): void {
+  console.log('\n' + '='.repeat(60))
+  console.log('M7: VALIDATION ANALYSIS SUMMARY')
+  console.log('='.repeat(60))
+
+  console.log(`\nGenerated: ${report.generatedAt}`)
+  console.log(`Repositories: ${report.totalRepos}`)
+  console.log(`Total scans: ${report.totalScans}`)
+
+  console.log('\n--- Aggregate Metrics ---')
+  console.log(`Total findings: ${report.aggregateMetrics.totalFindings}`)
+  console.log(`Medium+ findings: ${report.aggregateMetrics.totalMediumPlus} (to triage)`)
+  console.log(`Avg per repo: ${report.aggregateMetrics.avgFindingsPerRepo}`)
+
+  console.log('\n--- M5 Detector Performance ---')
+  console.log('Category                      | Total | Medium+')
+  console.log('-'.repeat(50))
+
+  for (const [category, stats] of Object.entries(report.aggregateMetrics.m5DetectorPerformance)) {
+    const catPadded = category.padEnd(30)
+    console.log(`${catPadded}| ${String(stats.total).padEnd(6)}| ${stats.mediumPlus}`)
+  }
+
+  console.log('\n--- Per-Repo Breakdown ---')
+  for (const analysis of report.analyses) {
+    console.log(`\n${analysis.repoName} (${analysis.scanDepth}):`)
+    console.log(`  Files: ${analysis.filesScanned}, Findings: ${analysis.totalFindings}, Medium+: ${analysis.mediumPlusFindings}`)
+    console.log(`  Code types: Library=${analysis.libraryCodeFindings}, Example=${analysis.exampleCodeFindings}, Test=${analysis.testCodeFindings}`)
+  }
+}
+
+/**
+ * Main entry point
+ */
+async function main() {
+  const args = process.argv.slice(2)
+  let outputPath: string | null = null
+
+  // Parse args
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--output' && args[i + 1]) {
+      outputPath = args[i + 1]
+      i++
+    } else if (args[i] === '--help' || args[i] === '-h') {
+      console.log(`
+M7: Results Analysis Script
+
+Usage:
+  npx tsx analyze-results.ts [options]
+
+Options:
+  --output <path>   Write markdown report to file
+  --help, -h        Show this help
+
+Examples:
+  npx tsx analyze-results.ts                                    # Print summary
+  npx tsx analyze-results.ts --output docs/RESULTSCOMPARISON.md # Generate report
+`)
+      process.exit(0)
+    }
+  }
+
+  // Load results
+  const results = loadResults()
+  console.log(`Loaded ${results.size} result files`)
+
+  // Analyze each result
+  const analyses: RepoAnalysis[] = []
+  for (const [key, result] of results) {
+    const analysis = analyzeResult(key, result)
+    analyses.push(analysis)
+  }
+
+  // Get unique repo count
+  const uniqueRepos = new Set(analyses.map(a => a.repoName))
+
+  // Build report
+  const report: ValidationReport = {
+    generatedAt: new Date().toISOString(),
+    totalRepos: uniqueRepos.size,
+    totalScans: analyses.length,
+    analyses,
+    aggregateMetrics: computeAggregateMetrics(analyses),
+  }
+
+  // Print console summary
+  printConsoleSummary(report)
+
+  // Generate markdown if output path specified
+  if (outputPath) {
+    const markdown = generateMarkdownReport(report)
+    const fullPath = path.isAbsolute(outputPath)
+      ? outputPath
+      : path.join(process.cwd(), outputPath)
+
+    // Ensure directory exists
+    const dir = path.dirname(fullPath)
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true })
+    }
+
+    fs.writeFileSync(fullPath, markdown)
+    console.log(`\nMarkdown report saved to: ${fullPath}`)
+  } else {
+    console.log('\nTip: Use --output docs/RESULTSCOMPARISON.md to generate full report')
+  }
+}
+
+// Run
+main().catch(err => {
+  console.error('Analysis failed:', err)
+  process.exit(1)
+})

package/src/__tests__/validation/extract-for-triage.ts

@@ -0,0 +1,146 @@
+#!/usr/bin/env npx tsx
+/**
+ * Extract medium+ findings for systematic triage
+ *
+ * Outputs findings grouped by category with file paths for code review
+ */
+
+import * as fs from 'fs'
+import * as path from 'path'
+import type { ScanResult, Vulnerability } from '../../types'
+
+const RESULTS_DIR = path.join(__dirname, '../../../validation-results')
+const REPOS_DIR = path.join(__dirname, '../../../validation-repos')
+
+interface TriageFinding {
+  id: string
+  repo: string
+  file: string
+  line: number
+  severity: string
+  category: string
+  title: string
+  description: string
+  lineContent: string
+  codeType: 'library' | 'example' | 'test' | 'other'
+}
+
+function classifyCodeType(filePath: string): 'library' | 'example' | 'test' | 'other' {
+  const lowerPath = filePath.toLowerCase()
+
+  if (lowerPath.includes('__tests__') || lowerPath.includes('/test/') ||
+      lowerPath.includes('/tests/') || lowerPath.includes('.test.') ||
+      lowerPath.includes('.spec.') || lowerPath.includes('/fixtures/')) {
+    return 'test'
+  }
+
+  if (lowerPath.includes('/examples/') || lowerPath.includes('/example/') ||
+      lowerPath.includes('/demos/') || lowerPath.includes('/demo/') ||
+      lowerPath.includes('/templates/') || lowerPath.includes('/cookbook/')) {
+    return 'example'
+  }
+
+  if (lowerPath.includes('/src/') || lowerPath.includes('/lib/') ||
+      lowerPath.includes('/libs/') || lowerPath.includes('/packages/')) {
+    return 'library'
+  }
+
+  return 'other'
+}
+
+function extractMediumPlusFindings(): TriageFinding[] {
+  const findings: TriageFinding[] = []
+
+  const files = fs.readdirSync(RESULTS_DIR).filter(f => f.endsWith('.json'))
+
+  for (const file of files) {
+    const repoName = file.replace('-cheap.json', '').replace('-validated.json', '')
+    const content = fs.readFileSync(path.join(RESULTS_DIR, file), 'utf-8')
+    const result = JSON.parse(content) as ScanResult
+
+    for (const vuln of result.vulnerabilities) {
+      if (['critical', 'high', 'medium'].includes(vuln.severity)) {
+        findings.push({
+          id: vuln.id,
+          repo: repoName,
+          file: vuln.filePath,
+          line: vuln.lineNumber,
+          severity: vuln.severity,
+          category: vuln.category,
+          title: vuln.title,
+          description: vuln.description,
+          lineContent: vuln.lineContent,
+          codeType: classifyCodeType(vuln.filePath),
+        })
+      }
+    }
+  }
+
+  return findings
+}
+
+function groupByCategory(findings: TriageFinding[]): Map<string, TriageFinding[]> {
+  const groups = new Map<string, TriageFinding[]>()
+
+  for (const f of findings) {
+    const existing = groups.get(f.category) || []
+    existing.push(f)
+    groups.set(f.category, existing)
+  }
+
+  return groups
+}
+
+function main() {
+  const findings = extractMediumPlusFindings()
+  const grouped = groupByCategory(findings)
+
+  console.log(`\n${'='.repeat(60)}`)
+  console.log('MEDIUM+ FINDINGS FOR TRIAGE')
+  console.log('='.repeat(60))
+  console.log(`Total: ${findings.length} findings\n`)
+
+  // Summary by category
+  console.log('By Category:')
+  const sortedCategories = Array.from(grouped.entries())
+    .sort((a, b) => b[1].length - a[1].length)
+
+  for (const [category, catFindings] of sortedCategories) {
+    const bySeverity = {
+      critical: catFindings.filter(f => f.severity === 'critical').length,
+      high: catFindings.filter(f => f.severity === 'high').length,
+      medium: catFindings.filter(f => f.severity === 'medium').length,
+    }
+    console.log(`  ${category}: ${catFindings.length} (C:${bySeverity.critical} H:${bySeverity.high} M:${bySeverity.medium})`)
+  }
+
+  // By code type
+  const byCodeType = {
+    library: findings.filter(f => f.codeType === 'library').length,
+    example: findings.filter(f => f.codeType === 'example').length,
+    test: findings.filter(f => f.codeType === 'test').length,
+    other: findings.filter(f => f.codeType === 'other').length,
+  }
+  console.log(`\nBy Code Type:`)
+  console.log(`  Library: ${byCodeType.library}`)
+  console.log(`  Example: ${byCodeType.example}`)
+  console.log(`  Test: ${byCodeType.test}`)
+  console.log(`  Other: ${byCodeType.other}`)
+
+  // Output JSON for further processing
+  const outputPath = path.join(RESULTS_DIR, 'medium-plus-findings.json')
+  fs.writeFileSync(outputPath, JSON.stringify(findings, null, 2))
+  console.log(`\nFindings exported to: ${outputPath}`)
+
+  // Output summary by repo
+  console.log('\nBy Repository:')
+  const byRepo = new Map<string, number>()
+  for (const f of findings) {
+    byRepo.set(f.repo, (byRepo.get(f.repo) || 0) + 1)
+  }
+  for (const [repo, count] of byRepo) {
+    console.log(`  ${repo}: ${count}`)
+  }
+}
+
+main()