@oculum/scanner 1.0.9 → 1.0.11

package/src/layer3/anthropic/utils/index.ts

@@ -0,0 +1,26 @@
+/**
+ * Utility Functions Index
+ *
+ * Re-exports all utility functions from the anthropic module.
+ */
+
+export {
+  normalizePathForComparison,
+  findMatchingFilePath,
+  getLanguageFromPath,
+} from './path-helpers'
+
+export {
+  makeAnthropicRequestWithRetry,
+  makeOpenAIRequestWithRetry,
+} from './retry'
+
+export {
+  parseAIResponse,
+  parseValidationResponse,
+  parseMultiFileValidationResponse,
+  applyValidationResults,
+  validateSeverity,
+  validateCategory,
+  getLineContent,
+} from './response-parser'

package/src/layer3/anthropic/utils/path-helpers.ts

@@ -0,0 +1,68 @@
+/**
+ * Path Normalization Helpers
+ *
+ * Functions for normalizing and matching file paths in AI responses.
+ * AI models may return paths in different formats than expected.
+ */
+
+/**
+ * Normalize a file path for comparison purposes.
+ * Handles common variations: ./src/file.ts, src/file.ts, /src/file.ts
+ */
+export function normalizePathForComparison(path: string): string {
+  return path
+    .replace(/^\.\//, '')  // Remove leading ./
+    .replace(/^\//, '')    // Remove leading /
+    .replace(/\\/g, '/')   // Normalize Windows backslashes
+}
+
+/**
+ * Find a matching file path from expected paths, handling path format variations.
+ * AI responses may use different path formats than what we sent.
+ */
+export function findMatchingFilePath(responsePath: string, expectedPaths: string[]): string | null {
+  // Exact match first
+  if (expectedPaths.includes(responsePath)) return responsePath
+
+  // Normalized match
+  const normalized = normalizePathForComparison(responsePath)
+  for (const expected of expectedPaths) {
+    if (normalizePathForComparison(expected) === normalized) {
+      console.log(`[AI Validation] Path fuzzy matched: "${responsePath}" -> "${expected}"`)
+      return expected
+    }
+  }
+
+  // Basename match (only if unique) - handles cases like "file.ts" matching "src/api/file.ts"
+  const basename = responsePath.split('/').pop() || responsePath
+  const matches = expectedPaths.filter(p => (p.split('/').pop() || p) === basename)
+  if (matches.length === 1) {
+    console.log(`[AI Validation] Path basename matched: "${responsePath}" -> "${matches[0]}"`)
+    return matches[0]
+  }
+
+  return null
+}
+
+/**
+ * Get language identifier from file path extension
+ */
+export function getLanguageFromPath(path: string): string {
+  const ext = path.split('.').pop()?.toLowerCase()
+  const langMap: Record<string, string> = {
+    ts: 'typescript',
+    tsx: 'tsx',
+    js: 'javascript',
+    jsx: 'jsx',
+    py: 'python',
+    rb: 'ruby',
+    go: 'go',
+    java: 'java',
+    php: 'php',
+    cs: 'csharp',
+    json: 'json',
+    yaml: 'yaml',
+    yml: 'yaml',
+  }
+  return langMap[ext || ''] || ext || 'text'
+}

package/src/layer3/anthropic/utils/response-parser.ts

@@ -0,0 +1,322 @@
+/**
+ * AI Response Parsing Utilities
+ *
+ * Functions for parsing validation responses from AI models.
+ */
+
+import type { VulnerabilitySeverity, VulnerabilityCategory, Vulnerability, ValidationStatus } from '../../../types'
+import type { ValidationResult, AIFinding } from '../types'
+import { findMatchingFilePath } from './path-helpers'
+
+/**
+ * Extract the first top-level JSON array from text.
+ * The model may include prose before/after the JSON.
+ */
+function extractTopLevelArray(text: string): string | null {
+  const startIndex = text.indexOf('[')
+  if (startIndex === -1) return null
+
+  let depth = 0
+  let inString = false
+  let stringChar: '"' | "'" | null = null
+  let escape = false
+
+  for (let i = startIndex; i < text.length; i++) {
+    const ch = text[i]
+
+    if (inString) {
+      if (escape) {
+        escape = false
+        continue
+      }
+
+      if (ch === '\\') {
+        escape = true
+        continue
+      }
+
+      if (stringChar && ch === stringChar) {
+        inString = false
+        stringChar = null
+      }
+      continue
+    }
+
+    if (ch === '"' || ch === "'") {
+      inString = true
+      stringChar = ch as '"' | "'"
+      continue
+    }
+
+    if (ch === '[') {
+      depth++
+    } else if (ch === ']') {
+      depth--
+      if (depth === 0) {
+        return text.slice(startIndex, i + 1)
+      }
+    }
+  }
+
+  return null
+}
+
+/**
+ * Parse AI response for single file validation
+ */
+export function parseAIResponse(response: string): AIFinding[] {
+  try {
+    // Try to extract JSON from the response
+    const jsonMatch = response.match(/\[[\s\S]*\]/)
+    if (!jsonMatch) {
+      return []
+    }
+
+    const parsed = JSON.parse(jsonMatch[0])
+
+    // Validate the structure
+    if (!Array.isArray(parsed)) {
+      return []
+    }
+
+    return parsed.filter(item =>
+      typeof item.lineNumber === 'number' &&
+      typeof item.severity === 'string' &&
+      typeof item.category === 'string' &&
+      typeof item.title === 'string' &&
+      typeof item.description === 'string'
+    ).map(item => ({
+      lineNumber: item.lineNumber,
+      severity: validateSeverity(item.severity),
+      category: validateCategory(item.category),
+      title: item.title,
+      description: item.description,
+      suggestedFix: item.suggestedFix || 'Review and fix the security issue',
+    }))
+  } catch (error) {
+    console.error('Failed to parse AI response:', error)
+    return []
+  }
+}
+
+/**
+ * Parse single-file validation response
+ */
+export function parseValidationResponse(response: string): ValidationResult[] {
+  try {
+    const jsonSlice = extractTopLevelArray(response)
+    if (!jsonSlice) return []
+
+    const parsed = JSON.parse(jsonSlice)
+    if (!Array.isArray(parsed)) return []
+
+    return parsed
+      .filter(item =>
+        typeof item.index === 'number' &&
+        typeof item.keep === 'boolean'
+      )
+      .map(item => {
+        // Normalize notes field: prefer new 'notes', fallback to legacy 'reason' or 'validationNotes'
+        const notes = item.notes || item.validationNotes || item.reason || undefined
+
+        return {
+          index: item.index,
+          keep: item.keep,
+          notes,
+          adjustedSeverity: item.adjustedSeverity || null,
+          // Keep legacy fields for backward compatibility
+          reason: item.reason,
+          validationNotes: item.validationNotes,
+          // Actionable output fields (PRO-82)
+          impact: item.impact || undefined,
+          fixSuggestion: item.fixSuggestion || undefined,
+        }
+      })
+  } catch (error) {
+    console.error('Failed to parse validation response:', error)
+    return []
+  }
+}
+
+/**
+ * Parse multi-file validation response (Phase 2)
+ * Returns a map of file path -> validation results
+ */
+export function parseMultiFileValidationResponse(
+  response: string,
+  expectedFiles: string[]
+): Map<string, ValidationResult[]> {
+  const resultMap = new Map<string, ValidationResult[]>()
+
+  try {
+    const jsonSlice = extractTopLevelArray(response)
+    if (!jsonSlice) {
+      console.error('[AI Validation] Multi-file: No JSON array found in response')
+      return resultMap
+    }
+
+    const parsed = JSON.parse(jsonSlice)
+    if (!Array.isArray(parsed)) {
+      console.error('[AI Validation] Multi-file: Parsed result is not an array')
+      return resultMap
+    }
+
+    // Process each file's results
+    for (const fileResult of parsed) {
+      if (!fileResult.file || !Array.isArray(fileResult.validations)) {
+        console.warn('[AI Validation] Multi-file: Invalid file result structure, skipping')
+        continue
+      }
+
+      // Use path normalization to match AI response paths to expected paths
+      const responsePath = fileResult.file
+      const matchedPath = findMatchingFilePath(responsePath, expectedFiles)
+
+      if (!matchedPath) {
+        console.warn(`[AI Validation] Multi-file: Could not match path "${responsePath}" to any expected file`)
+        continue
+      }
+
+      const validations: ValidationResult[] = fileResult.validations
+        .filter((item: any) =>
+          typeof item.index === 'number' &&
+          typeof item.keep === 'boolean'
+        )
+        .map((item: any) => {
+          // Normalize notes field: prefer new 'notes', fallback to legacy 'reason' or 'validationNotes'
+          const notes = item.notes || item.validationNotes || item.reason || undefined
+
+          return {
+            index: item.index,
+            keep: item.keep,
+            notes,
+            adjustedSeverity: item.adjustedSeverity || null,
+            // Keep legacy fields for backward compatibility
+            reason: item.reason,
+            validationNotes: item.validationNotes,
+            // Actionable output fields (PRO-82)
+            impact: item.impact || undefined,
+            fixSuggestion: item.fixSuggestion || undefined,
+          }
+        })
+
+      resultMap.set(matchedPath, validations)
+    }
+
+    // Log any files that weren't in the response (these will be REJECTED by default)
+    const missingFiles = expectedFiles.filter(f => !resultMap.has(f))
+    if (missingFiles.length > 0) {
+      console.warn(`[AI Validation] Multi-file: Missing ${missingFiles.length} files from response: ${missingFiles.join(', ')}`)
+    }
+
+  } catch (error) {
+    console.error('[AI Validation] Multi-file: Failed to parse response:', error)
+  }
+
+  return resultMap
+}
+
+/**
+ * Apply validation results to findings
+ */
+export function applyValidationResults(
+  findings: Vulnerability[],
+  validationResults: ValidationResult[]
+): { processed: Vulnerability[]; dismissedCount: number } {
+  const processed: Vulnerability[] = []
+  let dismissedCount = 0
+
+  for (let i = 0; i < findings.length; i++) {
+    const finding = findings[i]
+    const validation = validationResults.find(v => v.index === i)
+
+    if (!validation) {
+      // No validation result - REJECT by default (conservative approach)
+      // If AI doesn't explicitly validate a finding, assume it's a false positive
+      console.warn(`[AI Validation] No result for finding ${i}: ${finding.title} - REJECTING`)
+      dismissedCount++
+      continue  // Don't add to processed - finding is removed
+    }
+
+    if (validation.keep) {
+      // Keep the finding
+      const adjustedFinding: Vulnerability = {
+        ...finding,
+        validatedByAI: true,
+        confidence: 'high',
+      }
+
+      // Extract notes from optimized or legacy format
+      const validationNotes = validation.notes || validation.validationNotes || validation.reason || undefined
+
+      if (validation.adjustedSeverity && validation.adjustedSeverity !== finding.severity) {
+        // Severity was adjusted
+        adjustedFinding.originalSeverity = finding.severity
+        adjustedFinding.severity = validation.adjustedSeverity
+        adjustedFinding.validationStatus = 'downgraded' as ValidationStatus
+        adjustedFinding.validationNotes = validationNotes || 'Severity adjusted by AI validation'
+      } else {
+        // Confirmed at original severity
+        adjustedFinding.validationStatus = 'confirmed' as ValidationStatus
+        adjustedFinding.validationNotes = validationNotes
+      }
+
+      // Apply AI-generated actionable fields (PRO-82)
+      if (validation.impact) {
+        adjustedFinding.impact = validation.impact
+        adjustedFinding.aiEnhanced = true
+      }
+      if (validation.fixSuggestion) {
+        // AI-generated fix becomes the primary fix step
+        adjustedFinding.fixSteps = [validation.fixSuggestion]
+        adjustedFinding.aiEnhanced = true
+      }
+
+      processed.push(adjustedFinding)
+    } else {
+      // Finding was dismissed - only log in debug mode to reduce noise
+      if (process.env.DEBUG || process.env.OCULUM_DEBUG) {
+        console.log(`[AI Validation] Rejected: ${finding.title} at ${finding.filePath}:${finding.lineNumber}`)
+      }
+      dismissedCount++
+      // Don't add to processed - finding is removed
+    }
+  }
+
+  return { processed, dismissedCount }
+}
+
+/**
+ * Validate severity value from AI response
+ */
+export function validateSeverity(severity: string): VulnerabilitySeverity {
+  const valid: VulnerabilitySeverity[] = ['critical', 'high', 'medium', 'low', 'info']
+  return valid.includes(severity as VulnerabilitySeverity)
+    ? severity as VulnerabilitySeverity
+    : 'medium'
+}
+
+/**
+ * Validate category value from AI response
+ */
+export function validateCategory(category: string): VulnerabilityCategory {
+  const valid: VulnerabilityCategory[] = [
+    'sql_injection', 'xss', 'command_injection', 'missing_auth',
+    'dangerous_function', 'hardcoded_secret', 'high_entropy_string',
+    'sensitive_variable', 'security_bypass', 'insecure_config',
+    'suspicious_package', 'cors_misconfiguration', 'root_container',
+    'weak_crypto', 'sensitive_url', 'ai_pattern', 'dangerous_file',
+    'data_exposure',  // For logging/exposing sensitive data
+  ]
+  return valid.includes(category as VulnerabilityCategory)
+    ? category as VulnerabilityCategory
+    : 'dangerous_function'
+}
+
+/**
+ * Get line content from file content by line number
+ */
+export function getLineContent(content: string, lineNumber: number): string {
+  const lines = content.split('\n')
+  return lines[lineNumber - 1]?.trim() || ''
+}

package/src/layer3/anthropic/utils/retry.ts

@@ -0,0 +1,75 @@
+/**
+ * Retry Logic for AI API Calls
+ *
+ * Implements exponential backoff for rate limit handling.
+ */
+
+/**
+ * Helper function to make Anthropic API calls with retry logic for rate limiting
+ * Implements exponential backoff for 429 (rate limit) errors
+ */
+export async function makeAnthropicRequestWithRetry<T>(
+  requestFn: () => Promise<T>,
+  maxRetries: number = 3,
+  initialDelayMs: number = 1000
+): Promise<T> {
+  let lastError: Error | null = null
+
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      return await requestFn()
+    } catch (error: any) {
+      lastError = error
+
+      // Check if it's a rate limit error (429)
+      const isRateLimit = error?.status === 429 || error?.message?.includes('rate limit')
+
+      if (isRateLimit && attempt < maxRetries) {
+        // Exponential backoff: 1s, 2s, 4s
+        const delayMs = initialDelayMs * Math.pow(2, attempt)
+        console.log(`[AI Validation] Rate limit hit, retrying in ${delayMs}ms (attempt ${attempt + 1}/${maxRetries})`)
+        await new Promise(resolve => setTimeout(resolve, delayMs))
+        continue
+      }
+
+      // If not rate limit or max retries reached, throw
+      throw error
+    }
+  }
+
+  throw lastError || new Error('Max retries exceeded')
+}
+
+/**
+ * Helper to make OpenAI requests with retry logic for rate limits
+ */
+export async function makeOpenAIRequestWithRetry<T>(
+  requestFn: () => Promise<T>,
+  maxRetries = 3,
+  initialDelayMs = 1000
+): Promise<T> {
+  let lastError: Error | null = null
+
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      return await requestFn()
+    } catch (error: any) {
+      lastError = error
+
+      // Check if it's a rate limit error (429) - but NOT insufficient_quota
+      const isRateLimit = error?.status === 429 && error?.code !== 'insufficient_quota'
+
+      if (isRateLimit && attempt < maxRetries) {
+        const delayMs = initialDelayMs * Math.pow(2, attempt)
+        console.log(`[OpenAI Validation] Rate limit hit, retrying in ${delayMs}ms (attempt ${attempt + 1}/${maxRetries})`)
+        await new Promise(resolve => setTimeout(resolve, delayMs))
+        continue
+      }
+
+      // If it's a quota error or max retries reached, throw
+      throw error
+    }
+  }
+
+  throw lastError || new Error('Max retries exceeded')
+}

package/src/layer3/index.ts

@@ -6,6 +6,7 @@
 import type { Vulnerability, ScanFile, CancellationToken } from '../types'
 import { batchAnalyzeWithAI, type Layer3Context } from './anthropic'
 import { checkPackages } from './package-check'
+import { checkPackageAdvisories } from './osv-check'
 
 export interface Layer3Result {
   vulnerabilities: Vulnerability[]
@@ -58,14 +59,25 @@ export async function runLayer3Scan(
     }
   }
 
-  // 1. Check packages (always run, fast)
-  const packageFiles = files.filter(f => f.path.endsWith('package.json'))
-  for (const file of packageFiles) {
+  // 1. Check packages for hallucination/risk (package.json and requirements.txt)
+  const packageManifests = files.filter(f =>
+    f.path.endsWith('package.json') ||
+    f.path.endsWith('requirements.txt')
+  )
+
+  for (const file of packageManifests) {
     // Check for cancellation in package loop
     if (options.cancellationToken?.cancelled) break
 
-    const packageFindings = await checkPackages(file.content, file.path)
-    vulnerabilities.push(...packageFindings)
+    // Run hallucination/risk check (package-check only handles package.json)
+    if (file.path.endsWith('package.json')) {
+      const packageFindings = await checkPackages(file.content, file.path)
+      vulnerabilities.push(...packageFindings)
+    }
+
+    // Run OSV advisory check (handles both npm and Python)
+    const osvFindings = await checkPackageAdvisories(file.content, file.path)
+    vulnerabilities.push(...osvFindings)
   }
 
   // Check for cancellation before AI analysis
@@ -153,3 +165,4 @@ function getAIPriorityScore(file: ScanFile): number {
 
 export { analyzeWithAI, batchAnalyzeWithAI, type Layer3Context } from './anthropic'
 export { checkPackages } from './package-check'
+export { checkPackageAdvisories } from './osv-check'