@oculum/scanner 1.0.14 → 1.0.15

package/src/validate/providers/anthropic.ts

@@ -4,16 +4,29 @@
  * Validation using Anthropic Claude 3.5 Haiku model.
  */
 
-import type { Vulnerability, ScanFile, ValidationStatus } from '../../shared/types'
-import type { ContextEngineResult } from '../../model/taint-types'
-import type { ProjectContext } from '../../model/project-context'
-import { buildProjectContext } from '../../model/project-context'
-import type { ValidationStats, AIValidationResult } from '../types'
-import { getAnthropicClient, HAIKU_PRICING, FILES_PER_API_BATCH } from '../clients'
-import { makeAnthropicRequestWithRetry } from '../utils/retry'
-import { parseMultiFileValidationResponse, parseValidationResponse, applyValidationResults } from '../utils/response-parser'
-import { buildMultiFileValidationRequest } from '../request-builder'
-import { assembleValidationPrompt } from '../prompts/validation'
+import type {
+  Vulnerability,
+  ScanFile,
+  ValidationStatus,
+} from "../../shared/types";
+import type { ContextEngineResult } from "../../model/taint-types";
+import type { ProjectContext } from "../../model/project-context";
+import { buildProjectContext } from "../../model/project-context";
+import type { ValidationStats, AIValidationResult } from "../types";
+import {
+  getAnthropicClient,
+  HAIKU_PRICING,
+  FILES_PER_API_BATCH,
+} from "../clients";
+import { makeAnthropicRequestWithRetry } from "../utils/retry";
+import {
+  parseMultiFileValidationResponse,
+  parseValidationResponse,
+  applyValidationResults,
+  type DismissedFinding,
+} from "../utils/response-parser";
+import { buildMultiFileValidationRequest } from "../request-builder";
+import { assembleValidationPrompt } from "../prompts/validation";
 
 /**
  * Validate findings using Anthropic Claude 3.5 Haiku
@@ -23,52 +36,74 @@ export async function validateWithAnthropic(
   files: ScanFile[],
   ceResult: ContextEngineResult | undefined,
   stats: ValidationStats,
-  onProgress?: (progress: { filesProcessed: number; totalFiles: number; status: string }) => void
+  onProgress?: (progress: {
+    filesProcessed: number;
+    totalFiles: number;
+    status: string;
+  }) => void,
+  quiet: boolean = true,
 ): Promise<AIValidationResult> {
-  console.log('[AI Validation] Initializing Anthropic client...')
-  const client = getAnthropicClient()
+  const log = (msg: string, ...args: unknown[]) => {
+    if (!quiet) console.error(msg, ...args);
+  };
+  log("[AI Validation] Initializing Anthropic client...");
+  const client = getAnthropicClient();
 
   // Extract project context from CE result or build from scratch
-  const context: ProjectContext = ceResult?.projectContext ?? buildProjectContext(files)
-  const fileTaintAnalyses = ceResult?.fileTaintAnalyses
-  const routeMap = ceResult?.routeMap
-  const frameworkAnalyses = ceResult?.frameworkAnalyses
+  const context: ProjectContext =
+    ceResult?.projectContext ?? buildProjectContext(files);
+  const fileTaintAnalyses = ceResult?.fileTaintAnalyses;
+  const routeMap = ceResult?.routeMap;
+  const frameworkAnalyses = ceResult?.frameworkAnalyses;
   if (!ceResult) {
-    console.log('[AI Validation] No ContextEngineResult provided, built project context from scratch:', {
-      hasAuthMiddleware: context.auth.hasGlobalMiddleware,
-      authProvider: context.auth.authProvider,
-      orm: context.dataAccess.orm,
-      framework: context.frameworks.primary,
-    })
+    log(
+      "[AI Validation] No ContextEngineResult provided, built project context from scratch:",
+      {
+        hasAuthMiddleware: context.auth.hasGlobalMiddleware,
+        authProvider: context.auth.authProvider,
+        orm: context.dataAccess.orm,
+        framework: context.frameworks.primary,
+      },
+    );
   }
 
   // Group findings by file for efficient validation
-  const findingsByFile = new Map<string, Vulnerability[]>()
+  const findingsByFile = new Map<string, Vulnerability[]>();
   for (const finding of findings) {
-    const existing = findingsByFile.get(finding.filePath) || []
-    existing.push(finding)
-    findingsByFile.set(finding.filePath, existing)
+    const existing = findingsByFile.get(finding.filePath) || [];
+    existing.push(finding);
+    findingsByFile.set(finding.filePath, existing);
   }
 
-  const validatedFindings: Vulnerability[] = []
+  const validatedFindings: Vulnerability[] = [];
+  const allDismissed: DismissedFinding[] = [];
 
   // Phase 2: Multi-file batching
-  const fileEntries = Array.from(findingsByFile.entries())
+  const fileEntries = Array.from(findingsByFile.entries());
 
   // Track metrics
-  let totalBatchWaitTime = 0
-  let totalApiBatches = 0
+  let totalBatchWaitTime = 0;
+  let totalApiBatches = 0;
 
-  const totalFileBatches = Math.ceil(fileEntries.length / FILES_PER_API_BATCH)
-  console.log(`[AI Validation] Phase 2: Processing ${fileEntries.length} files in ${totalFileBatches} API batch(es) (${FILES_PER_API_BATCH} files/batch)`)
+  const totalFileBatches = Math.ceil(fileEntries.length / FILES_PER_API_BATCH);
+  log(
+    `[AI Validation] Phase 2: Processing ${fileEntries.length} files in ${totalFileBatches} API batch(es) (${FILES_PER_API_BATCH} files/batch)`,
+  );
 
   // Track files processed for progress reporting
-  let filesValidated = 0
+  let filesValidated = 0;
 
   // Process files in batches - each batch is ONE API call with multiple files
-  for (let batchStart = 0; batchStart < fileEntries.length; batchStart += FILES_PER_API_BATCH) {
-    const fileBatch = fileEntries.slice(batchStart, batchStart + FILES_PER_API_BATCH)
-    const batchNum = Math.floor(batchStart / FILES_PER_API_BATCH) + 1
+  for (
+    let batchStart = 0;
+    batchStart < fileEntries.length;
+    batchStart += FILES_PER_API_BATCH
+  ) {
+    const fileBatch = fileEntries.slice(
+      batchStart,
+      batchStart + FILES_PER_API_BATCH,
+    );
+    const batchNum = Math.floor(batchStart / FILES_PER_API_BATCH) + 1;
 
     // Report progress before processing batch
     if (onProgress) {
@@ -76,21 +111,30 @@ export async function validateWithAnthropic(
         filesProcessed: filesValidated,
         totalFiles: fileEntries.length,
         status: `AI validating batch ${batchNum}/${totalFileBatches}`,
-      })
+      });
     }
 
-    console.log(`[AI Validation] API Batch ${batchNum}/${totalFileBatches}: ${fileBatch.length} files`)
+    log(
+      `[AI Validation] API Batch ${batchNum}/${totalFileBatches}: ${fileBatch.length} files`,
+    );
 
     // Prepare file data for batch request
-    const fileDataList: Array<{ file: ScanFile; findings: Vulnerability[]; filePath: string }> = []
-    const filesWithoutContent: Array<{ filePath: string; findings: Vulnerability[] }> = []
+    const fileDataList: Array<{
+      file: ScanFile;
+      findings: Vulnerability[];
+      filePath: string;
+    }> = [];
+    const filesWithoutContent: Array<{
+      filePath: string;
+      findings: Vulnerability[];
+    }> = [];
 
     for (const [filePath, fileFindings] of fileBatch) {
-      const file = files.find(f => f.path === filePath)
+      const file = files.find((f) => f.path === filePath);
       if (!file) {
-        filesWithoutContent.push({ filePath, findings: fileFindings })
+        filesWithoutContent.push({ filePath, findings: fileFindings });
       } else {
-        fileDataList.push({ file, findings: fileFindings, filePath })
+        fileDataList.push({ file, findings: fileFindings, filePath });
       }
     }
 
@@ -100,167 +144,191 @@ export async function validateWithAnthropic(
         validatedFindings.push({
           ...f,
           validatedByAI: false,
-          validationStatus: 'not_validated' as ValidationStatus,
-          validationNotes: 'File content not available for validation',
-        })
+          validationStatus: "not_validated" as ValidationStatus,
+          validationNotes: "File content not available for validation",
+        });
       }
     }
 
     // Skip API call if no files with content
     if (fileDataList.length === 0) {
-      continue
+      continue;
     }
 
-    const batchStartTime = Date.now()
+    const batchStartTime = Date.now();
 
     try {
       // Build multi-file validation request with scoped context, taint data, and route map
       const validationRequest = buildMultiFileValidationRequest(
         fileDataList.map(({ file, findings }) => ({ file, findings })),
         context,
-        { contextMode: 'scoped' },
+        { contextMode: "scoped" },
         fileTaintAnalyses,
         routeMap,
-        ceResult?.crossFileTaintPaths,
-        frameworkAnalyses
-      )
+        frameworkAnalyses,
+      );
 
       // Assemble category-aware prompt for this batch
-      const batchCategories = [...new Set(
-        fileBatch.flatMap(([, fileFindings]) => fileFindings.map(f => f.category))
-      )]
-      const systemPrompt = assembleValidationPrompt(batchCategories)
+      const batchCategories = [
+        ...new Set(
+          fileBatch.flatMap(([, fileFindings]) =>
+            fileFindings.map((f) => f.category),
+          ),
+        ),
+      ];
+      const systemPrompt = assembleValidationPrompt(batchCategories);
 
       // Use Anthropic prompt caching with multi-file request
       const response = await makeAnthropicRequestWithRetry(() =>
         client.messages.create({
-          model: 'claude-3-5-haiku-20241022',
+          model: "claude-3-5-haiku-20241022",
           max_tokens: 16384, // Must accommodate 8 files × multiple findings with full validation fields
           system: [
             {
-              type: 'text',
+              type: "text",
               text: systemPrompt,
-              cache_control: { type: 'ephemeral' }, // Cache for 5 minutes
+              cache_control: { type: "ephemeral" }, // Cache for 5 minutes
             },
           ],
-          messages: [{ role: 'user', content: validationRequest }],
-        })
-      )
+          messages: [{ role: "user", content: validationRequest }],
+        }),
+      );
 
       // Track API call stats
-      stats.apiCalls++
-      totalApiBatches++
+      stats.apiCalls++;
+      totalApiBatches++;
 
       // Extract cache metrics from usage
-      const usage = response.usage
+      const usage = response.usage;
       if (usage) {
         // DEBUG: Log full usage object to understand token breakdown
-        console.log(`[DEBUG] Batch ${batchNum} - Full API Response Usage:`)
-        console.log(JSON.stringify(usage, null, 2))
-        console.log(`[DEBUG] Breakdown:`)
-        console.log(`  - input_tokens: ${usage.input_tokens || 0}`)
-        console.log(`  - output_tokens: ${usage.output_tokens || 0}`)
+        log(`[DEBUG] Batch ${batchNum} - Full API Response Usage:`);
+        log(JSON.stringify(usage, null, 2));
+        log(`[DEBUG] Breakdown:`);
+        log(`  - input_tokens: ${usage.input_tokens || 0}`);
+        log(`  - output_tokens: ${usage.output_tokens || 0}`);
         // @ts-ignore
-        console.log(`  - cache_creation_input_tokens: ${usage.cache_creation_input_tokens || 0}`)
+        log(
+          `  - cache_creation_input_tokens: ${usage.cache_creation_input_tokens || 0}`,
+        );
         // @ts-ignore
-        console.log(`  - cache_read_input_tokens: ${usage.cache_read_input_tokens || 0}`)
+        log(
+          `  - cache_read_input_tokens: ${usage.cache_read_input_tokens || 0}`,
+        );
 
-        stats.estimatedInputTokens += usage.input_tokens || 0
-        stats.estimatedOutputTokens += usage.output_tokens || 0
+        stats.estimatedInputTokens += usage.input_tokens || 0;
+        stats.estimatedOutputTokens += usage.output_tokens || 0;
 
         // @ts-ignore - cache fields not in types yet
-        const cacheCreation = usage.cache_creation_input_tokens || 0
+        const cacheCreation = usage.cache_creation_input_tokens || 0;
         // @ts-ignore
-        const cacheRead = usage.cache_read_input_tokens || 0
+        const cacheRead = usage.cache_read_input_tokens || 0;
 
-        stats.cacheCreationTokens += cacheCreation
-        stats.cacheReadTokens += cacheRead
+        stats.cacheCreationTokens += cacheCreation;
+        stats.cacheReadTokens += cacheRead;
       }
 
-      const textContent = response.content.find((block: { type: string }) => block.type === 'text')
-      if (!textContent || textContent.type !== 'text') {
+      const textContent = response.content.find(
+        (block: { type: string }) => block.type === "text",
+      );
+      if (!textContent || textContent.type !== "text") {
         // No valid response - mark all findings as not validated
         for (const { findings } of fileDataList) {
           for (const f of findings) {
             validatedFindings.push({
               ...f,
               validatedByAI: false,
-              validationStatus: 'not_validated' as ValidationStatus,
-              validationNotes: 'No valid response from AI',
-            })
+              validationStatus: "not_validated" as ValidationStatus,
+              validationNotes: "No valid response from AI",
+            });
           }
         }
-        continue
+        continue;
       }
 
       // Parse multi-file response
-      const expectedFiles = fileDataList.map(({ filePath }) => filePath)
-      const validationResultsMap = parseMultiFileValidationResponse(textContent.text, expectedFiles)
+      const expectedFiles = fileDataList.map(({ filePath }) => filePath);
+      const validationResultsMap = parseMultiFileValidationResponse(
+        textContent.text,
+        expectedFiles,
+      );
 
       // Apply results per file
       for (const { filePath, findings } of fileDataList) {
-        const fileResults = validationResultsMap.get(filePath)
+        const fileResults = validationResultsMap.get(filePath);
 
         if (!fileResults || fileResults.length === 0) {
           // No results for this file - try single-file parsing as fallback
-          const singleFileResults = parseValidationResponse(textContent.text)
+          const singleFileResults = parseValidationResponse(textContent.text);
 
           if (singleFileResults.length > 0 && fileDataList.length === 1) {
             // Single file in batch, use single-file parsing
-            const { processed: processedFindings, dismissedCount } = applyValidationResults(findings, singleFileResults)
-            stats.validatedFindings += processedFindings.length + dismissedCount
-            stats.dismissedFindings += dismissedCount
+            const {
+              processed: processedFindings,
+              dismissedCount,
+              dismissed,
+            } = applyValidationResults(findings, singleFileResults);
+            stats.validatedFindings +=
+              processedFindings.length + dismissedCount;
+            stats.dismissedFindings += dismissedCount;
+            allDismissed.push(...dismissed);
             for (const processed of processedFindings) {
-              if (processed.validationStatus === 'confirmed') {
-                stats.confirmedFindings++
-              } else if (processed.validationStatus === 'downgraded') {
-                stats.downgradedFindings++
+              if (processed.validationStatus === "confirmed") {
+                stats.confirmedFindings++;
+              } else if (processed.validationStatus === "downgraded") {
+                stats.downgradedFindings++;
               }
-              validatedFindings.push(processed)
+              validatedFindings.push(processed);
             }
           } else {
             // No validation results - REJECT all findings for this file (conservative approach)
-            console.warn(`[AI Validation] No results for ${filePath} - REJECTING ${findings.length} findings`)
-            stats.validatedFindings += findings.length
-            stats.dismissedFindings += findings.length
+            log(
+              `[AI Validation] No results for ${filePath} - REJECTING ${findings.length} findings`,
+            );
+            stats.validatedFindings += findings.length;
+            stats.dismissedFindings += findings.length;
             // Don't add to validatedFindings - findings are rejected
           }
         } else {
           // Apply validation results for this file
-          const { processed: processedFindings, dismissedCount } = applyValidationResults(findings, fileResults)
-          stats.validatedFindings += processedFindings.length + dismissedCount
-          stats.dismissedFindings += dismissedCount
+          const {
+            processed: processedFindings,
+            dismissedCount,
+            dismissed,
+          } = applyValidationResults(findings, fileResults);
+          stats.validatedFindings += processedFindings.length + dismissedCount;
+          stats.dismissedFindings += dismissedCount;
+          allDismissed.push(...dismissed);
           for (const processed of processedFindings) {
-            if (processed.validationStatus === 'confirmed') {
-              stats.confirmedFindings++
-            } else if (processed.validationStatus === 'downgraded') {
-              stats.downgradedFindings++
+            if (processed.validationStatus === "confirmed") {
+              stats.confirmedFindings++;
+            } else if (processed.validationStatus === "downgraded") {
+              stats.downgradedFindings++;
             }
-            validatedFindings.push(processed)
+            validatedFindings.push(processed);
           }
         }
       }
-
     } catch (error) {
-      console.error(`[AI Validation] Error in batch ${batchNum}:`, error)
+      console.error(`[AI Validation] Error in batch ${batchNum}:`, error);
       // Fallback: keep all findings but mark as not validated
       for (const { findings } of fileDataList) {
         for (const f of findings) {
           validatedFindings.push({
             ...f,
             validatedByAI: false,
-            validationStatus: 'not_validated' as ValidationStatus,
-            validationNotes: 'Validation failed due to API error',
-          })
+            validationStatus: "not_validated" as ValidationStatus,
+            validationNotes: "Validation failed due to API error",
+          });
         }
       }
     }
 
-    const batchDuration = Date.now() - batchStartTime
-    totalBatchWaitTime += batchDuration
+    const batchDuration = Date.now() - batchStartTime;
+    totalBatchWaitTime += batchDuration;
 
     // Update files validated counter
-    filesValidated += fileBatch.length
+    filesValidated += fileBatch.length;
 
     // Report progress after batch completion
     if (onProgress) {
@@ -268,44 +336,82 @@ export async function validateWithAnthropic(
         filesProcessed: filesValidated,
         totalFiles: fileEntries.length,
         status: `AI validation complete for batch ${batchNum}/${totalFileBatches}`,
-      })
+      });
     }
   }
 
+  // Build rejection breakdown from dismissed findings
+  if (allDismissed.length > 0) {
+    const breakdown: Record<
+      string,
+      Array<{
+        title: string;
+        filePath: string;
+        lineNumber: number;
+        severity: string;
+        confidenceScore?: number;
+        reason: string;
+      }>
+    > = {};
+    for (const d of allDismissed) {
+      if (!breakdown[d.category]) breakdown[d.category] = [];
+      breakdown[d.category].push({
+        title: d.title,
+        filePath: d.filePath,
+        lineNumber: d.lineNumber,
+        severity: d.severity,
+        confidenceScore: d.confidenceScore,
+        reason: d.rejectionReason,
+      });
+    }
+    stats.rejectionBreakdown = breakdown;
+  }
+
   // Calculate cache hit rate
-  const totalCacheableTokens = stats.cacheCreationTokens + stats.cacheReadTokens
-  stats.cacheHitRate = totalCacheableTokens > 0
-    ? stats.cacheReadTokens / totalCacheableTokens
-    : 0
+  const totalCacheableTokens =
+    stats.cacheCreationTokens + stats.cacheReadTokens;
+  stats.cacheHitRate =
+    totalCacheableTokens > 0 ? stats.cacheReadTokens / totalCacheableTokens : 0;
 
   // Calculate estimated cost with cache pricing
-  const freshInputCost = (stats.estimatedInputTokens * HAIKU_PRICING.input) / 1_000_000
-  const cacheWriteCost = (stats.cacheCreationTokens * HAIKU_PRICING.cacheWrite) / 1_000_000
-  const cacheReadCost = (stats.cacheReadTokens * HAIKU_PRICING.cacheRead) / 1_000_000
-  const outputCost = (stats.estimatedOutputTokens * HAIKU_PRICING.output) / 1_000_000
-
-  stats.estimatedCost = freshInputCost + cacheWriteCost + cacheReadCost + outputCost
+  const freshInputCost =
+    (stats.estimatedInputTokens * HAIKU_PRICING.input) / 1_000_000;
+  const cacheWriteCost =
+    (stats.cacheCreationTokens * HAIKU_PRICING.cacheWrite) / 1_000_000;
+  const cacheReadCost =
+    (stats.cacheReadTokens * HAIKU_PRICING.cacheRead) / 1_000_000;
+  const outputCost =
+    (stats.estimatedOutputTokens * HAIKU_PRICING.output) / 1_000_000;
+
+  stats.estimatedCost =
+    freshInputCost + cacheWriteCost + cacheReadCost + outputCost;
 
   // Log validation stats with cache metrics and performance
-  console.log(`[AI Validation] Stats:`)
-  console.log(`  - Total findings: ${stats.totalFindings}`)
-  console.log(`  - AI validated: ${stats.validatedFindings}`)
-  console.log(`  - Confirmed: ${stats.confirmedFindings}`)
-  console.log(`  - Dismissed: ${stats.dismissedFindings}`)
-  console.log(`  - Downgraded: ${stats.downgradedFindings}`)
-  console.log(`  - API calls: ${stats.apiCalls}`)
-  console.log(`  - Performance:`)
-  console.log(`    - Total duration: ${(totalBatchWaitTime / 1000).toFixed(1)}s`)
-  console.log(`    - Total API batches: ${totalApiBatches}`)
-  console.log(`    - Avg time per file: ${fileEntries.length > 0 ? (totalBatchWaitTime / fileEntries.length).toFixed(0) : 0}ms`)
-  console.log(`  - Cache metrics:`)
-  console.log(`    - Cache writes: ${stats.cacheCreationTokens.toLocaleString()} tokens`)
-  console.log(`    - Cache reads: ${stats.cacheReadTokens.toLocaleString()} tokens`)
-  console.log(`    - Cache hit rate: ${(stats.cacheHitRate * 100).toFixed(1)}%`)
-  console.log(`  - Token usage:`)
-  console.log(`    - Input (total): ${stats.estimatedInputTokens.toLocaleString()} tokens`)
-  console.log(`    - Output: ${stats.estimatedOutputTokens.toLocaleString()} tokens`)
-  console.log(`  - Estimated cost: $${stats.estimatedCost.toFixed(4)}`)
-
-  return { vulnerabilities: validatedFindings, stats }
+  log(`[AI Validation] Stats:`);
+  log(`  - Total findings: ${stats.totalFindings}`);
+  log(`  - AI validated: ${stats.validatedFindings}`);
+  log(`  - Confirmed: ${stats.confirmedFindings}`);
+  log(`  - Dismissed: ${stats.dismissedFindings}`);
+  log(`  - Downgraded: ${stats.downgradedFindings}`);
+  log(`  - API calls: ${stats.apiCalls}`);
+  log(`  - Performance:`);
+  log(`    - Total duration: ${(totalBatchWaitTime / 1000).toFixed(1)}s`);
+  log(`    - Total API batches: ${totalApiBatches}`);
+  log(
+    `    - Avg time per file: ${fileEntries.length > 0 ? (totalBatchWaitTime / fileEntries.length).toFixed(0) : 0}ms`,
+  );
+  log(`  - Cache metrics:`);
+  log(
+    `    - Cache writes: ${stats.cacheCreationTokens.toLocaleString()} tokens`,
+  );
+  log(`    - Cache reads: ${stats.cacheReadTokens.toLocaleString()} tokens`);
+  log(`    - Cache hit rate: ${(stats.cacheHitRate * 100).toFixed(1)}%`);
+  log(`  - Token usage:`);
+  log(
+    `    - Input (total): ${stats.estimatedInputTokens.toLocaleString()} tokens`,
+  );
+  log(`    - Output: ${stats.estimatedOutputTokens.toLocaleString()} tokens`);
+  log(`  - Estimated cost: $${stats.estimatedCost.toFixed(4)}`);
+
+  return { vulnerabilities: validatedFindings, stats };
 }