@oculum/scanner 1.0.0

package/src/utils/context-helpers.ts

@@ -0,0 +1,535 @@
+/**
+ * Shared Context Helpers
+ * Centralized utility functions for detecting file and code context
+ * Used across Layer 1 and Layer 2 scanners to reduce false positives
+ */
+
+// ============================================================================
+// File Path Context Detection
+// ============================================================================
+
+/**
+ * Check if file is server-only (not bundled to client)
+ * Server-only files can safely use service role keys and other admin secrets
+ */
+export function isServerOnlyFile(filePath: string): boolean {
+  const serverPatterns = [
+    /lib\/supabase\/(server|admin|middleware)\.(ts|js)$/i,
+    /\/api\//i,                    // Next.js API routes
+    /\/server\//i,                 // Server directories
+    /\.server\.(ts|js|tsx|jsx)$/i, // .server.ts files
+    /\/actions\//i,                // Server actions
+    /middleware\.(ts|js)$/i,       // Middleware files
+    /\/cron\//i,                   // Cron jobs
+    /\/workers?\//i,               // Worker files
+    /\/scripts?\//i,               // Scripts
+    /\/seed\//i,                   // Database seeds
+    /\/migrations?\//i,            // Database migrations
+    /\/lib\/[^/]+\/server/i,       // lib/*/server patterns
+    /\/utils\/server/i,            // utils/server
+    /\/helpers\/server/i,          // helpers/server
+    /\.action\.(ts|js)$/i,         // .action.ts files
+    /route\.(ts|js)$/i,            // Next.js route handlers
+  ]
+  return serverPatterns.some(pattern => pattern.test(filePath))
+}
+
+/**
+ * Check if file is a test, mock, or fixture file
+ * These files often contain fake secrets and should have lower severity
+ */
+export function isTestOrMockFile(filePath: string): boolean {
+  const testPatterns = [
+    /\.(test|spec)\.(ts|tsx|js|jsx)$/i,
+    /\/__tests__\//i,
+    /\/test\//i,
+    /\/tests\//i,
+    /\/mock/i,
+    /\/mocks\//i,
+    /\/fixtures?\//i,
+    /\.mock\.(ts|tsx|js|jsx)$/i,
+    /\.stub\.(ts|tsx|js|jsx)$/i,
+    /\.(stories|story)\.(ts|tsx|js|jsx)$/i,  // Storybook
+    /\/e2e\//i,                               // E2E tests
+    /\/cypress\//i,                           // Cypress tests
+    /\/playwright\//i,                        // Playwright tests
+    /\/vitest\//i,                            // Vitest
+    /\/jest\//i,                              // Jest
+  ]
+  return testPatterns.some(pattern => pattern.test(filePath))
+}
+
+/**
+ * Check if file is an example/sample/template file
+ * These files should be skipped or have significantly reduced severity
+ */
+export function isExampleFile(filePath: string): boolean {
+  return (
+    filePath.includes('.example') ||
+    filePath.includes('.sample') ||
+    filePath.includes('.template') ||
+    filePath.includes('README') ||
+    filePath.includes('/examples/') ||
+    filePath.includes('/example/') ||
+    filePath.includes('/demo/') ||
+    filePath.includes('/demos/')
+  )
+}
+
+/**
+ * Check if file is in an examples/demo directory
+ * Stronger check than isExampleFile - specifically for directories
+ * These are typically tutorial/demo code, not production patterns
+ */
+export function isExampleDirectory(filePath: string): boolean {
+  const examplePatterns = [
+    /\/examples?\//i,
+    /\/demos?\//i,
+    /\/templates?\//i,
+    /\/samples?\//i,
+    /\/tutorials?\//i,
+    /\/cookbook\//i,
+    /\/quickstart\//i,
+    /\/getting-started\//i,
+  ]
+  return examplePatterns.some(pattern => pattern.test(filePath))
+}
+
+/**
+ * Check if file is library/framework code (base classes, utilities)
+ * Library code is intentionally generic - consumers add security
+ * This applies to: langchain, vercel/ai, llamaindex, etc.
+ */
+export function isLibraryCode(filePath: string): boolean {
+  const libraryPatterns = [
+    // Package source directories in monorepos
+    /\/libs\/[^/]+\/src\//i,
+    /\/packages\/[^/]+\/src\//i,
+    // Common library patterns
+    /\/langchain-/i,
+    /\/llamaindex/i,
+    // Source files that aren't examples or tests
+    /\/src\/(?!.*(?:examples?|demos?|tests?)\/).*\.(ts|js)$/i,
+  ]
+
+  // Must match library pattern AND not be example/test
+  return (
+    libraryPatterns.some(pattern => pattern.test(filePath)) &&
+    !isExampleDirectory(filePath) &&
+    !isTestOrMockFile(filePath)
+  )
+}
+
+/**
+ * Check if file is a fixture file (test data, mock responses)
+ * Fixtures contain fake data and should have reduced severity
+ */
+export function isFixtureFile(filePath: string): boolean {
+  const fixturePatterns = [
+    /__fixtures__\//i,
+    /\.fixture\./i,
+    /fixtures?\//i,
+    /testdata\//i,
+    /test-data\//i,
+    /test_data\//i,
+    /mock-data\//i,
+    /mockdata\//i,
+    /\.mock\./i,
+    /\.stub\./i,
+  ]
+  return fixturePatterns.some(pattern => pattern.test(filePath))
+}
+
+/**
+ * Check if file is documentation (README, CHANGELOG, etc.)
+ * These files should typically be skipped for security scanning
+ */
+export function isDocumentationFile(filePath: string): boolean {
+  const docPatterns = [
+    /README/i,
+    /CHANGELOG/i,
+    /CONTRIBUTING/i,
+    /LICENSE/i,
+    /\.md$/i,
+    /\.mdx$/i,
+    /\/docs\//i,
+    /\/documentation\//i,
+  ]
+  return docPatterns.some(pattern => pattern.test(filePath))
+}
+
+/**
+ * Check if file is scanner code, fixture, or rule definition
+ * Avoid flagging the scanner's own code/test cases
+ */
+export function isScannerOrFixtureFile(filePath: string): boolean {
+  const scannerPatterns = [
+    /\/scanner\//i,
+    /\/detector\//i,
+    /\/security\//i,
+    /\/rules?\//i,
+    /\/patterns?\//i,
+    /\/fixtures?\//i,
+    /\/testdata\//i,
+    /\/test-data\//i,
+    /\/test_data\//i,
+  ]
+  return scannerPatterns.some(pattern => pattern.test(filePath))
+}
+
+/**
+ * Check if file is likely client-bundled (exposed to browser)
+ */
+export function isClientBundledFile(filePath: string): boolean {
+  // Files in these locations are typically client-bundled
+  const clientPatterns = [
+    /\/components\//i,
+    /\/pages\//i,           // Next.js pages (can be SSR, but code visible)
+    /\/app\/.*page\.(ts|tsx|js|jsx)$/i,  // Next.js app router pages
+    /\/hooks\//i,
+    /\/contexts?\//i,
+    /\/providers?\//i,
+    /\/stores?\//i,         // State management
+    /\.client\.(ts|js|tsx|jsx)$/i,  // .client.ts files
+  ]
+
+  // But not if they're also server files
+  if (isServerOnlyFile(filePath)) {
+    return false
+  }
+
+  return clientPatterns.some(pattern => pattern.test(filePath))
+}
+
+// ============================================================================
+// Code Line Context Detection
+// ============================================================================
+
+/**
+ * Check if line uses environment variable reference (not hardcoded)
+ */
+export function isEnvVarReference(line: string): boolean {
+  return (
+    /process\.env\.[A-Z_]+/.test(line) ||
+    /\$\{?[A-Z_]+\}?/.test(line) ||
+    /import\.meta\.env\.[A-Z_]+/.test(line) ||
+    /Deno\.env\.get\(/.test(line) ||
+    /os\.environ\[/.test(line) ||      // Python
+    /os\.getenv\(/.test(line) ||       // Python
+    /ENV\[['"]/.test(line) ||          // Ruby
+    /env\(["']/.test(line)             // Laravel PHP
+  )
+}
+
+/**
+ * Check if line uses NEXT_PUBLIC_ prefix (client-exposed)
+ */
+export function isNextPublicEnvVar(line: string): boolean {
+  return /NEXT_PUBLIC_[A-Z_]+/.test(line)
+}
+
+/**
+ * Check if line is a comment
+ */
+export function isComment(lineContent: string): boolean {
+  const trimmed = lineContent.trim()
+  return (
+    trimmed.startsWith('//') ||
+    trimmed.startsWith('#') ||
+    trimmed.startsWith('*') ||
+    trimmed.startsWith('/*') ||
+    trimmed.startsWith('"""') ||
+    trimmed.startsWith("'''") ||
+    trimmed.startsWith('<!--')
+  )
+}
+
+/**
+ * Check if value/line appears to be a placeholder
+ */
+export function isPlaceholderValue(value: string, line: string): boolean {
+  const placeholderPatterns = [
+    /xxx/i,
+    /your[-_]?/i,
+    /YOUR[-_]?/i,
+    /placeholder/i,
+    /example/i,
+    /REPLACE[-_]?/i,
+    /CHANGEME/i,
+    /<[a-z_-]+>/i,           // <your-api-key>
+    /\[\s*[a-z_-]+\s*\]/i,   // [API_KEY]
+    /todo/i,
+    /fixme/i,
+  ]
+
+  return placeholderPatterns.some(pattern =>
+    pattern.test(value) || pattern.test(line)
+  )
+}
+
+// ============================================================================
+// Security Context Detection
+// ============================================================================
+
+/**
+ * Check if line/path indicates a public endpoint (health, webhook, cron)
+ * These don't need authentication
+ */
+export function isPublicEndpoint(lineContent: string, filePath: string): boolean {
+  // Health check patterns
+  const healthCheckPatterns = [
+    /\/health\/?["'`]?/i,
+    /\/healthz\/?["'`]?/i,
+    /\/ready\/?["'`]?/i,
+    /\/readyz\/?["'`]?/i,
+    /\/live\/?["'`]?/i,
+    /\/livez\/?["'`]?/i,
+    /\/ping\/?["'`]?/i,
+    /\/status\/?["'`]?/i,
+    /\/api\/health/i,
+    /\/api\/status/i,
+    /\/_health/i,
+  ]
+
+  // Webhook patterns
+  const webhookPatterns = [
+    /\/webhook/i,
+    /\/webhooks\//i,
+    /\/callback/i,
+    /\/stripe\/webhook/i,
+    /\/github\/webhook/i,
+    /\/clerk\/webhook/i,
+  ]
+
+  // Cron/scheduled job patterns
+  const cronPatterns = [
+    /\/cron\//i,
+    /\/scheduled\//i,
+    /\/tasks?\//i,
+    /\/jobs?\//i,
+  ]
+
+  // Check line content
+  const allPatterns = [...healthCheckPatterns, ...webhookPatterns, ...cronPatterns]
+  if (allPatterns.some(pattern => pattern.test(lineContent))) {
+    return true
+  }
+
+  // Check file path
+  if (filePath.includes('/health') ||
+      filePath.includes('/webhook') ||
+      filePath.includes('/cron') ||
+      filePath.includes('/scheduled')) {
+    return true
+  }
+
+  return false
+}
+
+/**
+ * Check if webhook has signature verification nearby
+ */
+export function hasWebhookSignatureVerification(lines: string[], lineIndex: number, windowSize: number = 15): boolean {
+  const signaturePatterns = [
+    /verifySignature/i,
+    /validateSignature/i,
+    /checkSignature/i,
+    /signature.*verify/i,
+    /verify.*signature/i,
+    /hmac/i,
+    /x-hub-signature/i,
+    /stripe-signature/i,
+    /svix-signature/i,
+    /webhook.*secret/i,
+    /constructEvent/i,      // Stripe webhook verification
+    /Webhook\.verify/i,     // Generic webhook verify
+  ]
+
+  const start = Math.max(0, lineIndex - windowSize)
+  const end = Math.min(lines.length, lineIndex + windowSize)
+
+  for (let i = start; i < end; i++) {
+    if (signaturePatterns.some(pattern => pattern.test(lines[i]))) {
+      return true
+    }
+  }
+
+  return false
+}
+
+/**
+ * Check if there's an auth check nearby (bidirectional search)
+ */
+export function hasAuthCheckNearby(lines: string[], lineIndex: number, windowSize: number = 20): boolean {
+  const authPatterns = [
+    /authorization/i,
+    /bearer\s+token/i,
+    /req\.user/i,
+    /request\.user/i,
+    /\.user\s*[=!]/,
+    /isAuthenticated/i,
+    /requireAuth/i,
+    /ensureAuth/i,
+    /checkAuth/i,
+    /verifyToken/i,
+    /validateToken/i,
+    /checkPermission/i,
+    /getServerSession/i,
+    /middleware.*auth/i,
+    /session\.user/i,
+    /currentUser/i,
+    /getSession\(/i,
+    /useSession\(/i,
+    /auth\(\)/i,             // Next-Auth auth()
+    /withAuth/i,
+    /protected/i,
+    /verifySignature/i,      // Webhook signature
+    /checkApiKey/i,
+    /validateApiKey/i,
+    /requireRole/i,
+    /hasRole/i,
+    /isAdmin/i,
+  ]
+
+  // Search bidirectionally
+  const start = Math.max(0, lineIndex - windowSize)
+  const end = Math.min(lines.length, lineIndex + windowSize)
+
+  for (let i = start; i < end; i++) {
+    if (authPatterns.some(pattern => pattern.test(lines[i]))) {
+      return true
+    }
+  }
+
+  return false
+}
+
+// ============================================================================
+// BYOK (Bring Your Own Key) Context Detection
+// ============================================================================
+
+/**
+ * Check if this appears to be a BYOK (user-provided key) context
+ * BYOK is a feature, not a vulnerability, unless improperly handled
+ */
+export function isBYOKContext(lineContent: string, filePath: string): boolean {
+  // Common BYOK patterns
+  const byokPatterns = [
+    /user.*api.*key/i,
+    /customer.*key/i,
+    /your.*api.*key/i,
+    /provide.*key/i,
+    /enter.*key/i,
+    /input.*key/i,
+    /form.*key/i,
+    /settings.*key/i,
+    /config.*key.*user/i,
+    /BYOK/i,
+    /bring.*your.*own/i,
+  ]
+
+  // Form/input contexts
+  const inputPatterns = [
+    /input.*type/i,
+    /onChange/i,
+    /onSubmit/i,
+    /handleSubmit/i,
+    /useState.*key/i,
+    /form.*data/i,
+  ]
+
+  // Settings/config UI patterns
+  const settingsPatterns = [
+    /\/settings\//i,
+    /\/config\//i,
+    /\/preferences\//i,
+    /\/profile\//i,
+  ]
+
+  // Check line content
+  if (byokPatterns.some(p => p.test(lineContent)) ||
+      inputPatterns.some(p => p.test(lineContent))) {
+    return true
+  }
+
+  // Check file path
+  if (settingsPatterns.some(p => p.test(filePath))) {
+    // In settings files, look for user input context
+    if (inputPatterns.some(p => p.test(lineContent))) {
+      return true
+    }
+  }
+
+  return false
+}
+
+/**
+ * Check if key is being stored/handled properly (not exposed)
+ */
+export function isKeyProperlyHandled(lineContent: string, lines: string[], lineIndex: number): boolean {
+  // Proper handling patterns (encryption, secure storage, etc.)
+  const properHandlingPatterns = [
+    /encrypt/i,
+    /hash/i,
+    /secure.*storage/i,
+    /keychain/i,
+    /vault/i,
+    /secretsManager/i,
+    /kms/i,
+    /\.env/i,
+  ]
+
+  // Check current line
+  if (properHandlingPatterns.some(p => p.test(lineContent))) {
+    return true
+  }
+
+  // Check nearby lines (5 lines before and after)
+  const start = Math.max(0, lineIndex - 5)
+  const end = Math.min(lines.length, lineIndex + 5)
+
+  for (let i = start; i < end; i++) {
+    if (properHandlingPatterns.some(p => p.test(lines[i]))) {
+      return true
+    }
+  }
+
+  return false
+}
+
+// ============================================================================
+// Service Role Key Context
+// ============================================================================
+
+/**
+ * Check if this is a service role key usage that's acceptable
+ * Server-only + env var = acceptable
+ * Client exposure = critical
+ */
+export function getServiceRoleKeyContext(
+  lineContent: string,
+  filePath: string
+): 'safe_server' | 'needs_review' | 'client_exposure' {
+  const isServer = isServerOnlyFile(filePath)
+  const usesEnvVar = isEnvVarReference(lineContent)
+  const isClientFile = isClientBundledFile(filePath)
+  const isNextPublic = isNextPublicEnvVar(lineContent)
+
+  // NEXT_PUBLIC_ service role key = always critical (client exposure)
+  if (isNextPublic) {
+    return 'client_exposure'
+  }
+
+  // Server-only file using env var = safe
+  if (isServer && usesEnvVar) {
+    return 'safe_server'
+  }
+
+  // Client-bundled file = exposure risk
+  if (isClientFile) {
+    return 'client_exposure'
+  }
+
+  // Hardcoded or ambiguous = needs review
+  return 'needs_review'
+}

package/src/utils/diff-detector.ts

@@ -0,0 +1,135 @@
+/**
+ * Diff Detection Utility
+ * Compares current repository tree against previous scan data to detect changed files
+ * Used for incremental scanning (Story D)
+ */
+
+export interface TreeFileInfo {
+  path: string
+  sha: string
+}
+
+export interface DiffResult {
+  /** Files that are new (not in previous scan) */
+  added: string[]
+  /** Files whose SHA changed (content modified) */
+  modified: string[]
+  /** Files that were deleted (in previous scan but not in current) */
+  deleted: string[]
+  /** Files that haven't changed */
+  unchanged: string[]
+  /** Total count of changed files (added + modified) */
+  changedCount: number
+  /** Whether this is a significant change warranting incremental scan */
+  shouldUseIncremental: boolean
+}
+
+/**
+ * Maximum number of changed files before falling back to full scan
+ * Beyond this threshold, incremental mode provides diminishing returns
+ */
+const MAX_CHANGED_FILES_FOR_INCREMENTAL = 50
+
+/**
+ * Detect which files have changed between the current tree and previous scan
+ *
+ * @param currentTree - Array of files from current GitHub tree (path + sha)
+ * @param previousFileShas - Map of {file_path: sha} from previous scan
+ * @returns DiffResult with categorized file changes
+ */
+export function detectChangedFiles(
+  currentTree: TreeFileInfo[],
+  previousFileShas: Record<string, string> | null | undefined
+): DiffResult {
+  // If no previous data, everything is "new"
+  if (!previousFileShas || Object.keys(previousFileShas).length === 0) {
+    return {
+      added: currentTree.map(f => f.path),
+      modified: [],
+      deleted: [],
+      unchanged: [],
+      changedCount: currentTree.length,
+      shouldUseIncremental: false, // No baseline = full scan
+    }
+  }
+
+  const added: string[] = []
+  const modified: string[] = []
+  const unchanged: string[] = []
+  const currentPaths = new Set<string>()
+
+  // Compare current files against previous
+  for (const file of currentTree) {
+    currentPaths.add(file.path)
+    const previousSha = previousFileShas[file.path]
+
+    if (!previousSha) {
+      // File didn't exist before
+      added.push(file.path)
+    } else if (previousSha !== file.sha) {
+      // File content changed (SHA differs)
+      modified.push(file.path)
+    } else {
+      // Same SHA = unchanged
+      unchanged.push(file.path)
+    }
+  }
+
+  // Find deleted files (in previous but not in current)
+  const deleted: string[] = []
+  for (const path of Object.keys(previousFileShas)) {
+    if (!currentPaths.has(path)) {
+      deleted.push(path)
+    }
+  }
+
+  const changedCount = added.length + modified.length
+  const shouldUseIncremental = changedCount > 0 && changedCount <= MAX_CHANGED_FILES_FOR_INCREMENTAL
+
+  return {
+    added,
+    modified,
+    deleted,
+    unchanged,
+    changedCount,
+    shouldUseIncremental,
+  }
+}
+
+/**
+ * Build a file SHA map from a GitHub tree response
+ * Filters to only include blob (file) entries
+ *
+ * @param tree - GitHub tree items with path and sha
+ * @returns Map of {file_path: sha}
+ */
+export function buildFileShaMap(
+  tree: Array<{ path: string; sha: string; type: 'blob' | 'tree' }>
+): Record<string, string> {
+  const map: Record<string, string> = {}
+
+  for (const item of tree) {
+    // Only include files (blobs), not directories (trees)
+    if (item.type === 'blob') {
+      map[item.path] = item.sha
+    }
+  }
+
+  return map
+}
+
+/**
+ * Check if two tree SHAs are different
+ * Quick check before doing detailed file comparison
+ *
+ * @param currentTreeSha - SHA of current tree
+ * @param previousTreeSha - SHA from previous scan
+ * @returns true if trees are different (need detailed comparison)
+ */
+export function hasTreeChanged(
+  currentTreeSha: string,
+  previousTreeSha: string | null | undefined
+): boolean {
+  if (!previousTreeSha) return true
+  return currentTreeSha !== previousTreeSha
+}