@oculum/scanner 1.0.0

package/src/layer1/patterns.ts

@@ -0,0 +1,516 @@
+/**
+ * Layer 1: Known Pattern Matching
+ * Curated library of high-fidelity regex patterns for detecting secrets
+ */
+
+import type { SecretPattern, Vulnerability } from '../types'
+import {
+  isServerOnlyFile,
+  isExampleFile,
+  isEnvVarReference,
+  isNextPublicEnvVar,
+  isComment,
+  isPlaceholderValue,
+  isTestOrMockFile,
+  isScannerOrFixtureFile,
+  isBYOKContext,
+  getServiceRoleKeyContext,
+} from '../utils/context-helpers'
+
+// Check if file is documentation/README
+function isDocumentationFile(filePath: string): boolean {
+  const docPatterns = [
+    /README/i,
+    /CHANGELOG/i,
+    /CONTRIBUTING/i,
+    /LICENSE/i,
+    /CODE_OF_CONDUCT/i,
+    /SECURITY/i,
+    /AUTHORS/i,
+    /HISTORY/i,
+    /\.md$/i,
+    /\.mdx$/i,
+    /\.rst$/i,        // reStructuredText
+    /\.adoc$/i,       // AsciiDoc
+    /\.txt$/i,        // Plain text docs
+    /\/docs\//i,
+    /\/documentation\//i,
+    /\/wiki\//i,
+    /\/guides?\//i,
+    /\/tutorials?\//i,
+    /\/examples?\//i,  // Example directories often have sample configs
+  ]
+  return docPatterns.some(p => p.test(filePath))
+}
+
+// Check if line contains example/sample placeholder values (not real secrets)
+function isExamplePlaceholder(line: string, value: string): boolean {
+  const placeholderPatterns = [
+    // Common placeholder indicators in the line context
+    /example/i,
+    /sample/i,
+    /demo/i,
+    /placeholder/i,
+    /your[_-]?api[_-]?key/i,
+    /your[_-]?secret/i,
+    /replace[_-]?with/i,
+    /insert[_-]?here/i,
+    /xxx+/i,
+    /yyy+/i,
+    /todo/i,
+    /fixme/i,
+  ]
+
+  // Check if the value itself looks like a placeholder
+  const valuePlaceholderPatterns = [
+    /^your[_-]/i,
+    /^my[_-]/i,
+    /^test[_-]/i,
+    /^sample[_-]/i,
+    /^example[_-]/i,
+    /^demo[_-]/i,
+    /^placeholder/i,
+    /^xxx+$/i,
+    /^yyy+$/i,
+    /^\*+$/,           // Just asterisks
+    /^\.{3,}$/,        // Just dots
+    /^<.*>$/,          // Angle bracket placeholders like <YOUR_KEY>
+    /^\[.*\]$/,        // Square bracket placeholders like [YOUR_KEY]
+    /^\{.*\}$/,        // Curly bracket placeholders like {YOUR_KEY}
+  ]
+
+  return placeholderPatterns.some(p => p.test(line)) ||
+         valuePlaceholderPatterns.some(p => p.test(value))
+}
+
+// Check if the variable name indicates test/mock data
+function hasTestVariableName(line: string): boolean {
+  const varNamePatterns = [
+    // JS/TS variable declarations: const TEST_API_KEY = "..."
+    /(?:const|let|var|export\s+const|export\s+let)\s+([A-Z_][A-Z0-9_]*)\s*=/i,
+    // Object property shorthand or assignment: TEST_KEY: "..." or "testKey": "..."
+    /([A-Z_][A-Z0-9_]*)\s*:\s*['"`]/,
+    // JSON-style keys: "test_key": or 'testKey':
+    /['"]([a-zA-Z_][a-zA-Z0-9_]*)['"]\s*:/,
+  ]
+
+  // Keywords that indicate test/mock data when in variable names
+  const testKeywords = /^(TEST|MOCK|EXAMPLE|DUMMY|FAKE|SAMPLE|PLACEHOLDER|DEMO)[_A-Z0-9]*$/i
+  const testSuffixes = /_?(TEST|MOCK|EXAMPLE|DUMMY|FAKE|SAMPLE)$/i
+
+  for (const pattern of varNamePatterns) {
+    const match = line.match(pattern)
+    if (match && match[1]) {
+      const varName = match[1]
+      if (testKeywords.test(varName) || testSuffixes.test(varName)) {
+        return true
+      }
+    }
+  }
+  return false
+}
+
+// Comprehensive list of secret patterns with specific prefixes
+export const SECRET_PATTERNS: SecretPattern[] = [
+  // API Keys with known prefixes
+  {
+    name: 'OpenAI API Key',
+    pattern: /sk-[a-zA-Z0-9]{20,}/g,
+    severity: 'critical',
+    description: 'OpenAI API key detected',
+  },
+  {
+    name: 'OpenAI Project API Key',
+    pattern: /sk-proj-[a-zA-Z0-9]{48,}/g,
+    severity: 'critical',
+    description: 'OpenAI project API key detected (new format)',
+  },
+  {
+    name: 'Anthropic API Key',
+    pattern: /sk-ant-[a-zA-Z0-9-]{20,}/g,
+    severity: 'critical',
+    description: 'Anthropic API key detected',
+  },
+  {
+    name: 'Anthropic API Key (Full)',
+    pattern: /sk-ant-api03-[a-zA-Z0-9_-]{90,}/g,
+    severity: 'critical',
+    description: 'Anthropic API key detected (full format)',
+  },
+  {
+    name: 'GitHub Token',
+    pattern: /ghp_[a-zA-Z0-9]{36,}/g,
+    severity: 'critical',
+    description: 'GitHub personal access token detected',
+  },
+  {
+    name: 'GitHub OAuth Token',
+    pattern: /gho_[a-zA-Z0-9]{36,}/g,
+    severity: 'critical',
+    description: 'GitHub OAuth token detected',
+  },
+  {
+    name: 'GitHub App Token',
+    pattern: /ghu_[a-zA-Z0-9]{36,}/g,
+    severity: 'critical',
+    description: 'GitHub App user token detected',
+  },
+  {
+    name: 'GitHub Refresh Token',
+    pattern: /ghr_[a-zA-Z0-9]{36,}/g,
+    severity: 'critical',
+    description: 'GitHub refresh token detected',
+  },
+  {
+    name: 'Stripe Secret Key',
+    pattern: /sk_live_[a-zA-Z0-9]{24,}/g,
+    severity: 'critical',
+    description: 'Stripe live secret key detected',
+  },
+  {
+    name: 'Stripe Test Key',
+    pattern: /sk_test_[a-zA-Z0-9]{24,}/g,
+    severity: 'medium',
+    description: 'Stripe test secret key detected',
+  },
+  {
+    name: 'Stripe Publishable Key',
+    pattern: /pk_(live|test)_[a-zA-Z0-9]{24,}/g,
+    severity: 'low',
+    description: 'Stripe publishable key detected (usually safe but verify)',
+  },
+  {
+    name: 'Square Access Token',
+    pattern: /sq0csp-[a-zA-Z0-9-_]{43}/g,
+    severity: 'critical',
+    description: 'Square access token detected',
+  },
+  {
+    name: 'AWS Access Key ID',
+    pattern: /AKIA[0-9A-Z]{16}/g,
+    severity: 'critical',
+    description: 'AWS Access Key ID detected',
+  },
+  {
+    name: 'AWS Secret Access Key',
+    // AWS secret keys are exactly 40 chars, base64-like, and typically appear near AWS context
+    // The pattern requires it to be in an AWS-related context (variable name, nearby AKIA, etc.)
+    pattern: /(?:aws[_-]?secret[_-]?(?:access[_-]?)?key|secret[_-]?access[_-]?key|AWS_SECRET)\s*[:=]\s*['"`]?([a-zA-Z0-9/+=]{40})['"`]?/gi,
+    severity: 'critical',
+    description: 'AWS Secret Access Key detected',
+  },
+  {
+    name: 'Google API Key',
+    pattern: /AIza[0-9A-Za-z-_]{35}/g,
+    severity: 'high',
+    description: 'Google API key detected',
+  },
+  {
+    name: 'Slack Token',
+    pattern: /xox[baprs]-[0-9a-zA-Z]{10,}/g,
+    severity: 'critical',
+    description: 'Slack token detected',
+  },
+  {
+    name: 'Slack Webhook',
+    pattern: /https:\/\/hooks\.slack\.com\/services\/T[a-zA-Z0-9_]+\/B[a-zA-Z0-9_]+\/[a-zA-Z0-9_]+/g,
+    severity: 'high',
+    description: 'Slack webhook URL detected',
+  },
+  {
+    name: 'Discord Webhook',
+    pattern: /https:\/\/discord(?:app)?\.com\/api\/webhooks\/[0-9]+\/[a-zA-Z0-9_-]+/g,
+    severity: 'high',
+    description: 'Discord webhook URL detected',
+  },
+  {
+    name: 'Twilio API Key',
+    pattern: /SK[a-f0-9]{32}/g,
+    severity: 'critical',
+    description: 'Twilio API key detected',
+  },
+  {
+    name: 'SendGrid API Key',
+    pattern: /SG\.[a-zA-Z0-9_-]{22}\.[a-zA-Z0-9_-]{43}/g,
+    severity: 'critical',
+    description: 'SendGrid API key detected',
+  },
+  {
+    name: 'Mailgun API Key',
+    pattern: /key-[a-zA-Z0-9]{32}/g,
+    severity: 'critical',
+    description: 'Mailgun API key detected',
+  },
+  {
+    name: 'Firebase API Key',
+    pattern: /AIza[0-9A-Za-z-_]{35}/g,
+    severity: 'high',
+    description: 'Firebase API key detected',
+  },
+  {
+    name: 'Supabase Anon Key',
+    pattern: /eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\.[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+/g,
+    severity: 'low',
+    description: 'Supabase anon key detected (usually safe for client-side)',
+  },
+  {
+    name: 'Supabase Service Role Key',
+    pattern: /eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\.[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+/g,
+    severity: 'critical',
+    description: 'JWT token detected - verify if this is a service role key',
+  },
+  {
+    name: 'Heroku API Key',
+    pattern: /[h|H][e|E][r|R][o|O][k|K][u|U].*[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}/gi,
+    severity: 'critical',
+    description: 'Heroku API key detected',
+  },
+  {
+    name: 'NPM Token',
+    pattern: /npm_[a-zA-Z0-9]{36}/g,
+    severity: 'critical',
+    description: 'NPM access token detected',
+  },
+  {
+    name: 'PyPI Token',
+    pattern: /pypi-[a-zA-Z0-9]{32,}/g,
+    severity: 'critical',
+    description: 'PyPI API token detected',
+  },
+  {
+    name: 'Private Key',
+    pattern: /-----BEGIN (RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----/g,
+    severity: 'critical',
+    description: 'Private key detected',
+  },
+  {
+    name: 'Generic Password in URL',
+    pattern: /[a-zA-Z]{3,10}:\/\/[^:]+:[^@]+@/g,
+    severity: 'high',
+    description: 'Password in URL detected',
+  },
+  {
+    name: 'Database Connection String',
+    pattern: /(mongodb|postgres|mysql|redis|amqp):\/\/[^\s"']+/gi,
+    severity: 'high',
+    description: 'Database connection string detected - may contain credentials',
+  },
+  // Additional patterns from checklist
+  {
+    name: 'Hardcoded JWT Token',
+    pattern: /eyJ[a-zA-Z0-9_-]{10,}\.eyJ[a-zA-Z0-9_-]{10,}\.[a-zA-Z0-9_-]{10,}/g,
+    severity: 'high',
+    description: 'Hardcoded JWT token detected',
+  },
+  {
+    name: 'PostgreSQL Connection String',
+    pattern: /postgres:\/\/[^:]+:[^@]+@[^\s"']+/gi,
+    severity: 'critical',
+    description: 'PostgreSQL connection string with credentials detected',
+  },
+  {
+    name: 'MongoDB Connection String',
+    pattern: /mongodb(\+srv)?:\/\/[^:]+:[^@]+@[^\s"']+/gi,
+    severity: 'critical',
+    description: 'MongoDB connection string with credentials detected',
+  },
+  {
+    name: 'MySQL Connection String',
+    pattern: /mysql:\/\/[^:]+:[^@]+@[^\s"']+/gi,
+    severity: 'critical',
+    description: 'MySQL connection string with credentials detected',
+  },
+  {
+    name: 'Generic API Key Assignment',
+    pattern: /['"']?api[_-]?key['"']?\s*[:=]\s*['"][a-zA-Z0-9_-]{20,}['"]/gi,
+    severity: 'medium',
+    description: 'Possible API key assignment detected - requires validation',
+  },
+  {
+    name: 'Generic Secret Key Assignment',
+    pattern: /['"']?secret[_-]?key['"']?\s*[:=]\s*['"][a-zA-Z0-9_-]{20,}['"]/gi,
+    severity: 'medium',
+    description: 'Possible secret key assignment detected - requires validation',
+  },
+  {
+    name: 'Vercel Token',
+    pattern: /vercel_[a-zA-Z0-9]{24,}/gi,
+    severity: 'critical',
+    description: 'Vercel API token detected',
+  },
+  {
+    name: 'Netlify Token',
+    pattern: /nfp_[a-zA-Z0-9]{40,}/gi,
+    severity: 'critical',
+    description: 'Netlify personal access token detected',
+  },
+  {
+    name: 'Cloudflare API Token',
+    pattern: /[a-zA-Z0-9_-]{40}(?=.*cloudflare)/gi,
+    severity: 'high',
+    description: 'Potential Cloudflare API token detected',
+  },
+]
+
+export function detectKnownPatterns(
+  content: string,
+  filePath: string
+): Vulnerability[] {
+  const vulnerabilities: Vulnerability[] = []
+  const lines = content.split('\n')
+
+  // Skip example files entirely
+  if (isExampleFile(filePath)) {
+    return vulnerabilities
+  }
+
+  // Skip scanner/fixture files to avoid self-detection
+  if (isScannerOrFixtureFile(filePath)) {
+    return vulnerabilities
+  }
+
+  // Skip documentation/README files
+  if (isDocumentationFile(filePath)) {
+    return vulnerabilities
+  }
+
+  // Check context for file-level decisions
+  const isServerFile = isServerOnlyFile(filePath)
+  const isTestFile = isTestOrMockFile(filePath)
+
+  for (const secretPattern of SECRET_PATTERNS) {
+    lines.forEach((line, index) => {
+      // Skip comments
+      if (isComment(line)) return
+
+      // Reset regex state
+      const regex = new RegExp(secretPattern.pattern.source, secretPattern.pattern.flags)
+      let match
+
+      while ((match = regex.exec(line)) !== null) {
+        const value = match[0]
+
+        // Skip placeholder values
+        if (isPlaceholderValue(value, line)) {
+          continue
+        }
+
+        // Skip obvious example/sample values
+        if (/example|sample|demo|test|dummy|fake|mock/i.test(value)) {
+          continue
+        }
+
+        // Skip values that look like format descriptions
+        if (/^[a-z]+_[a-z]+_[a-z]+$/i.test(value) || /^your[_-]/i.test(value)) {
+          continue
+        }
+
+        // Skip example/placeholder values (more comprehensive check)
+        if (isExamplePlaceholder(line, value)) {
+          continue
+        }
+
+        // Skip secrets in variables with test/mock names (e.g., TEST_API_KEY, MOCK_SECRET)
+        if (hasTestVariableName(line)) {
+          continue
+        }
+
+        // Check for BYOK (Bring Your Own Key) context - this is a feature, not a vulnerability
+        if (isBYOKContext(line, filePath)) {
+          // Skip BYOK patterns entirely if they're properly handled in server context
+          if (isServerFile && isEnvVarReference(line)) {
+            continue
+          }
+          // For client-side BYOK forms, we still don't flag - it's user input
+          // Only flag if it's a hardcoded key being exposed
+          if (!isEnvVarReference(line) && line.includes('=') && /['"`][a-zA-Z0-9_-]{20,}['"`]/.test(line)) {
+            // This might be a hardcoded default key in a BYOK context - needs review
+          } else {
+            continue
+          }
+        }
+
+        // Determine severity based on context
+        let adjustedSeverity = secretPattern.severity
+        let requiresAIValidation = false
+        let adjustedDescription = secretPattern.description
+        let adjustedConfidence: 'high' | 'medium' | 'low' = 'high'
+
+        // Check if this is a Supabase service role key or JWT
+        const isSupabaseOrJWT =
+          secretPattern.name.includes('Supabase') ||
+          secretPattern.name.includes('JWT') ||
+          /eyJ[a-zA-Z0-9_-]+\.eyJ/.test(value)
+
+        if (isSupabaseOrJWT) {
+          // Use the comprehensive service role context checker
+          const serviceRoleContext = getServiceRoleKeyContext(line, filePath)
+
+          if (serviceRoleContext === 'safe_server') {
+            // Server-only + env var = expected pattern, skip entirely
+            continue
+          } else if (serviceRoleContext === 'client_exposure') {
+            // Client exposure = critical
+            adjustedSeverity = 'critical'
+            requiresAIValidation = true
+            adjustedDescription = isNextPublicEnvVar(line)
+              ? `${secretPattern.description} - EXPOSED via NEXT_PUBLIC_ prefix (client-accessible)`
+              : `${secretPattern.description} - may be exposed to client bundle`
+          } else {
+            // needs_review - check more context
+            if (isEnvVarReference(line)) {
+              // Using env var but context unclear - validate
+              adjustedSeverity = 'medium'
+              requiresAIValidation = true
+              adjustedDescription = `${secretPattern.description} - verify this is not exposed to client`
+            } else if (isServerFile) {
+              // Hardcoded in server file - bad but not critical
+              adjustedSeverity = 'high'
+              requiresAIValidation = true
+              adjustedDescription = `${secretPattern.description} - hardcoded in server file, should use env var`
+            } else {
+              // Hardcoded in unknown context - critical + needs validation
+              adjustedSeverity = 'critical'
+              requiresAIValidation = true
+              adjustedDescription = `${secretPattern.description} - hardcoded secret may be exposed`
+            }
+          }
+        }
+
+        // Downgrade test file severity
+        if (isTestFile) {
+          if (adjustedSeverity === 'critical') {
+            adjustedSeverity = 'medium'
+          } else if (adjustedSeverity === 'high') {
+            adjustedSeverity = 'low'
+          } else {
+            adjustedSeverity = 'info'
+          }
+          adjustedConfidence = 'low'
+          adjustedDescription = `${adjustedDescription} (in test file)`
+        }
+
+        // Generic patterns always require AI validation
+        const isGenericPattern = secretPattern.name.includes('Generic')
+        const finalRequiresAIValidation = requiresAIValidation || isGenericPattern
+
+        vulnerabilities.push({
+          id: `pattern-${filePath}-${index + 1}-${secretPattern.name}`,
+          filePath,
+          lineNumber: index + 1,
+          lineContent: line.trim(),
+          severity: adjustedSeverity,
+          category: 'hardcoded_secret',
+          title: secretPattern.name,
+          description: adjustedDescription,
+          suggestedFix: 'Move this secret to an environment variable. Never commit secrets to version control.',
+          confidence: adjustedConfidence,
+          layer: 1,
+          requiresAIValidation: finalRequiresAIValidation,
+        })
+      }
+    })
+  }
+
+  return vulnerabilities
+}