@oculum/scanner 1.0.13 → 1.0.15

package/src/detect/secrets/entropy.ts

@@ -69,6 +69,11 @@ function extractStringLiterals(content: string): Array<{ value: string; line: nu
   return strings
 }
 
+// Check if line is a package.json metadata field (author, description, etc.) — not secrets
+function isPackageJsonMetadataField(lineContent: string): boolean {
+  return /^\s*"(author|description|license|homepage|keywords|repository|bugs|funding|contributors|maintainers|engines|os|cpu)"/.test(lineContent)
+}
+
 // Check if string looks like a known safe pattern (URLs, paths, etc.)
 function isSafePattern(str: string): boolean {
   const safePatterns = [
@@ -420,6 +425,121 @@ function isSQLQuery(value: string, lineContent: string): boolean {
   )
 }
 
+/**
+ * Check if string is a DynamoDB/NoSQL expression (not a secret)
+ * DynamoDB expressions have high entropy due to placeholder syntax and mixed-case keywords
+ */
+function isNoSQLExpression(value: string, lineContent: string): boolean {
+  // DynamoDB expression keywords
+  const dynamoKeywords = [
+    /\bKeyConditionExpression\b/,
+    /\bFilterExpression\b/,
+    /\bProjectionExpression\b/,
+    /\bConditionExpression\b/,
+    /\bUpdateExpression\b/,
+    /\bExpressionAttributeNames\b/,
+    /\bExpressionAttributeValues\b/,
+  ]
+
+  // DynamoDB expression syntax patterns
+  const dynamoSyntax = [
+    /:[a-zA-Z_]\w*/,              // :placeholder patterns
+    /#[a-zA-Z_]\w*/,              // #attribute patterns
+    /\bbegins_with\s*\(/i,
+    /\battribute_exists\s*\(/i,
+    /\battribute_not_exists\s*\(/i,
+    /\bcontains\s*\(/i,
+  ]
+
+  // DynamoDB key construction patterns (PK/SK/GSI with interpolation)
+  const keyConstructionPatterns = [
+    /\b(PK|SK|GSI)\b.*\$\{/,
+    /\$\{.*\b(PK|SK|GSI)\b/,
+    /['"`].*\b(PK|SK|GSI)[#|_].*['"`]/,
+  ]
+
+  // Repository/Data-access file context
+  const dataAccessFilePatterns = [
+    /Repository\.(cs|ts|js)$/i,
+    /Repository\./i,
+    /DynamoDB/i,
+    /dynamo/i,
+  ]
+
+  const combined = value + ' ' + lineContent
+
+  // Check for DynamoDB keywords in the line context
+  if (dynamoKeywords.some(p => p.test(lineContent))) return true
+
+  // Check for DynamoDB expression syntax in the value
+  const syntaxMatches = dynamoSyntax.filter(p => p.test(value)).length
+  if (syntaxMatches >= 2) return true
+
+  // Check for key construction patterns
+  if (keyConstructionPatterns.some(p => p.test(combined))) return true
+
+  // Single syntax match in a data-access file context
+  if (syntaxMatches >= 1 && dataAccessFilePatterns.some(p => p.test(lineContent))) return true
+
+  return false
+}
+
+/**
+ * Check if string is a DateTime format or format string (not a secret)
+ * Date/time format strings and .NET/Python format placeholders have high entropy
+ */
+function isDateTimeOrFormatString(value: string, lineContent: string): boolean {
+  // ISO date format patterns
+  const dateFormatPatterns = [
+    /yyyy[-/]MM[-/]dd/,
+    /MM[-/]dd[-/]yyyy/,
+    /dd[-/]MM[-/]yyyy/,
+    /HH:mm:ss/,
+    /hh:mm:ss/,
+    /\.ToString\s*\(\s*["']/i,
+  ]
+
+  // C# DateTime formatting
+  const csharpDateTimePatterns = [
+    /\{0:yyyy/,
+    /\{0:dd/,
+    /\{0:MM/,
+    /\{0:HH/,
+    /\bDateTime\.Parse\b/,
+    /\bDateTime\.TryParse\b/,
+    /\bDateTimeOffset\b/,
+    /\bDateTime\.Now\b/,
+    /\bDateTime\.UtcNow\b/,
+  ]
+
+  // Python format strings
+  const pythonFormatPatterns = [
+    /\.strftime\s*\(/,
+    /\.strptime\s*\(/,
+    /%Y[-%]%m[-%]%d/,
+    /%H:%M:%S/,
+    /%d\/%m\/%Y/,
+  ]
+
+  // .NET format placeholders: {0}, {1}, {2}
+  const formatPlaceholderPattern = /\{\d+\}/g
+  const placeholderMatches = value.match(formatPlaceholderPattern)
+  if (placeholderMatches && placeholderMatches.length >= 3) return true
+
+  // General format specifiers: %s, %d, %f mixed in
+  const generalFormatPattern = /%[sdifFeEgGxXocrpn%]/g
+  const generalMatches = value.match(generalFormatPattern)
+  if (generalMatches && generalMatches.length >= 3) return true
+
+  const combined = value + ' ' + lineContent
+
+  return (
+    dateFormatPatterns.some(p => p.test(combined)) ||
+    csharpDateTimePatterns.some(p => p.test(combined)) ||
+    pythonFormatPatterns.some(p => p.test(combined))
+  )
+}
+
 /**
  * Check if string is an i18n/translation string (not a secret)
  * Internationalization strings can have high entropy
@@ -694,6 +814,52 @@ function isMinifiedFile(filePath: string): boolean {
   return minifiedPatterns.some(p => p.test(filePath))
 }
 
+/**
+ * Check if file is in a static asset directory (images, fonts, etc.)
+ * These files contain resource hashes/names, not secrets
+ */
+function isStaticAssetDirectory(filePath: string): boolean {
+  return /\/(assets|public|static|resources|images|img|fonts|icons|media)\//i.test(filePath)
+}
+
+/**
+ * Check if file is in an educational/challenge directory
+ * CTF challenges and hacking tutorials contain intentional secrets
+ */
+function isEducationalOrChallengeDirectory(filePath: string): boolean {
+  return /\/(challenges?|hacking[-_]?instructor|ctf|capture-the-flag|vulnerable-apps?|attack[-_]lab)\//i.test(filePath)
+}
+
+/**
+ * Check if value is an alphabet/charset constant (sequential chars used for encoding)
+ */
+function isAlphabetOrCharsetConstant(value: string, lineContent: string): boolean {
+  // Variable name contains alphabet/chars/charset etc.
+  if (/\b(alphabet|chars|charset|charSet|characters|base64[_-]?chars|allowed[_-]?chars|valid[_-]?chars|char[_-]?set|char[_-]?pool|key[_-]?space|hex[_-]?chars|alphanum)\b/i.test(lineContent)) return true
+  // Value contains sequential alpha/digit runs (ABCDEFGHIJKLMNOPQRSTUVWXYZabc... or 0123456789)
+  if (/abcdefghij|ABCDEFGHIJ|0123456789/.test(value) && /^[A-Za-z0-9+/=_-]+$/.test(value)) return true
+  return false
+}
+
+/**
+ * Check if value is a public OAuth client ID (not a secret)
+ * Google OAuth client IDs (*.apps.googleusercontent.com) are public identifiers
+ */
+function isPublicOAuthClientID(value: string): boolean {
+  return /\.apps\.googleusercontent\.com$/.test(value)
+}
+
+/**
+ * Check if value is an SPDX license expression or license text (not a secret)
+ */
+function isLicenseOrSPDX(value: string, lineContent: string): boolean {
+  if (/\bSPDX-License-Identifier\b/i.test(lineContent)) return true
+  if (/^(MIT|Apache-2\.0|GPL-[23]\.0|BSD-[23]-Clause|ISC|MPL-2\.0|LGPL|AGPL|Unlicense)(\s+(OR|AND|WITH)\s+\w+)*$/i.test(value)) return true
+  if (/\b(Permission is hereby granted|THE SOFTWARE IS PROVIDED|WITHOUT WARRANTY|all rights reserved)\b/i.test(value)) return true
+  if (/["'](license|licence|spdx)["']\s*:/i.test(lineContent)) return true
+  return false
+}
+
 export function detectHighEntropyStrings(
   content: string,
   filePath: string,
@@ -706,6 +872,16 @@ export function detectHighEntropyStrings(
     return vulnerabilities
   }
 
+  // Skip static asset directories (images, fonts, etc.)
+  if (isStaticAssetDirectory(filePath)) {
+    return vulnerabilities
+  }
+
+  // Skip educational/challenge directories (intentional secrets)
+  if (isEducationalOrChallengeDirectory(filePath)) {
+    return vulnerabilities
+  }
+
   // Skip scanner/fixture files to avoid self-detection
   if (isScannerOrFixtureFile(filePath)) {
     return vulnerabilities
@@ -731,9 +907,13 @@ export function detectHighEntropyStrings(
     return vulnerabilities
   }
 
+  const isPackageJson = filePath.endsWith('package.json')
   const strings = extractStringLiterals(content)
 
   for (const { value, line, lineContent } of strings) {
+    // Skip package.json metadata fields (author, description, etc.)
+    if (isPackageJson && isPackageJsonMetadataField(lineContent)) continue
+
     // Skip comments
     if (isComment(lineContent)) continue
 
@@ -776,6 +956,12 @@ export function detectHighEntropyStrings(
     // Skip SQL queries (SELECT, INSERT, etc. have high entropy but aren't secrets)
     if (isSQLQuery(value, lineContent)) continue
 
+    // Skip DynamoDB/NoSQL expressions (placeholder syntax creates high entropy)
+    if (isNoSQLExpression(value, lineContent)) continue
+
+    // Skip DateTime format strings and .NET/Python format placeholders
+    if (isDateTimeOrFormatString(value, lineContent)) continue
+
     // Skip i18n/translation strings
     if (isI18nString(value, lineContent)) continue
 
@@ -809,6 +995,15 @@ export function detectHighEntropyStrings(
     // Skip natural language strings (high ratio of common English words)
     if (isNaturalLanguage(value)) continue
 
+    // Skip alphabet/charset constants (sequential chars for encoding)
+    if (isAlphabetOrCharsetConstant(value, lineContent)) continue
+
+    // Skip public OAuth client IDs (not secrets)
+    if (isPublicOAuthClientID(value)) continue
+
+    // Skip SPDX license expressions and license text
+    if (isLicenseOrSPDX(value, lineContent)) continue
+
     // Calculate entropy
     const entropy = calculateEntropy(value)
 

package/src/detect/secrets/index.ts

@@ -7,14 +7,13 @@
 import type { Vulnerability, ScanFile, CancellationToken } from '../../shared/types'
 import type { ProgressCallback } from '../../shared/types'
 import { detectHighEntropyStrings } from './entropy'
-import { detectKnownPatterns } from './patterns'
 import { auditConfiguration } from './config-audit'
 import { detectDangerousFiles } from '../config/file-flags'
 import { detectAICommentPatterns } from '../config/comments'
 import { detectSensitiveURLs } from '../config/urls'
-import { detectWeakCrypto } from './weak-crypto'
 import { detectMCPConfigIssues } from './config-mcp-audit'
 import { detectAgentSkillInjection } from '../config/agent-skill-injection'
+import { detectRulesFileBackdoor } from '../config/rules-file-backdoor'
 // Tier system removed in Phase 2B — confidence scoring handles routing
 import {
   filterFindingsByPath,
@@ -45,8 +44,6 @@ export interface Layer1Result {
 
 // Layer 1 detector stat keys
 type Layer1StatsRecord = {
-  known_secrets: number
-  weak_crypto: number
   sensitive_urls: number
   entropy: number
   config_audit: number
@@ -54,6 +51,7 @@ type Layer1StatsRecord = {
   ai_comments: number
   mcp_config: number
   agent_skill: number
+  rules_file_backdoor: number
 }
 
 // Process a single file through all Layer 1 detectors
@@ -62,8 +60,6 @@ function processFileLayer1(file: ScanFile): {
   stats: Layer1StatsRecord
 } {
   const stats: Layer1StatsRecord = {
-    known_secrets: 0,
-    weak_crypto: 0,
     sensitive_urls: 0,
     entropy: 0,
     config_audit: 0,
@@ -71,42 +67,40 @@ function processFileLayer1(file: ScanFile): {
     ai_comments: 0,
     mcp_config: 0,
     agent_skill: 0,
+    rules_file_backdoor: 0,
   }
 
   // Create ParsedFile once for all detectors to share
   const parsed = ParsedFile.from(file)
 
   const entropyFindings = detectHighEntropyStrings(file.content, file.path, { parsed })
-  const patternFindings = detectKnownPatterns(file.content, file.path, { parsed })
   const configFindings = auditConfiguration(file.content, file.path, { parsed })
   const fileFlags = detectDangerousFiles(file.content, file.path, { parsed })
   const commentFindings = detectAICommentPatterns(file.content, file.path, { parsed })
   const urlFindings = detectSensitiveURLs(file.content, file.path, { parsed })
-  const cryptoFindings = detectWeakCrypto(file.content, file.path, { parsed })
   const mcpConfigFindings = detectMCPConfigIssues(file.content, file.path, { parsed })
   const agentSkillFindings = detectAgentSkillInjection(file.content, file.path, { parsed })
+  const rulesFileFindings = detectRulesFileBackdoor(file.content, file.path, { parsed })
 
   stats.entropy = entropyFindings.length
-  stats.known_secrets = patternFindings.length
   stats.config_audit = configFindings.length
   stats.file_flags = fileFlags.length
   stats.ai_comments = commentFindings.length
   stats.sensitive_urls = urlFindings.length
-  stats.weak_crypto = cryptoFindings.length
   stats.mcp_config = mcpConfigFindings.length
   stats.agent_skill = agentSkillFindings.length
+  stats.rules_file_backdoor = rulesFileFindings.length
 
   return {
     findings: [
       ...entropyFindings,
-      ...patternFindings,
       ...configFindings,
       ...fileFlags,
       ...commentFindings,
       ...urlFindings,
-      ...cryptoFindings,
       ...mcpConfigFindings,
       ...agentSkillFindings,
+      ...rulesFileFindings,
     ],
     stats,
   }
@@ -127,8 +121,6 @@ export async function runLayer1Scan(
 
   // Track raw counts per detector (before dedupe)
   const rawStats: Layer1StatsRecord = {
-    known_secrets: 0,
-    weak_crypto: 0,
     sensitive_urls: 0,
     entropy: 0,
     config_audit: 0,
@@ -136,6 +128,7 @@ export async function runLayer1Scan(
     ai_comments: 0,
     mcp_config: 0,
     agent_skill: 0,
+    rules_file_backdoor: 0,
   }
 
   // Track progress for frequent updates
@@ -220,11 +213,9 @@ function deduplicateFindings(vulnerabilities: Vulnerability[]): Vulnerability[]
 // severityRank imported from utils/parsed-file
 
 export { detectHighEntropyStrings } from './entropy'
-export { detectKnownPatterns } from './patterns'
 export { auditConfiguration } from './config-audit'
 export { detectDangerousFiles } from '../config/file-flags'
 export { detectAICommentPatterns } from '../config/comments'
 export { detectSensitiveURLs } from '../config/urls'
-export { detectWeakCrypto } from './weak-crypto'
 export { detectMCPConfigIssues } from './config-mcp-audit'
 export { detectAgentSkillInjection } from '../config/agent-skill-injection'