@complior/engine 0.9.0

package/src/domain/scanner/rules/comment-filter.ts

@@ -0,0 +1,297 @@
+/**
+ * Strip comments and string literals from source code.
+ * Replaces them with whitespace to preserve line numbers.
+ */
+export const stripCommentsAndStrings = (content: string, extension: string): string => {
+  if (['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.go', '.java', '.rs'].includes(extension)) {
+    return stripCStyleComments(content);
+  }
+  if (extension === '.py') {
+    return stripPythonComments(content);
+  }
+  return content;
+};
+
+/**
+ * Strip comments only (preserve string literals).
+ * Use for pattern matching where compliance patterns may be in string values.
+ */
+export const stripCommentsOnly = (content: string, extension: string): string => {
+  if (['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.go', '.java', '.rs'].includes(extension)) {
+    return stripCStyleCommentsOnly(content);
+  }
+  if (extension === '.py') {
+    return stripPythonCommentsOnly(content);
+  }
+  return content;
+};
+
+/** Strip C-style comments only — skip over string literals without removing them */
+const stripCStyleCommentsOnly = (content: string): string => {
+  let result = '';
+  let i = 0;
+
+  while (i < content.length) {
+    // Line comment
+    if (content[i] === '/' && content[i + 1] === '/') {
+      while (i < content.length && content[i] !== '\n') {
+        result += ' ';
+        i++;
+      }
+      continue;
+    }
+
+    // Block comment
+    if (content[i] === '/' && content[i + 1] === '*') {
+      result += ' ';
+      i += 2;
+      while (i < content.length) {
+        if (content[i] === '*' && content[i + 1] === '/') {
+          result += '  ';
+          i += 2;
+          break;
+        }
+        result += content[i] === '\n' ? '\n' : ' ';
+        i++;
+      }
+      continue;
+    }
+
+    // Skip over strings without stripping them (to not enter comment-like chars inside strings)
+    if (content[i] === '`' || content[i] === '"' || content[i] === "'") {
+      const quote = content[i];
+      result += content[i];
+      i++;
+      if (quote === '`') {
+        while (i < content.length && content[i] !== '`') {
+          if (content[i] === '\\') {
+            result += content[i] + (content[i + 1] ?? '');
+            i += 2;
+            continue;
+          }
+          result += content[i];
+          i++;
+        }
+      } else {
+        while (i < content.length && content[i] !== quote) {
+          if (content[i] === '\\') {
+            result += content[i] + (content[i + 1] ?? '');
+            i += 2;
+            continue;
+          }
+          result += content[i];
+          i++;
+        }
+      }
+      if (i < content.length) { result += content[i]; i++; }
+      continue;
+    }
+
+    result += content[i];
+    i++;
+  }
+
+  return result;
+};
+
+/** Strip Python comments only — preserve regular strings */
+const stripPythonCommentsOnly = (content: string): string => {
+  let result = '';
+  let i = 0;
+
+  while (i < content.length) {
+    // Line comment
+    if (content[i] === '#') {
+      while (i < content.length && content[i] !== '\n') {
+        result += ' ';
+        i++;
+      }
+      continue;
+    }
+
+    // Skip over strings without stripping
+    if (content[i] === '"' || content[i] === "'") {
+      // Check triple-quote
+      if (content[i + 1] === content[i] && content[i + 2] === content[i]) {
+        const q = content.slice(i, i + 3);
+        result += q;
+        i += 3;
+        while (i < content.length) {
+          if (content[i] === q[0] && content[i + 1] === q[1] && content[i + 2] === q[2]) {
+            result += q;
+            i += 3;
+            break;
+          }
+          result += content[i];
+          i++;
+        }
+        continue;
+      }
+      // Single-line string
+      const quote = content[i];
+      result += content[i];
+      i++;
+      while (i < content.length && content[i] !== quote && content[i] !== '\n') {
+        if (content[i] === '\\') {
+          result += content[i] + (content[i + 1] ?? '');
+          i += 2;
+          continue;
+        }
+        result += content[i];
+        i++;
+      }
+      if (i < content.length && content[i] === quote) { result += content[i]; i++; }
+      continue;
+    }
+
+    result += content[i];
+    i++;
+  }
+
+  return result;
+};
+
+/**
+ * Strip C-style comments (// and /* *​/) and string literals.
+ */
+const stripCStyleComments = (content: string): string => {
+  let result = '';
+  let i = 0;
+
+  while (i < content.length) {
+    // Line comment
+    if (content[i] === '/' && content[i + 1] === '/') {
+      while (i < content.length && content[i] !== '\n') {
+        result += ' ';
+        i++;
+      }
+      continue;
+    }
+
+    // Block comment
+    if (content[i] === '/' && content[i + 1] === '*') {
+      result += ' ';
+      i += 2;
+      while (i < content.length) {
+        if (content[i] === '*' && content[i + 1] === '/') {
+          result += '  ';
+          i += 2;
+          break;
+        }
+        result += content[i] === '\n' ? '\n' : ' ';
+        i++;
+      }
+      continue;
+    }
+
+    // Template literal
+    if (content[i] === '`') {
+      result += ' ';
+      i++;
+      while (i < content.length && content[i] !== '`') {
+        // Handle template literal expressions ${...}
+        if (content[i] === '$' && content[i + 1] === '{') {
+          result += '  ';
+          i += 2;
+          let depth = 1;
+          while (i < content.length && depth > 0) {
+            if (content[i] === '{') depth++;
+            if (content[i] === '}') depth--;
+            result += content[i] === '\n' ? '\n' : ' ';
+            i++;
+          }
+          continue;
+        }
+        result += content[i] === '\n' ? '\n' : ' ';
+        i++;
+      }
+      if (i < content.length) { result += ' '; i++; }
+      continue;
+    }
+
+    // String literals (single and double quotes)
+    if (content[i] === '"' || content[i] === "'") {
+      const quote = content[i];
+      result += ' ';
+      i++;
+      while (i < content.length && content[i] !== quote) {
+        if (content[i] === '\\') {
+          result += '  ';
+          i += 2;
+          continue;
+        }
+        result += content[i] === '\n' ? '\n' : ' ';
+        i++;
+      }
+      if (i < content.length) { result += ' '; i++; }
+      continue;
+    }
+
+    result += content[i];
+    i++;
+  }
+
+  return result;
+};
+
+/**
+ * Strip Python comments (# and triple-quoted strings).
+ */
+const stripPythonComments = (content: string): string => {
+  let result = '';
+  let i = 0;
+
+  while (i < content.length) {
+    // Triple-quoted strings (""" or ''')
+    if (
+      (content[i] === '"' && content[i + 1] === '"' && content[i + 2] === '"') ||
+      (content[i] === "'" && content[i + 1] === "'" && content[i + 2] === "'")
+    ) {
+      const quote = content.slice(i, i + 3);
+      result += '   ';
+      i += 3;
+      while (i < content.length) {
+        if (content[i] === quote[0] && content[i + 1] === quote[1] && content[i + 2] === quote[2]) {
+          result += '   ';
+          i += 3;
+          break;
+        }
+        result += content[i] === '\n' ? '\n' : ' ';
+        i++;
+      }
+      continue;
+    }
+
+    // Line comment
+    if (content[i] === '#') {
+      while (i < content.length && content[i] !== '\n') {
+        result += ' ';
+        i++;
+      }
+      continue;
+    }
+
+    // Regular strings
+    if (content[i] === '"' || content[i] === "'") {
+      const quote = content[i];
+      result += ' ';
+      i++;
+      while (i < content.length && content[i] !== quote && content[i] !== '\n') {
+        if (content[i] === '\\') {
+          result += '  ';
+          i += 2;
+          continue;
+        }
+        result += ' ';
+        i++;
+      }
+      if (i < content.length && content[i] === quote) { result += ' '; i++; }
+      continue;
+    }
+
+    result += content[i];
+    i++;
+  }
+
+  return result;
+};

package/src/domain/scanner/rules/index.ts

@@ -0,0 +1,9 @@
+export { BANNED_PACKAGES, PROHIBITED_PATTERNS } from './banned-packages-data.js';
+export type { BannedPackage, ProhibitedPattern } from './banned-packages-data.js';
+export { AI_SDK_PACKAGES, BIAS_TESTING_PACKAGES } from './banned-packages-sdk.js';
+export { loadCustomBannedPackages, isBannedPackage, isAiSdkPackage, matchProhibitedPattern } from './banned-packages.js';
+export { stripCommentsAndStrings, stripCommentsOnly } from './comment-filter.js';
+export { NHI_PATTERNS, NHI_EXCLUDE_PATTERNS, shouldScanFile } from './nhi-patterns.js';
+export type { NhiCategory, NhiPattern } from './nhi-patterns.js';
+export { PATTERN_RULES } from './pattern-rules.js';
+export type { PatternCategory, PatternType, PatternRule } from './pattern-rules.js';

package/src/domain/scanner/rules/nhi-patterns.test.ts

@@ -0,0 +1,128 @@
+import { describe, it, expect } from 'vitest';
+import { NHI_PATTERNS, shouldScanFile } from './nhi-patterns.js';
+
+// Build test tokens at runtime to avoid triggering GitHub push protection.
+// These are NOT real secrets — they are synthetic values for regex validation.
+const fake = (prefix: string, suffix: string) => prefix + suffix;
+
+describe('NHI_PATTERNS', () => {
+  it('has at least 20 patterns', () => {
+    expect(NHI_PATTERNS.length).toBeGreaterThanOrEqual(20);
+  });
+
+  it('detects OpenAI key', () => {
+    const p = NHI_PATTERNS.find(p => p.id === 'nhi-openai-key')!;
+    expect(p.pattern.test(fake('sk-', 'abcdefghijklmnopqrstuvwx'))).toBe(true);
+    expect(p.pattern.test('sk-short')).toBe(false);
+  });
+
+  it('detects Anthropic key', () => {
+    const p = NHI_PATTERNS.find(p => p.id === 'nhi-anthropic-key')!;
+    expect(p.pattern.test(fake('sk-ant-api03-', 'abcdefghijklmnopqrst'))).toBe(true);
+    expect(p.pattern.test('sk-ant-short')).toBe(false);
+  });
+
+  it('detects AWS access key', () => {
+    const p = NHI_PATTERNS.find(p => p.id === 'nhi-aws-access-key')!;
+    expect(p.pattern.test(fake('AKIA', 'IOSFODNN7EXAMPLE'))).toBe(true);
+    expect(p.pattern.test('AKIA12345')).toBe(false);
+  });
+
+  it('detects GitHub PAT', () => {
+    const p = NHI_PATTERNS.find(p => p.id === 'nhi-github-pat')!;
+    expect(p.pattern.test(fake('ghp_', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij'))).toBe(true);
+  });
+
+  it('detects Google API key', () => {
+    const p = NHI_PATTERNS.find(p => p.id === 'nhi-google-api-key')!;
+    expect(p.pattern.test(fake('AIzaSy', 'A1234567890abcdefghijklmnopqrstuvw'))).toBe(true);
+  });
+
+  it('detects Slack bot token', () => {
+    const p = NHI_PATTERNS.find(p => p.id === 'nhi-slack-bot')!;
+    expect(p.pattern.test(fake('xoxb-', '0000000000-0000000000-TESTFAKETESTFAKETESTFAKE'))).toBe(true);
+  });
+
+  it('detects Stripe live key', () => {
+    const p = NHI_PATTERNS.find(p => p.id === 'nhi-stripe-key')!;
+    expect(p.pattern.test(fake('sk_live_', 'TESTFAKETESTFAKETESTFAKE'))).toBe(true);
+  });
+
+  it('detects private key header', () => {
+    const p = NHI_PATTERNS.find(p => p.id === 'nhi-private-key')!;
+    expect(p.pattern.test('-----BEGIN RSA PRIVATE KEY-----')).toBe(true);
+    expect(p.pattern.test('-----BEGIN EC PRIVATE KEY-----')).toBe(true);
+    expect(p.pattern.test('-----BEGIN PRIVATE KEY-----')).toBe(true);
+  });
+
+  it('detects GCP service account', () => {
+    const p = NHI_PATTERNS.find(p => p.id === 'nhi-gcp-service-account')!;
+    expect(p.pattern.test('"type": "service_account"')).toBe(true);
+    expect(p.pattern.test('"type" : "service_account"')).toBe(true);
+  });
+
+  it('detects AWS secret key assignment', () => {
+    const p = NHI_PATTERNS.find(p => p.id === 'nhi-aws-secret')!;
+    expect(p.pattern.test(fake('aws_secret_access_key = "', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"'))).toBe(true);
+  });
+
+  it('detects hardcoded password', () => {
+    const p = NHI_PATTERNS.find(p => p.id === 'nhi-password-assign')!;
+    expect(p.pattern.test('password = "my-secret-password-123"')).toBe(true);
+    expect(p.pattern.test("password = 'short'")).toBe(false); // too short
+  });
+
+  it('detects connection string', () => {
+    const p = NHI_PATTERNS.find(p => p.id === 'nhi-connection-string')!;
+    expect(p.pattern.test('mongodb://admin:secret@localhost:27017')).toBe(true);
+    expect(p.pattern.test('postgres://user:pass@db.example.com/mydb')).toBe(true);
+  });
+
+  it('detects JWT token', () => {
+    const p = NHI_PATTERNS.find(p => p.id === 'nhi-jwt')!;
+    expect(p.pattern.test('eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.abc123def456')).toBe(true);
+  });
+
+  it('detects NPM token', () => {
+    const p = NHI_PATTERNS.find(p => p.id === 'nhi-npm-token')!;
+    expect(p.pattern.test(fake('npm_', 'abcdefghijklmnopqrstuvwxyz1234567890'))).toBe(true);
+  });
+
+  it('detects SendGrid API key', () => {
+    const p = NHI_PATTERNS.find(p => p.id === 'nhi-sendgrid-key')!;
+    expect(p.pattern.test(fake('SG.', 'TESTFAKETESTFAKETESTFA.TESTFAKETESTFAKETESTFAKETESTFAKETESTFA12TEST'))).toBe(true);
+  });
+
+  it('all patterns have required fields', () => {
+    for (const p of NHI_PATTERNS) {
+      expect(p.id).toBeTruthy();
+      expect(p.name).toBeTruthy();
+      expect(p.category).toMatch(/^(api_key|service_account|secret|token)$/);
+      expect(p.severity).toMatch(/^(critical|high|medium)$/);
+      expect(p.pattern).toBeInstanceOf(RegExp);
+      expect(p.description).toBeTruthy();
+    }
+  });
+});
+
+describe('shouldScanFile', () => {
+  it('excludes .env.example files', () => {
+    expect(shouldScanFile('.env.example')).toBe(false);
+  });
+
+  it('excludes test files', () => {
+    expect(shouldScanFile('src/scanner.test.ts')).toBe(false);
+    expect(shouldScanFile('src/scanner.spec.ts')).toBe(false);
+    expect(shouldScanFile('__tests__/foo.ts')).toBe(false);
+  });
+
+  it('excludes snapshots', () => {
+    expect(shouldScanFile('src/__snapshots__/foo.snap')).toBe(false);
+  });
+
+  it('includes regular source files', () => {
+    expect(shouldScanFile('src/config.ts')).toBe(true);
+    expect(shouldScanFile('lib/app.js')).toBe(true);
+    expect(shouldScanFile('.env')).toBe(true);
+  });
+});

package/src/domain/scanner/rules/nhi-patterns.ts

@@ -0,0 +1,60 @@
+export type NhiCategory = 'api_key' | 'service_account' | 'secret' | 'token';
+
+export interface NhiPattern {
+  readonly id: string;
+  readonly name: string;
+  readonly category: NhiCategory;
+  readonly pattern: RegExp;
+  readonly severity: 'critical' | 'high' | 'medium';
+  readonly description: string;
+}
+
+export const NHI_PATTERNS: readonly NhiPattern[] = [
+  // API Keys
+  { id: 'nhi-openai-key', name: 'OpenAI API Key', category: 'api_key', pattern: /sk-[a-zA-Z0-9]{20,}/, severity: 'critical', description: 'OpenAI API key detected' },
+  { id: 'nhi-anthropic-key', name: 'Anthropic API Key', category: 'api_key', pattern: /sk-ant-[a-zA-Z0-9\-]{20,}/, severity: 'critical', description: 'Anthropic API key detected' },
+  { id: 'nhi-aws-access-key', name: 'AWS Access Key', category: 'api_key', pattern: /AKIA[0-9A-Z]{16}/, severity: 'critical', description: 'AWS access key ID detected' },
+  { id: 'nhi-github-pat', name: 'GitHub PAT', category: 'api_key', pattern: /ghp_[a-zA-Z0-9]{36}/, severity: 'high', description: 'GitHub personal access token detected' },
+  { id: 'nhi-github-oauth', name: 'GitHub OAuth', category: 'api_key', pattern: /gho_[a-zA-Z0-9]{36}/, severity: 'high', description: 'GitHub OAuth token detected' },
+  { id: 'nhi-google-api-key', name: 'Google API Key', category: 'api_key', pattern: /AIzaSy[a-zA-Z0-9\-_]{33}/, severity: 'high', description: 'Google API key detected' },
+  { id: 'nhi-slack-bot', name: 'Slack Bot Token', category: 'api_key', pattern: /xoxb-[0-9]{10,}-[0-9]{10,}-[a-zA-Z0-9]{20,}/, severity: 'high', description: 'Slack bot token detected' },
+  { id: 'nhi-slack-user', name: 'Slack User Token', category: 'api_key', pattern: /xoxp-[0-9]{10,}-[0-9]{10,}-[a-zA-Z0-9]{20,}/, severity: 'high', description: 'Slack user token detected' },
+  { id: 'nhi-stripe-key', name: 'Stripe API Key', category: 'api_key', pattern: /sk_live_[a-zA-Z0-9]{24,}/, severity: 'critical', description: 'Stripe live API key detected' },
+  { id: 'nhi-sendgrid-key', name: 'SendGrid API Key', category: 'api_key', pattern: /SG\.[a-zA-Z0-9\-_]{22}\.[a-zA-Z0-9\-_]{43}/, severity: 'high', description: 'SendGrid API key detected' },
+  { id: 'nhi-twilio-key', name: 'Twilio API Key', category: 'api_key', pattern: /SK[0-9a-fA-F]{32}/, severity: 'high', description: 'Twilio API key detected' },
+  // Service Accounts
+  { id: 'nhi-gcp-service-account', name: 'GCP Service Account', category: 'service_account', pattern: /"type"\s*:\s*"service_account"/, severity: 'critical', description: 'GCP service account JSON key file detected' },
+  // Secrets
+  { id: 'nhi-private-key', name: 'Private Key', category: 'secret', pattern: /-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----/, severity: 'critical', description: 'Private key detected in source code' },
+  { id: 'nhi-aws-secret', name: 'AWS Secret Key', category: 'secret', pattern: /(?:aws_secret_access_key|AWS_SECRET_ACCESS_KEY)\s*[=:]\s*['"]?[A-Za-z0-9/+=]{40}['"]?/, severity: 'critical', description: 'AWS secret access key detected' },
+  { id: 'nhi-password-assign', name: 'Hardcoded Password', category: 'secret', pattern: /(?:password|passwd|pwd)\s*[=:]\s*['"][^'"]{8,}['"]/, severity: 'high', description: 'Hardcoded password detected' },
+  { id: 'nhi-connection-string', name: 'Connection String', category: 'secret', pattern: /(?:mongodb|postgres|mysql|redis):\/\/[^:]+:[^@]+@[^/\s]+/, severity: 'high', description: 'Database connection string with credentials detected' },
+  // Tokens
+  { id: 'nhi-jwt', name: 'JWT Token', category: 'token', pattern: /eyJ[a-zA-Z0-9\-_]+\.eyJ[a-zA-Z0-9\-_]+\.[a-zA-Z0-9\-_]+/, severity: 'medium', description: 'JWT token detected in source code' },
+  { id: 'nhi-bearer-token', name: 'Bearer Token', category: 'token', pattern: /[Bb]earer\s+[a-zA-Z0-9\-_.~+/]+=*/, severity: 'medium', description: 'Bearer token detected in source code' },
+  { id: 'nhi-npm-token', name: 'NPM Token', category: 'token', pattern: /npm_[a-zA-Z0-9]{36}/, severity: 'high', description: 'NPM access token detected' },
+  { id: 'nhi-pypi-token', name: 'PyPI Token', category: 'token', pattern: /pypi-[a-zA-Z0-9\-_]{50,}/, severity: 'high', description: 'PyPI access token detected' },
+] as const;
+
+// File extensions/paths to skip during NHI scanning
+export const NHI_EXCLUDE_PATTERNS = [
+  /\.env\.example$/,
+  /\.env\.sample$/,
+  /\.env\.template$/,
+  /\.test\./,
+  /\.spec\./,
+  /__tests__\//,
+  /test\//,
+  /tests\//,
+  /\.snap$/,
+  /\.lock$/,
+  /node_modules\//,
+  /\.git\//,
+  /\.md$/,           // Documentation files (contain example headers/tokens)
+  /nhi-patterns\.ts$/, // Self-exclusion (pattern definitions, not real secrets)
+  /last-scan\.json$/,  // Cached scan results (contain quoted findings, not real secrets)
+  /\.complior\//,      // Complior internal data directory
+] as const;
+
+export const shouldScanFile = (path: string): boolean =>
+  !NHI_EXCLUDE_PATTERNS.some(p => p.test(path));