@complior/engine 0.9.0

package/src/domain/scanner/external/checks.ts

@@ -0,0 +1,321 @@
+import type { ExternalCheck, PageData } from './types.js';
+import { CORE_DISCLOSURE_PATTERNS } from '../../shared/disclosure-patterns.js';
+
+/** Page-specific disclosure patterns (beyond core). */
+const PAGE_SPECIFIC_PATTERNS: readonly RegExp[] = [
+  /generated by ai/i,
+  /language model/i,
+  /chatbot/i,
+  /ai[- ]?enabled/i,
+];
+
+const AI_DISCLOSURE_PATTERNS: readonly RegExp[] = [
+  ...CORE_DISCLOSURE_PATTERNS,
+  ...PAGE_SPECIFIC_PATTERNS,
+];
+
+export const checkAiDisclosure = (page: PageData): ExternalCheck => {
+  const html = page.html.toLowerCase();
+  const matched = AI_DISCLOSURE_PATTERNS.find((p) => p.test(html));
+
+  if (matched) {
+    const match = html.match(matched);
+    return {
+      name: 'AI Disclosure Text',
+      status: 'PASS',
+      obligation: 'OBL-015',
+      article: 'Art. 50(1)',
+      evidence: `AI disclosure found: "${match?.[0] ?? 'detected'}"`,
+      confidence: 85,
+    };
+  }
+
+  return {
+    name: 'AI Disclosure Text',
+    status: 'FAIL',
+    obligation: 'OBL-015',
+    article: 'Art. 50(1)',
+    evidence: 'No AI disclosure text found on page',
+    confidence: 80,
+  };
+};
+
+export const checkWellKnown = (page: PageData): ExternalCheck => {
+  if (page.wellKnownAiCompliance) {
+    try {
+      const data = JSON.parse(page.wellKnownAiCompliance);
+      const hasRequired = data.aiProvider || data.aiSystem || data.complianceStatus;
+      return {
+        name: '.well-known/ai-compliance.json',
+        status: hasRequired ? 'PASS' : 'PARTIAL',
+        obligation: 'OBL-021',
+        article: 'Art. 50',
+        evidence: hasRequired
+          ? 'Compliance metadata present with required fields'
+          : 'Compliance metadata found but missing required fields',
+        confidence: hasRequired ? 95 : 60,
+      };
+    } catch {
+      return {
+        name: '.well-known/ai-compliance.json',
+        status: 'PARTIAL',
+        obligation: 'OBL-021',
+        article: 'Art. 50',
+        evidence: 'File exists but contains invalid JSON',
+        confidence: 40,
+      };
+    }
+  }
+
+  return {
+    name: '.well-known/ai-compliance.json',
+    status: 'FAIL',
+    obligation: 'OBL-021',
+    article: 'Art. 50',
+    evidence: 'No .well-known/ai-compliance.json found (404)',
+    confidence: 90,
+  };
+};
+
+export const checkMetaTags = (page: PageData): ExternalCheck => {
+  const aiMeta = page.metaTags.filter((t) =>
+    t.name.toLowerCase().includes('ai-') ||
+    t.name.toLowerCase().includes('ai_') ||
+    t.name.toLowerCase() === 'generator',
+  );
+
+  if (aiMeta.length > 0) {
+    const names = aiMeta.map((t) => t.name).join(', ');
+    return {
+      name: 'Meta Tags',
+      status: 'PASS',
+      obligation: 'OBL-015',
+      article: 'Art. 50(1)',
+      evidence: `AI-related meta tags found: ${names}`,
+      confidence: 90,
+    };
+  }
+
+  return {
+    name: 'Meta Tags',
+    status: 'FAIL',
+    obligation: 'OBL-015',
+    article: 'Art. 50(1)',
+    evidence: 'No AI-related meta tags found',
+    confidence: 75,
+  };
+};
+
+const PRIVACY_AI_PATTERNS = [
+  /artificial intelligence/i,
+  /machine learning/i,
+  /automated decision/i,
+  /ai[- ]processing/i,
+  /algorithmic/i,
+  /ai[- ]system/i,
+  /language model/i,
+];
+
+export const checkPrivacyPolicy = (page: PageData): ExternalCheck => {
+  if (!page.privacyPolicyUrl) {
+    return {
+      name: 'Privacy Policy AI Section',
+      status: 'FAIL',
+      obligation: 'GDPR + AI Act',
+      article: 'Art. 50',
+      evidence: 'No privacy policy link found on page',
+      confidence: 70,
+    };
+  }
+
+  if (!page.privacyPolicyText) {
+    return {
+      name: 'Privacy Policy AI Section',
+      status: 'PARTIAL',
+      obligation: 'GDPR + AI Act',
+      article: 'Art. 50',
+      evidence: `Privacy policy found at ${page.privacyPolicyUrl} but content not accessible`,
+      confidence: 40,
+    };
+  }
+
+  const hasAiSection = PRIVACY_AI_PATTERNS.some((p) => p.test(page.privacyPolicyText!));
+
+  return {
+    name: 'Privacy Policy AI Section',
+    status: hasAiSection ? 'PASS' : 'FAIL',
+    obligation: 'GDPR + AI Act',
+    article: 'Art. 50',
+    evidence: hasAiSection
+      ? 'AI-specific section found in privacy policy'
+      : 'No AI-specific section in privacy policy',
+    confidence: hasAiSection ? 85 : 75,
+  };
+};
+
+export const checkApiHeaders = (page: PageData): ExternalCheck => {
+  const aiHeaders = Object.entries(page.headers).filter(([key]) => {
+    const lower = key.toLowerCase();
+    return lower.includes('x-ai-') || lower.includes('x-content-marking') || lower.includes('x-generated-by');
+  });
+
+  if (aiHeaders.length > 0) {
+    const names = aiHeaders.map(([k]) => k).join(', ');
+    return {
+      name: 'API Response Headers',
+      status: 'PASS',
+      obligation: 'OBL-021',
+      article: 'Art. 50',
+      evidence: `AI compliance headers found: ${names}`,
+      confidence: 95,
+    };
+  }
+
+  return {
+    name: 'API Response Headers',
+    status: 'FAIL',
+    obligation: 'OBL-021',
+    article: 'Art. 50',
+    evidence: 'No AI compliance headers (X-AI-Disclosure, X-Content-Marking) found',
+    confidence: 80,
+  };
+};
+
+export const checkChatbotDetection = (page: PageData): ExternalCheck => {
+  let confidence = 0;
+  const evidence: string[] = [];
+
+  if (page.hasWebSocket) {
+    confidence += 30;
+    evidence.push('WebSocket detected (likely real-time chat)');
+  }
+
+  if (page.chatInputs.length > 0) {
+    confidence += 40;
+    const placeholders = page.chatInputs.map((i) => i.placeholder).filter(Boolean);
+    evidence.push(`Chat input found${placeholders.length ? `: "${placeholders[0]}"` : ''}`);
+  }
+
+  const chatPatterns = /chat|message|ask|conversation|assistant/i;
+  if (chatPatterns.test(page.html)) {
+    confidence += 20;
+    evidence.push('Chat-related keywords found in DOM');
+  }
+
+  if (confidence >= 50) {
+    return {
+      name: 'Chatbot Detection',
+      status: 'PASS',
+      obligation: 'Detection',
+      article: 'Art. 50',
+      evidence: `AI chatbot detected (confidence ${confidence}%): ${evidence.join('; ')}`,
+      confidence,
+    };
+  }
+
+  if (confidence > 0) {
+    return {
+      name: 'Chatbot Detection',
+      status: 'PARTIAL',
+      obligation: 'Detection',
+      article: 'Art. 50',
+      evidence: `Possible AI interface (confidence ${confidence}%): ${evidence.join('; ')}`,
+      confidence,
+    };
+  }
+
+  return {
+    name: 'Chatbot Detection',
+    status: 'N_A',
+    obligation: 'Detection',
+    article: 'Art. 50',
+    evidence: 'No AI chatbot interface detected',
+    confidence: 60,
+  };
+};
+
+export const checkImageMetadata = (page: PageData): ExternalCheck => {
+  if (page.images.length === 0) {
+    return {
+      name: 'Image Metadata (C2PA/IPTC)',
+      status: 'N_A',
+      obligation: 'OBL-016',
+      article: 'Art. 50(2)',
+      evidence: 'No images found on page',
+      confidence: 100,
+    };
+  }
+
+  // In a real implementation, we'd download images and check C2PA.
+  // For L1 passive crawl, we check for alt-text AI indicators.
+  const aiImages = page.images.filter((img) =>
+    /ai[- ]generated|generated by|dall-?e|midjourney|stable diffusion/i.test(img.alt || img.src),
+  );
+
+  if (aiImages.length === 0) {
+    return {
+      name: 'Image Metadata (C2PA/IPTC)',
+      status: 'PARTIAL',
+      obligation: 'OBL-016',
+      article: 'Art. 50(2)',
+      evidence: `${page.images.length} images found, none indicate AI generation in metadata/alt`,
+      confidence: 50,
+    };
+  }
+
+  return {
+    name: 'Image Metadata (C2PA/IPTC)',
+    status: 'FAIL',
+    obligation: 'OBL-016',
+    article: 'Art. 50(2)',
+    evidence: `${aiImages.length} AI-generated images detected without C2PA provenance`,
+    confidence: 70,
+  };
+};
+
+const HUMAN_ESCALATION_PATTERNS = [
+  /talk to (?:a )?human/i,
+  /contact support/i,
+  /speak (?:to|with) (?:a )?(?:person|agent|representative)/i,
+  /human (?:agent|support|assistance)/i,
+  /live (?:chat|agent|support)/i,
+  /request (?:a )?callback/i,
+  /escalat/i,
+];
+
+export const checkHumanEscalation = (page: PageData): ExternalCheck => {
+  const htmlAndLinks = page.html + ' ' + page.links.map((l) => l.text).join(' ');
+  const matched = HUMAN_ESCALATION_PATTERNS.find((p) => p.test(htmlAndLinks));
+
+  if (matched) {
+    const match = htmlAndLinks.match(matched);
+    return {
+      name: 'Human Escalation',
+      status: 'PASS',
+      obligation: 'OBL-008',
+      article: 'Art. 14',
+      evidence: `Human escalation option found: "${match?.[0] ?? 'detected'}"`,
+      confidence: 85,
+    };
+  }
+
+  return {
+    name: 'Human Escalation',
+    status: 'FAIL',
+    obligation: 'OBL-008',
+    article: 'Art. 14',
+    evidence: 'No human escalation option (talk to human, contact support) found',
+    confidence: 70,
+  };
+};
+
+export const ALL_L1_CHECKS = [
+  checkAiDisclosure,
+  checkWellKnown,
+  checkMetaTags,
+  checkPrivacyPolicy,
+  checkApiHeaders,
+  checkChatbotDetection,
+  checkImageMetadata,
+  checkHumanEscalation,
+] as const;

package/src/domain/scanner/external/dedup.test.ts

@@ -0,0 +1,79 @@
+import { describe, it, expect } from 'vitest';
+import { deduplicateFindings, mergeFindings } from './dedup.js';
+import type { Finding } from '../../../types/common.types.js';
+
+const mkFinding = (overrides: Partial<Finding>): Finding => ({
+  checkId: 'test',
+  type: 'fail',
+  message: 'test finding',
+  severity: 'medium',
+  ...overrides,
+});
+
+describe('deduplicateFindings', () => {
+  it('removes detect-secrets findings when NHI scanner found same file:line', () => {
+    const base = [
+      mkFinding({ checkId: 'l4-nhi-api-key', file: 'config.js', line: 5 }),
+    ];
+    const external = [
+      mkFinding({ checkId: 'ext-detect-secrets-HexHighEntropy', file: 'config.js', line: 5 }),
+      mkFinding({ checkId: 'ext-detect-secrets-Base64', file: 'other.js', line: 10 }),
+    ];
+
+    const result = deduplicateFindings(base, external);
+    expect(result).toHaveLength(1);
+    expect(result[0]!.file).toBe('other.js');
+  });
+
+  it('keeps Semgrep findings even when L4 found same location', () => {
+    const base = [
+      mkFinding({ checkId: 'l4-bare-llm', type: 'info', severity: 'info', file: 'ai.ts', line: 42 }),
+    ];
+    const external = [
+      mkFinding({ checkId: 'ext-semgrep-complior-bare-call', type: 'info', severity: 'info', file: 'ai.ts', line: 42 }),
+    ];
+
+    const result = deduplicateFindings(base, external);
+    expect(result).toHaveLength(1);
+    expect(result[0]!.checkId).toBe('ext-semgrep-complior-bare-call');
+  });
+
+  it('deduplicates same-tool findings at same file:line', () => {
+    const external = [
+      mkFinding({ checkId: 'ext-semgrep-rule1', file: 'a.ts', line: 1 }),
+      mkFinding({ checkId: 'ext-semgrep-rule1', file: 'a.ts', line: 1 }),
+    ];
+
+    const result = deduplicateFindings([], external);
+    expect(result).toHaveLength(1);
+  });
+});
+
+describe('mergeFindings', () => {
+  it('removes L4 findings superseded by Semgrep at same location', () => {
+    const base = [
+      mkFinding({ checkId: 'l4-bare-llm', type: 'info', severity: 'info', file: 'ai.ts', line: 42 }),
+      mkFinding({ checkId: 'l1-docs-missing', message: 'keep this' }),
+    ];
+    const external = [
+      mkFinding({ checkId: 'ext-semgrep-complior-bare-call', type: 'info', severity: 'info', file: 'ai.ts', line: 42 }),
+    ];
+
+    const result = mergeFindings(base, external);
+    expect(result).toHaveLength(2);
+    expect(result.some((f) => f.checkId === 'l4-bare-llm')).toBe(false);
+    expect(result.some((f) => f.checkId === 'ext-semgrep-complior-bare-call')).toBe(true);
+    expect(result.some((f) => f.checkId === 'l1-docs-missing')).toBe(true);
+  });
+
+  it('keeps L4 findings without Semgrep equivalent', () => {
+    const base = [
+      mkFinding({ checkId: 'l4-bare-llm', type: 'info', severity: 'info', file: 'ai.ts', line: 42 }),
+    ];
+    const external: Finding[] = [];
+
+    const result = mergeFindings(base, external);
+    expect(result).toHaveLength(1);
+    expect(result[0]!.checkId).toBe('l4-bare-llm');
+  });
+});

package/src/domain/scanner/external/dedup.ts

@@ -0,0 +1,94 @@
+import type { Finding } from '../../../types/common.types.js';
+
+/**
+ * Deduplicate findings from external tools against existing L1-L4 findings.
+ *
+ * Rules:
+ * 1. detect-secrets vs NHI scanner: same file+line → keep NHI (richer metadata)
+ * 2. Semgrep vs L4 regex: same file+line+similar checkId → keep Semgrep (more precise)
+ * 3. Cross-tool: same file+line+similar message → keep first
+ */
+export const deduplicateFindings = (
+  baseFindings: readonly Finding[],
+  externalFindings: readonly Finding[],
+): readonly Finding[] => {
+  // Build a set of base finding keys for O(1) lookup
+  const baseKeys = new Set<string>();
+  for (const f of baseFindings) {
+    if (f.file && f.line !== undefined) {
+      baseKeys.add(`${f.file}:${f.line}`);
+    }
+  }
+
+  const result: Finding[] = [];
+  const seenExternal = new Set<string>();
+
+  for (const ext of externalFindings) {
+    const fileLineKey = ext.file && ext.line !== undefined
+      ? `${ext.file}:${ext.line}`
+      : null;
+
+    // Rule 1: detect-secrets vs NHI — skip if NHI already found this file:line
+    if (ext.checkId.startsWith('ext-detect-secrets-') && fileLineKey) {
+      const nhiDuplicate = baseFindings.some((base) =>
+        base.checkId.startsWith('l4-nhi-')
+        && base.file === ext.file
+        && base.line === ext.line,
+      );
+      if (nhiDuplicate) continue;
+    }
+
+    // Rule 2: Skip non-Semgrep, non-detect-secrets findings if base already has same file:line.
+    // Semgrep supersedes L4 in merge step. detect-secrets is handled by Rule 1 (NHI dedup).
+    if (fileLineKey && baseKeys.has(fileLineKey)) {
+      if (!ext.checkId.startsWith('ext-semgrep-') && !ext.checkId.startsWith('ext-detect-secrets-')) continue;
+    }
+
+    // Rule 3: Cross-tool dedup — skip if same tool already reported same file:line
+    if (fileLineKey) {
+      const dedupKey = `${ext.checkId}:${fileLineKey}`;
+      if (seenExternal.has(dedupKey)) continue;
+      seenExternal.add(dedupKey);
+    }
+
+    result.push(ext);
+  }
+
+  return result;
+};
+
+/**
+ * Merge base L1-L4 findings with deduplicated external findings.
+ * Removes L4 findings that have a more precise Semgrep equivalent,
+ * transferring fixDiff/codeContext from the superseded L4 finding.
+ */
+export const mergeFindings = (
+  baseFindings: readonly Finding[],
+  externalFindings: readonly Finding[],
+): readonly Finding[] => {
+  // Build map of Semgrep file:line keys → index in externalFindings
+  const semgrepKeys = new Map<string, number>();
+  for (let i = 0; i < externalFindings.length; i++) {
+    const f = externalFindings[i]!;
+    if (f.checkId.startsWith('ext-semgrep-') && f.file && f.line !== undefined) {
+      semgrepKeys.set(`${f.file}:${f.line}`, i);
+    }
+  }
+
+  // Transfer fixDiff from superseded L4 findings to their semgrep replacements
+  const enrichedExternal = [...externalFindings];
+  const filteredBase = baseFindings.filter((f) => {
+    if (!f.checkId.startsWith('l4-') || !f.file || f.line === undefined) return true;
+    const key = `${f.file}:${f.line}`;
+    const extIdx = semgrepKeys.get(key);
+    if (extIdx === undefined) return true; // Keep — no semgrep equivalent
+    // Transfer fixDiff and codeContext to the semgrep finding
+    const ext = enrichedExternal[extIdx]!;
+    if (f.fixDiff && !ext.fixDiff) {
+      enrichedExternal[extIdx] = { ...ext, fixDiff: f.fixDiff, codeContext: f.codeContext ?? ext.codeContext };
+    }
+    return false; // Remove — superseded by semgrep
+  });
+
+  return [...filteredBase, ...enrichedExternal];
+};

package/src/domain/scanner/external/detect-secrets-runner.test.ts

@@ -0,0 +1,58 @@
+import { describe, it, expect } from 'vitest';
+import { parseDetectSecretsOutput } from './detect-secrets-runner.js';
+
+describe('parseDetectSecretsOutput', () => {
+  it('parses valid detect-secrets JSON output', () => {
+    const output = JSON.stringify({
+      version: '1.4.0',
+      results: {
+        '/project/config.js': [
+          {
+            type: 'Hex High Entropy String',
+            line_number: 5,
+            hashed_secret: 'abc123',
+            is_verified: false,
+          },
+        ],
+        '/project/src/api.ts': [
+          {
+            type: 'Base64 High Entropy String',
+            line_number: 12,
+            hashed_secret: 'def456',
+            is_verified: true,
+          },
+          {
+            type: 'Private Key',
+            line_number: 20,
+            hashed_secret: 'ghi789',
+            is_verified: false,
+          },
+        ],
+      },
+    });
+
+    const findings = parseDetectSecretsOutput(output, '/project');
+    expect(findings).toHaveLength(3);
+
+    expect(findings[0]!.ruleId).toBe('Hex High Entropy String');
+    expect(findings[0]!.file).toBe('config.js');
+    expect(findings[0]!.line).toBe(5);
+    expect(findings[0]!.message).not.toContain('verified');
+
+    expect(findings[1]!.ruleId).toBe('Base64 High Entropy String');
+    expect(findings[1]!.file).toBe('src/api.ts');
+    expect(findings[1]!.message).toContain('verified');
+
+    expect(findings[2]!.file).toBe('src/api.ts');
+    expect(findings[2]!.line).toBe(20);
+  });
+
+  it('returns empty array for empty output', () => {
+    expect(parseDetectSecretsOutput('', '/project')).toEqual([]);
+  });
+
+  it('returns empty array for empty results', () => {
+    const output = JSON.stringify({ version: '1.4.0', results: {} });
+    expect(parseDetectSecretsOutput(output, '/project')).toEqual([]);
+  });
+});

package/src/domain/scanner/external/detect-secrets-runner.ts

@@ -0,0 +1,81 @@
+import type { ExternalRunner, ExternalRunnerDeps, ExternalRunnerResult, RawExternalFinding } from './runner-port.js';
+import { normalizeFilePath } from './path-utils.js';
+
+interface DetectSecretsResult {
+  readonly results?: Record<string, readonly DetectSecretsFinding[]>;
+  readonly version?: string;
+}
+
+interface DetectSecretsFinding {
+  readonly type: string;
+  readonly line_number: number;
+  readonly hashed_secret: string;
+  readonly is_verified?: boolean;
+}
+
+export const createDetectSecretsRunner = (): ExternalRunner => Object.freeze({
+  name: 'detect-secrets',
+  run: async (deps: ExternalRunnerDeps): Promise<ExternalRunnerResult> => {
+    const startTime = Date.now();
+    const toolCmd = deps.toolPath ?? 'detect-secrets';
+
+    const result = await deps.runProcess(toolCmd, [
+      'scan',
+      '--all-files',
+      '--exclude-files', '\\.complior/|node_modules/|dist/|\\.git/',
+    ], { timeout: 120_000, cwd: deps.projectPath });
+
+    const duration = Date.now() - startTime;
+
+    if (result.exitCode !== 0) {
+      return {
+        tool: 'detect-secrets',
+        version: '',
+        rawFindings: [],
+        duration,
+        exitCode: result.exitCode,
+        error: result.stderr.slice(0, 500),
+      };
+    }
+
+    const rawFindings = parseDetectSecretsOutput(result.stdout, deps.projectPath);
+    return {
+      tool: 'detect-secrets',
+      version: '',
+      rawFindings,
+      duration,
+      exitCode: result.exitCode,
+    };
+  },
+});
+
+export const parseDetectSecretsOutput = (stdout: string, projectPath: string): RawExternalFinding[] => {
+  if (!stdout.trim()) return [];
+
+  let parsed: DetectSecretsResult;
+  try {
+    parsed = JSON.parse(stdout) as DetectSecretsResult;
+  } catch {
+    return [];
+  }
+
+  if (!parsed.results) return [];
+
+  const findings: RawExternalFinding[] = [];
+  for (const [filePath, secrets] of Object.entries(parsed.results)) {
+    const relativePath = normalizeFilePath(filePath, projectPath);
+
+    for (const secret of secrets) {
+      findings.push({
+        ruleId: secret.type,
+        message: `${secret.type} detected${secret.is_verified ? ' (verified)' : ''}`,
+        severity: 'high',
+        file: relativePath,
+        line: secret.line_number,
+        category: 'secrets',
+      });
+    }
+  }
+
+  return findings;
+};