@complior/engine 0.9.0

package/src/domain/scanner/layers/layer5-docs.test.ts

@@ -0,0 +1,99 @@
+import { describe, it, expect } from 'vitest';
+import { DOCUMENT_CHECKLISTS, buildDocValidationPrompt, docValidationToFindings, getChecklist, detectDocType } from './layer5-docs.js';
+import type { DocValidationResult } from './layer5-docs.js';
+
+describe('DOCUMENT_CHECKLISTS', () => {
+  it('has checklists for 4 document types', () => {
+    expect(DOCUMENT_CHECKLISTS.length).toBe(4);
+    expect(DOCUMENT_CHECKLISTS.map((c) => c.docType)).toEqual([
+      'fria', 'technical-documentation', 'transparency-notice', 'risk-management',
+    ]);
+  });
+
+  it('FRIA has 8 elements with 5 required', () => {
+    const fria = DOCUMENT_CHECKLISTS.find((c) => c.docType === 'fria')!;
+    expect(fria.elements.length).toBe(8);
+    expect(fria.elements.filter((e) => e.required).length).toBe(5);
+  });
+});
+
+describe('buildDocValidationPrompt', () => {
+  it('includes document content and checklist', () => {
+    const checklist = DOCUMENT_CHECKLISTS[0]!;
+    const prompt = buildDocValidationPrompt('# FRIA\n## Risk Assessment\nHigh risk identified.', checklist);
+    expect(prompt).toContain('FRIA');
+    expect(prompt).toContain('affected-rights');
+    expect(prompt).toContain('REQUIRED');
+    expect(prompt).toContain('Risk Assessment');
+  });
+
+  it('truncates very long documents', () => {
+    const longDoc = 'x'.repeat(20000);
+    const checklist = DOCUMENT_CHECKLISTS[0]!;
+    const prompt = buildDocValidationPrompt(longDoc, checklist);
+    expect(prompt.length).toBeLessThan(15000); // truncated to 8000 chars
+  });
+});
+
+describe('docValidationToFindings', () => {
+  it('creates pass finding for high quality docs', () => {
+    const results: DocValidationResult[] = [{
+      docType: 'fria',
+      file: 'docs/fria.md',
+      qualityScore: 85,
+      elementsPresent: 7,
+      elementsTotal: 8,
+      missingElements: ['consultation'],
+      feedback: [],
+      severity: 'info',
+    }];
+    const findings = docValidationToFindings(results);
+    expect(findings[0]?.type).toBe('pass');
+    expect(findings[0]?.checkId).toBe('l5-doc-fria');
+  });
+
+  it('creates fail finding for low quality docs', () => {
+    const results: DocValidationResult[] = [{
+      docType: 'fria',
+      file: 'docs/fria.md',
+      qualityScore: 40,
+      elementsPresent: 3,
+      elementsTotal: 8,
+      missingElements: ['quantitative-assessment', 'affected-population', 'monitoring-plan', 'proportionality', 'alternatives'],
+      feedback: ['Add quantitative risk assessment', 'Describe affected population'],
+      severity: 'high',
+    }];
+    const findings = docValidationToFindings(results);
+    expect(findings[0]?.type).toBe('fail');
+    expect(findings[0]?.message).toContain('quantitative-assessment');
+  });
+});
+
+describe('getChecklist', () => {
+  it('returns checklist by doc type', () => {
+    expect(getChecklist('fria')?.article).toBe('Art. 27');
+    expect(getChecklist('risk-management')?.article).toBe('Art. 9');
+    expect(getChecklist('unknown')).toBeUndefined();
+  });
+});
+
+describe('detectDocType', () => {
+  it('detects FRIA from filename', () => {
+    expect(detectDocType('docs/fria.md')).toBe('fria');
+    expect(detectDocType('docs/FRIA.md')).toBe('fria');
+  });
+
+  it('detects technical documentation', () => {
+    expect(detectDocType('docs/technical-documentation.md')).toBe('technical-documentation');
+    expect(detectDocType('docs/tech-documentation.md')).toBe('technical-documentation');
+  });
+
+  it('detects risk management', () => {
+    expect(detectDocType('docs/risk-management.md')).toBe('risk-management');
+    expect(detectDocType('docs/risk-assessment.md')).toBe('risk-management');
+  });
+
+  it('returns undefined for unknown docs', () => {
+    expect(detectDocType('docs/readme.md')).toBeUndefined();
+  });
+});

package/src/domain/scanner/layers/layer5-docs.ts

@@ -0,0 +1,250 @@
+/**
+ * L5 Document Validation: LLM checks document CONTENT against
+ * regulation-specific checklists (Art. 27 FRIA, Art. 11 Tech Doc, etc.).
+ */
+
+import type { Finding } from '../../../types/common.types.js';
+
+export interface DocumentChecklist {
+  readonly docType: string;
+  readonly article: string;
+  readonly elements: readonly ChecklistElement[];
+}
+
+export interface ChecklistElement {
+  readonly id: string;
+  readonly description: string;
+  readonly required: boolean;
+}
+
+export interface DocValidationResult {
+  readonly docType: string;
+  readonly file: string;
+  readonly qualityScore: number;          // 0-100
+  readonly elementsPresent: number;
+  readonly elementsTotal: number;
+  readonly missingElements: readonly string[];
+  readonly feedback: readonly string[];
+  readonly severity: 'high' | 'medium' | 'low' | 'info';
+}
+
+/** Regulation checklists per document type */
+export const DOCUMENT_CHECKLISTS: readonly DocumentChecklist[] = [
+  {
+    docType: 'fria',
+    article: 'Art. 27',
+    elements: [
+      { id: 'affected-rights', description: 'Specific fundamental rights affected (Art. 6-50 EU Charter)', required: true },
+      { id: 'quantitative-assessment', description: 'Quantitative risk assessment (probability × impact)', required: true },
+      { id: 'mitigation-measures', description: 'Concrete mitigation measures (not generic)', required: true },
+      { id: 'affected-population', description: 'Description of affected population', required: true },
+      { id: 'monitoring-plan', description: 'Monitoring plan for rights impact', required: true },
+      { id: 'proportionality', description: 'Proportionality analysis', required: false },
+      { id: 'alternatives', description: 'Alternatives considered', required: false },
+      { id: 'consultation', description: 'Stakeholder consultation process', required: false },
+    ],
+  },
+  {
+    docType: 'technical-documentation',
+    article: 'Art. 11',
+    elements: [
+      { id: 'system-description', description: 'General description of the AI system', required: true },
+      { id: 'intended-purpose', description: 'Intended purpose and intended use', required: true },
+      { id: 'hardware-requirements', description: 'Hardware and software requirements', required: true },
+      { id: 'training-data', description: 'Training data description and methodology', required: true },
+      { id: 'performance-metrics', description: 'Performance metrics and benchmarks', required: true },
+      { id: 'limitations', description: 'Known limitations and conditions of use', required: true },
+      { id: 'human-oversight', description: 'Human oversight measures', required: true },
+      { id: 'risk-management', description: 'Risk management measures', required: true },
+      { id: 'testing-validation', description: 'Testing and validation procedures', required: true },
+      { id: 'logging-capability', description: 'Logging capabilities description', required: false },
+      { id: 'lifecycle-changes', description: 'Changes throughout lifecycle', required: false },
+      { id: 'standards', description: 'Applied standards and conformity', required: false },
+    ],
+  },
+  {
+    docType: 'transparency-notice',
+    article: 'Art. 13',
+    elements: [
+      { id: 'ai-disclosure', description: 'Clear disclosure that system uses AI', required: true },
+      { id: 'capabilities', description: 'System capabilities description', required: true },
+      { id: 'limitations', description: 'System limitations description', required: true },
+      { id: 'human-contact', description: 'Human contact information', required: true },
+      { id: 'opt-out', description: 'Opt-out mechanism description', required: false },
+      { id: 'data-usage', description: 'Data usage explanation', required: false },
+    ],
+  },
+  {
+    docType: 'risk-management',
+    article: 'Art. 9',
+    elements: [
+      { id: 'risk-identification', description: 'Risk identification methodology', required: true },
+      { id: 'probability-assessment', description: 'Probability assessment for each risk', required: true },
+      { id: 'impact-assessment', description: 'Impact assessment for each risk', required: true },
+      { id: 'mitigation-measures', description: 'Mitigation measures per risk', required: true },
+      { id: 'residual-risks', description: 'Residual risk assessment', required: true },
+      { id: 'monitoring-plan', description: 'Risk monitoring plan', required: false },
+      { id: 'review-schedule', description: 'Review schedule', required: false },
+      { id: 'responsible-persons', description: 'Responsible persons', required: false },
+    ],
+  },
+];
+
+/**
+ * Build LLM prompt for document validation.
+ */
+export const buildDocValidationPrompt = (
+  docContent: string,
+  checklist: DocumentChecklist,
+): string => {
+  const elementList = checklist.elements
+    .map((e, i) => `${i + 1}. [${e.required ? 'REQUIRED' : 'OPTIONAL'}] ${e.id}: ${e.description}`)
+    .join('\n');
+
+  return `You are a compliance auditor validating a ${checklist.docType} document against EU AI Act ${checklist.article}.
+
+Analyze the document below and determine which required elements are adequately covered.
+An element is "present" if it contains SPECIFIC, CONCRETE information (not generic placeholder text).
+Generic phrases like "we comply with all laws" or "risks will be assessed" are NOT adequate.
+
+Document content:
+---
+${docContent.slice(0, 8000)}
+---
+
+Required elements to check:
+${elementList}
+
+Respond ONLY in this JSON format:
+{
+  "elements": [
+    { "id": "element-id", "present": true/false, "adequate": true/false, "feedback": "specific feedback" }
+  ],
+  "overallScore": 0-100,
+  "summary": "1-2 sentence summary"
+}`;
+};
+
+/**
+ * Convert document validation results into scanner findings.
+ */
+export const docValidationToFindings = (
+  results: readonly DocValidationResult[],
+): readonly Finding[] => {
+  const findings: Finding[] = [];
+
+  for (const result of results) {
+    if (result.qualityScore >= 80) {
+      findings.push({
+        checkId: `l5-doc-${result.docType}`,
+        type: 'pass',
+        message: `${result.docType} content validation: ${result.elementsPresent}/${result.elementsTotal} elements adequate (score: ${result.qualityScore}/100)`,
+        severity: 'info',
+        confidence: 85,
+        confidenceLevel: 'LIKELY_PASS',
+        l5Analyzed: true,
+      });
+    } else {
+      const missing = result.missingElements.join(', ');
+      findings.push({
+        checkId: `l5-doc-${result.docType}`,
+        type: 'fail',
+        message: `${result.docType} content validation: missing/inadequate elements: ${missing} (score: ${result.qualityScore}/100)`,
+        severity: result.severity,
+        obligationId: `eu-ai-act-doc-${result.docType}`,
+        fix: result.feedback.join('; '),
+        file: result.file,
+        confidence: 85,
+        confidenceLevel: 'LIKELY_FAIL',
+        l5Analyzed: true,
+      });
+    }
+  }
+
+  return findings;
+};
+
+/** Type guard for record-shaped objects (used instead of `as` assertions). */
+const isRecord = (v: unknown): v is Record<string, unknown> =>
+  typeof v === 'object' && v !== null;
+
+/**
+ * Parse LLM JSON response into a DocValidationResult.
+ * Returns undefined if the response cannot be parsed.
+ */
+export const parseDocValidationResponse = (
+  llmText: string,
+  docType: string,
+  filePath: string,
+  totalElements: number,
+): DocValidationResult | undefined => {
+  const jsonMatch = llmText.match(/\{[\s\S]*\}/);
+  if (!jsonMatch) return undefined;
+
+  let raw: unknown;
+  try {
+    raw = JSON.parse(jsonMatch[0]);
+  } catch {
+    return undefined;
+  }
+  if (!isRecord(raw)) return undefined;
+
+  const overallScore = typeof raw['overallScore'] === 'number' ? raw['overallScore'] : 50;
+  const elements = Array.isArray(raw['elements']) ? raw['elements'] : [];
+
+  const present = elements.filter((e: unknown) => {
+    if (!isRecord(e)) return false;
+    return e['present'] === true && e['adequate'] === true;
+  }).length;
+
+  const missing = elements
+    .filter((e: unknown) => {
+      if (!isRecord(e)) return true;
+      return e['present'] !== true || e['adequate'] !== true;
+    })
+    .map((e: unknown) => {
+      if (!isRecord(e)) return 'unknown';
+      return typeof e['id'] === 'string' ? e['id'] : 'unknown';
+    });
+
+  const feedback: string[] = [];
+  for (const e of elements) {
+    if (!isRecord(e)) continue;
+    const fb = e['feedback'];
+    if (typeof fb === 'string' && fb.length > 0) {
+      feedback.push(fb);
+    }
+  }
+
+  const severity: DocValidationResult['severity'] =
+    overallScore < 40 ? 'high' : overallScore < 60 ? 'medium' : overallScore < 80 ? 'low' : 'info';
+
+  return {
+    docType,
+    file: filePath,
+    qualityScore: overallScore,
+    elementsPresent: present,
+    elementsTotal: totalElements,
+    missingElements: missing,
+    feedback,
+    severity,
+  };
+};
+
+/**
+ * Get checklist for a document type.
+ */
+export const getChecklist = (docType: string): DocumentChecklist | undefined =>
+  DOCUMENT_CHECKLISTS.find((c) => c.docType === docType);
+
+/**
+ * Determine document type from file path.
+ */
+export const detectDocType = (filePath: string): string | undefined => {
+  const filename = filePath.split('/').pop()?.toLowerCase() ?? '';
+  if (/fria/i.test(filename)) return 'fria';
+  if (/tech.*doc|technical.*doc/i.test(filename)) return 'technical-documentation';
+  if (/transparency/i.test(filename)) return 'transparency-notice';
+  if (/risk.*manage|risk.*assess/i.test(filename)) return 'risk-management';
+  return undefined;
+};

package/src/domain/scanner/layers/layer5-llm.test.ts

@@ -0,0 +1,146 @@
+import { describe, it, expect } from 'vitest';
+import { selectPromptType, extractSnippets, buildL5Prompt, createLayer5 } from './layer5-llm.js';
+import type { Finding } from '../../../types/common.types.js';
+
+const makeFinding = (overrides: Partial<Finding> = {}): Finding => ({
+  checkId: 'ai-disclosure',
+  type: 'fail',
+  message: 'No AI disclosure found',
+  severity: 'high',
+  confidence: 55,
+  confidenceLevel: 'UNCERTAIN',
+  ...overrides,
+});
+
+describe('Scanner L5 — LLM Deep Analysis', () => {
+  it('selectPromptType maps disclosure to code_pattern_check', () => {
+    expect(selectPromptType(makeFinding({ checkId: 'ai-disclosure' }))).toBe('code_pattern_check');
+  });
+
+  it('selectPromptType maps documentation to documentation_check', () => {
+    expect(selectPromptType(makeFinding({ checkId: 'documentation', message: 'Missing documentation' }))).toBe('documentation_check');
+  });
+
+  it('selectPromptType maps monitoring to architecture_check', () => {
+    expect(selectPromptType(makeFinding({ checkId: 'monitoring', message: 'No monitoring found' }))).toBe('architecture_check');
+  });
+
+  it('selectPromptType maps data-related to data_handling_check', () => {
+    expect(selectPromptType(makeFinding({ checkId: 'data-retention', message: 'No data retention policy' }))).toBe('data_handling_check');
+  });
+
+  it('extractSnippets returns relevant code', () => {
+    const files = new Map([
+      ['src/app.tsx', 'export function App() { return <AIDisclosure />; }'],
+      ['src/utils.ts', 'export const add = (a: number, b: number) => a + b;'],
+      ['src/compliance.ts', 'export const disclosure = "This service uses AI";'],
+    ]);
+
+    const snippets = extractSnippets(makeFinding(), files);
+    expect(snippets.length).toBeGreaterThan(0);
+    expect(snippets[0].relevance).toBeGreaterThan(0);
+  });
+
+  it('extractSnippets limits to 500 lines', () => {
+    const longFile = Array(600).fill('const x = 1;').join('\n');
+    const files = new Map([['src/long.ts', longFile]]);
+
+    const snippets = extractSnippets(makeFinding(), files);
+    if (snippets.length > 0) {
+      expect(snippets[0].endLine).toBeLessThanOrEqual(500);
+    }
+  });
+
+  it('buildL5Prompt includes finding info and snippets', () => {
+    const snippets = [{ file: 'src/app.tsx', startLine: 1, endLine: 5, content: 'const x = 1;', relevance: 0.8 }];
+    const prompt = buildL5Prompt('code_pattern_check', makeFinding(), snippets);
+
+    expect(prompt).toContain('ai-disclosure');
+    expect(prompt).toContain('No AI disclosure found');
+    expect(prompt).toContain('src/app.tsx');
+    expect(prompt).toContain('implementation patterns');
+  });
+
+  it('isUncertain identifies uncertain findings', () => {
+    const l5 = createLayer5({
+      callLlm: async () => ({ text: '{}', inputTokens: 0, outputTokens: 0 }),
+      readFile: async () => '',
+      calculateCost: () => 0,
+    });
+
+    expect(l5.isUncertain(makeFinding({ confidence: 55 }))).toBe(true);
+    expect(l5.isUncertain(makeFinding({ confidence: 95 }))).toBe(false);
+    expect(l5.isUncertain(makeFinding({ confidence: 20 }))).toBe(false);
+  });
+
+  it('analyzeFindings processes uncertain findings', async () => {
+    const l5 = createLayer5({
+      callLlm: async () => ({
+        text: JSON.stringify({
+          verdict: 'pass',
+          confidence: 85,
+          reasoning: 'Disclosure component found',
+          evidence: ['src/app.tsx:1 — AIDisclosure component'],
+        }),
+        inputTokens: 500,
+        outputTokens: 200,
+      }),
+      readFile: async () => '',
+      calculateCost: () => 0.01,
+    });
+
+    const findings: Finding[] = [
+      makeFinding({ confidence: 55 }),
+      makeFinding({ checkId: 'logging', confidence: 45, message: 'No logging found' }),
+      makeFinding({ checkId: 'certain', confidence: 95, message: 'Already certain' }),
+    ];
+
+    const files = new Map([['src/app.tsx', 'const disclosure = true;']]);
+    const results = await l5.analyzeFindings(findings, files);
+
+    // Only 2 uncertain findings should be analyzed (confidence 40-70)
+    expect(results).toHaveLength(2);
+    expect(results[0].verdict).toBe('pass');
+    expect(results[0].newConfidence).toBe(85);
+  });
+
+  it('applyResults updates findings', () => {
+    const l5 = createLayer5({
+      callLlm: async () => ({ text: '{}', inputTokens: 0, outputTokens: 0 }),
+      readFile: async () => '',
+      calculateCost: () => 0,
+    });
+
+    const findings: Finding[] = [makeFinding({ confidence: 55 })];
+    const l5Results = [{
+      findingId: 'ai-disclosure',
+      originalConfidence: 55,
+      newConfidence: 85,
+      verdict: 'pass' as const,
+      reasoning: 'Found',
+      evidence: [],
+      promptType: 'code_pattern_check' as const,
+      cost: 0.01,
+    }];
+
+    const updated = l5.applyResults(findings, l5Results);
+    expect(updated[0].type).toBe('pass');
+    expect(updated[0].confidence).toBe(85);
+    expect(updated[0].severity).toBe('info');
+  });
+
+  it('handles LLM errors gracefully', async () => {
+    const l5 = createLayer5({
+      callLlm: async () => { throw new Error('LLM timeout'); },
+      readFile: async () => '',
+      calculateCost: () => 0,
+    });
+
+    const findings: Finding[] = [makeFinding({ confidence: 55 })];
+    const results = await l5.analyzeFindings(findings, new Map());
+
+    expect(results).toHaveLength(1);
+    expect(results[0].verdict).toBe('uncertain');
+    expect(results[0].reasoning).toContain('failed');
+  });
+});