@complior/engine 0.9.0

package/src/domain/scanner/layers/layer5-llm.ts

@@ -0,0 +1,262 @@
+import type { Finding } from '../../../types/common.types.js';
+
+export type PromptType = 'code_pattern_check' | 'documentation_check' | 'config_check' | 'architecture_check' | 'data_handling_check';
+
+export interface CodeSnippet {
+  readonly file: string;
+  readonly startLine: number;
+  readonly endLine: number;
+  readonly content: string;
+  readonly relevance: number;
+}
+
+export interface L5Result {
+  readonly findingId: string;
+  readonly originalConfidence: number;
+  readonly newConfidence: number;
+  readonly verdict: 'pass' | 'fail' | 'uncertain';
+  readonly reasoning: string;
+  readonly evidence: readonly string[];
+  readonly promptType: PromptType;
+  readonly cost: number;
+}
+
+export interface L5AnalyzerDeps {
+  readonly callLlm: (prompt: string) => Promise<{ text: string; inputTokens: number; outputTokens: number }>;
+  readonly readFile: (path: string) => Promise<string>;
+  readonly calculateCost: (model: string, inputTokens: number, outputTokens: number) => number;
+}
+
+const MAX_FINDINGS_PER_SCAN = 20;
+const MAX_SNIPPET_LINES = 500;
+const UNCERTAIN_MIN = 40;
+const UNCERTAIN_MAX = 70;
+
+const PROMPT_KEYWORDS: Record<PromptType, readonly string[]> = {
+  data_handling_check: ['data', 'privacy', 'retention', 'biometric', 'consent'],
+  code_pattern_check: ['disclosure', 'logging', 'transparency', 'interaction', 'content-marking'],
+  documentation_check: ['documentation', 'literacy', 'policy', 'report', 'conformity'],
+  config_check: ['config', 'metadata', 'environment', 'deployment'],
+  architecture_check: ['monitoring', 'oversight', 'audit', 'kill-switch'],
+};
+
+export const selectPromptType = (finding: Finding): PromptType => {
+  const text = `${finding.checkId} ${finding.message}`.toLowerCase();
+
+  const isPromptType = (s: string): s is PromptType =>
+    s in PROMPT_KEYWORDS;
+
+  for (const [type, keywords] of Object.entries(PROMPT_KEYWORDS)) {
+    if (keywords.some((kw) => text.includes(kw)) && isPromptType(type)) {
+      return type;
+    }
+  }
+  return 'code_pattern_check';
+};
+
+export const buildL5Prompt = (promptType: PromptType, finding: Finding, snippets: readonly CodeSnippet[]): string => {
+  const snippetText = snippets
+    .map((s) => `--- ${s.file}:${s.startLine}-${s.endLine} ---\n${s.content}`)
+    .join('\n\n');
+
+  const base = `You are a compliance auditor analyzing an AI project for EU AI Act compliance.
+
+Finding: ${finding.checkId} — ${finding.message}
+Severity: ${finding.severity}
+Article: ${finding.articleReference ?? 'N/A'}
+Current confidence: ${finding.confidence ?? 50}%
+
+Analyze the following code snippets and determine if this compliance requirement is met.
+
+${snippetText}
+
+Respond in JSON format:
+{
+  "verdict": "pass" | "fail" | "uncertain",
+  "confidence": 0-100,
+  "reasoning": "brief explanation",
+  "evidence": ["file:line — description"]
+}`;
+
+  const typeInstructions: Record<PromptType, string> = {
+    code_pattern_check: 'Focus on: implementation patterns, components, middleware, function calls.',
+    documentation_check: 'Focus on: document structure, required sections, completeness.',
+    config_check: 'Focus on: configuration values, environment variables, security settings.',
+    architecture_check: 'Focus on: system design, monitoring, logging, human oversight.',
+    data_handling_check: 'Focus on: data flow, storage, consent, retention policies.',
+  };
+
+  return `${base}\n\n${typeInstructions[promptType]}`;
+};
+
+export const extractSnippets = (
+  finding: Finding,
+  fileContents: ReadonlyMap<string, string>,
+): readonly CodeSnippet[] => {
+  const keywords = finding.checkId.split('-').concat(
+    finding.message.toLowerCase().split(/\s+/).filter((w) => w.length > 4),
+  );
+
+  const snippets: CodeSnippet[] = [];
+  let totalLines = 0;
+
+  for (const [file, content] of fileContents) {
+    if (totalLines >= MAX_SNIPPET_LINES) break;
+
+    const lines = content.split('\n');
+    let relevance = 0;
+    const lowerContent = content.toLowerCase();
+
+    for (const kw of keywords) {
+      if (lowerContent.includes(kw.toLowerCase())) relevance += 0.2;
+    }
+
+    if (relevance > 0) {
+      const endLine = Math.min(lines.length, MAX_SNIPPET_LINES - totalLines);
+      snippets.push({
+        file,
+        startLine: 1,
+        endLine,
+        content: lines.slice(0, endLine).join('\n'),
+        relevance: Math.min(1, relevance),
+      });
+      totalLines += endLine;
+    }
+  }
+
+  return snippets
+    .sort((a, b) => b.relevance - a.relevance)
+    .slice(0, 5);
+};
+
+export const createLayer5 = (deps: L5AnalyzerDeps) => {
+  const { callLlm, calculateCost } = deps;
+
+  const isUncertain = (finding: Finding): boolean => {
+    const confidence = finding.confidence ?? 50;
+    return confidence >= UNCERTAIN_MIN && confidence <= UNCERTAIN_MAX;
+  };
+
+  const analyzeFindings = async (
+    findings: readonly Finding[],
+    fileContents: ReadonlyMap<string, string>,
+  ): Promise<readonly L5Result[]> => {
+    const uncertainFindings = findings.filter(isUncertain).slice(0, MAX_FINDINGS_PER_SCAN);
+    const results: L5Result[] = [];
+
+    for (const finding of uncertainFindings) {
+      const promptType = selectPromptType(finding);
+      const snippets = extractSnippets(finding, fileContents);
+      const prompt = buildL5Prompt(promptType, finding, snippets);
+
+      try {
+        const response = await callLlm(prompt);
+        const cost = calculateCost('claude-sonnet-4', response.inputTokens, response.outputTokens);
+
+        // Parse LLM response
+        const jsonMatch = response.text.match(/\{[\s\S]*\}/);
+        if (jsonMatch) {
+          const isRec = (v: unknown): v is Record<string, unknown> =>
+            typeof v === 'object' && v !== null;
+          const raw: unknown = JSON.parse(jsonMatch[0]);
+          const parsed = isRec(raw) ? raw : {};
+
+          const rawVerdict = typeof parsed['verdict'] === 'string' ? parsed['verdict'] : '';
+          const verdict = rawVerdict === 'pass' ? 'pass'
+            : rawVerdict === 'fail' ? 'fail'
+            : 'uncertain';
+          const rawEvidence = Array.isArray(parsed['evidence'])
+            ? parsed['evidence'].filter((e): e is string => typeof e === 'string')
+            : [];
+          results.push({
+            findingId: finding.checkId,
+            originalConfidence: finding.confidence ?? 50,
+            newConfidence: typeof parsed['confidence'] === 'number' ? parsed['confidence'] : (finding.confidence ?? 50),
+            verdict,
+            reasoning: typeof parsed['reasoning'] === 'string' ? parsed['reasoning'] : 'Could not parse LLM response',
+            evidence: rawEvidence,
+            promptType,
+            cost,
+          });
+        } else {
+          results.push({
+            findingId: finding.checkId,
+            originalConfidence: finding.confidence ?? 50,
+            newConfidence: finding.confidence ?? 50,
+            verdict: 'uncertain',
+            reasoning: 'Could not parse LLM response',
+            evidence: [],
+            promptType,
+            cost,
+          });
+        }
+      } catch {
+        // On error, keep original status
+        results.push({
+          findingId: finding.checkId,
+          originalConfidence: finding.confidence ?? 50,
+          newConfidence: finding.confidence ?? 50,
+          verdict: 'uncertain',
+          reasoning: 'LLM analysis failed',
+          evidence: [],
+          promptType,
+          cost: 0,
+        });
+      }
+    }
+
+    return results;
+  };
+
+  const applyResults = (
+    findings: readonly Finding[],
+    l5Results: readonly L5Result[],
+  ): readonly Finding[] => {
+    const resultMap = new Map(l5Results.map((r) => [r.findingId, r]));
+
+    return findings.map((f) => {
+      const result = resultMap.get(f.checkId);
+      if (!result) return f;
+
+      return {
+        ...f,
+        type: result.verdict === 'pass' ? 'pass' : result.verdict === 'fail' ? 'fail' : f.type,
+        confidence: result.newConfidence,
+        confidenceLevel: result.newConfidence >= 95 ? 'PASS'
+          : result.newConfidence >= 70 ? 'LIKELY_PASS'
+          : result.newConfidence >= 40 ? 'UNCERTAIN'
+          : result.newConfidence >= 5 ? 'LIKELY_FAIL'
+          : 'FAIL',
+        severity: result.verdict === 'pass' ? 'info' : f.severity,
+        // Populate fix from L5 reasoning when verdict is fail and finding lacks a fix
+        fix: (result.verdict === 'fail' && !f.fix && result.reasoning)
+          ? result.reasoning
+          : f.fix,
+        l5Analyzed: true,
+      };
+    });
+  };
+
+  const getTotalCost = (results: readonly L5Result[]): number =>
+    results.reduce((sum, r) => sum + r.cost, 0);
+
+  /** Raw LLM call for custom prompts (e.g. doc validation). */
+  const callRaw = async (prompt: string): Promise<{ text: string; cost: number }> => {
+    const response = await callLlm(prompt);
+    const cost = calculateCost('claude-sonnet-4', response.inputTokens, response.outputTokens);
+    return { text: response.text, cost };
+  };
+
+  return Object.freeze({
+    isUncertain,
+    analyzeFindings,
+    applyResults,
+    getTotalCost,
+    selectPromptType,
+    extractSnippets,
+    buildL5Prompt,
+    callRaw,
+  });
+};
+
+export type Layer5Analyzer = ReturnType<typeof createLayer5>;

package/src/domain/scanner/layers/layer5-targeted.test.ts

@@ -0,0 +1,93 @@
+import { describe, it, expect } from 'vitest';
+import { selectUncertainFindings, buildTargetedPrompts, applyTargetedResults, estimateTargetedCost } from './layer5-targeted.js';
+import type { Finding } from '../../../types/common.types.js';
+
+const makeFinding = (overrides: Partial<Finding> = {}): Finding => ({
+  checkId: 'l4-kill-switch',
+  type: 'fail',
+  message: 'No kill switch found',
+  severity: 'medium',
+  confidence: 65,
+  confidenceLevel: 'UNCERTAIN',
+  ...overrides,
+});
+
+describe('selectUncertainFindings', () => {
+  it('selects findings with confidence 50-80', () => {
+    const findings = [
+      makeFinding({ confidence: 30 }), // too low
+      makeFinding({ confidence: 65 }), // in range
+      makeFinding({ confidence: 90 }), // too high
+      makeFinding({ confidence: 75 }), // in range
+    ];
+    const selected = selectUncertainFindings(findings);
+    expect(selected.length).toBe(2);
+  });
+
+  it('respects maxFindings limit', () => {
+    const findings = Array.from({ length: 30 }, (_, i) =>
+      makeFinding({ checkId: `l4-test-${i}`, confidence: 60 }),
+    );
+    const selected = selectUncertainFindings(findings, { maxFindings: 5, budgetLimit: 0.05, confidenceMin: 50, confidenceMax: 80 });
+    expect(selected.length).toBe(5);
+  });
+
+  it('treats undefined confidence as 50', () => {
+    const findings = [makeFinding({ confidence: undefined })];
+    const selected = selectUncertainFindings(findings);
+    expect(selected.length).toBe(1);
+  });
+});
+
+describe('buildTargetedPrompts', () => {
+  it('builds obligation-specific prompts', () => {
+    const findings = [makeFinding({ checkId: 'l4-kill-switch', file: 'src/safety.ts', line: 15 })];
+    const files = new Map([['src/safety.ts', 'const killSwitch = () => process.exit(0);']]);
+    const prompts = buildTargetedPrompts(findings, files);
+    expect(prompts.length).toBe(1);
+    expect(prompts[0]?.prompt).toContain('Art. 14(4)');
+    expect(prompts[0]?.prompt).toContain('kill switch');
+  });
+
+  it('includes code context in prompt', () => {
+    const findings = [makeFinding({ file: 'src/chat.ts', line: 5 })];
+    const files = new Map([['src/chat.ts', 'line1\nline2\nline3\nline4\nconst x = 1;\nline6']]);
+    const prompts = buildTargetedPrompts(findings, files);
+    expect(prompts[0]?.prompt).toContain('line1');
+    expect(prompts[0]?.contextFiles).toContain('src/chat.ts');
+  });
+});
+
+describe('applyTargetedResults', () => {
+  it('flips fail to pass when L5 disproves', () => {
+    const findings = [makeFinding({ type: 'fail', confidence: 65 })];
+    const results = [{ findingId: 'l4-kill-switch', confirmed: false, newConfidence: 90, explanation: 'This is a real kill switch', cost: 0.001 }];
+    const updated = applyTargetedResults(findings, results);
+    expect(updated[0]?.type).toBe('pass');
+    expect(updated[0]?.confidence).toBe(90);
+  });
+
+  it('increases confidence when L5 confirms fail', () => {
+    const findings = [makeFinding({ type: 'fail', confidence: 65 })];
+    const results = [{ findingId: 'l4-kill-switch', confirmed: true, newConfidence: 95, explanation: 'Confirmed missing', cost: 0.001 }];
+    const updated = applyTargetedResults(findings, results);
+    expect(updated[0]?.type).toBe('fail');
+    expect(updated[0]?.confidence).toBe(95);
+  });
+
+  it('leaves unmatched findings unchanged', () => {
+    const findings = [makeFinding({ checkId: 'l4-other' })];
+    const results = [{ findingId: 'l4-kill-switch', confirmed: true, newConfidence: 95, explanation: 'test', cost: 0 }];
+    const updated = applyTargetedResults(findings, results);
+    expect(updated[0]).toEqual(findings[0]);
+  });
+});
+
+describe('estimateTargetedCost', () => {
+  it('estimates cost for prompts', () => {
+    const prompts = [{ findingId: 'test', prompt: 'x'.repeat(4000), contextFiles: [], estimatedTokens: 1000 }];
+    const cost = estimateTargetedCost(prompts);
+    expect(cost).toBeGreaterThan(0);
+    expect(cost).toBeLessThan(0.01); // Should be cheap
+  });
+});

package/src/domain/scanner/layers/layer5-targeted.ts

@@ -0,0 +1,233 @@
+/**
+ * Targeted L5: Sends ONLY findings with confidence 50-80% to LLM
+ * with structured, obligation-specific questions.
+ * Cost: ~$0.01 vs $0.10 for full deep scan.
+ */
+
+import type { Finding } from '../../../types/common.types.js';
+import type { ImportGraph } from '../import-graph.js';
+
+export interface TargetedL5Config {
+  readonly maxFindings: number;      // default 20
+  readonly budgetLimit: number;      // max $ per scan, default 0.05
+  readonly confidenceMin: number;    // default 50
+  readonly confidenceMax: number;    // default 80
+}
+
+export interface TargetedL5Prompt {
+  readonly findingId: string;
+  readonly prompt: string;
+  readonly contextFiles: readonly string[];
+  readonly estimatedTokens: number;
+}
+
+export interface TargetedL5Result {
+  readonly findingId: string;
+  readonly confirmed: boolean;
+  readonly newConfidence: number;
+  readonly explanation: string;
+  readonly cost: number;
+}
+
+const DEFAULT_CONFIG: TargetedL5Config = {
+  maxFindings: 20,
+  budgetLimit: 0.05,
+  confidenceMin: 50,
+  confidenceMax: 80,
+};
+
+/**
+ * Obligation-specific prompt templates.
+ * Each template asks a concrete yes/no question about a specific EU AI Act article.
+ */
+const OBLIGATION_PROMPTS: Record<string, string> = {
+  'kill-switch': `L4 scanner found a potential kill switch / emergency stop mechanism.
+Question: Does this code implement a genuine kill switch that satisfies EU AI Act Art. 14(4) —
+allowing a human to interrupt, stop, or disable the AI system?
+A kill switch must: (1) be accessible to authorized personnel, (2) work reliably under all conditions,
+(3) not require AI involvement to function.`,
+
+  'disclosure': `L4 scanner found a potential AI disclosure mechanism.
+Question: Does this code implement AI disclosure that satisfies EU AI Act Art. 50(1) —
+informing natural persons that they are interacting with an AI system?
+Disclosure must be: (1) clear and understandable, (2) provided before or at the start of interaction,
+(3) include the AI system's capabilities and limitations.`,
+
+  'human-oversight': `L4 scanner found a potential human oversight mechanism.
+Question: Does this code implement human oversight that satisfies EU AI Act Art. 14 —
+enabling effective human oversight of the AI system during operation?
+Oversight must include: (1) ability for human to understand AI decisions, (2) ability to intervene,
+(3) ability to override or reverse AI decisions.`,
+
+  'logging': `L4 scanner found a potential interaction logging mechanism.
+Question: Does this code implement logging that satisfies EU AI Act Art. 12 —
+automatic recording of AI system events (logs) during operation?
+Logging must: (1) capture input/output of AI interactions, (2) be traceable,
+(3) have defined retention period (≥180 days for high-risk).`,
+
+  'content-marking': `L4 scanner found a potential content marking mechanism.
+Question: Does this code implement content marking that satisfies EU AI Act Art. 50(2) —
+marking AI-generated content as artificially generated or manipulated?
+Marking must be: (1) machine-readable where technically feasible, (2) detectable and interoperable.`,
+
+  'data-governance': `L4 scanner found a potential data governance mechanism.
+Question: Does this code implement data governance that satisfies EU AI Act Art. 10 —
+ensuring training and validation data meets quality criteria?
+Must include: (1) data quality examination, (2) bias assessment, (3) data provenance.`,
+
+  'bare-llm': `L4 scanner detected a potential bare LLM API call without compliance wrapper.
+Question: Is this a direct LLM API call without transparency, logging, or safety controls?
+A bare call is non-compliant if it: (1) lacks disclosure to users, (2) has no logging,
+(3) has no content safety filtering.`,
+
+  'deployer-monitoring': `L4 scanner found a potential deployer monitoring mechanism.
+Question: Does this code implement monitoring that satisfies EU AI Act Art. 26(5) —
+deployer monitoring of AI system operation and reporting incidents?
+Must include: (1) performance monitoring, (2) anomaly detection, (3) incident reporting.`,
+
+  'security-risk': `L4 scanner detected a potential security vulnerability.
+Question: Does this code introduce a cybersecurity risk per EU AI Act Art. 15(4)?
+Check for: (1) unsafe deserialization, (2) code injection vectors, (3) missing input validation.`,
+};
+
+/**
+ * Select uncertain findings for targeted L5 analysis.
+ */
+export const selectUncertainFindings = (
+  findings: readonly Finding[],
+  config: TargetedL5Config = DEFAULT_CONFIG,
+): readonly Finding[] => {
+  return findings
+    .filter((f) => {
+      const conf = f.confidence ?? 50;
+      return conf >= config.confidenceMin && conf <= config.confidenceMax;
+    })
+    .slice(0, config.maxFindings);
+};
+
+/**
+ * Build structured prompts for targeted L5 analysis.
+ * Each prompt is obligation-specific with concrete questions.
+ */
+export const buildTargetedPrompts = (
+  findings: readonly Finding[],
+  fileContents: ReadonlyMap<string, string>,
+  importGraph?: ImportGraph,
+): readonly TargetedL5Prompt[] => {
+  return findings.map((finding) => {
+    // Determine category for prompt template
+    const category = finding.checkId.replace(/^l4-/, '').replace(/-\d+$/, '');
+    const template = OBLIGATION_PROMPTS[category] ?? OBLIGATION_PROMPTS['bare-llm'] ?? '';
+
+    // Gather context: the finding's file + its imports from import-graph
+    const contextFiles: string[] = [];
+    if (finding.file) {
+      contextFiles.push(finding.file);
+
+      // Add imported files from import-graph (1 level deep)
+      if (importGraph) {
+        const node = importGraph.nodes.get(finding.file);
+        if (node) {
+          for (const imp of node.imports.slice(0, 3)) { // Max 3 related files
+            contextFiles.push(imp);
+          }
+        }
+      }
+    }
+
+    // Build code context
+    const codeSnippets = contextFiles
+      .map((file) => {
+        const content = fileContents.get(file);
+        if (!content) return '';
+        // Extract relevant portion (around the finding line, or first 100 lines)
+        const lines = content.split('\n');
+        const targetLine = finding.line ?? 1;
+        const start = Math.max(0, targetLine - 20);
+        const end = Math.min(lines.length, targetLine + 30);
+        return `--- ${file}:${start + 1}-${end} ---\n${lines.slice(start, end).join('\n')}`;
+      })
+      .filter(Boolean)
+      .join('\n\n');
+
+    const prompt = `${template}
+
+Finding: ${finding.checkId} — ${finding.message}
+Severity: ${finding.severity}
+Article: ${finding.articleReference ?? 'N/A'}
+Current confidence: ${finding.confidence ?? 50}%
+
+Code context:
+${codeSnippets || 'No code context available'}
+
+Respond ONLY in this JSON format:
+{
+  "confirmed": true/false,
+  "confidence": 0-100,
+  "explanation": "brief explanation why this is/isn't compliant"
+}`;
+
+    return {
+      findingId: finding.checkId,
+      prompt,
+      contextFiles,
+      estimatedTokens: Math.ceil(prompt.length / 4), // rough estimate
+    };
+  });
+};
+
+/**
+ * Apply targeted L5 results back to findings.
+ */
+export const applyTargetedResults = (
+  findings: readonly Finding[],
+  results: readonly TargetedL5Result[],
+): readonly Finding[] => {
+  const resultMap = new Map(results.map((r) => [r.findingId, r]));
+
+  return findings.map((f) => {
+    const result = resultMap.get(f.checkId);
+    if (!result) return f;
+
+    // If L5 confirms it's a real finding (fail), keep as fail with higher confidence
+    if (result.confirmed && f.type === 'fail') {
+      return {
+        ...f,
+        confidence: result.newConfidence,
+        confidenceLevel: result.newConfidence >= 95 ? 'FAIL' : 'LIKELY_FAIL',
+      };
+    }
+
+    // If L5 says it's actually compliant, flip to pass
+    if (!result.confirmed && f.type === 'fail') {
+      return {
+        ...f,
+        type: 'pass' as const,
+        confidence: result.newConfidence,
+        confidenceLevel: result.newConfidence >= 95 ? 'PASS' : 'LIKELY_PASS',
+        message: `${f.message} [L5 verified: ${result.explanation}]`,
+      };
+    }
+
+    // If L5 confirms pass, increase confidence
+    if (result.confirmed && f.type === 'pass') {
+      return {
+        ...f,
+        confidence: result.newConfidence,
+        confidenceLevel: result.newConfidence >= 95 ? 'PASS' : 'LIKELY_PASS',
+      };
+    }
+
+    return f;
+  });
+};
+
+/**
+ * Estimate total cost for targeted L5 analysis.
+ */
+export const estimateTargetedCost = (prompts: readonly TargetedL5Prompt[]): number => {
+  // Approximate: $0.003/1K input tokens + $0.015/1K output tokens (Claude Sonnet)
+  const inputCost = prompts.reduce((sum, p) => sum + p.estimatedTokens, 0) * 0.003 / 1000;
+  const outputCost = prompts.length * 200 * 0.015 / 1000; // ~200 tokens per response
+  return Math.round((inputCost + outputCost) * 10000) / 10000;
+};