@complior/engine 0.9.0

package/src/domain/scanner/confidence.test.ts

@@ -0,0 +1,235 @@
+import { describe, it, expect } from 'vitest';
+import {
+  getConfidenceLevel,
+  l1Confidence,
+  l2Confidence,
+  l3Confidence,
+  l4Confidence,
+  aggregateConfidence,
+  aggregateLevel,
+  confidenceScoreMultiplier,
+} from './confidence.js';
+import type { CheckWithConfidence } from './confidence.js';
+import { createScanner } from './create-scanner.js';
+import { createScanFile, createScanCtx } from '../../test-helpers/factories.js';
+
+describe('getConfidenceLevel', () => {
+  it('returns PASS for high-confidence pass', () => {
+    expect(getConfidenceLevel(95, true)).toBe('PASS');
+    expect(getConfidenceLevel(100, true)).toBe('PASS');
+  });
+
+  it('returns LIKELY_PASS for medium-confidence pass', () => {
+    expect(getConfidenceLevel(70, true)).toBe('LIKELY_PASS');
+    expect(getConfidenceLevel(85, true)).toBe('LIKELY_PASS');
+  });
+
+  it('returns UNCERTAIN for low-confidence pass', () => {
+    expect(getConfidenceLevel(50, true)).toBe('UNCERTAIN');
+    expect(getConfidenceLevel(69, true)).toBe('UNCERTAIN');
+  });
+
+  it('returns FAIL for high-confidence fail', () => {
+    expect(getConfidenceLevel(95, false)).toBe('FAIL');
+    expect(getConfidenceLevel(100, false)).toBe('FAIL');
+  });
+
+  it('returns LIKELY_FAIL for medium-confidence fail', () => {
+    expect(getConfidenceLevel(70, false)).toBe('LIKELY_FAIL');
+    expect(getConfidenceLevel(80, false)).toBe('LIKELY_FAIL');
+  });
+});
+
+describe('per-layer confidence', () => {
+  it('L1: file found → PASS (95%)', () => {
+    const result = l1Confidence(true);
+    expect(result.level).toBe('PASS');
+    expect(result.confidence).toBe(95);
+    expect(result.layer).toBe('L1');
+  });
+
+  it('L1: file not found → FAIL (98%)', () => {
+    const result = l1Confidence(false);
+    expect(result.level).toBe('FAIL');
+    expect(result.confidence).toBe(98);
+  });
+
+  it('L2: VALID → PASS (95%)', () => {
+    expect(l2Confidence('VALID').level).toBe('PASS');
+    expect(l2Confidence('VALID').confidence).toBe(95);
+  });
+
+  it('L2: PARTIAL → LIKELY_PASS (75%)', () => {
+    expect(l2Confidence('PARTIAL').level).toBe('LIKELY_PASS');
+    expect(l2Confidence('PARTIAL').confidence).toBe(75);
+  });
+
+  it('L2: EMPTY → FAIL (95%)', () => {
+    expect(l2Confidence('EMPTY').level).toBe('FAIL');
+    expect(l2Confidence('EMPTY').confidence).toBe(95);
+  });
+
+  it('L3: PROHIBITED → FAIL (99%)', () => {
+    expect(l3Confidence('PROHIBITED').level).toBe('FAIL');
+    expect(l3Confidence('PROHIBITED').confidence).toBe(99);
+  });
+
+  it('L3: OK → LIKELY_PASS (80%)', () => {
+    expect(l3Confidence('OK').level).toBe('LIKELY_PASS');
+    expect(l3Confidence('OK').confidence).toBe(80);
+  });
+
+  it('L4: negative FOUND → LIKELY_FAIL (80%)', () => {
+    expect(l4Confidence('negative', 'FOUND').level).toBe('LIKELY_FAIL');
+    expect(l4Confidence('negative', 'FOUND').confidence).toBe(80);
+  });
+
+  it('L4: positive FOUND → LIKELY_PASS (75%)', () => {
+    expect(l4Confidence('positive', 'FOUND').level).toBe('LIKELY_PASS');
+    expect(l4Confidence('positive', 'FOUND').confidence).toBe(75);
+  });
+});
+
+describe('aggregateConfidence', () => {
+  it('returns weighted average for checks from different layers', () => {
+    const checks: CheckWithConfidence[] = [
+      { layer: 'L1', confidence: 95, level: 'PASS' },    // weight 1.0
+      { layer: 'L2', confidence: 75, level: 'LIKELY_PASS' }, // weight 0.95
+      { layer: 'L4', confidence: 80, level: 'LIKELY_FAIL' }, // weight 0.75
+    ];
+
+    const result = aggregateConfidence(checks);
+
+    // (95*1.0 + 75*0.95 + 80*0.75) / (1.0+0.95+0.75)
+    // = (95 + 71.25 + 60) / 2.70
+    // = 226.25 / 2.70 ≈ 83.80
+    expect(result).toBeCloseTo(83.80, 1);
+  });
+
+  it('returns 0 for empty checks', () => {
+    expect(aggregateConfidence([])).toBe(0);
+  });
+
+  it('returns single check confidence for one check', () => {
+    const checks: CheckWithConfidence[] = [
+      { layer: 'L1', confidence: 98, level: 'FAIL' },
+    ];
+    expect(aggregateConfidence(checks)).toBe(98);
+  });
+});
+
+describe('aggregateLevel', () => {
+  it('returns UNCERTAIN for contradictory signals', () => {
+    const checks: CheckWithConfidence[] = [
+      { layer: 'L1', confidence: 95, level: 'PASS' },
+      { layer: 'L4', confidence: 80, level: 'LIKELY_FAIL' },
+    ];
+
+    const result = aggregateLevel(checks);
+    expect(result).toBe('UNCERTAIN');
+  });
+
+  it('returns PASS for all-pass checks', () => {
+    const checks: CheckWithConfidence[] = [
+      { layer: 'L1', confidence: 95, level: 'PASS' },
+      { layer: 'L2', confidence: 95, level: 'PASS' },
+    ];
+
+    const result = aggregateLevel(checks);
+    expect(result).toBe('PASS');
+  });
+
+  it('returns FAIL for all-fail checks', () => {
+    const checks: CheckWithConfidence[] = [
+      { layer: 'L1', confidence: 98, level: 'FAIL' },
+      { layer: 'L3', confidence: 99, level: 'FAIL' },
+    ];
+
+    const result = aggregateLevel(checks);
+    expect(result).toBe('FAIL');
+  });
+});
+
+describe('confidenceScoreMultiplier', () => {
+  it('returns correct multiplier for each level', () => {
+    expect(confidenceScoreMultiplier('PASS')).toBe(1.0);
+    expect(confidenceScoreMultiplier('LIKELY_PASS')).toBe(0.75);
+    expect(confidenceScoreMultiplier('UNCERTAIN')).toBe(0.5);
+    expect(confidenceScoreMultiplier('LIKELY_FAIL')).toBe(0.25);
+    expect(confidenceScoreMultiplier('FAIL')).toBe(0.0);
+  });
+});
+
+describe('scanner integration — confidence in findings', () => {
+  it('attaches confidence to findings in full scan (all PASS)', () => {
+    const scanner = createScanner();
+    const ctx = createScanCtx([
+      createScanFile('AI-LITERACY.md', '# AI Literacy\n## Training Program\n## Training Levels\n## Assessment Methods'),
+      createScanFile('ART5-SCREENING.md', '# Screening\n## Prohibited Practices\n## Screening Results\n## Mitigations'),
+      createScanFile('COMPLIANCE.md', '# EU AI Act Compliance\nRisk assessment documentation'),
+      createScanFile('.complior/config.json', '{"version":"1.0"}'),
+      createScanFile('.well-known/ai-compliance.json', '{"compliant":true}'),
+    ]);
+
+    const result = scanner.scan(ctx);
+
+    // Some findings should have confidence
+    const withConfidence = result.findings.filter((f) => f.confidence !== undefined);
+    expect(withConfidence.length).toBeGreaterThan(0);
+
+    // High-confidence L1/L2 passes should be 95%
+    const passes = withConfidence.filter((f) => f.type === 'pass' && f.confidence === 95);
+    expect(passes.length).toBeGreaterThan(0);
+
+    // Confidence summary should be present
+    expect(result.score.confidenceSummary).toBeDefined();
+    expect(result.score.confidenceSummary!.total).toBeGreaterThan(0);
+  });
+
+  it('attaches confidence to mixed findings', () => {
+    const scanner = createScanner();
+    const ctx = createScanCtx([
+      createScanFile('src/chat.tsx', `
+import OpenAI from 'openai';
+try {
+  const response = await openai.chat.completions.create({ model: 'gpt-4' });
+} catch (err) {
+  console.error('LLM call failed:', err);
+  throw err;
+}
+      `),
+      createScanFile('package.json', '{"dependencies":{"openai":"^4.0.0"}}'),
+    ]);
+
+    const result = scanner.scan(ctx);
+
+    const withConfidence = result.findings.filter((f) => f.confidence !== undefined);
+    expect(withConfidence.length).toBeGreaterThan(0);
+
+    // Should have mix of levels
+    const levels = [...new Set(withConfidence.map((f) => f.confidenceLevel))];
+    expect(levels.length).toBeGreaterThan(1); // At least PASS + FAIL/LIKELY_FAIL
+  });
+
+  it('confidence summary counts all levels correctly', () => {
+    const scanner = createScanner();
+    const ctx = createScanCtx([
+      createScanFile('AI-LITERACY.md', '# Policy\n## Training Program\n## Training Levels\n## Assessment Methods'),
+      createScanFile('package.json', '{"dependencies":{"openai":"^4.0.0","express":"^4.0.0"}}'),
+      createScanFile('src/app.ts', `
+try {
+  const res = await openai.chat.completions.create({ model: 'gpt-4' });
+} catch (err) {
+  console.error('LLM call failed:', err);
+  throw err;
+}
+      `),
+    ]);
+
+    const result = scanner.scan(ctx);
+    const summary = result.score.confidenceSummary!;
+
+    expect(summary.total).toBeGreaterThan(0);
+    expect(summary.pass + summary.likelyPass + summary.uncertain + summary.likelyFail + summary.fail).toBe(summary.total);
+  });
+});

package/src/domain/scanner/confidence.ts

@@ -0,0 +1,156 @@
+// --- 5-Tier Confidence Levels ---
+
+import confidenceData from '../../../data/scanner/confidence-params.json' with { type: 'json' };
+
+export type ConfidenceLevel = 'PASS' | 'LIKELY_PASS' | 'UNCERTAIN' | 'LIKELY_FAIL' | 'FAIL';
+
+export type ScannerLayer = 'L1' | 'L2' | 'L3' | 'L4' | 'L5';
+
+export interface CheckWithConfidence {
+  readonly layer: ScannerLayer;
+  readonly confidence: number; // 0-100
+  readonly level: ConfidenceLevel;
+  readonly obligationId?: string;
+}
+
+// --- Layer Weights ---
+
+const LAYER_WEIGHTS = confidenceData.layer_weights as Readonly<Record<ScannerLayer, number>>;
+
+// --- Confidence Level Determination ---
+
+export const getConfidenceLevel = (confidence: number, isPass: boolean): ConfidenceLevel => {
+  if (isPass) {
+    if (confidence >= 95) return 'PASS';
+    if (confidence >= 70) return 'LIKELY_PASS';
+    return 'UNCERTAIN';
+  }
+
+  // Fail direction
+  if (confidence >= 95) return 'FAIL';
+  if (confidence >= 70) return 'LIKELY_FAIL';
+  return 'UNCERTAIN';
+};
+
+// --- Per-Layer Confidence Assignment ---
+
+// L1: File presence (deterministic, high confidence)
+export const l1Confidence = (found: boolean): CheckWithConfidence => ({
+  layer: 'L1',
+  confidence: found ? 95 : 98,
+  level: found ? 'PASS' : 'FAIL',
+});
+
+// L2: Document structure (deterministic, high confidence)
+export const l2Confidence = (status: 'VALID' | 'PARTIAL' | 'SHALLOW' | 'EMPTY'): CheckWithConfidence => {
+  switch (status) {
+    case 'VALID':
+      return { layer: 'L2', confidence: 95, level: 'PASS' };
+    case 'PARTIAL':
+      return { layer: 'L2', confidence: 75, level: 'LIKELY_PASS' };
+    case 'SHALLOW':
+      return { layer: 'L2', confidence: 65, level: 'UNCERTAIN' };
+    case 'EMPTY':
+      return { layer: 'L2', confidence: 95, level: 'FAIL' };
+  }
+};
+
+// L3: Config analysis (medium-high confidence)
+export const l3Confidence = (status: 'OK' | 'WARNING' | 'FAIL' | 'PROHIBITED'): CheckWithConfidence => {
+  switch (status) {
+    case 'OK':
+      return { layer: 'L3', confidence: 80, level: 'LIKELY_PASS' };
+    case 'WARNING':
+      return { layer: 'L3', confidence: 80, level: 'LIKELY_FAIL' };
+    case 'FAIL':
+      return { layer: 'L3', confidence: 80, level: 'LIKELY_FAIL' };
+    case 'PROHIBITED':
+      return { layer: 'L3', confidence: 99, level: 'FAIL' };
+  }
+};
+
+// L4: Pattern matching (heuristic, medium confidence)
+export const l4Confidence = (
+  patternType: 'positive' | 'negative',
+  status: 'FOUND' | 'NOT_FOUND',
+): CheckWithConfidence => {
+  if (patternType === 'negative' && status === 'FOUND') {
+    return { layer: 'L4', confidence: 80, level: 'LIKELY_FAIL' };
+  }
+  if (patternType === 'positive' && status === 'FOUND') {
+    return { layer: 'L4', confidence: 75, level: 'LIKELY_PASS' };
+  }
+  if (patternType === 'positive' && status === 'NOT_FOUND') {
+    return { layer: 'L4', confidence: 80, level: 'LIKELY_FAIL' };
+  }
+  // negative NOT_FOUND → good (absence of bad)
+  return { layer: 'L4', confidence: 70, level: 'LIKELY_PASS' };
+};
+
+// --- Aggregation ---
+
+export const aggregateConfidence = (checks: readonly CheckWithConfidence[]): number => {
+  if (checks.length === 0) return 0;
+
+  let totalWeight = 0;
+  let weightedSum = 0;
+
+  for (const check of checks) {
+    const weight = LAYER_WEIGHTS[check.layer];
+    totalWeight += weight;
+    weightedSum += check.confidence * weight;
+  }
+
+  return totalWeight === 0 ? 0 : Math.round((weightedSum / totalWeight) * 100) / 100;
+};
+
+export const aggregateLevel = (checks: readonly CheckWithConfidence[]): ConfidenceLevel => {
+  if (checks.length === 0) return 'UNCERTAIN';
+
+  const confidence = aggregateConfidence(checks);
+
+  // Determine direction: majority of checks
+  const passDirection = checks.filter((c) =>
+    c.level === 'PASS' || c.level === 'LIKELY_PASS',
+  ).length;
+  const failDirection = checks.filter((c) =>
+    c.level === 'FAIL' || c.level === 'LIKELY_FAIL',
+  ).length;
+
+  // If contradictory signals → UNCERTAIN
+  if (passDirection > 0 && failDirection > 0) {
+    // Check if one side is clearly dominant
+    const ratio = Math.max(passDirection, failDirection) / checks.length;
+    if (ratio < 0.7) return 'UNCERTAIN';
+  }
+
+  const isPass = passDirection >= failDirection;
+  return getConfidenceLevel(confidence, isPass);
+};
+
+// --- Score Impact ---
+
+const CONFIDENCE_SCORE_MULTIPLIER = confidenceData.score_multipliers as Readonly<Record<ConfidenceLevel, number>>;
+
+export const confidenceScoreMultiplier = (level: ConfidenceLevel): number =>
+  CONFIDENCE_SCORE_MULTIPLIER[level];
+
+// --- Summary ---
+
+export interface ConfidenceSummary {
+  readonly pass: number;
+  readonly likelyPass: number;
+  readonly uncertain: number;
+  readonly likelyFail: number;
+  readonly fail: number;
+  readonly total: number;
+}
+
+export const summarizeConfidence = (checks: readonly CheckWithConfidence[]): ConfidenceSummary => ({
+  pass: checks.filter((c) => c.level === 'PASS').length,
+  likelyPass: checks.filter((c) => c.level === 'LIKELY_PASS').length,
+  uncertain: checks.filter((c) => c.level === 'UNCERTAIN').length,
+  likelyFail: checks.filter((c) => c.level === 'LIKELY_FAIL').length,
+  fail: checks.filter((c) => c.level === 'FAIL').length,
+  total: checks.length,
+});

package/src/domain/scanner/constants.ts

@@ -0,0 +1,13 @@
+/**
+ * Re-export from data/scanner-constants.ts (architecturally neutral location).
+ * Domain modules import from here; infra modules import directly from data/.
+ */
+export {
+  CODE_EXTENSIONS,
+  AST_SUPPORTED_EXTENSIONS,
+  DOC_EXTENSIONS,
+  CONFIG_EXTENSIONS,
+  STYLE_EXTENSIONS,
+  ALL_SCANNABLE_EXTENSIONS,
+  EXCLUDED_DIRS,
+} from '../../data/scanner-constants.js';