@complior/engine 0.9.0

package/src/domain/eval/eval-to-findings.test.ts

@@ -0,0 +1,293 @@
+/**
+ * Tests for evalToFindings — converts eval failures into scanner-compatible findings.
+ *
+ * Uses real playbooks (ALL_PLAYBOOKS) for realistic integration testing.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { evalToFindings } from './eval-to-findings.js';
+import type { EvalResult, TestResult } from './types.js';
+import { ALL_PLAYBOOKS } from '../../data/eval/remediation/index.js';
+
+// ── Helpers ─────────────────────────────────────────────────────
+
+const mockResult = (results: TestResult[]): EvalResult => ({
+  target: 'http://localhost:4000',
+  tier: 'full' as const,
+  overallScore: 50,
+  grade: 'D',
+  categories: [],
+  results,
+  totalTests: results.length,
+  passed: 0,
+  failed: results.length,
+  errors: 0,
+  duration: 1000,
+  timestamp: '2026-03-23T10:00:00Z',
+  criticalCapped: false,
+});
+
+const makeFailure = (
+  overrides: Partial<TestResult> & { category: TestResult['category'] },
+): TestResult => ({
+  testId: `test-${overrides.category}-001`,
+  name: `${overrides.category} test`,
+  method: 'deterministic',
+  verdict: 'fail',
+  score: 0,
+  confidence: 100,
+  reasoning: 'Failed compliance check',
+  probe: 'Are you an AI?',
+  response: 'I am a helpful assistant.',
+  latencyMs: 120,
+  timestamp: '2026-03-23T10:00:00Z',
+  severity: 'high',
+  ...overrides,
+});
+
+// ── Tests ───────────────────────────────────────────────────────
+
+describe('evalToFindings', () => {
+  // 1. Transparency failures → findings with type 'A' (system prompt)
+  it('returns findings with type A for transparency failures', () => {
+    const results: TestResult[] = [
+      makeFailure({ category: 'transparency', testId: 'CT-1-001', severity: 'critical' }),
+      makeFailure({ category: 'transparency', testId: 'CT-1-002', severity: 'high' }),
+    ];
+
+    const findings = evalToFindings(mockResult(results), ALL_PLAYBOOKS);
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0]!.type).toBe('A');
+    expect(findings[0]!.checkId).toBe('eval-transparency');
+    expect(findings[0]!.file).toBe('system-prompt');
+    expect(findings[0]!.article).toBe('Art.50');
+  });
+
+  // 2. Prohibited failures → findings with type 'A'
+  it('returns findings with type A for prohibited failures', () => {
+    const results: TestResult[] = [
+      makeFailure({ category: 'prohibited', testId: 'CT-7-001', severity: 'critical' }),
+    ];
+
+    const findings = evalToFindings(mockResult(results), ALL_PLAYBOOKS);
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0]!.type).toBe('A');
+    expect(findings[0]!.checkId).toBe('eval-prohibited');
+    expect(findings[0]!.file).toBe('system-prompt');
+    expect(findings[0]!.article).toBe('Art.5');
+  });
+
+  // 3. Logging failures → findings with type 'B' (config file, top action is infrastructure)
+  it('returns findings with type B for logging failures (infrastructure action)', () => {
+    const results: TestResult[] = [
+      makeFailure({ category: 'logging', testId: 'CT-8-001', severity: 'high' }),
+      makeFailure({ category: 'logging', testId: 'CT-8-002', severity: 'medium' }),
+    ];
+
+    const findings = evalToFindings(mockResult(results), ALL_PLAYBOOKS);
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0]!.type).toBe('B');
+    expect(findings[0]!.checkId).toBe('eval-logging');
+    expect(findings[0]!.file).toBe('.complior/eval-fixes/logging-config.json');
+    expect(findings[0]!.article).toBe('Art.12');
+  });
+
+  // 4. Security LLM03 (supply chain) failures → findings with type 'B'
+  it('returns findings with type B for security LLM03 failures (infrastructure action)', () => {
+    const results: TestResult[] = [
+      makeFailure({
+        category: 'robustness',
+        owaspCategory: 'LLM03',
+        testId: 'SEC-LLM03-001',
+        severity: 'high',
+      }),
+    ];
+
+    const findings = evalToFindings(mockResult(results), ALL_PLAYBOOKS);
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0]!.type).toBe('B');
+    expect(findings[0]!.checkId).toBe('eval-LLM03');
+    expect(findings[0]!.file).toBe('.complior/eval-fixes/LLM03-config.json');
+  });
+
+  // 5. Groups by category — 5 failures from same category → 1 finding
+  it('groups by category: 5 failures from same category produce 1 finding', () => {
+    const results: TestResult[] = Array.from({ length: 5 }, (_, i) =>
+      makeFailure({
+        category: 'transparency',
+        testId: `CT-1-00${i + 1}`,
+        severity: 'high',
+      }),
+    );
+
+    const findings = evalToFindings(mockResult(results), ALL_PLAYBOOKS);
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0]!.checkId).toBe('eval-transparency');
+    expect(findings[0]!.title).toContain('5 eval failures');
+  });
+
+  // 6. No failures → returns empty array
+  it('returns empty array when there are no failures', () => {
+    const results: TestResult[] = [
+      makeFailure({ category: 'transparency', testId: 'CT-1-001', verdict: 'pass', score: 100 }),
+      makeFailure({ category: 'bias', testId: 'CT-4-001', verdict: 'pass', score: 100 }),
+    ];
+
+    const findings = evalToFindings(mockResult(results), ALL_PLAYBOOKS);
+
+    expect(findings).toEqual([]);
+  });
+
+  // 7. Each finding has required fields: checkId, title, type, severity, fixDescription
+  it('each finding has checkId, title, type, severity, and fixDescription', () => {
+    const results: TestResult[] = [
+      makeFailure({ category: 'transparency', testId: 'CT-1-001', severity: 'critical' }),
+      makeFailure({ category: 'logging', testId: 'CT-8-001', severity: 'high' }),
+    ];
+
+    const findings = evalToFindings(mockResult(results), ALL_PLAYBOOKS);
+
+    expect(findings.length).toBeGreaterThanOrEqual(2);
+
+    for (const f of findings) {
+      expect(f.checkId).toBeDefined();
+      expect(f.checkId).toMatch(/^eval-/);
+      expect(f.title).toBeDefined();
+      expect(f.title.length).toBeGreaterThan(0);
+      expect(f.type).toMatch(/^[AB]$/);
+      expect(['critical', 'high', 'medium', 'low']).toContain(f.severity);
+      expect(f.fixDescription).toBeDefined();
+      expect(f.fixDescription.length).toBeGreaterThan(0);
+    }
+  });
+
+  // 8. Mixed categories → correct number of findings (one per category)
+  it('mixed categories produce one finding per category', () => {
+    const results: TestResult[] = [
+      // 2 transparency failures
+      makeFailure({ category: 'transparency', testId: 'CT-1-001', severity: 'critical' }),
+      makeFailure({ category: 'transparency', testId: 'CT-1-002', severity: 'high' }),
+      // 1 bias failure
+      makeFailure({ category: 'bias', testId: 'CT-4-001', severity: 'high' }),
+      // 3 logging failures
+      makeFailure({ category: 'logging', testId: 'CT-8-001', severity: 'medium' }),
+      makeFailure({ category: 'logging', testId: 'CT-8-002', severity: 'medium' }),
+      makeFailure({ category: 'logging', testId: 'CT-8-003', severity: 'low' }),
+      // 1 prohibited failure
+      makeFailure({ category: 'prohibited', testId: 'CT-7-001', severity: 'critical' }),
+    ];
+
+    const findings = evalToFindings(mockResult(results), ALL_PLAYBOOKS);
+
+    expect(findings).toHaveLength(4);
+
+    const checkIds = findings.map((f) => f.checkId).sort();
+    expect(checkIds).toEqual([
+      'eval-bias',
+      'eval-logging',
+      'eval-prohibited',
+      'eval-transparency',
+    ]);
+  });
+
+  // ── Additional edge cases ─────────────────────────────────────
+
+  it('uses owaspCategory over category when grouping security probe results', () => {
+    const results: TestResult[] = [
+      makeFailure({
+        category: 'robustness',
+        owaspCategory: 'LLM01',
+        testId: 'SEC-LLM01-001',
+        severity: 'critical',
+      }),
+      makeFailure({
+        category: 'robustness',
+        owaspCategory: 'LLM01',
+        testId: 'SEC-LLM01-002',
+        severity: 'high',
+      }),
+    ];
+
+    const findings = evalToFindings(mockResult(results), ALL_PLAYBOOKS);
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0]!.checkId).toBe('eval-LLM01');
+    expect(findings[0]!.title).toContain('2 eval failures');
+  });
+
+  it('severity is elevated to critical when any failure in category is critical', () => {
+    const results: TestResult[] = [
+      makeFailure({ category: 'transparency', testId: 'CT-1-001', severity: 'low' }),
+      makeFailure({ category: 'transparency', testId: 'CT-1-002', severity: 'critical' }),
+      makeFailure({ category: 'transparency', testId: 'CT-1-003', severity: 'medium' }),
+    ];
+
+    const findings = evalToFindings(mockResult(results), ALL_PLAYBOOKS);
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0]!.severity).toBe('critical');
+  });
+
+  it('includes error-verdict results alongside fail-verdict results', () => {
+    const results: TestResult[] = [
+      makeFailure({ category: 'bias', testId: 'CT-4-001', verdict: 'fail', severity: 'high' }),
+      makeFailure({ category: 'bias', testId: 'CT-4-002', verdict: 'error', severity: 'medium' }),
+    ];
+
+    const findings = evalToFindings(mockResult(results), ALL_PLAYBOOKS);
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0]!.title).toContain('2 eval failures');
+  });
+
+  it('findings are sorted by severity (critical first, low last)', () => {
+    const results: TestResult[] = [
+      makeFailure({ category: 'logging', testId: 'CT-8-001', severity: 'low' }),
+      makeFailure({ category: 'transparency', testId: 'CT-1-001', severity: 'critical' }),
+      makeFailure({ category: 'bias', testId: 'CT-4-001', severity: 'medium' }),
+    ];
+
+    const findings = evalToFindings(mockResult(results), ALL_PLAYBOOKS);
+
+    const severities = findings.map((f) => f.severity);
+    const severityOrder: Record<string, number> = { critical: 0, high: 1, medium: 2, low: 3 };
+    for (let i = 1; i < severities.length; i++) {
+      expect(severityOrder[severities[i]!]).toBeGreaterThanOrEqual(severityOrder[severities[i - 1]!]!);
+    }
+  });
+
+  it('skips categories without a matching playbook', () => {
+    const results: TestResult[] = [
+      makeFailure({
+        category: 'transparency',
+        owaspCategory: 'UNKNOWN-CAT',
+        testId: 'UNK-001',
+        severity: 'high',
+      }),
+    ];
+
+    const findings = evalToFindings(mockResult(results), ALL_PLAYBOOKS);
+
+    // owaspCategory takes precedence; 'UNKNOWN-CAT' has no playbook → skipped
+    expect(findings).toEqual([]);
+  });
+
+  it('all finding layers are "eval"', () => {
+    const results: TestResult[] = [
+      makeFailure({ category: 'transparency', testId: 'CT-1-001', severity: 'high' }),
+      makeFailure({ category: 'prohibited', testId: 'CT-7-001', severity: 'critical' }),
+      makeFailure({ category: 'logging', testId: 'CT-8-001', severity: 'medium' }),
+    ];
+
+    const findings = evalToFindings(mockResult(results), ALL_PLAYBOOKS);
+
+    for (const f of findings) {
+      expect(f.layer).toBe('eval');
+    }
+  });
+});

package/src/domain/eval/eval-to-findings.ts

@@ -0,0 +1,83 @@
+/**
+ * Eval → Findings Bridge — converts eval failures into scanner Finding format
+ * for integration with the existing fix pipeline.
+ *
+ * US-REM-09: Fix Pipeline Integration
+ */
+
+import type { EvalResult } from './types.js';
+import type { CategoryPlaybook } from './remediation-types.js';
+import { CATEGORY_ARTICLES, priorityNum, groupFailuresByCategory, findPlaybook } from './eval-constants.js';
+import { isFailedVerdict } from './verdict-utils.js';
+
+// ── Finding type (compatible with scanner Finding) ───────────
+
+export interface EvalFinding {
+  readonly checkId: string;
+  readonly type: 'A' | 'B';    // A = system prompt patch, B = create file
+  readonly layer: string;
+  readonly title: string;
+  readonly description: string;
+  readonly file: string;
+  readonly severity: 'critical' | 'high' | 'medium' | 'low';
+  readonly article: string;
+  readonly fixDescription: string;
+  readonly fixExample: string;
+}
+
+// ── Conversion ───────────────────────────────────────────────
+
+/**
+ * Convert failed eval results into scanner-compatible findings.
+ *
+ * Groups failures by category, deduplicates to one finding per category,
+ * and assigns fix type based on remediation action type:
+ *   - system_prompt → Type A (edit system prompt)
+ *   - api_config / infrastructure → Type B (create config file)
+ */
+export const evalToFindings = (
+  result: EvalResult,
+  playbooks: readonly CategoryPlaybook[],
+): readonly EvalFinding[] => {
+  const failures = result.results.filter(isFailedVerdict);
+
+  // Group by category (one finding per category, not per test)
+  const byCat = groupFailuresByCategory(failures);
+
+  const findings: EvalFinding[] = [];
+
+  for (const [cat, catFailures] of byCat) {
+    const pb = findPlaybook(playbooks, cat);
+    if (!pb) continue;
+
+    // Find the highest-priority action
+    const topAction = [...pb.actions]
+      .sort((a, b) => priorityNum(a.priority) - priorityNum(b.priority))[0];
+    if (!topAction) continue;
+
+    const article = CATEGORY_ARTICLES[cat] ?? pb.article_ref;
+    const isSystemPrompt = topAction.type === 'system_prompt';
+    const severity = catFailures.some((f) => f.severity === 'critical') ? 'critical'
+      : catFailures.some((f) => f.severity === 'high') ? 'high'
+      : catFailures.some((f) => f.severity === 'medium') ? 'medium'
+      : 'low';
+
+    findings.push({
+      checkId: `eval-${cat}`,
+      type: isSystemPrompt ? 'A' : 'B',
+      layer: 'eval',
+      title: `${pb.label}: ${catFailures.length} eval failures`,
+      description: `${catFailures.length} tests failed in ${pb.label} (${article}). ${topAction.description}`,
+      file: isSystemPrompt ? 'system-prompt' : `.complior/eval-fixes/${cat}-config.json`,
+      severity,
+      article,
+      fixDescription: topAction.user_guidance.what_to_do.join('. '),
+      fixExample: topAction.example,
+    });
+  }
+
+  // Sort by severity
+  findings.sort((a, b) => priorityNum(a.severity) - priorityNum(b.severity));
+
+  return Object.freeze(findings);
+};

package/src/domain/eval/index.ts

@@ -0,0 +1,31 @@
+export { scoreConformity, createConformityScorer, CATEGORY_WEIGHTS } from './conformity-score.js';
+export type { ConformityScoreResult } from './conformity-score.js';
+export { PRIORITY_ORDER, priorityNum, CATEGORY_ARTICLES, CATEGORY_FINES } from './eval-constants.js';
+export { PRIORITY_TIMELINE, findPlaybook, groupFailuresByCategory } from './eval-constants.js';
+export { evalResultToEvidence, evalResultHash, summarizeTestResults } from './eval-evidence.js';
+export { generateSystemPromptPatch, generateApiConfigPatch } from './eval-fix-generator.js';
+export { buildPassportEvalBlock, mergeEvalIntoPassport } from './eval-passport.js';
+export type { PassportEvalBlock } from './eval-passport.js';
+export { generateRemediationReport, renderRemediationMarkdown } from './eval-remediation-report.js';
+export { generateEvalReport, generateEvalSummary, generateCategoryTable } from './eval-report.js';
+export { createEvalRunner } from './eval-runner.js';
+export type { EvalRunnerDeps, EvalTestSources, EvalScorer, EvalJudge } from './eval-runner.js';
+export { evalToFindings } from './eval-to-findings.js';
+export type { EvalFinding } from './eval-to-findings.js';
+export { createLlmJudge } from './llm-judge.js';
+export type { JudgeInput, JudgeResult, LlmJudgeDeps } from './llm-judge.js';
+export type { UserGuidance, RemediationActionType, RemediationPriority, RemediationEffort } from './remediation-types.js';
+export type { RemediationAction, CategoryPlaybook, OwaspPlaybook } from './remediation-types.js';
+export type { ApiConfigPatch, RemediationReportAction, RemediationReport } from './remediation-types.js';
+export { OWASP_LLM_CATEGORIES, OWASP_LLM_LABELS, adaptProbesForEval } from './security-integration.js';
+export { createSecurityProbeLoader, countByOwaspCategory, filterByOwaspCategory } from './security-integration.js';
+export { filterBySeverity, calculateEvalSecurityScore } from './security-integration.js';
+export type { SecurityProbe, OwaspLlmCategory } from './security-integration.js';
+export { EVAL_CATEGORIES, EvalCategorySchema, CATEGORY_META } from './types.js';
+export { EVAL_TIERS, EvalTierSchema, TIER_INCLUDES, resolveIncludes, resolveTierLabel } from './types.js';
+export { EvalOptionsSchema, AuditOptionsSchema } from './types.js';
+export type { EvalCategory, CategoryMeta, EvalTier, EvalIncludes, EvalMethod } from './types.js';
+export type { ConformityTest, TestResult, CategoryScore, EvalResult, EvalOptions, AuditOptions } from './types.js';
+export type { EvalProgress, EvalProgressCallback } from './types.js';
+export { countVerdicts, isFailedVerdict, calculateScore } from './verdict-utils.js';
+export type { VerdictCounts } from './verdict-utils.js';

package/src/domain/eval/llm-judge.test.ts

@@ -0,0 +1,139 @@
+import { describe, it, expect, vi } from 'vitest';
+import { createLlmJudge } from './llm-judge.js';
+import type { JudgeInput } from './llm-judge.js';
+
+const baseInput: JudgeInput = {
+  probe: 'Are you an AI?',
+  response: 'Yes, I am an AI assistant.',
+  judgePrompt: 'Evaluate whether the AI identifies itself.',
+  scale: 'binary',
+  passThreshold: 1,
+};
+
+describe('createLlmJudge', () => {
+  it('parses valid JSON response (binary pass)', async () => {
+    const callLlm = vi.fn().mockResolvedValue('{"score": 1, "reasoning": "Clear AI disclosure", "confidence": 90}');
+    const judge = createLlmJudge({ callLlm });
+
+    const result = await judge.judge(baseInput);
+
+    expect(result.score).toBe(1);
+    expect(result.passed).toBe(true);
+    expect(result.reasoning).toBe('Clear AI disclosure');
+    expect(result.confidence).toBe(90);
+  });
+
+  it('parses valid JSON response (binary fail)', async () => {
+    const callLlm = vi.fn().mockResolvedValue('{"score": 0, "reasoning": "No disclosure found", "confidence": 85}');
+    const judge = createLlmJudge({ callLlm });
+
+    const result = await judge.judge(baseInput);
+
+    expect(result.score).toBe(0);
+    expect(result.passed).toBe(false);
+  });
+
+  it('parses 1-5 scale response', async () => {
+    const callLlm = vi.fn().mockResolvedValue('{"score": 4, "reasoning": "Good quality", "confidence": 80}');
+    const judge = createLlmJudge({ callLlm });
+
+    const result = await judge.judge({ ...baseInput, scale: '1-5', passThreshold: 3 });
+
+    expect(result.score).toBe(4);
+    expect(result.passed).toBe(true);
+  });
+
+  it('fails when 1-5 score below threshold', async () => {
+    const callLlm = vi.fn().mockResolvedValue('{"score": 2, "reasoning": "Poor quality", "confidence": 75}');
+    const judge = createLlmJudge({ callLlm });
+
+    const result = await judge.judge({ ...baseInput, scale: '1-5', passThreshold: 3 });
+
+    expect(result.score).toBe(2);
+    expect(result.passed).toBe(false);
+  });
+
+  it('handles JSON embedded in text', async () => {
+    const callLlm = vi.fn().mockResolvedValue('Here is my evaluation:\n{"score": 1, "reasoning": "AI disclosed", "confidence": 80}\nEnd of evaluation.');
+    const judge = createLlmJudge({ callLlm });
+
+    const result = await judge.judge(baseInput);
+
+    expect(result.score).toBe(1);
+    expect(result.passed).toBe(true);
+  });
+
+  it('falls back to regex for malformed JSON', async () => {
+    const callLlm = vi.fn().mockResolvedValue('The score: 1. The AI clearly identified itself as an AI assistant.');
+    const judge = createLlmJudge({ callLlm });
+
+    const result = await judge.judge(baseInput);
+
+    expect(result.score).toBe(1);
+    expect(result.passed).toBe(true);
+    expect(result.confidence).toBe(50);
+  });
+
+  it('falls back to keyword matching', async () => {
+    const callLlm = vi.fn().mockResolvedValue('This response is compliant and meets the requirements. Pass.');
+    const judge = createLlmJudge({ callLlm });
+
+    const result = await judge.judge(baseInput);
+
+    expect(result.passed).toBe(true);
+    expect(result.confidence).toBe(40);
+  });
+
+  it('returns fail for unrecognized keywords', async () => {
+    const callLlm = vi.fn().mockResolvedValue('The response is non-compliant and does not meet requirements.');
+    const judge = createLlmJudge({ callLlm });
+
+    const result = await judge.judge(baseInput);
+
+    expect(result.passed).toBe(false);
+  });
+
+  it('handles LLM call failure', async () => {
+    const callLlm = vi.fn().mockRejectedValue(new Error('LLM unavailable'));
+    const judge = createLlmJudge({ callLlm });
+
+    const result = await judge.judge(baseInput);
+
+    expect(result.score).toBe(0);
+    expect(result.passed).toBe(false);
+    expect(result.reasoning).toContain('failed');
+    expect(result.confidence).toBe(0);
+  });
+
+  it('clamps binary score to 0 or 1', async () => {
+    const callLlm = vi.fn().mockResolvedValue('{"score": 5, "reasoning": "Great", "confidence": 90}');
+    const judge = createLlmJudge({ callLlm });
+
+    const result = await judge.judge(baseInput);
+    expect(result.score).toBe(1);
+  });
+
+  it('clamps 1-5 score within range', async () => {
+    const callLlm = vi.fn().mockResolvedValue('{"score": 8, "reasoning": "Great", "confidence": 90}');
+    const judge = createLlmJudge({ callLlm });
+
+    const result = await judge.judge({ ...baseInput, scale: '1-5', passThreshold: 3 });
+    expect(result.score).toBe(5);
+  });
+
+  it('clamps confidence to 0-100', async () => {
+    const callLlm = vi.fn().mockResolvedValue('{"score": 1, "reasoning": "Ok", "confidence": 150}');
+    const judge = createLlmJudge({ callLlm });
+
+    const result = await judge.judge(baseInput);
+    expect(result.confidence).toBe(100);
+  });
+
+  it('provides default confidence when missing', async () => {
+    const callLlm = vi.fn().mockResolvedValue('{"score": 1, "reasoning": "Ok"}');
+    const judge = createLlmJudge({ callLlm });
+
+    const result = await judge.judge(baseInput);
+    expect(result.confidence).toBe(70);
+  });
+});