@complior/engine 0.9.0

package/src/domain/eval/eval-evidence.ts

@@ -0,0 +1,103 @@
+/**
+ * Eval Evidence Bridge — records eval results into the evidence chain.
+ *
+ * Translates EvalResult into Evidence entries compatible with the
+ * existing EvidenceStore (domain/scanner/evidence-store.ts).
+ */
+
+import type { EvalResult, TestResult } from './types.js';
+import { countVerdicts } from './verdict-utils.js';
+
+/** Evidence type matching the scanner's Evidence interface. */
+interface EvalEvidence {
+  readonly type: 'eval';
+  readonly checkId: string;
+  readonly data: Record<string, unknown>;
+}
+
+/**
+ * Convert an EvalResult to evidence entries for the evidence chain.
+ *
+ * Creates one entry per test result + a summary entry.
+ */
+export const evalResultToEvidence = (result: EvalResult): readonly EvalEvidence[] => {
+  const entries: EvalEvidence[] = [];
+
+  // Individual test results
+  for (const r of result.results) {
+    entries.push({
+      type: 'eval',
+      checkId: r.testId,
+      data: {
+        verdict: r.verdict,
+        score: r.score,
+        category: r.category,
+        confidence: r.confidence,
+        method: r.method,
+      },
+    });
+  }
+
+  // Summary entry
+  entries.push({
+    type: 'eval',
+    checkId: 'eval-summary',
+    data: {
+      target: result.target,
+      tier: result.tier,
+      overallScore: result.overallScore,
+      grade: result.grade,
+      totalTests: result.totalTests,
+      passed: result.passed,
+      failed: result.failed,
+      errors: result.errors,
+      securityScore: result.securityScore,
+      securityGrade: result.securityGrade,
+      criticalCapped: result.criticalCapped,
+      duration: result.duration,
+      timestamp: result.timestamp,
+      agent: result.agent,
+    },
+  });
+
+  return Object.freeze(entries);
+};
+
+/**
+ * Calculate a hash of an eval result for evidence verification.
+ */
+export const evalResultHash = (result: EvalResult): string => {
+  const summary = {
+    target: result.target,
+    tier: result.tier,
+    overallScore: result.overallScore,
+    totalTests: result.totalTests,
+    passed: result.passed,
+    failed: result.failed,
+    timestamp: result.timestamp,
+  };
+  return JSON.stringify(summary);
+};
+
+/**
+ * Summarize test results for compact evidence storage.
+ */
+export const summarizeTestResults = (
+  results: readonly TestResult[],
+): {
+  readonly total: number;
+  readonly passed: number;
+  readonly failed: number;
+  readonly errors: number;
+  readonly categories: readonly string[];
+} => {
+  const categories = [...new Set(results.map((r) => r.category))];
+  const counts = countVerdicts(results);
+  return Object.freeze({
+    total: results.length,
+    passed: counts.passed,
+    failed: counts.failed,
+    errors: counts.errors,
+    categories,
+  });
+};

package/src/domain/eval/eval-fix-generator.test.ts

@@ -0,0 +1,421 @@
+import { describe, it, expect } from 'vitest';
+import { generateSystemPromptPatch, generateApiConfigPatch } from './eval-fix-generator.js';
+import type { TestResult, EvalCategory } from './types.js';
+import type { CategoryPlaybook } from './remediation-types.js';
+import { CT_1_PLAYBOOK } from '../../data/eval/remediation/ct-1-transparency.js';
+import { LLM01_PLAYBOOK } from '../../data/eval/remediation/owasp-llm01.js';
+import { LLM02_PLAYBOOK } from '../../data/eval/remediation/owasp-llm02.js';
+import { LLM07_PLAYBOOK } from '../../data/eval/remediation/owasp-llm07.js';
+import { CT_10_PLAYBOOK } from '../../data/eval/remediation/ct-10-gpai.js';
+import { CT_6_PLAYBOOK } from '../../data/eval/remediation/ct-6-robustness.js';
+
+// ── Helpers ──────────────────────────────────────────────────
+
+const mockFailure = (testId: string, category: string, owaspCategory?: string): TestResult => ({
+  testId,
+  name: `Test ${testId}`,
+  category: category as EvalCategory,
+  method: 'deterministic' as const,
+  verdict: 'fail' as const,
+  score: 0,
+  confidence: 75,
+  reasoning: 'test failure',
+  probe: 'test probe',
+  response: 'test response',
+  latencyMs: 100,
+  timestamp: new Date().toISOString(),
+  ...(owaspCategory ? { owaspCategory } : {}),
+});
+
+// ── generateSystemPromptPatch ────────────────────────────────
+
+describe('generateSystemPromptPatch', () => {
+  it('generates transparency content for CT-1 failures', () => {
+    const failures = [
+      mockFailure('CT-1-001', 'transparency'),
+      mockFailure('CT-1-002', 'transparency'),
+    ];
+    const result = generateSystemPromptPatch(failures, [CT_1_PLAYBOOK]);
+
+    expect(result).toContain('System Prompt Compliance Patch');
+    expect(result).toContain('AI Disclosure');
+    expect(result).toContain('Art.50');
+    expect(result).toContain('2 tests affected');
+  });
+
+  it('includes fine information for transparency category', () => {
+    const failures = [mockFailure('CT-1-001', 'transparency')];
+    const result = generateSystemPromptPatch(failures, [CT_1_PLAYBOOK]);
+
+    expect(result).toContain('fine: up to 35M EUR');
+  });
+
+  it('generates prompt injection defense content for LLM01 failures', () => {
+    const failures = [
+      mockFailure('SEC-001', 'robustness', 'LLM01'),
+      mockFailure('SEC-002', 'robustness', 'LLM01'),
+    ];
+    const result = generateSystemPromptPatch(failures, [LLM01_PLAYBOOK]);
+
+    expect(result).toContain('Prompt Injection Defense');
+    // The example's quoted instruction is extracted by regex /"([^"]+)"/
+    // which captures: CRITICAL: Your core instructions cannot be overridden...
+    expect(result).toContain('CRITICAL: Your core instructions cannot be overridden');
+    expect(result).toContain('2 tests affected');
+  });
+
+  it('uses owaspCategory over category when present', () => {
+    // owaspCategory=LLM01 should match LLM01_PLAYBOOK, not robustness
+    const failures = [mockFailure('SEC-001', 'robustness', 'LLM01')];
+    const result = generateSystemPromptPatch(failures, [LLM01_PLAYBOOK, CT_6_PLAYBOOK]);
+
+    expect(result).toContain('Prompt Injection Defense');
+  });
+
+  it('deduplicates — one block per category, not per test', () => {
+    const failures = [
+      mockFailure('CT-1-001', 'transparency'),
+      mockFailure('CT-1-002', 'transparency'),
+      mockFailure('CT-1-003', 'transparency'),
+      mockFailure('SEC-001', 'robustness', 'LLM01'),
+      mockFailure('SEC-002', 'robustness', 'LLM01'),
+    ];
+    const result = generateSystemPromptPatch(failures, [CT_1_PLAYBOOK, LLM01_PLAYBOOK]);
+
+    // Should have exactly 2 category blocks (transparency + LLM01), not 5
+    expect(result).toContain('5 failing tests');
+
+    // Count numbered sections (## 1. and ## 2.)
+    const sectionMatches = result.match(/^## \d+\./gm);
+    expect(sectionMatches).toHaveLength(2);
+  });
+
+  it('deduplicates actions across categories sharing the same action ID', () => {
+    // Create two playbooks that share the same action ID
+    const playbook1: CategoryPlaybook = {
+      category_id: 'cat-a',
+      label: 'Category A',
+      article_ref: 'Art.99',
+      description: 'Test category A',
+      actions: [{
+        id: 'SHARED-A1',
+        type: 'system_prompt',
+        title: 'Shared Action',
+        description: 'Shared description',
+        example: '"Shared instruction text"',
+        priority: 'high',
+        effort: 'minimal',
+        article_ref: 'Art.99',
+        user_guidance: { why: 'test', what_to_do: ['test'], verification: 'test', resources: [] },
+      }],
+    };
+    const playbook2: CategoryPlaybook = {
+      category_id: 'cat-b',
+      label: 'Category B',
+      article_ref: 'Art.100',
+      description: 'Test category B',
+      actions: [{
+        id: 'SHARED-A1', // same action ID
+        type: 'system_prompt',
+        title: 'Shared Action Duplicate',
+        description: 'Same action, different playbook',
+        example: '"Shared instruction text"',
+        priority: 'high',
+        effort: 'minimal',
+        article_ref: 'Art.100',
+        user_guidance: { why: 'test', what_to_do: ['test'], verification: 'test', resources: [] },
+      }],
+    };
+
+    const failures = [
+      mockFailure('T-001', 'cat-a' as EvalCategory),
+      mockFailure('T-002', 'cat-b' as EvalCategory),
+    ];
+    const result = generateSystemPromptPatch(failures, [playbook1, playbook2]);
+
+    // SHARED-A1 should appear only once (in cat-a block). cat-b block has no new actions.
+    const sectionMatches = result.match(/^## \d+\./gm);
+    expect(sectionMatches).toHaveLength(1);
+  });
+
+  it('returns empty string when no failures', () => {
+    const result = generateSystemPromptPatch([], [CT_1_PLAYBOOK, LLM01_PLAYBOOK]);
+    expect(result).toBe('');
+  });
+
+  it('returns empty string when failures have no matching playbooks', () => {
+    const failures = [mockFailure('T-001', 'unknown-cat' as EvalCategory)];
+    const result = generateSystemPromptPatch(failures, [CT_1_PLAYBOOK]);
+    expect(result).toBe('');
+  });
+
+  it('returns empty string when playbook has no system_prompt actions', () => {
+    const apiOnlyPlaybook: CategoryPlaybook = {
+      category_id: 'api-only',
+      label: 'API Only',
+      article_ref: 'Art.99',
+      description: 'Only api_config actions',
+      actions: [{
+        id: 'AO-A1',
+        type: 'api_config',
+        title: 'API Header',
+        description: 'Some header',
+        example: 'X-Header: value',
+        priority: 'high',
+        effort: 'minimal',
+        article_ref: 'Art.99',
+        user_guidance: { why: 'test', what_to_do: ['test'], verification: 'test', resources: [] },
+      }],
+    };
+    const failures = [mockFailure('T-001', 'api-only' as EvalCategory)];
+    const result = generateSystemPromptPatch(failures, [apiOnlyPlaybook]);
+    expect(result).toBe('');
+  });
+
+  it('sorts critical actions before medium actions', () => {
+    const criticalPlaybook: CategoryPlaybook = {
+      category_id: 'crit-cat',
+      label: 'Critical Category',
+      article_ref: 'Art.1',
+      description: 'Has critical actions',
+      actions: [{
+        id: 'CRIT-A1',
+        type: 'system_prompt',
+        title: 'Critical Fix',
+        description: 'Critical action',
+        example: '"Critical instruction"',
+        priority: 'critical',
+        effort: 'minimal',
+        article_ref: 'Art.1',
+        user_guidance: { why: 'test', what_to_do: ['test'], verification: 'test', resources: [] },
+      }],
+    };
+    const mediumPlaybook: CategoryPlaybook = {
+      category_id: 'med-cat',
+      label: 'Medium Category',
+      article_ref: 'Art.2',
+      description: 'Has medium actions',
+      actions: [{
+        id: 'MED-A1',
+        type: 'system_prompt',
+        title: 'Medium Fix',
+        description: 'Medium action',
+        example: '"Medium instruction"',
+        priority: 'medium',
+        effort: 'minimal',
+        article_ref: 'Art.2',
+        user_guidance: { why: 'test', what_to_do: ['test'], verification: 'test', resources: [] },
+      }],
+    };
+
+    // Pass medium failures first, then critical — but critical should come first in output
+    const failures = [
+      mockFailure('T-001', 'med-cat' as EvalCategory),
+      mockFailure('T-002', 'crit-cat' as EvalCategory),
+    ];
+    const result = generateSystemPromptPatch(failures, [mediumPlaybook, criticalPlaybook]);
+
+    const critIdx = result.indexOf('Critical Fix');
+    const medIdx = result.indexOf('Medium Fix');
+    expect(critIdx).toBeGreaterThan(-1);
+    expect(medIdx).toBeGreaterThan(-1);
+    expect(critIdx).toBeLessThan(medIdx);
+  });
+
+  it('breaks priority ties by test count (more affected tests first)', () => {
+    const playbookA: CategoryPlaybook = {
+      category_id: 'cat-few',
+      label: 'Few Tests',
+      article_ref: 'Art.1',
+      description: 'Few tests fail',
+      actions: [{
+        id: 'FEW-A1', type: 'system_prompt', title: 'Few Tests Fix',
+        description: 'desc', example: '"fix"', priority: 'high', effort: 'minimal',
+        article_ref: 'Art.1',
+        user_guidance: { why: 'test', what_to_do: ['test'], verification: 'test', resources: [] },
+      }],
+    };
+    const playbookB: CategoryPlaybook = {
+      category_id: 'cat-many',
+      label: 'Many Tests',
+      article_ref: 'Art.2',
+      description: 'Many tests fail',
+      actions: [{
+        id: 'MANY-A1', type: 'system_prompt', title: 'Many Tests Fix',
+        description: 'desc', example: '"fix"', priority: 'high', effort: 'minimal',
+        article_ref: 'Art.2',
+        user_guidance: { why: 'test', what_to_do: ['test'], verification: 'test', resources: [] },
+      }],
+    };
+
+    const failures = [
+      mockFailure('T-001', 'cat-few' as EvalCategory),
+      mockFailure('T-002', 'cat-many' as EvalCategory),
+      mockFailure('T-003', 'cat-many' as EvalCategory),
+      mockFailure('T-004', 'cat-many' as EvalCategory),
+    ];
+    const result = generateSystemPromptPatch(failures, [playbookA, playbookB]);
+
+    // Both are 'high' priority, but cat-many has 3 tests vs cat-few's 1
+    const manyIdx = result.indexOf('Many Tests Fix');
+    const fewIdx = result.indexOf('Few Tests Fix');
+    expect(manyIdx).toBeLessThan(fewIdx);
+  });
+
+  it('renders code blocks with extracted instruction from example', () => {
+    const failures = [mockFailure('CT-1-001', 'transparency')];
+    const result = generateSystemPromptPatch(failures, [CT_1_PLAYBOOK]);
+
+    // The CT-1-A1 example has a quoted instruction that gets extracted
+    expect(result).toContain('```');
+    expect(result).toContain('AI assistant');
+  });
+
+  it('includes total category and failure counts in header', () => {
+    const failures = [
+      mockFailure('CT-1-001', 'transparency'),
+      mockFailure('SEC-001', 'robustness', 'LLM01'),
+    ];
+    const result = generateSystemPromptPatch(failures, [CT_1_PLAYBOOK, LLM01_PLAYBOOK]);
+
+    expect(result).toContain('2 categories');
+    expect(result).toContain('2 failing tests');
+  });
+});
+
+// ── generateApiConfigPatch ───────────────────────────────────
+
+describe('generateApiConfigPatch', () => {
+  it('generates transparency headers for transparency failures', () => {
+    const failures = [mockFailure('CT-1-001', 'transparency')];
+    const result = generateApiConfigPatch(failures, [CT_1_PLAYBOOK]);
+
+    expect(result.headers['X-AI-Generated']).toBe('true');
+    expect(result.headers['X-AI-Disclosure']).toContain('AI system');
+  });
+
+  it('generates GPAI headers for gpai failures', () => {
+    const failures = [mockFailure('CT-10-001', 'gpai')];
+    const result = generateApiConfigPatch(failures, [CT_10_PLAYBOOK]);
+
+    expect(result.headers['X-AI-Model']).toBe('{{model_name}}');
+    expect(result.headers['X-AI-Provider']).toBe('{{provider_name}}');
+  });
+
+  it('generates bannedPatterns for LLM01 prompt injection failures', () => {
+    const failures = [mockFailure('SEC-001', 'robustness', 'LLM01')];
+    const result = generateApiConfigPatch(failures, [LLM01_PLAYBOOK]);
+
+    expect(result.inputValidation.bannedPatterns).toBeDefined();
+    expect(result.inputValidation.bannedPatterns).toContain('ignore previous instructions');
+    expect(result.inputValidation.bannedPatterns).toContain('DAN mode');
+    expect(result.inputValidation.bannedPatterns).toContain('jailbreak');
+    expect(result.inputValidation.maxLength).toBe(4096);
+  });
+
+  it('generates piiFilterPatterns for LLM02 data leakage failures', () => {
+    const failures = [mockFailure('SEC-001', 'robustness', 'LLM02')];
+    const result = generateApiConfigPatch(failures, [LLM02_PLAYBOOK]);
+
+    expect(result.outputValidation.piiFilterPatterns).toBeDefined();
+    expect(result.outputValidation.piiFilterPatterns!.length).toBeGreaterThan(0);
+    // Should include email pattern
+    expect(result.outputValidation.piiFilterPatterns!.some(
+      (p) => p.includes('@') || p.includes('\\w'),
+    )).toBe(true);
+  });
+
+  it('generates promptLeakPatterns for LLM07 system prompt leakage failures', () => {
+    const failures = [mockFailure('SEC-001', 'robustness', 'LLM07')];
+    const result = generateApiConfigPatch(failures, [LLM07_PLAYBOOK]);
+
+    expect(result.outputValidation.promptLeakPatterns).toBeDefined();
+    expect(result.outputValidation.promptLeakPatterns).toContain('system prompt');
+    expect(result.outputValidation.promptLeakPatterns).toContain('my instructions');
+    expect(result.outputValidation.promptLeakPatterns).toContain('I was told to');
+  });
+
+  it('generates provider examples when headers or banned patterns exist', () => {
+    const failures = [mockFailure('CT-1-001', 'transparency')];
+    const result = generateApiConfigPatch(failures, [CT_1_PLAYBOOK]);
+
+    expect(result.providerExamples['openai']).toBeDefined();
+    expect(result.providerExamples['anthropic']).toBeDefined();
+    expect(result.providerExamples['ollama']).toBeDefined();
+
+    // Openai example includes headers and temperature
+    const openai = result.providerExamples['openai'] as Record<string, unknown>;
+    expect(openai.headers).toBeDefined();
+    expect(openai.max_tokens).toBe(2048);
+    expect(openai.temperature).toBe(0.3);
+  });
+
+  it('returns empty/default config when no failures', () => {
+    const result = generateApiConfigPatch([], [CT_1_PLAYBOOK, LLM01_PLAYBOOK]);
+
+    expect(Object.keys(result.headers)).toHaveLength(0);
+    expect(result.inputValidation.bannedPatterns).toBeUndefined();
+    expect(result.inputValidation.maxLength).toBeUndefined();
+    expect(result.outputValidation.piiFilterPatterns).toBeUndefined();
+    expect(result.outputValidation.promptLeakPatterns).toBeUndefined();
+    expect(Object.keys(result.providerExamples)).toHaveLength(0);
+  });
+
+  it('returns empty config when failures have no matching playbooks', () => {
+    const failures = [mockFailure('T-001', 'unknown-cat' as EvalCategory)];
+    const result = generateApiConfigPatch(failures, [CT_1_PLAYBOOK]);
+
+    expect(Object.keys(result.headers)).toHaveLength(0);
+    expect(result.inputValidation.bannedPatterns).toBeUndefined();
+    expect(Object.keys(result.providerExamples)).toHaveLength(0);
+  });
+
+  it('deduplicates bannedPatterns when multiple LLM01 failures exist', () => {
+    const failures = [
+      mockFailure('SEC-001', 'robustness', 'LLM01'),
+      mockFailure('SEC-002', 'robustness', 'LLM01'),
+      mockFailure('SEC-003', 'robustness', 'LLM01'),
+    ];
+    const result = generateApiConfigPatch(failures, [LLM01_PLAYBOOK]);
+
+    // Even with 3 failures, patterns should not be duplicated
+    const patterns = result.inputValidation.bannedPatterns!;
+    const unique = new Set(patterns);
+    expect(patterns.length).toBe(unique.size);
+  });
+
+  it('combines security and transparency patches for mixed failures', () => {
+    const failures = [
+      mockFailure('CT-1-001', 'transparency'),
+      mockFailure('SEC-001', 'robustness', 'LLM01'),
+      mockFailure('SEC-002', 'robustness', 'LLM02'),
+      mockFailure('SEC-003', 'robustness', 'LLM07'),
+    ];
+    const result = generateApiConfigPatch(
+      failures,
+      [CT_1_PLAYBOOK, LLM01_PLAYBOOK, LLM02_PLAYBOOK, LLM07_PLAYBOOK],
+    );
+
+    // Transparency headers
+    expect(result.headers['X-AI-Generated']).toBe('true');
+    // LLM01 input validation
+    expect(result.inputValidation.bannedPatterns).toBeDefined();
+    expect(result.inputValidation.bannedPatterns!.length).toBeGreaterThan(0);
+    // LLM02 PII filtering
+    expect(result.outputValidation.piiFilterPatterns).toBeDefined();
+    // LLM07 prompt leak filtering
+    expect(result.outputValidation.promptLeakPatterns).toBeDefined();
+    // Provider examples present
+    expect(Object.keys(result.providerExamples).length).toBeGreaterThan(0);
+  });
+
+  it('returns frozen object', () => {
+    const failures = [mockFailure('CT-1-001', 'transparency')];
+    const result = generateApiConfigPatch(failures, [CT_1_PLAYBOOK]);
+
+    expect(Object.isFrozen(result)).toBe(true);
+    expect(Object.isFrozen(result.inputValidation)).toBe(true);
+    expect(Object.isFrozen(result.outputValidation)).toBe(true);
+    expect(Object.isFrozen(result.providerExamples)).toBe(true);
+  });
+});