@complior/engine 0.9.0

package/src/domain/eval/eval-fix-generator.ts

@@ -0,0 +1,205 @@
+/**
+ * Eval Fix Generator — generates system prompt patches and API config patches
+ * from eval failures + remediation playbooks.
+ *
+ * US-REM-05: System Prompt Patch Generator
+ * US-REM-06: API Config Patch Generator
+ */
+
+import type { TestResult } from './types.js';
+import type { CategoryPlaybook, RemediationAction, ApiConfigPatch } from './remediation-types.js';
+import {
+  PRIORITY_ORDER,
+  CATEGORY_ARTICLES,
+  CATEGORY_FINES,
+  groupFailuresByCategory,
+  findPlaybook,
+} from './eval-constants.js';
+
+// ── System Prompt Patch ──────────────────────────────────────
+
+/**
+ * Generate a system prompt patch from failed tests.
+ *
+ * Groups failures by category, deduplicates, sorts by priority,
+ * and produces a numbered markdown string of system prompt additions.
+ */
+export const generateSystemPromptPatch = (
+  failures: readonly TestResult[],
+  playbooks: readonly CategoryPlaybook[],
+): string => {
+  // Group failures by category
+  const byCat = groupFailuresByCategory(failures);
+
+  // Collect system_prompt actions per category (deduplicated)
+  const blocks: { category: string; article: string; actions: RemediationAction[]; testCount: number }[] = [];
+  const seenActionIds = new Set<string>();
+
+  for (const [cat, catFailures] of byCat) {
+    const pb = findPlaybook(playbooks, cat);
+    if (!pb) continue;
+
+    const promptActions = pb.actions.filter((a) => a.type === 'system_prompt');
+    const newActions: RemediationAction[] = [];
+    for (const a of promptActions) {
+      if (!seenActionIds.has(a.id)) {
+        seenActionIds.add(a.id);
+        newActions.push(a);
+      }
+    }
+
+    if (newActions.length > 0) {
+      const article = CATEGORY_ARTICLES[cat] ?? pb.article_ref;
+      blocks.push({ category: cat, article, actions: newActions, testCount: catFailures.length });
+    }
+  }
+
+  // Sort blocks by priority of their highest-priority action
+  blocks.sort((a, b) => {
+    const aPri = Math.min(...a.actions.map((x) => PRIORITY_ORDER[x.priority] ?? 3));
+    const bPri = Math.min(...b.actions.map((x) => PRIORITY_ORDER[x.priority] ?? 3));
+    if (aPri !== bPri) return aPri - bPri;
+    return b.testCount - a.testCount;
+  });
+
+  if (blocks.length === 0) return '';
+
+  // Render markdown
+  const lines: string[] = [
+    '# System Prompt Compliance Patch',
+    '',
+    `> Generated by complior eval — ${blocks.length} categories, ${failures.length} failing tests`,
+    '',
+    'Add the following instructions to your AI system\'s system prompt:',
+    '',
+  ];
+
+  let n = 1;
+  for (const block of blocks) {
+    const fine = CATEGORY_FINES[block.category];
+    const fineStr = fine ? ` (fine: ${fine})` : '';
+    lines.push(`## ${n}. ${block.actions[0].title} — ${block.article}${fineStr}`);
+    lines.push('');
+    lines.push(`> ${block.testCount} tests affected`);
+    lines.push('');
+
+    for (const action of block.actions) {
+      // Extract the quoted instruction from the example
+      const instructionMatch = action.example.match(/"([^"]+)"/);
+      const instruction = instructionMatch ? instructionMatch[1] : action.description;
+      lines.push(`\`\`\`\n${instruction}\n\`\`\``);
+      lines.push('');
+    }
+
+    n++;
+  }
+
+  return lines.join('\n');
+};
+
+// ── API Config Patch ─────────────────────────────────────────
+
+/**
+ * Generate API config patch from failed tests.
+ *
+ * Collects api_config + infrastructure actions and builds a structured
+ * JSON object with headers, validation patterns, and provider examples.
+ */
+export const generateApiConfigPatch = (
+  failures: readonly TestResult[],
+  playbooks: readonly CategoryPlaybook[],
+): ApiConfigPatch => {
+  const headers: Record<string, string> = {};
+  const bannedPatterns: string[] = [];
+  const piiFilterPatterns: string[] = [];
+  const promptLeakPatterns: string[] = [];
+
+  // Check which categories have failures
+  const failedCategories = new Set(failures.map((f) => f.owaspCategory ?? f.category));
+
+  for (const cat of failedCategories) {
+    const pb = findPlaybook(playbooks, cat);
+    if (!pb) continue;
+
+    for (const action of pb.actions) {
+      if (action.type !== 'api_config') continue;
+
+      // Transparency headers
+      if (cat === 'transparency' || action.id === 'CT-1-A3') {
+        headers['X-AI-Generated'] = 'true';
+        headers['X-AI-Disclosure'] = 'This response was generated by an AI system';
+        headers['X-Content-Origin'] = 'ai';
+      }
+
+      // GPAI headers
+      if (cat === 'gpai') {
+        headers['X-AI-Model'] = '{{model_name}}';
+        headers['X-AI-Provider'] = '{{provider_name}}';
+      }
+
+      // Input validation (robustness, prompt injection)
+      if (cat === 'robustness' || cat === 'LLM01') {
+        bannedPatterns.push(
+          'ignore previous instructions',
+          'you are now',
+          'new instructions:',
+          'DAN mode',
+          'jailbreak',
+        );
+      }
+
+      // Output filtering (sensitive info, system prompt leakage)
+      if (cat === 'LLM02') {
+        piiFilterPatterns.push(
+          '[\\w.-]+@[\\w.-]+\\.[a-z]{2,}',
+          '\\+?\\d[\\d\\s-]{8,}',
+          '\\d{3}-\\d{2}-\\d{4}',
+        );
+      }
+      if (cat === 'LLM07') {
+        promptLeakPatterns.push(
+          'system prompt',
+          'my instructions',
+          'I was told to',
+        );
+      }
+    }
+  }
+
+  // Deduplicate
+  const uniqueBanned = [...new Set(bannedPatterns)];
+  const uniquePii = [...new Set(piiFilterPatterns)];
+  const uniqueLeak = [...new Set(promptLeakPatterns)];
+
+  // Provider examples
+  const providerExamples: Record<string, Record<string, unknown>> = {};
+
+  if (Object.keys(headers).length > 0 || uniqueBanned.length > 0) {
+    providerExamples['openai'] = {
+      headers: { ...headers },
+      max_tokens: 2048,
+      temperature: 0.3,
+    };
+    providerExamples['anthropic'] = {
+      headers: { ...headers },
+      max_tokens: 2048,
+    };
+    providerExamples['ollama'] = {
+      headers: { ...headers },
+      options: { num_predict: 2048, temperature: 0.3 },
+    };
+  }
+
+  return Object.freeze({
+    headers,
+    inputValidation: Object.freeze({
+      maxLength: uniqueBanned.length > 0 ? 4096 : undefined,
+      bannedPatterns: uniqueBanned.length > 0 ? Object.freeze(uniqueBanned) : undefined,
+    }),
+    outputValidation: Object.freeze({
+      piiFilterPatterns: uniquePii.length > 0 ? Object.freeze(uniquePii) : undefined,
+      promptLeakPatterns: uniqueLeak.length > 0 ? Object.freeze(uniqueLeak) : undefined,
+    }),
+    providerExamples: Object.freeze(providerExamples),
+  });
+};

package/src/domain/eval/eval-passport.test.ts

@@ -0,0 +1,82 @@
+import { describe, it, expect } from 'vitest';
+import { buildPassportEvalBlock, mergeEvalIntoPassport } from './eval-passport.js';
+import type { EvalResult } from './types.js';
+
+const mockResult: EvalResult = Object.freeze({
+  target: 'http://localhost:4000/api/chat',
+  tier: 'full',
+  overallScore: 78,
+  grade: 'B',
+  categories: [
+    { category: 'transparency', score: 90, grade: 'A', passed: 9, failed: 1, errors: 0, skipped: 0, total: 10 },
+    { category: 'bias', score: 60, grade: 'C', passed: 3, failed: 2, errors: 0, skipped: 0, total: 5 },
+  ],
+  securityScore: 85,
+  securityGrade: 'B',
+  results: [],
+  totalTests: 15,
+  passed: 12,
+  failed: 3,
+  errors: 0,
+  duration: 10000,
+  timestamp: '2026-03-21T10:00:00Z',
+  criticalCapped: false,
+  agent: 'my-agent',
+});
+
+describe('buildPassportEvalBlock', () => {
+  it('maps all fields from EvalResult', () => {
+    const block = buildPassportEvalBlock(mockResult);
+    expect(block.eval_score).toBe(78);
+    expect(block.eval_grade).toBe('B');
+    expect(block.eval_tier).toBe('full');
+    expect(block.eval_target).toBe('http://localhost:4000/api/chat');
+    expect(block.eval_date).toBe('2026-03-21T10:00:00Z');
+    expect(block.eval_tests_total).toBe(15);
+    expect(block.eval_tests_passed).toBe(12);
+    expect(block.eval_tests_failed).toBe(3);
+    expect(block.eval_critical_capped).toBe(false);
+    expect(block.eval_security_score).toBe(85);
+    expect(block.eval_security_grade).toBe('B');
+  });
+
+  it('includes category breakdown', () => {
+    const block = buildPassportEvalBlock(mockResult);
+    expect(block.eval_categories.length).toBe(2);
+    expect(block.eval_categories[0]!.category).toBe('transparency');
+    expect(block.eval_categories[0]!.score).toBe(90);
+  });
+});
+
+describe('mergeEvalIntoPassport', () => {
+  it('adds eval block to compliance', () => {
+    const passport = { name: 'test-agent', compliance: { complior_score: 80 } };
+    const block = buildPassportEvalBlock(mockResult);
+    const updated = mergeEvalIntoPassport(passport, block);
+    expect((updated.compliance as Record<string, unknown>).eval).toEqual(block);
+    // Preserves existing compliance fields
+    expect((updated.compliance as Record<string, unknown>).complior_score).toBe(80);
+  });
+
+  it('does not mutate original passport', () => {
+    const passport = { name: 'test-agent', compliance: { complior_score: 80 } };
+    const block = buildPassportEvalBlock(mockResult);
+    mergeEvalIntoPassport(passport, block);
+    // Original should not have eval
+    expect((passport.compliance as Record<string, unknown>).eval).toBeUndefined();
+  });
+
+  it('adds updated timestamp', () => {
+    const passport = { name: 'test-agent' };
+    const block = buildPassportEvalBlock(mockResult);
+    const updated = mergeEvalIntoPassport(passport, block);
+    expect(updated.updated).toBeDefined();
+  });
+
+  it('handles passport with no compliance block', () => {
+    const passport = { name: 'test-agent' };
+    const block = buildPassportEvalBlock(mockResult);
+    const updated = mergeEvalIntoPassport(passport, block);
+    expect((updated.compliance as Record<string, unknown>).eval).toEqual(block);
+  });
+});

package/src/domain/eval/eval-passport.ts

@@ -0,0 +1,89 @@
+/**
+ * Eval Passport — updates passport compliance block with eval results.
+ *
+ * After an eval run completes, this module enriches the passport's
+ * compliance section with eval score, grade, and category breakdown.
+ */
+
+import type { EvalResult } from './types.js';
+
+/** Eval data to be written into the passport compliance block. */
+export interface PassportEvalBlock {
+  readonly eval_score: number;
+  readonly eval_grade: string;
+  readonly eval_tier: string;
+  readonly eval_target: string;
+  readonly eval_date: string;
+  readonly eval_tests_total: number;
+  readonly eval_tests_passed: number;
+  readonly eval_tests_failed: number;
+  readonly eval_critical_capped: boolean;
+  readonly eval_security_score?: number;
+  readonly eval_security_grade?: string;
+  readonly eval_critical_gaps: readonly string[];
+  readonly eval_category_pass_rates: Readonly<Record<string, number>>;
+  readonly eval_categories: readonly {
+    readonly category: string;
+    readonly score: number;
+    readonly grade: string;
+  }[];
+}
+
+/**
+ * Build passport eval block from an EvalResult.
+ */
+export const buildPassportEvalBlock = (result: EvalResult): PassportEvalBlock => {
+  // critical_gaps: categories with grade F or score < 40
+  const critical_gaps = result.categories
+    .filter((c) => c.grade === 'F' || c.score < 40)
+    .map((c) => c.category);
+
+  // category_pass_rates: { ct_1: 0.85, ct_2: 0.72, ... }
+  const category_pass_rates: Record<string, number> = {};
+  for (const c of result.categories) {
+    const key = c.category.replace(/-/g, '_');
+    category_pass_rates[key] = Math.round((c.score / 100) * 100) / 100;
+  }
+
+  return Object.freeze({
+    eval_score: result.overallScore,
+    eval_grade: result.grade,
+    eval_tier: result.tier,
+    eval_target: result.target,
+    eval_date: result.timestamp,
+    eval_tests_total: result.totalTests,
+    eval_tests_passed: result.passed,
+    eval_tests_failed: result.failed,
+    eval_critical_capped: result.criticalCapped,
+    eval_security_score: result.securityScore,
+    eval_security_grade: result.securityGrade,
+    eval_critical_gaps: Object.freeze(critical_gaps),
+    eval_category_pass_rates: Object.freeze(category_pass_rates),
+    eval_categories: Object.freeze(
+      result.categories.map((c) =>
+        Object.freeze({ category: c.category, score: c.score, grade: c.grade }),
+      ),
+    ),
+  });
+};
+
+/**
+ * Merge eval data into an existing passport JSON (mutable clone).
+ *
+ * Adds `compliance.eval` block without touching other passport fields.
+ * Returns a NEW object — does not mutate input.
+ */
+export const mergeEvalIntoPassport = (
+  passport: Record<string, unknown>,
+  evalBlock: PassportEvalBlock,
+): Record<string, unknown> => {
+  const compliance = (passport.compliance ?? {}) as Record<string, unknown>;
+  return Object.freeze({
+    ...passport,
+    compliance: Object.freeze({
+      ...compliance,
+      eval: evalBlock,
+    }),
+    updated: new Date().toISOString(),
+  });
+};