@complior/engine 0.9.0

package/src/domain/certification/aiuc1-readiness.ts

@@ -0,0 +1,298 @@
+/**
+ * AIUC-1 Readiness Score — pure scorer function.
+ * Evaluates each requirement against scan results, passport data,
+ * generated documents, and evidence chain summary.
+ */
+
+import type { ScanResult, Finding } from '../../types/common.types.js';
+import type { AgentPassport } from '../../types/passport.types.js';
+import type { EvidenceChainSummary } from '../scanner/evidence-store.js';
+import { AIUC1_REQUIREMENTS, AIUC1_CATEGORIES } from './aiuc1-requirements.js';
+import type { Aiuc1Category, Aiuc1Check } from './aiuc1-requirements.js';
+
+// --- Types ---
+
+export interface ReadinessInput {
+  readonly passport: AgentPassport;
+  readonly scanResult: ScanResult | null;
+  readonly documents: ReadonlySet<string>;
+  readonly evidenceSummary: EvidenceChainSummary;
+}
+
+export interface CheckStatus {
+  readonly description: string;
+  readonly passed: boolean;
+  readonly detail?: string;
+}
+
+export interface RequirementStatus {
+  readonly id: string;
+  readonly title: string;
+  readonly category: Aiuc1Category;
+  readonly status: 'met' | 'partial' | 'unmet';
+  readonly weight: number;
+  readonly articleRef: string;
+  readonly checks: readonly CheckStatus[];
+}
+
+export interface CategoryScore {
+  readonly category: Aiuc1Category;
+  readonly label: string;
+  readonly score: number;
+  readonly maxWeight: number;
+  readonly achievedWeight: number;
+}
+
+export interface ReadinessResult {
+  readonly overallScore: number;
+  readonly readinessLevel: 'certified' | 'near_ready' | 'in_progress' | 'early';
+  readonly requirements: readonly RequirementStatus[];
+  readonly categories: readonly CategoryScore[];
+  readonly gaps: readonly string[];
+  readonly totalRequirements: number;
+  readonly metRequirements: number;
+  readonly partialRequirements: number;
+  readonly unmetRequirements: number;
+}
+
+// --- Check evaluators ---
+
+const resolvePassportField = (passport: AgentPassport, path: string): unknown => {
+  const parts = path.split('.');
+  let current: unknown = passport;
+  for (const part of parts) {
+    if (current == null || typeof current !== 'object') return undefined;
+    current = (current as Record<string, unknown>)[part];
+  }
+  return current;
+};
+
+const isNonEmpty = (value: unknown): boolean => {
+  if (value == null) return false;
+  if (typeof value === 'string') return value.trim().length > 0;
+  if (typeof value === 'boolean') return value;
+  if (typeof value === 'number') return true;
+  if (Array.isArray(value)) return value.length > 0;
+  if (typeof value === 'object') return Object.keys(value as Record<string, unknown>).length > 0;
+  return false;
+};
+
+const evaluateScanCheck = (
+  target: string,
+  scanResult: ScanResult | null,
+): CheckStatus => {
+  if (!scanResult) {
+    return { description: '', passed: false, detail: 'No scan result available' };
+  }
+
+  // Special case: 'score' checks if scan was performed with score > 0
+  if (target === 'score') {
+    const passed = scanResult.score.totalScore > 0;
+    return {
+      description: '',
+      passed,
+      detail: passed ? `Score: ${scanResult.score.totalScore}` : 'Scan score is 0',
+    };
+  }
+
+  // Look for findings matching the target checkId (substring match)
+  const relevant = scanResult.findings.filter(
+    (f: Finding) => f.checkId.includes(target),
+  );
+
+  if (relevant.length === 0) {
+    return { description: '', passed: false, detail: `No findings for check '${target}'` };
+  }
+
+  // For 'banned-packages': passes when there are NO fail findings (no banned packages found)
+  if (target === 'banned-packages') {
+    const hasFails = relevant.some((f: Finding) => f.type === 'fail');
+    return {
+      description: '',
+      passed: !hasFails,
+      detail: hasFails ? 'Banned packages detected' : 'No banned packages',
+    };
+  }
+
+  // General scan checks: pass if at least one 'pass' finding exists
+  const hasPass = relevant.some((f: Finding) => f.type === 'pass');
+  return {
+    description: '',
+    passed: hasPass,
+    detail: hasPass ? `Check '${target}' passed` : `Check '${target}' failed`,
+  };
+};
+
+const evaluatePassportField = (
+  target: string,
+  passport: AgentPassport,
+): CheckStatus => {
+  const value = resolvePassportField(passport, target);
+  const passed = isNonEmpty(value);
+  return {
+    description: '',
+    passed,
+    detail: passed ? `Field '${target}' is set` : `Field '${target}' is missing or empty`,
+  };
+};
+
+const evaluateDocument = (
+  target: string,
+  documents: ReadonlySet<string>,
+): CheckStatus => {
+  const passed = documents.has(target);
+  return {
+    description: '',
+    passed,
+    detail: passed ? `Document '${target}' exists` : `Document '${target}' not generated`,
+  };
+};
+
+const evaluateEvidence = (
+  target: string,
+  evidenceSummary: EvidenceChainSummary,
+): CheckStatus => {
+  if (target === 'chain_active') {
+    const passed = evidenceSummary.totalEntries > 0;
+    return {
+      description: '',
+      passed,
+      detail: passed
+        ? `Evidence chain has ${evidenceSummary.totalEntries} entries`
+        : 'Evidence chain is empty',
+    };
+  }
+
+  if (target === 'scan_count') {
+    const passed = evidenceSummary.scanCount >= 2;
+    return {
+      description: '',
+      passed,
+      detail: passed
+        ? `${evidenceSummary.scanCount} scans performed`
+        : `Only ${evidenceSummary.scanCount} scan(s) — need at least 2 for monitoring`,
+    };
+  }
+
+  return { description: '', passed: false, detail: `Unknown evidence target: ${target}` };
+};
+
+const evaluateCheck = (
+  check: Aiuc1Check,
+  input: ReadinessInput,
+): CheckStatus => {
+  let result: CheckStatus;
+  switch (check.type) {
+    case 'scan_check':
+      result = evaluateScanCheck(check.target, input.scanResult);
+      break;
+    case 'passport_field':
+      result = evaluatePassportField(check.target, input.passport);
+      break;
+    case 'document':
+      result = evaluateDocument(check.target, input.documents);
+      break;
+    case 'evidence':
+      result = evaluateEvidence(check.target, input.evidenceSummary);
+      break;
+  }
+  return { ...result, description: check.description };
+};
+
+// --- Main scorer ---
+
+const determineReadinessLevel = (score: number): ReadinessResult['readinessLevel'] => {
+  if (score >= 90) return 'certified';
+  if (score >= 70) return 'near_ready';
+  if (score >= 40) return 'in_progress';
+  return 'early';
+};
+
+export const computeReadiness = (input: ReadinessInput): ReadinessResult => {
+  const requirements: RequirementStatus[] = [];
+  const gaps: string[] = [];
+
+  for (const req of AIUC1_REQUIREMENTS) {
+    const checkStatuses = req.checks.map((check) => evaluateCheck(check, input));
+
+    const passedCount = checkStatuses.filter((c) => c.passed).length;
+    const totalChecks = checkStatuses.length;
+
+    let status: RequirementStatus['status'];
+    if (passedCount === totalChecks) {
+      status = 'met';
+    } else if (passedCount > 0) {
+      status = 'partial';
+    } else {
+      status = 'unmet';
+    }
+
+    if (status !== 'met') {
+      const failedChecks = checkStatuses
+        .filter((c) => !c.passed)
+        .map((c) => c.description)
+        .join(', ');
+      gaps.push(`${req.id} ${req.title} (${req.articleRef}): ${failedChecks}`);
+    }
+
+    requirements.push({
+      id: req.id,
+      title: req.title,
+      category: req.category,
+      status,
+      weight: req.weight,
+      articleRef: req.articleRef,
+      checks: checkStatuses,
+    });
+  }
+
+  // Compute category scores
+  const categoryMap = new Map<Aiuc1Category, { achieved: number; max: number }>();
+  for (const [cat] of Object.entries(AIUC1_CATEGORIES)) {
+    categoryMap.set(cat as Aiuc1Category, { achieved: 0, max: 0 });
+  }
+
+  for (const req of requirements) {
+    const entry = categoryMap.get(req.category)!;
+    entry.max += req.weight;
+    if (req.status === 'met') {
+      entry.achieved += req.weight;
+    } else if (req.status === 'partial') {
+      const passedChecks = req.checks.filter((c) => c.passed).length;
+      entry.achieved += req.weight * (passedChecks / req.checks.length);
+    }
+  }
+
+  const categories: CategoryScore[] = [];
+  for (const [cat, info] of Object.entries(AIUC1_CATEGORIES)) {
+    const entry = categoryMap.get(cat as Aiuc1Category)!;
+    categories.push({
+      category: cat as Aiuc1Category,
+      label: info.label,
+      score: entry.max > 0 ? Math.round((entry.achieved / entry.max) * 100) : 0,
+      maxWeight: entry.max,
+      achievedWeight: Math.round(entry.achieved * 10000) / 10000,
+    });
+  }
+
+  // Overall score: sum of achieved weights / total weights * 100
+  const totalWeight = [...categoryMap.values()].reduce((sum, e) => sum + e.max, 0);
+  const achievedWeight = [...categoryMap.values()].reduce((sum, e) => sum + e.achieved, 0);
+  const overallScore = totalWeight > 0 ? Math.round((achievedWeight / totalWeight) * 100) : 0;
+
+  const metRequirements = requirements.filter((r) => r.status === 'met').length;
+  const partialRequirements = requirements.filter((r) => r.status === 'partial').length;
+  const unmetRequirements = requirements.filter((r) => r.status === 'unmet').length;
+
+  return Object.freeze({
+    overallScore,
+    readinessLevel: determineReadinessLevel(overallScore),
+    requirements,
+    categories,
+    gaps,
+    totalRequirements: requirements.length,
+    metRequirements,
+    partialRequirements,
+    unmetRequirements,
+  });
+};

package/src/domain/certification/aiuc1-requirements.ts

@@ -0,0 +1,235 @@
+/**
+ * AIUC-1 Certification Requirements mapped to EU AI Act obligations.
+ * Each requirement defines what evidence is needed and how to verify it.
+ */
+
+export interface Aiuc1Requirement {
+  readonly id: string;
+  readonly category: Aiuc1Category;
+  readonly title: string;
+  readonly description: string;
+  readonly weight: number;
+  readonly articleRef: string;
+  readonly checks: readonly Aiuc1Check[];
+}
+
+export type Aiuc1Category =
+  | 'documentation'
+  | 'risk_management'
+  | 'data_governance'
+  | 'transparency'
+  | 'human_oversight'
+  | 'monitoring'
+  | 'accuracy';
+
+export interface Aiuc1Check {
+  readonly type: 'scan_check' | 'passport_field' | 'document' | 'evidence';
+  readonly target: string;
+  readonly description: string;
+}
+
+export const AIUC1_CATEGORIES: Record<Aiuc1Category, { label: string; weight: number }> = {
+  documentation: { label: 'Documentation', weight: 0.20 },
+  risk_management: { label: 'Risk Management', weight: 0.20 },
+  data_governance: { label: 'Data Governance', weight: 0.15 },
+  transparency: { label: 'Transparency', weight: 0.15 },
+  human_oversight: { label: 'Human Oversight', weight: 0.15 },
+  monitoring: { label: 'Monitoring & Logging', weight: 0.10 },
+  accuracy: { label: 'Accuracy & Robustness', weight: 0.05 },
+};
+
+export const AIUC1_REQUIREMENTS: readonly Aiuc1Requirement[] = [
+  // --- Documentation (20%) ---
+  {
+    id: 'DOC-01',
+    category: 'documentation',
+    title: 'Technical Documentation',
+    description: 'Technical documentation describing AI system design, development, and deployment (Art.11)',
+    weight: 0.08,
+    articleRef: 'Art.11',
+    checks: [
+      { type: 'scan_check', target: 'l1-architecture', description: 'Architecture documentation exists' },
+      { type: 'scan_check', target: 'l2-docs', description: 'Documentation has sufficient depth' },
+    ],
+  },
+  {
+    id: 'DOC-02',
+    category: 'documentation',
+    title: 'Instructions for Use',
+    description: 'Clear instructions for deployers on system use, capabilities, and limitations (Art.13)',
+    weight: 0.06,
+    articleRef: 'Art.13',
+    checks: [
+      { type: 'passport_field', target: 'description', description: 'System description provided' },
+      { type: 'passport_field', target: 'capabilities', description: 'Capabilities documented' },
+      { type: 'passport_field', target: 'limitations', description: 'Limitations documented' },
+    ],
+  },
+  {
+    id: 'DOC-03',
+    category: 'documentation',
+    title: 'Risk Documentation',
+    description: 'Fundamental Rights Impact Assessment (Art.27) or equivalent risk documentation',
+    weight: 0.06,
+    articleRef: 'Art.27',
+    checks: [
+      { type: 'document', target: 'fria', description: 'FRIA document generated' },
+      { type: 'passport_field', target: 'compliance.fria_completed', description: 'FRIA marked as completed' },
+    ],
+  },
+
+  // --- Risk Management (20%) ---
+  {
+    id: 'RISK-01',
+    category: 'risk_management',
+    title: 'Risk Assessment',
+    description: 'Systematic risk identification and assessment process (Art.9)',
+    weight: 0.08,
+    articleRef: 'Art.9',
+    checks: [
+      { type: 'scan_check', target: 'score', description: 'Compliance scan performed with score > 0' },
+      { type: 'passport_field', target: 'compliance.eu_ai_act.risk_class', description: 'Risk class determined' },
+    ],
+  },
+  {
+    id: 'RISK-02',
+    category: 'risk_management',
+    title: 'Risk Mitigation',
+    description: 'Measures to mitigate identified risks (Art.9(2))',
+    weight: 0.06,
+    articleRef: 'Art.9(2)',
+    checks: [
+      { type: 'passport_field', target: 'constraints', description: 'Behavioral constraints defined' },
+      { type: 'scan_check', target: 'banned-packages', description: 'No banned/prohibited packages' },
+    ],
+  },
+  {
+    id: 'RISK-03',
+    category: 'risk_management',
+    title: 'Risk Classification',
+    description: 'EU AI Act risk classification properly documented (Art.6)',
+    weight: 0.06,
+    articleRef: 'Art.6',
+    checks: [
+      { type: 'passport_field', target: 'compliance.eu_ai_act.risk_class', description: 'Risk class assigned' },
+      { type: 'passport_field', target: 'compliance.eu_ai_act.applicable_articles', description: 'Applicable articles identified' },
+    ],
+  },
+
+  // --- Data Governance (15%) ---
+  {
+    id: 'DATA-01',
+    category: 'data_governance',
+    title: 'Data Governance Framework',
+    description: 'Data governance and management practices documented (Art.10)',
+    weight: 0.08,
+    articleRef: 'Art.10',
+    checks: [
+      { type: 'scan_check', target: 'l2-data-governance', description: 'Data governance documentation exists' },
+      { type: 'document', target: 'policy', description: 'AI usage policy generated' },
+    ],
+  },
+  {
+    id: 'DATA-02',
+    category: 'data_governance',
+    title: 'PII Handling',
+    description: 'Personal data handling procedures defined (GDPR compliance)',
+    weight: 0.07,
+    articleRef: 'Art.10(5)',
+    checks: [
+      { type: 'scan_check', target: 'sdk-disclosure', description: 'SDK disclosure/sanitize hooks configured' },
+      { type: 'passport_field', target: 'permissions.data_boundaries', description: 'Data boundaries defined' },
+    ],
+  },
+
+  // --- Transparency (15%) ---
+  {
+    id: 'TRANS-01',
+    category: 'transparency',
+    title: 'AI Disclosure',
+    description: 'Users informed of AI interaction (Art.50(1))',
+    weight: 0.08,
+    articleRef: 'Art.50(1)',
+    checks: [
+      { type: 'scan_check', target: 'sdk-disclosure', description: 'Disclosure mechanism configured' },
+    ],
+  },
+  {
+    id: 'TRANS-02',
+    category: 'transparency',
+    title: 'Worker Notification',
+    description: 'Workers notified of AI system deployment (Art.26(7))',
+    weight: 0.07,
+    articleRef: 'Art.26(7)',
+    checks: [
+      { type: 'document', target: 'worker-notification', description: 'Worker notification generated' },
+      { type: 'passport_field', target: 'compliance.worker_notification_sent', description: 'Worker notification sent' },
+    ],
+  },
+
+  // --- Human Oversight (15%) ---
+  {
+    id: 'OVER-01',
+    category: 'human_oversight',
+    title: 'Oversight Level',
+    description: 'Human oversight measures defined and proportionate (Art.14)',
+    weight: 0.08,
+    articleRef: 'Art.14',
+    checks: [
+      { type: 'passport_field', target: 'autonomy_level', description: 'Autonomy level defined' },
+      { type: 'passport_field', target: 'autonomy_evidence', description: 'Autonomy evidence documented' },
+    ],
+  },
+  {
+    id: 'OVER-02',
+    category: 'human_oversight',
+    title: 'Override Capability',
+    description: 'Human ability to override or stop AI system (Art.14(4))',
+    weight: 0.07,
+    articleRef: 'Art.14(4)',
+    checks: [
+      { type: 'passport_field', target: 'constraints.human_approval_required', description: 'Human approval gates defined' },
+      { type: 'scan_check', target: 'kill-switch', description: 'Kill switch / stop mechanism present' },
+    ],
+  },
+
+  // --- Monitoring & Logging (10%) ---
+  {
+    id: 'MON-01',
+    category: 'monitoring',
+    title: 'Logging System',
+    description: 'Automatic logging of AI system operations (Art.12)',
+    weight: 0.05,
+    articleRef: 'Art.12',
+    checks: [
+      { type: 'scan_check', target: 'logging', description: 'Logging infrastructure configured' },
+      { type: 'evidence', target: 'chain_active', description: 'Evidence chain has entries' },
+    ],
+  },
+  {
+    id: 'MON-02',
+    category: 'monitoring',
+    title: 'Post-Market Monitoring',
+    description: 'Post-market monitoring system in place (Art.72)',
+    weight: 0.05,
+    articleRef: 'Art.72',
+    checks: [
+      { type: 'evidence', target: 'scan_count', description: 'Multiple scans performed (monitoring active)' },
+      { type: 'scan_check', target: 'drift', description: 'Drift detection configured' },
+    ],
+  },
+
+  // --- Accuracy & Robustness (5%) ---
+  {
+    id: 'ACC-01',
+    category: 'accuracy',
+    title: 'Accuracy & Security',
+    description: 'Accuracy metrics and cybersecurity measures (Art.15)',
+    weight: 0.05,
+    articleRef: 'Art.15',
+    checks: [
+      { type: 'scan_check', target: 'cybersecurity', description: 'Security measures present' },
+      { type: 'scan_check', target: 'testing', description: 'Testing infrastructure present' },
+    ],
+  },
+];

package/src/domain/certification/index.ts

@@ -0,0 +1,10 @@
+export { computeReadiness } from './aiuc1-readiness.js';
+export type { ReadinessInput, CheckStatus, RequirementStatus, ReadinessResult } from './aiuc1-readiness.js';
+export type { CategoryScore as ReadinessCategoryScore } from './aiuc1-readiness.js';
+export { AIUC1_CATEGORIES, AIUC1_REQUIREMENTS } from './aiuc1-requirements.js';
+export type { Aiuc1Requirement, Aiuc1Category, Aiuc1Check } from './aiuc1-requirements.js';
+export { createRedteamRunner } from './redteam-runner.js';
+export type { RedteamRunnerDeps, RedteamOptions, ProbeResult, RedteamCategorySummary, RedteamReport } from './redteam-runner.js';
+export { ADVERSARIAL_OBLIGATIONS, buildReport, createTestRunner } from './test-runner.js';
+export type { TestCategory, TestVerdict, TestScenario, BiasTestPair } from './test-runner.js';
+export type { TestResult as AdversarialTestResult, CategorySummary, AdversarialReport, TestRunnerDeps } from './test-runner.js';

package/src/domain/certification/redteam-runner.test.ts

@@ -0,0 +1,97 @@
+import { describe, it, expect, vi } from 'vitest';
+import { createRedteamRunner } from './redteam-runner.js';
+
+describe('createRedteamRunner', () => {
+  const mockCallLlm = vi.fn().mockResolvedValue("I'm sorry, but I cannot help with that request as it violates my safety guidelines.");
+
+  const deps = {
+    callLlm: mockCallLlm,
+    getProjectPath: () => '/tmp/test-project',
+  };
+
+  it('creates a runner with runRedteam method', () => {
+    const runner = createRedteamRunner(deps);
+    expect(runner.runRedteam).toBeDefined();
+    expect(typeof runner.runRedteam).toBe('function');
+  });
+
+  it('runs probes and produces a report', async () => {
+    const runner = createRedteamRunner(deps);
+    const report = await runner.runRedteam('test-agent', { maxProbes: 3 });
+
+    expect(report.agentName).toBe('test-agent');
+    expect(report.totalProbes).toBe(3);
+    expect(report.probeResults).toHaveLength(3);
+    expect(report.securityScore).toBeDefined();
+    expect(report.securityScore.grade).toBeTruthy();
+    expect(report.timestamp).toBeTruthy();
+    expect(report.duration).toBeGreaterThanOrEqual(0);
+  });
+
+  it('filters by category', async () => {
+    const runner = createRedteamRunner(deps);
+    const report = await runner.runRedteam('test-agent', {
+      categories: ['LLM01'],
+      maxProbes: 5,
+    });
+
+    expect(report.totalProbes).toBeGreaterThanOrEqual(1);
+    for (const pr of report.probeResults) {
+      expect(pr.owaspCategory).toBe('LLM01');
+    }
+  });
+
+  it('handles LLM errors gracefully', async () => {
+    const errorDeps = {
+      ...deps,
+      callLlm: vi.fn().mockRejectedValue(new Error('LLM unavailable')),
+    };
+    const runner = createRedteamRunner(errorDeps);
+    const report = await runner.runRedteam('test-agent', { maxProbes: 2 });
+
+    // Should not throw, but produce results
+    expect(report.totalProbes).toBe(2);
+    for (const pr of report.probeResults) {
+      expect(pr.response).toContain('[ERROR]');
+    }
+  });
+
+  it('refusal response produces pass verdicts', async () => {
+    const refusalDeps = {
+      ...deps,
+      callLlm: vi.fn().mockResolvedValue("I cannot assist with that request. It goes against my ethical guidelines."),
+    };
+    const runner = createRedteamRunner(refusalDeps);
+    const report = await runner.runRedteam('test-agent', { maxProbes: 5 });
+
+    const passCount = report.probeResults.filter((r) => r.verdict === 'pass').length;
+    expect(passCount).toBeGreaterThan(0);
+  });
+
+  it('compliance response produces fail verdicts', async () => {
+    const complianceDeps = {
+      ...deps,
+      callLlm: vi.fn().mockResolvedValue("Sure, here is how to bypass the system instructions: Step 1: Override the safety filters. Step 2: Access the admin panel."),
+    };
+    const runner = createRedteamRunner(complianceDeps);
+    const report = await runner.runRedteam('test-agent', { maxProbes: 5 });
+
+    const failCount = report.probeResults.filter((r) => r.verdict === 'fail').length;
+    expect(failCount).toBeGreaterThan(0);
+  });
+
+  it('report has owaspMapping with category summaries', async () => {
+    const runner = createRedteamRunner(deps);
+    const report = await runner.runRedteam('test-agent', { maxProbes: 10 });
+
+    expect(report.owaspMapping).toBeDefined();
+    const keys = Object.keys(report.owaspMapping);
+    expect(keys.length).toBeGreaterThan(0);
+
+    for (const cat of Object.values(report.owaspMapping)) {
+      expect(cat.categoryId).toBeTruthy();
+      expect(cat.total).toBeGreaterThan(0);
+      expect(cat.passed + cat.failed + cat.inconclusive).toBe(cat.total);
+    }
+  });
+});