@complior/engine 0.9.0

package/src/domain/documents/ai-enricher.test.ts

@@ -0,0 +1,120 @@
+/**
+ * Tests for detectWeakSections() and buildL2Feedback() pure functions.
+ */
+import { describe, it, expect } from 'vitest';
+import { detectWeakSections, buildL2Feedback } from './ai-enricher.js';
+
+const STRONG_DOC = `## Risk Identification
+
+We have identified 15 key risks through our systematic risk assessment process conducted in Q1 2026.
+Each risk was evaluated using the ISO 31000:2018 framework, with probability scores assigned on a 1-5 scale.
+The top 3 risks relate to bias (P=0.4, I=High), data quality (P=0.3, I=Medium), and model drift (P=0.25, I=High).
+Under Art. 9 of the EU AI Act (Regulation 2024/1689), providers must maintain a risk management system.
+
+## Mitigation Measures
+
+For each identified risk, we implement specific mitigation controls:
+- Bias: Monthly fairness audits using Aequitas toolkit, targeting <2% demographic parity gap
+- Data quality: Automated data validation pipeline with 99.5% accuracy threshold
+- Model drift: Weekly model monitoring with KL-divergence alerts at >0.05
+
+## Monitoring Plan
+
+Continuous monitoring via Prometheus dashboards with the following KPIs:
+1. False positive rate: target <5% (current: 3.2%)
+2. Latency p99: target <200ms (current: 145ms)
+3. Drift score: target <0.05 (current: 0.02)
+Reviews conducted quarterly per Art. 9(8) requirements.
+`;
+
+const WEAK_DOC = `## Risk Identification
+
+We assess risks.
+
+## Mitigation Measures
+
+[TODO: Add mitigation measures]
+
+## Monitoring Plan
+
+Monitoring will be done.
+`;
+
+const MIXED_DOC = `## System Description
+
+Our AI system processes customer support tickets using a fine-tuned GPT-4 model.
+It classifies tickets into 12 categories with 94.2% accuracy (validated on 10,000 test samples).
+The system handles approximately 50,000 tickets per day with p99 latency of 180ms.
+Deployed on AWS eu-west-1 region per Art. 11 requirements.
+
+## Intended Purpose
+
+This system helps with customer support.
+
+## Performance Metrics
+
+[TODO: Fill in performance benchmarks]
+`;
+
+describe('detectWeakSections', () => {
+  it('returns empty array for fully-populated document', () => {
+    const weak = detectWeakSections(STRONG_DOC);
+    expect(weak).toEqual([]);
+  });
+
+  it('detects weak sections in placeholder-heavy document', () => {
+    const weak = detectWeakSections(WEAK_DOC);
+    expect(weak.length).toBeGreaterThan(0);
+    // Should detect sections with placeholders or very short content
+    expect(weak.some(s => s.toLowerCase().includes('mitigation'))).toBe(true);
+  });
+
+  it('detects mix of strong and weak sections', () => {
+    const weak = detectWeakSections(MIXED_DOC);
+    // System Description is strong, others are weak
+    expect(weak.some(s => s.toLowerCase().includes('intended purpose') || s.toLowerCase().includes('performance'))).toBe(true);
+    // System Description should NOT be in weak list
+    expect(weak.some(s => s.toLowerCase() === 'system description')).toBe(false);
+  });
+
+  it('returns empty for empty content', () => {
+    const weak = detectWeakSections('');
+    expect(weak).toEqual([]);
+  });
+
+  it('returns empty for content with no headings', () => {
+    const weak = detectWeakSections('Just a paragraph with no markdown headings at all.');
+    expect(weak).toEqual([]);
+  });
+});
+
+describe('buildL2Feedback', () => {
+  it('returns "All sections appear adequate" for strong document', () => {
+    const feedback = buildL2Feedback(STRONG_DOC);
+    expect(feedback).toBe('All sections appear adequate.');
+  });
+
+  it('lists specific issues for weak document', () => {
+    const feedback = buildL2Feedback(WEAK_DOC);
+    expect(feedback).not.toBe('All sections appear adequate.');
+    // Should contain bullet points with section names
+    expect(feedback).toContain('-');
+    expect(feedback).toContain('"');
+  });
+
+  it('mentions placeholder count for sections with TODOs', () => {
+    const feedback = buildL2Feedback(WEAK_DOC);
+    expect(feedback).toMatch(/placeholder/i);
+  });
+
+  it('mentions word count for very short sections', () => {
+    const feedback = buildL2Feedback(WEAK_DOC);
+    expect(feedback).toMatch(/words/i);
+  });
+
+  it('returns feedback only for weak sections in mixed doc', () => {
+    const feedback = buildL2Feedback(MIXED_DOC);
+    // Should have feedback for weak sections but not System Description
+    expect(feedback).not.toBe('All sections appear adequate.');
+  });
+});

package/src/domain/documents/ai-enricher.ts

@@ -0,0 +1,159 @@
+/**
+ * LLM-enriched document generation.
+ * Two modes:
+ *   1. Scaffold enrichment — fill [TODO] placeholders in freshly generated docs
+ *   2. Draft enhancement — improve existing user-edited docs using L2 quality feedback
+ * Extracted from document-generator.ts for SRP compliance.
+ */
+import type { LanguageModel } from 'ai';
+import type { AgentPassport } from '../../types/passport.types.js';
+import type { DocResult } from './document-generator.js';
+import { extractSectionContents, measureSemanticDepth } from '../scanner/layers/layer2-parsing.js';
+
+export interface AiEnrichInput {
+  readonly baseResult: DocResult;
+  readonly manifest: AgentPassport;
+  readonly model: LanguageModel;
+}
+
+export interface AiEnrichedResult extends DocResult {
+  readonly aiEnriched: true;
+  readonly aiFieldsCount: number;
+}
+
+/**
+ * Analyze existing document content to find weak sections via L2 semantic depth.
+ * Returns section names + actionable feedback for sections scoring below threshold.
+ */
+export const detectWeakSections = (content: string): readonly string[] => {
+  const sections = extractSectionContents(content);
+  const weak: string[] = [];
+  for (const [heading, body] of sections) {
+    const depth = measureSemanticDepth(body, heading);
+    if (depth.isShallow || depth.placeholderCount > 0) {
+      weak.push(heading);
+    }
+  }
+  return weak;
+};
+
+/**
+ * Build L2-informed feedback for the LLM prompt.
+ * Tells the LLM exactly which sections need improvement and why.
+ */
+export const buildL2Feedback = (content: string): string => {
+  const sections = extractSectionContents(content);
+  const feedback: string[] = [];
+  for (const [heading, body] of sections) {
+    const depth = measureSemanticDepth(body, heading);
+    if (depth.isShallow || depth.placeholderCount > 0) {
+      const reasons: string[] = [];
+      if (depth.placeholderCount > 0) reasons.push(`${depth.placeholderCount} placeholder(s) to fill`);
+      if (depth.wordCount < 50) reasons.push(`only ${depth.wordCount} words (min 50)`);
+      if (!depth.hasNumericMetrics) reasons.push('no numeric metrics');
+      if (!depth.hasLegalReferences) reasons.push('no legal references');
+      if (!depth.hasMeasurableTargets) reasons.push('no measurable targets');
+      feedback.push(`- "${heading}": ${reasons.join(', ')}`);
+    }
+  }
+  return feedback.length > 0 ? feedback.join('\n') : 'All sections appear adequate.';
+};
+
+const buildPassportContext = (manifest: AgentPassport): string =>
+  [
+    `System: ${manifest.display_name}`,
+    `Description: ${manifest.description}`,
+    `Risk class: ${manifest.compliance?.eu_ai_act?.risk_class ?? 'unknown'}`,
+    `Autonomy: ${manifest.autonomy_level}`,
+    `Provider: ${manifest.model?.provider ?? 'unknown'}`,
+    `Model: ${manifest.model?.model_id ?? 'unknown'}`,
+    manifest.owner?.team ? `Organization: ${manifest.owner.team}` : '',
+    manifest.oversight?.responsible_person ? `Oversight: ${manifest.oversight.responsible_person} (${manifest.oversight.role})` : '',
+    manifest.permissions?.tools ? `Tools: ${(manifest.permissions.tools as readonly string[]).join(', ')}` : '',
+    manifest.constraints?.prohibited_actions ? `Prohibited: ${(manifest.constraints.prohibited_actions as readonly string[]).join(', ')}` : '',
+  ].filter(Boolean).join('\n');
+
+const buildSystemPrompt = (docType: string, mode: 'scaffold' | 'enhance'): string => {
+  const modeRules = mode === 'scaffold'
+    ? `- Fill in placeholder sections (marked with <!-- GUIDANCE --> comments or containing generic "[...]" placeholders) with specific content based on the passport data
+- Keep all existing filled content intact — do NOT modify already pre-filled fields`
+    : `- The document has been partially filled by the user — preserve ALL existing content
+- Strengthen weak sections identified in the quality feedback below
+- Add numeric metrics, legal references (Art. numbers), measurable targets, and specific details
+- Expand thin sections with concrete, actionable content based on the passport data`;
+
+  return `You are a compliance document specialist for the EU AI Act (Regulation 2024/1689).
+You are ${mode === 'scaffold' ? 'enriching' : 'strengthening'} a ${docType} document based on the AI system's passport data.
+
+Rules:
+${modeRules}
+- For legal assertions that require human sign-off (signatures, legal declarations, notified body details), replace with [REVIEW REQUIRED: brief description of what's needed]
+- Use professional regulatory language appropriate for EU AI Act compliance documentation
+- Be specific and actionable — reference the actual system name, risk class, and capabilities
+- Do NOT invent data that isn't in the passport (dates, test results, metrics) — use [TO BE COMPLETED: description] for those
+- Keep the exact Markdown structure and heading hierarchy`;
+};
+
+/**
+ * Enriches a compliance document via LLM.
+ * Two modes detected automatically:
+ *   - Scaffold: document has [TODO]/placeholder brackets → fill them
+ *   - Enhance: document has real content but weak sections → strengthen with specifics
+ * Human-required legal assertions are left with [REVIEW REQUIRED].
+ */
+export const enrichDocumentWithAI = async (input: AiEnrichInput): Promise<AiEnrichedResult> => {
+  const { baseResult, manifest, model } = input;
+  const { generateText } = await import('ai');
+
+  const passportContext = buildPassportContext(manifest);
+
+  // Auto-detect mode: scaffold (has placeholders) vs enhance (user-edited, no placeholders)
+  const hasPlaceholders = /\[[^\]]{2,60}\](?!\()/.test(baseResult.markdown);
+  const mode: 'scaffold' | 'enhance' = hasPlaceholders ? 'scaffold' : 'enhance';
+
+  const systemPrompt = buildSystemPrompt(baseResult.docType, mode);
+  const l2Feedback = buildL2Feedback(baseResult.markdown);
+
+  const prompt = mode === 'scaffold'
+    ? `Here is the AI system passport data:
+${passportContext}
+
+Here are the manual fields that need enrichment: ${baseResult.manualFields.join(', ')}
+
+Here is the document to enrich:
+${baseResult.markdown}
+
+Return the complete enriched document (full Markdown). Keep all existing content, enrich unfilled sections.`
+    : `Here is the AI system passport data:
+${passportContext}
+
+Quality analysis of weak sections that need strengthening:
+${l2Feedback}
+
+Here is the document to improve:
+${baseResult.markdown}
+
+Return the complete improved document (full Markdown). Preserve all existing content. Strengthen the weak sections identified above with specific details, metrics, and legal references.`;
+
+  try {
+    const result = await generateText({ model, system: systemPrompt, prompt });
+    const marker = `\n\n<!-- complior:reviewed ${new Date().toISOString()} -->`;
+    const enrichedMarkdown = result.text.trim() + marker;
+
+    // Count how many fields were likely addressed
+    const remainingPlaceholders = (enrichedMarkdown.match(/\[(?:TO BE COMPLETED|REVIEW REQUIRED)[^\]]*\]/g) ?? []).length;
+    const aiFieldsCount = Math.max(0, baseResult.manualFields.length - remainingPlaceholders);
+
+    return Object.freeze({
+      markdown: enrichedMarkdown,
+      docType: baseResult.docType,
+      prefilledFields: Object.freeze([...baseResult.prefilledFields]),
+      manualFields: Object.freeze([...baseResult.manualFields]),
+      aiEnriched: true as const,
+      aiFieldsCount,
+    });
+  } catch (err) {
+    // Re-throw so caller (fix-service) can handle and log the error
+    throw new Error(`AI enrichment failed: ${err instanceof Error ? err.message : String(err)}`);
+  }
+};

package/src/domain/documents/document-generator.test.ts

@@ -0,0 +1,318 @@
+import { describe, it, expect } from 'vitest';
+import { generateDocument, ALL_DOC_TYPES, TEMPLATE_FILE_MAP } from './document-generator.js';
+import type { DocType } from './document-generator.js';
+import type { AgentPassport } from '../../types/passport.types.js';
+
+const createMockPassport = (overrides?: Partial<AgentPassport>): AgentPassport =>
+  ({
+    agent_id: 'test-agent-001',
+    name: 'test-agent',
+    display_name: 'Test AI Agent',
+    version: '1.0.0',
+    description: 'A test AI agent for compliance scanning',
+    type: 'assistant',
+    framework: 'openai',
+    autonomy_level: 'L2',
+    model: {
+      provider: 'OpenAI',
+      model_id: 'gpt-4o',
+    },
+    owner: {
+      team: 'Acme Corp',
+      responsible_person: 'Jane Doe',
+    },
+    compliance: {
+      eu_ai_act: {
+        risk_class: 'high',
+      },
+      complior_score: 72,
+    },
+    constraints: {
+      human_approval_required: ['deploy', 'delete'],
+      escalation_rules: [],
+    },
+    autonomy_evidence: {
+      human_approval_gates: 3,
+      unsupervised_actions: 1,
+      no_logging_actions: 0,
+      auto_rated: false,
+    },
+    permissions: {
+      tools: ['read_file', 'write_file'],
+      denied: ['execute_shell'],
+    },
+    source_files: [],
+    signature: { value: '', publicKey: '', algorithm: 'ed25519' },
+    created: '2026-01-01T00:00:00Z',
+    updated: '2026-01-01T00:00:00Z',
+    ...overrides,
+  }) as unknown as AgentPassport;
+
+const SIMPLE_TEMPLATE = `# Test Document
+| Company | [Company Name] |
+| Date | [Date] |
+| System | [AI System Name] |
+| Provider | [Provider] |
+| Version | [X.Y] |
+| Risk | [Risk Class] |
+`;
+
+describe('document-generator', () => {
+  describe('ALL_DOC_TYPES', () => {
+    it('contains exactly 14 document types', () => {
+      expect(ALL_DOC_TYPES).toHaveLength(14);
+    });
+
+    it('maps each type to a template file', () => {
+      for (const dt of ALL_DOC_TYPES) {
+        expect(TEMPLATE_FILE_MAP[dt]).toBeDefined();
+        expect(TEMPLATE_FILE_MAP[dt]).toMatch(/\.md$/);
+      }
+    });
+  });
+
+  describe('generateDocument — common placeholders', () => {
+    it('replaces [Company Name] with organization override', () => {
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template: SIMPLE_TEMPLATE,
+        docType: 'ai-literacy',
+        organization: 'Override Corp',
+      });
+      expect(result.markdown).toContain('Override Corp');
+      expect(result.markdown).not.toContain('[Company Name]');
+      expect(result.prefilledFields).toContain('Company Name');
+    });
+
+    it('replaces [Company Name] with manifest.owner.team when no organization', () => {
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template: SIMPLE_TEMPLATE,
+        docType: 'ai-literacy',
+      });
+      expect(result.markdown).toContain('Acme Corp');
+    });
+
+    it('marks Company Name as manual when missing', () => {
+      const passport = createMockPassport({ owner: { team: '' } } as any);
+      const result = generateDocument({
+        manifest: passport,
+        template: SIMPLE_TEMPLATE,
+        docType: 'ai-literacy',
+      });
+      expect(result.manualFields).toContain('Company Name');
+    });
+
+    it('replaces [Date] with today ISO', () => {
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template: SIMPLE_TEMPLATE,
+        docType: 'ai-literacy',
+      });
+      const today = new Date().toISOString().slice(0, 10);
+      expect(result.markdown).toContain(today);
+      expect(result.markdown).not.toContain('[Date]');
+    });
+
+    it('replaces [AI System Name]', () => {
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template: SIMPLE_TEMPLATE,
+        docType: 'ai-literacy',
+      });
+      expect(result.markdown).toContain('Test AI Agent');
+    });
+
+    it('replaces [Provider]', () => {
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template: SIMPLE_TEMPLATE,
+        docType: 'ai-literacy',
+      });
+      expect(result.markdown).toContain('OpenAI');
+      expect(result.markdown).not.toContain('| Provider | [Provider] |');
+    });
+
+    it('replaces [X.Y] with version', () => {
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template: SIMPLE_TEMPLATE,
+        docType: 'ai-literacy',
+      });
+      expect(result.markdown).toContain('1.0.0');
+    });
+
+    it('replaces [Risk Class]', () => {
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template: SIMPLE_TEMPLATE,
+        docType: 'ai-literacy',
+      });
+      expect(result.markdown).toContain('high');
+      expect(result.markdown).not.toContain('[Risk Class]');
+    });
+  });
+
+  describe('generateDocument — document ID replacement', () => {
+    it('replaces ALP-[YYYY]-[NNN] for ai-literacy', () => {
+      const template = '| Document ID | ALP-[YYYY]-[NNN] |';
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template,
+        docType: 'ai-literacy',
+      });
+      expect(result.markdown).toMatch(/ALP-\d{4}-\d{3}/);
+      expect(result.markdown).not.toContain('[YYYY]');
+      expect(result.prefilledFields).toContain('Document ID');
+    });
+
+    it('replaces ART5-[YYYY]-[NNN] for art5-screening', () => {
+      const template = '| Report ID | ART5-[YYYY]-[NNN] |';
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template,
+        docType: 'art5-screening',
+      });
+      expect(result.markdown).toMatch(/ART5-\d{4}-\d{3}/);
+    });
+
+    it('does not add Document ID to prefilled if pattern not in template', () => {
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template: SIMPLE_TEMPLATE,
+        docType: 'technical-documentation',
+      });
+      expect(result.prefilledFields).not.toContain('Document ID');
+    });
+  });
+
+  describe('generateDocument — type-specific manual fields', () => {
+    it('ai-literacy has training-related manual fields', () => {
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template: SIMPLE_TEMPLATE,
+        docType: 'ai-literacy',
+      });
+      expect(result.manualFields).toContain('Training levels configuration');
+      expect(result.manualFields).toContain('Sign-off signatures');
+    });
+
+    it('art5-screening has prohibited practice manual fields', () => {
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template: SIMPLE_TEMPLATE,
+        docType: 'art5-screening',
+      });
+      expect(result.manualFields).toContain('Prohibited practice details');
+      expect(result.manualFields).toContain('Decision and justification');
+    });
+
+    it('technical-documentation has architecture manual fields', () => {
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template: SIMPLE_TEMPLATE,
+        docType: 'technical-documentation',
+      });
+      expect(result.manualFields).toContain('System architecture details');
+      expect(result.manualFields).toContain('Performance metrics');
+    });
+
+    it('incident-report has incident manual fields', () => {
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template: SIMPLE_TEMPLATE,
+        docType: 'incident-report',
+      });
+      expect(result.manualFields).toContain('Incident description');
+      expect(result.manualFields).toContain('Root cause analysis');
+    });
+
+    it('declaration-of-conformity has conformity manual fields', () => {
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template: SIMPLE_TEMPLATE,
+        docType: 'declaration-of-conformity',
+      });
+      expect(result.manualFields).toContain('Harmonised standards used');
+      expect(result.manualFields).toContain('Signatory');
+    });
+
+    it('monitoring-policy has monitoring manual fields', () => {
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template: SIMPLE_TEMPLATE,
+        docType: 'monitoring-policy',
+      });
+      expect(result.manualFields).toContain('Human oversight assignments');
+      expect(result.manualFields).toContain('Log retention schedule');
+    });
+  });
+
+  describe('generateDocument — all 6 types produce valid results', () => {
+    for (const docType of ALL_DOC_TYPES) {
+      it(`generates valid result for ${docType}`, () => {
+        const result = generateDocument({
+          manifest: createMockPassport(),
+          template: SIMPLE_TEMPLATE,
+          docType,
+          organization: 'Test Corp',
+        });
+        expect(result.docType).toBe(docType);
+        expect(result.markdown).toBeTruthy();
+        expect(result.prefilledFields.length).toBeGreaterThan(0);
+        expect(result.manualFields.length).toBeGreaterThan(0);
+      });
+    }
+  });
+
+  describe('generateDocument — result is frozen', () => {
+    it('returns a frozen object', () => {
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template: SIMPLE_TEMPLATE,
+        docType: 'ai-literacy',
+      });
+      expect(Object.isFrozen(result)).toBe(true);
+      expect(Object.isFrozen(result.prefilledFields)).toBe(true);
+      expect(Object.isFrozen(result.manualFields)).toBe(true);
+    });
+  });
+
+  describe('generateDocument — edge cases', () => {
+    it('handles passport with minimal fields gracefully', () => {
+      const minimal = createMockPassport({
+        model: { provider: '', model_id: '' } as any,
+        compliance: { eu_ai_act: { risk_class: '' } } as any,
+      });
+      const result = generateDocument({
+        manifest: minimal,
+        template: SIMPLE_TEMPLATE,
+        docType: 'ai-literacy',
+      });
+      expect(result.markdown).toBeTruthy();
+      expect(result.manualFields).toContain('Provider');
+      expect(result.manualFields).toContain('Risk Class');
+    });
+
+    it('replaces multiple occurrences of same placeholder', () => {
+      const template = '[Company Name] is great. [Company Name] is the best.';
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template,
+        docType: 'ai-literacy',
+        organization: 'Multi Corp',
+      });
+      expect(result.markdown).toBe('Multi Corp is great. Multi Corp is the best.');
+    });
+
+    it('preserves GUIDANCE comments in output', () => {
+      const template = '<!-- GUIDANCE: This is guidance -->\n[Company Name]';
+      const result = generateDocument({
+        manifest: createMockPassport(),
+        template,
+        docType: 'ai-literacy',
+      });
+      expect(result.markdown).toContain('<!-- GUIDANCE:');
+    });
+  });
+});