@ansvar/us-regulations-mcp 1.0.0 → 1.2.1

package/src/ingest/adapters/utah-xcode.ts

@@ -0,0 +1,143 @@
+/**
+ * Utah XCode Adapter
+ *
+ * Fetches Utah UCPA from le.utah.gov with version resolution.
+ * Source: Utah Code Ann. § 13-61-101 to 13-61-404
+ */
+
+import {
+  SourceAdapter,
+  RegulationMetadata,
+  Section,
+  Definition,
+  UpdateStatus,
+} from '../framework.js';
+import * as cheerio from 'cheerio';
+
+class ScrapingError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'ScrapingError';
+  }
+}
+
+export class UtahXcodeAdapter implements SourceAdapter {
+  private readonly regulationId = 'UTAH_UCPA';
+  private readonly sections = [
+    '101', '102', '103', '104', '201', '202', '203', '301', '302', '303', '304', '401', '402', '403', '404'
+  ];
+
+  async fetchMetadata(): Promise<RegulationMetadata> {
+    return {
+      id: 'UTAH_UCPA',
+      full_name: 'Utah Consumer Privacy Act',
+      citation: 'Utah Code Ann. §13-61-101 to 13-61-404',
+      effective_date: '2023-12-31',
+      last_amended: '2024-05-01',
+      source_url: 'https://le.utah.gov/xcode/Title13/Chapter61/',
+      jurisdiction: 'utah',
+      regulation_type: 'statute',
+    };
+  }
+
+  async *fetchSections(): AsyncGenerator<Section[]> {
+    const sections: Section[] = [];
+    let totalCount = 0;
+
+    console.log(`Fetching Utah UCPA sections (two-step version resolution)...`);
+
+    for (const sectionNum of this.sections) {
+      try {
+        // Step 1: Get version reference
+        const indexUrl = `https://le.utah.gov/xcode/Title13/Chapter61/13-61-S${sectionNum}.html`;
+
+        await this.sleep(500);
+        const indexResponse = await fetch(indexUrl);
+        if (!indexResponse.ok) {
+          console.warn(`  Skipping § 13-61-${sectionNum} (HTTP ${indexResponse.status})`);
+          continue;
+        }
+
+        const indexHtml = await indexResponse.text();
+
+        // Extract version filename from JavaScript
+        const versionMatch = indexHtml.match(/versionArr\s*=\s*\[\s*\['([^']+)'/);
+        if (!versionMatch) {
+          console.warn(`  Skipping § 13-61-${sectionNum} (no version found)`);
+          continue;
+        }
+
+        const versionFile = versionMatch[1];
+
+        // Step 2: Fetch versioned HTML
+        const versionUrl = `https://le.utah.gov/xcode/Title13/Chapter61/${versionFile}`;
+        await this.sleep(500);
+        const versionResponse = await fetch(versionUrl);
+
+        if (!versionResponse.ok) {
+          console.warn(`  Skipping § 13-61-${sectionNum} versioned file (HTTP ${versionResponse.status})`);
+          continue;
+        }
+
+        const versionHtml = await versionResponse.text();
+        const $ = cheerio.load(versionHtml);
+
+        const section = this.parseSection($, sectionNum);
+        if (section) {
+          sections.push(section);
+          totalCount++;
+          console.log(`  Fetched § 13-61-${sectionNum}`);
+        }
+
+        if (sections.length >= 10) {
+          yield sections.splice(0, 10);
+        }
+      } catch (error) {
+        console.warn(`  Failed to fetch § 13-61-${sectionNum}, continuing...`, error);
+      }
+    }
+
+    if (sections.length > 0) {
+      yield sections;
+    }
+
+    if (totalCount < 10) {
+      throw new ScrapingError(`Expected at least 10 sections, got ${totalCount}`);
+    }
+  }
+
+  private parseSection($: cheerio.Root, sectionNum: string): Section | null {
+    // Utah uses a main content area
+    const content = $('body').text().trim();
+
+    if (!content || content.length < 200) {
+      return null;
+    }
+
+    // Extract title - format: "13-61-101 Definitions."
+    const titleMatch = content.match(/13-61-\d+\s+(.+?)\.(?:\n|$)/);
+    const title = titleMatch ? titleMatch[1].trim() : `Section 13-61-${sectionNum}`;
+
+    return {
+      sectionNumber: `13-61-${sectionNum}`,
+      title: title,
+      text: content,
+    };
+  }
+
+  private async sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async extractDefinitions(): Promise<Definition[]> {
+    return [];
+  }
+
+  async checkForUpdates(lastFetched: Date): Promise<UpdateStatus> {
+    return { hasChanges: false };
+  }
+}
+
+export function createUtahAdapter(): SourceAdapter {
+  return new UtahXcodeAdapter();
+}

package/src/ingest/adapters/virginia-law.ts

@@ -0,0 +1,140 @@
+/**
+ * Virginia Legislative Information System Adapter
+ *
+ * Fetches Virginia CDPA from Virginia LIS.
+ * Source: Va. Code Ann. § 59.1-575 to 59.1-585
+ */
+
+import {
+  SourceAdapter,
+  RegulationMetadata,
+  Section,
+  Definition,
+  UpdateStatus,
+} from '../framework.js';
+import * as cheerio from 'cheerio';
+
+class ScrapingError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'ScrapingError';
+  }
+}
+
+export class VirginiaLawAdapter implements SourceAdapter {
+  private readonly regulationId = 'VIRGINIA_CDPA';
+  private readonly sectionStart = 575;
+  private readonly sectionEnd = 585;
+
+  async fetchMetadata(): Promise<RegulationMetadata> {
+    return {
+      id: 'VIRGINIA_CDPA',
+      full_name: 'Virginia Consumer Data Protection Act',
+      citation: 'Va. Code Ann. §59.1-575 to 59.1-585',
+      effective_date: '2023-01-01',
+      last_amended: '2026-01-01',
+      source_url: 'https://law.lis.virginia.gov/vacode/title59.1/chapter53/',
+      jurisdiction: 'virginia',
+      regulation_type: 'statute',
+    };
+  }
+
+  async *fetchSections(): AsyncGenerator<Section[]> {
+    const sections: Section[] = [];
+    let totalCount = 0;
+
+    console.log(`Fetching Virginia CDPA sections ${this.sectionStart}-${this.sectionEnd}...`);
+
+    for (let num = this.sectionStart; num <= this.sectionEnd; num++) {
+      try {
+        const url = `https://law.lis.virginia.gov/vacode/title59.1/chapter53/section59.1-${num}/`;
+
+        await this.sleep(500); // Polite delay
+        const response = await fetch(url);
+        if (!response.ok) {
+          console.warn(`  Skipping § 59.1-${num} (HTTP ${response.status})`);
+          continue;
+        }
+
+        const html = await response.text();
+        const $ = cheerio.load(html);
+
+        this.validateDOM($);
+
+        const section = this.parseSection($, num);
+        if (section) {
+          sections.push(section);
+          totalCount++;
+          console.log(`  Fetched § 59.1-${num}`);
+        }
+
+        if (sections.length >= 10) {
+          yield sections.splice(0, 10);
+        }
+      } catch (error) {
+        if (error instanceof ScrapingError) {
+          throw error; // Fail fast on DOM issues
+        }
+        console.warn(`  Failed to fetch § 59.1-${num}, continuing...`, error);
+      }
+    }
+
+    if (sections.length > 0) {
+      yield sections;
+    }
+
+    // Validation
+    if (totalCount < 10) {
+      throw new ScrapingError(`Expected at least 10 sections, got ${totalCount}`);
+    }
+  }
+
+  private validateDOM($: cheerio.Root): void {
+    const codeContent = $('#va_code');
+    if (codeContent.length === 0) {
+      throw new ScrapingError('DOM structure changed: no #va_code element found');
+    }
+  }
+
+  private parseSection($: cheerio.Root, num: number): Section | null {
+    // Extract section text from #va_code
+    const textEl = $('#va_code');
+    const text = textEl.text().trim();
+
+    if (!text || text.length < 100) {
+      return null; // Skip sections with insufficient content
+    }
+
+    // Extract title from h2 within #va_code
+    const titleEl = textEl.find('h2').first();
+    const titleRaw = titleEl.text().trim();
+
+    // Parse title to extract just the description
+    // Format: "§ 59.1-575. (Effective January 1, 2026) Definitions."
+    const titleMatch = titleRaw.match(/§\s*59\.1-\d+\.?\s*(?:\([^)]+\))?\s*(.+?)\.?\s*$/);
+    const title = titleMatch ? titleMatch[1].trim() : titleRaw;
+
+    return {
+      sectionNumber: `59.1-${num}`,
+      title: title,
+      text: text,
+    };
+  }
+
+  private async sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async extractDefinitions(): Promise<Definition[]> {
+    // No separate definitions for Virginia CDPA
+    return [];
+  }
+
+  async checkForUpdates(lastFetched: Date): Promise<UpdateStatus> {
+    return { hasChanges: false };
+  }
+}
+
+export function createVirginiaAdapter(): SourceAdapter {
+  return new VirginiaLawAdapter();
+}

package/scripts/quality-test.ts

@@ -1,346 +0,0 @@
-#!/usr/bin/env npx tsx
-
-/**
- * Comprehensive Quality Test for US Compliance MCP
- * Tests various compliance queries to assess quality and coverage
- */
-
-import Database from 'better-sqlite3';
-import { join, dirname } from 'path';
-import { fileURLToPath } from 'url';
-import { searchRegulations } from '../src/tools/search.js';
-import { getSection } from '../src/tools/section.js';
-import { compareRequirements } from '../src/tools/compare.js';
-import { mapControls } from '../src/tools/map.js';
-import { checkApplicability } from '../src/tools/applicability.js';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-const DB_PATH = join(__dirname, '..', 'data', 'regulations.db');
-
-const db = new Database(DB_PATH, { readonly: true });
-
-interface TestCase {
-  name: string;
-  category: string;
-  test: () => Promise<any>;
-}
-
-const tests: TestCase[] = [];
-let passed = 0;
-let failed = 0;
-
-function addTest(category: string, name: string, test: () => Promise<any>) {
-  tests.push({ category, name, test });
-}
-
-// ===== HIPAA QUERIES =====
-addTest('HIPAA', 'Security rule access controls', async () => {
-  const result = await searchRegulations(db, {
-    query: 'access control security',
-    regulation: 'HIPAA',
-    limit: 5
-  });
-  if (!result || result.length === 0) throw new Error('No results');
-  console.log(`   📋 Found ${result.length} sections`);
-  console.log(`   📝 Top result: ${result[0].section} - ${result[0].title}`);
-  return result;
-});
-
-addTest('HIPAA', 'Audit logs and ePHI access', async () => {
-  const result = await searchRegulations(db, {
-    query: 'audit log electronic protected health',
-    regulation: 'HIPAA',
-    limit: 5
-  });
-  if (!result || result.length === 0) throw new Error('No results');
-  console.log(`   📋 Found ${result.length} sections`);
-  console.log(`   📝 Top result: ${result[0].section}`);
-  return result;
-});
-
-addTest('HIPAA', 'Encryption requirements', async () => {
-  const result = await searchRegulations(db, {
-    query: 'encryption transmission storage',
-    regulation: 'HIPAA',
-    limit: 5
-  });
-  if (!result || result.length === 0) throw new Error('No results');
-  console.log(`   📋 Found ${result.length} sections`);
-  console.log(`   📝 Top result: ${result[0].section}`);
-  return result;
-});
-
-addTest('HIPAA', 'Breach notification requirements', async () => {
-  const result = await searchRegulations(db, {
-    query: 'breach notification timeline',
-    regulation: 'HIPAA',
-    limit: 5
-  });
-  if (!result || result.length === 0) throw new Error('No results');
-  console.log(`   📋 Found ${result.length} sections`);
-  console.log(`   📝 Top result: ${result[0].section}`);
-  return result;
-});
-
-addTest('HIPAA', 'Risk assessment requirements', async () => {
-  const result = await searchRegulations(db, {
-    query: 'risk assessment analysis',
-    regulation: 'HIPAA',
-    limit: 5
-  });
-  if (!result || result.length === 0) throw new Error('No results');
-  console.log(`   📋 Found ${result.length} sections`);
-  console.log(`   📝 Top result: ${result[0].section}`);
-  return result;
-});
-
-// ===== CCPA QUERIES =====
-addTest('CCPA', 'Consumer rights to data deletion', async () => {
-  const result = await searchRegulations(db, {
-    query: 'consumer right delete personal information',
-    regulation: 'CCPA',
-    limit: 5
-  });
-  if (!result || result.length === 0) throw new Error('No results');
-  console.log(`   📋 Found ${result.length} sections`);
-  console.log(`   📝 Top result: ${result[0].section}`);
-  return result;
-});
-
-addTest('CCPA', 'Data disclosure requirements', async () => {
-  const result = await searchRegulations(db, {
-    query: 'disclosure categories personal information',
-    regulation: 'CCPA',
-    limit: 5
-  });
-  if (!result || result.length === 0) throw new Error('No results');
-  console.log(`   📋 Found ${result.length} sections`);
-  console.log(`   📝 Top result: ${result[0].section}`);
-  return result;
-});
-
-addTest('CCPA', 'Do Not Sell opt-out', async () => {
-  const result = await searchRegulations(db, {
-    query: 'do not sell opt-out',
-    regulation: 'CCPA',
-    limit: 5
-  });
-  if (!result || result.length === 0) throw new Error('No results');
-  console.log(`   📋 Found ${result.length} sections`);
-  console.log(`   📝 Top result: ${result[0].section}`);
-  return result;
-});
-
-// ===== SOX QUERIES =====
-addTest('SOX', 'Section 404 IT controls', async () => {
-  const result = await searchRegulations(db, {
-    query: 'internal control information technology',
-    regulation: 'SOX',
-    limit: 5
-  });
-  if (!result || result.length === 0) throw new Error('No results');
-  console.log(`   📋 Found ${result.length} sections`);
-  console.log(`   📝 Top result: ${result[0].section}`);
-  return result;
-});
-
-addTest('SOX', 'Financial records retention', async () => {
-  const result = await searchRegulations(db, {
-    query: 'retention records audit',
-    regulation: 'SOX',
-    limit: 5
-  });
-  if (!result || result.length === 0) throw new Error('No results');
-  console.log(`   📋 Found ${result.length} sections`);
-  console.log(`   📝 Top result: ${result[0].section}`);
-  return result;
-});
-
-// ===== CROSS-REGULATION QUERIES =====
-addTest('Cross-Regulation', 'Compare breach notification timelines', async () => {
-  const result = await compareRequirements(db, {
-    topic: 'breach notification timeline',
-    regulations: ['HIPAA', 'CCPA']
-  });
-  if (!result || !result.comparisons) throw new Error('No comparison results');
-  console.log(`   📊 Compared ${result.comparisons.length} regulations`);
-  for (const comp of result.comparisons) {
-    console.log(`   - ${comp.regulation}: ${comp.matches?.length || 0} matches`);
-  }
-  return result;
-});
-
-addTest('Cross-Regulation', 'Compare incident response requirements', async () => {
-  const result = await compareRequirements(db, {
-    topic: 'incident response',
-    regulations: ['HIPAA', 'CCPA', 'SOX']
-  });
-  if (!result || !result.comparisons) throw new Error('No comparison results');
-  console.log(`   📊 Compared ${result.comparisons.length} regulations`);
-  for (const comp of result.comparisons) {
-    console.log(`   - ${comp.regulation}: ${comp.matches?.length || 0} matches`);
-  }
-  return result;
-});
-
-addTest('Cross-Regulation', 'Data protection across all regulations', async () => {
-  const result = await compareRequirements(db, {
-    topic: 'data protection security',
-    regulations: ['HIPAA', 'CCPA', 'SOX']
-  });
-  if (!result || !result.comparisons) throw new Error('No comparison results');
-  console.log(`   📊 Compared ${result.comparisons.length} regulations`);
-  for (const comp of result.comparisons) {
-    console.log(`   - ${comp.regulation}: ${comp.matches?.length || 0} matches`);
-  }
-  return result;
-});
-
-// ===== CONTROL MAPPINGS =====
-addTest('Control Mappings', 'NIST 800-53 to HIPAA', async () => {
-  const result = await mapControls(db, {
-    framework: 'NIST_800_53_R5',
-    regulation: 'HIPAA'
-  });
-  if (!result.mappings || result.mappings.length === 0) throw new Error('No mappings');
-  console.log(`   🔗 Found ${result.mappings.length} mappings`);
-  console.log(`   📝 Sample: ${result.mappings[0].control_id} → ${result.mappings[0].regulation_sections?.slice(0, 3).join(', ')}`);
-  return result;
-});
-
-addTest('Control Mappings', 'NIST CSF to CCPA', async () => {
-  const result = await mapControls(db, {
-    framework: 'NIST_CSF_2_0',
-    regulation: 'CCPA'
-  });
-  if (!result.mappings || result.mappings.length === 0) throw new Error('No mappings');
-  console.log(`   🔗 Found ${result.mappings.length} mappings`);
-  console.log(`   📝 Sample: ${result.mappings[0].control_id}`);
-  return result;
-});
-
-// ===== APPLICABILITY CHECKS =====
-addTest('Applicability', 'Healthcare sector', async () => {
-  const result = await checkApplicability(db, { sector: 'healthcare' });
-  if (!result.applicable_regulations) throw new Error('No results');
-  console.log(`   ✅ ${result.applicable_regulations.length} applicable regulations`);
-  for (const reg of result.applicable_regulations) {
-    console.log(`   - ${reg.regulation}: ${reg.confidence}`);
-  }
-  return result;
-});
-
-addTest('Applicability', 'Financial services', async () => {
-  const result = await checkApplicability(db, { sector: 'financial' });
-  if (!result.applicable_regulations) throw new Error('No results');
-  console.log(`   ✅ ${result.applicable_regulations.length} applicable regulations`);
-  for (const reg of result.applicable_regulations) {
-    console.log(`   - ${reg.regulation}: ${reg.confidence}`);
-  }
-  return result;
-});
-
-addTest('Applicability', 'E-commerce (California)', async () => {
-  const result = await checkApplicability(db, {
-    sector: 'technology',
-    details: 'e-commerce selling to California consumers'
-  });
-  if (!result.applicable_regulations) throw new Error('No results');
-  console.log(`   ✅ ${result.applicable_regulations.length} applicable regulations`);
-  for (const reg of result.applicable_regulations) {
-    console.log(`   - ${reg.regulation}: ${reg.confidence}`);
-  }
-  return result;
-});
-
-// ===== SECTION RETRIEVAL =====
-addTest('Section Retrieval', 'HIPAA Security Rule 164.312', async () => {
-  const result = await getSection(db, { regulation: 'HIPAA', section: '164.312' });
-  if (!result || !result.text) throw new Error('Section not found');
-  console.log(`   📄 ${result.section_number}: ${result.title}`);
-  console.log(`   📊 Text length: ${result.text.length} chars`);
-  return result;
-});
-
-addTest('Section Retrieval', 'CCPA 1798.100', async () => {
-  const result = await getSection(db, { regulation: 'CCPA', section: '1798.100' });
-  if (!result || !result.text) throw new Error('Section not found');
-  console.log(`   📄 ${result.section_number}: ${result.title}`);
-  console.log(`   📊 Text length: ${result.text.length} chars`);
-  return result;
-});
-
-// ===== EDGE CASES =====
-addTest('Edge Cases', 'Empty query handling', async () => {
-  try {
-    await searchRegulations(db, { query: '', limit: 5 });
-    throw new Error('Should have rejected empty query');
-  } catch (error: any) {
-    if (error.message.includes('Query cannot be empty')) {
-      console.log('   ✅ Correctly rejected empty query');
-      return { success: true };
-    }
-    throw error;
-  }
-});
-
-addTest('Edge Cases', 'Invalid regulation ID', async () => {
-  const result = await searchRegulations(db, {
-    query: 'test',
-    regulation: 'INVALID',
-    limit: 5
-  });
-  // Should return empty or handle gracefully
-  console.log(`   ✅ Handled invalid regulation: ${result.length} results`);
-  return result;
-});
-
-// ===== RUN TESTS =====
-async function runAllTests() {
-  console.log('🔍 US Compliance MCP - Comprehensive Quality Assessment\n');
-  console.log('=' .repeat(70));
-
-  let currentCategory = '';
-
-  for (const testCase of tests) {
-    if (testCase.category !== currentCategory) {
-      currentCategory = testCase.category;
-      console.log(`\n📂 ${currentCategory.toUpperCase()}`);
-      console.log('-'.repeat(70));
-    }
-
-    try {
-      console.log(`\n✨ ${testCase.name}`);
-      await testCase.test();
-      console.log('   ✅ PASSED');
-      passed++;
-    } catch (error: any) {
-      console.log(`   ❌ FAILED: ${error.message}`);
-      failed++;
-    }
-  }
-
-  // Final Summary
-  console.log('\n' + '='.repeat(70));
-  console.log('📊 FINAL RESULTS');
-  console.log('='.repeat(70));
-  console.log(`Total Tests: ${tests.length}`);
-  console.log(`✅ Passed: ${passed} (${Math.round(passed/tests.length * 100)}%)`);
-  console.log(`❌ Failed: ${failed} (${Math.round(failed/tests.length * 100)}%)`);
-
-  if (failed === 0) {
-    console.log('\n🎉 All quality tests passed! MCP is production-ready.\n');
-  } else {
-    console.log(`\n⚠️  ${failed} test(s) failed. Review errors above.\n`);
-    process.exit(1);
-  }
-
-  db.close();
-}
-
-runAllTests().catch(error => {
-  console.error('Fatal error:', error);
-  db.close();
-  process.exit(1);
-});