@ansvar/us-regulations-mcp 1.0.0

package/scripts/build-db.ts

@@ -0,0 +1,268 @@
+#!/usr/bin/env npx tsx
+
+/**
+ * Build the regulations.db SQLite database for US regulations.
+ * Run with: npm run build:db
+ */
+
+import Database from 'better-sqlite3';
+import { readFileSync, existsSync, mkdirSync, unlinkSync, readdirSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const DATA_DIR = join(__dirname, '..', 'data');
+const SEED_DIR = join(DATA_DIR, 'seed');
+const DB_PATH = join(DATA_DIR, 'regulations.db');
+
+const SCHEMA = `
+-- Core regulation metadata
+CREATE TABLE IF NOT EXISTS regulations (
+  id TEXT PRIMARY KEY,
+  full_name TEXT NOT NULL,
+  short_name TEXT,
+  citation TEXT NOT NULL,
+  effective_date TEXT,
+  last_amended TEXT,
+  source_url TEXT,
+  jurisdiction TEXT,
+  regulation_type TEXT
+);
+
+-- Sections table (equivalent to articles in EU regulations)
+CREATE TABLE IF NOT EXISTS sections (
+  rowid INTEGER PRIMARY KEY,
+  regulation TEXT NOT NULL REFERENCES regulations(id),
+  section_number TEXT NOT NULL,
+  title TEXT,
+  text TEXT NOT NULL,
+  part TEXT,
+  subpart TEXT,
+  chapter TEXT,
+  parent_section TEXT,
+  cross_references TEXT,
+  UNIQUE(regulation, section_number)
+);
+
+-- FTS5 virtual table for full-text search with Porter stemming
+CREATE VIRTUAL TABLE IF NOT EXISTS sections_fts USING fts5(
+  regulation,
+  section_number,
+  title,
+  text,
+  content='sections',
+  content_rowid='rowid',
+  tokenize='porter unicode61'
+);
+
+-- FTS5 triggers
+CREATE TRIGGER IF NOT EXISTS sections_ai AFTER INSERT ON sections BEGIN
+  INSERT INTO sections_fts(rowid, regulation, section_number, title, text)
+  VALUES (new.rowid, new.regulation, new.section_number, new.title, new.text);
+END;
+
+CREATE TRIGGER IF NOT EXISTS sections_ad AFTER DELETE ON sections BEGIN
+  INSERT INTO sections_fts(sections_fts, rowid, regulation, section_number, title, text)
+  VALUES('delete', old.rowid, old.regulation, old.section_number, old.title, old.text);
+END;
+
+CREATE TRIGGER IF NOT EXISTS sections_au AFTER UPDATE ON sections BEGIN
+  INSERT INTO sections_fts(sections_fts, rowid, regulation, section_number, title, text)
+  VALUES('delete', old.rowid, old.regulation, old.section_number, old.title, old.text);
+  INSERT INTO sections_fts(rowid, regulation, section_number, title, text)
+  VALUES (new.rowid, new.regulation, new.section_number, new.title, new.text);
+END;
+
+-- Definitions
+CREATE TABLE IF NOT EXISTS definitions (
+  id INTEGER PRIMARY KEY,
+  regulation TEXT NOT NULL REFERENCES regulations(id),
+  term TEXT NOT NULL,
+  definition TEXT NOT NULL,
+  section TEXT NOT NULL,
+  UNIQUE(regulation, term)
+);
+
+-- Control mappings (e.g., to NIST, ISO, etc.)
+CREATE TABLE IF NOT EXISTS control_mappings (
+  id INTEGER PRIMARY KEY,
+  framework TEXT NOT NULL DEFAULT 'NIST_CSF',
+  control_id TEXT NOT NULL,
+  control_name TEXT NOT NULL,
+  regulation TEXT NOT NULL REFERENCES regulations(id),
+  sections TEXT NOT NULL,
+  coverage TEXT CHECK(coverage IN ('full', 'partial', 'related')),
+  notes TEXT,
+  confidence INTEGER,
+  generated_by TEXT,
+  UNIQUE(framework, control_id, regulation, sections)
+);
+
+-- Applicability rules (which sectors/industries each regulation applies to)
+CREATE TABLE IF NOT EXISTS applicability_rules (
+  id INTEGER PRIMARY KEY,
+  regulation TEXT NOT NULL REFERENCES regulations(id),
+  sector TEXT NOT NULL,
+  subsector TEXT,
+  applies INTEGER NOT NULL,
+  confidence TEXT CHECK(confidence IN ('definite', 'likely', 'possible')),
+  basis_section TEXT,
+  notes TEXT,
+  rationale TEXT,
+  UNIQUE(regulation, sector, subsector)
+);
+
+-- Source registry for tracking data quality
+CREATE TABLE IF NOT EXISTS source_registry (
+  regulation TEXT PRIMARY KEY REFERENCES regulations(id),
+  source_type TEXT NOT NULL,
+  source_url TEXT NOT NULL,
+  api_endpoint TEXT,
+  last_fetched TEXT,
+  sections_expected INTEGER,
+  sections_parsed INTEGER,
+  quality_status TEXT CHECK(quality_status IN ('complete', 'review', 'incomplete')),
+  notes TEXT
+);
+`;
+
+interface RegulationSeed {
+  id: string;
+  full_name: string;
+  short_name?: string;
+  citation: string;
+  effective_date?: string;
+  source_url?: string;
+  jurisdiction?: string;
+  regulation_type?: string;
+  sections: Array<{
+    number: string;
+    title?: string;
+    text: string;
+    part?: string;
+    subpart?: string;
+    chapter?: string;
+    parent_section?: string;
+    cross_references?: string[];
+  }>;
+  definitions?: Array<{
+    term: string;
+    definition: string;
+    section: string;
+  }>;
+}
+
+function buildDatabase() {
+  console.log('Building US regulations database...');
+
+  // Ensure data directory exists
+  if (!existsSync(DATA_DIR)) {
+    mkdirSync(DATA_DIR, { recursive: true });
+  }
+
+  // Delete existing database
+  if (existsSync(DB_PATH)) {
+    console.log('Removing existing database...');
+    unlinkSync(DB_PATH);
+  }
+
+  // Create new database
+  const db = new Database(DB_PATH);
+  db.pragma('foreign_keys = ON');
+
+  // Create schema
+  console.log('Creating schema...');
+  db.exec(SCHEMA);
+
+  // Load and insert seed files
+  if (existsSync(SEED_DIR)) {
+    const seedFiles = readdirSync(SEED_DIR).filter((f: string) => f.endsWith('.json'));
+
+    for (const file of seedFiles) {
+      if (file.startsWith('mappings')) continue;
+
+      console.log(`Loading ${file}...`);
+      const content = readFileSync(join(SEED_DIR, file), 'utf-8');
+      const regulation: RegulationSeed = JSON.parse(content);
+
+      // Insert regulation
+      db.prepare(`
+        INSERT INTO regulations (id, full_name, short_name, citation, effective_date, source_url, jurisdiction, regulation_type)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+      `).run(
+        regulation.id,
+        regulation.full_name,
+        regulation.short_name || null,
+        regulation.citation,
+        regulation.effective_date || null,
+        regulation.source_url || null,
+        regulation.jurisdiction || null,
+        regulation.regulation_type || null
+      );
+
+      // Insert sections
+      const insertSection = db.prepare(`
+        INSERT INTO sections (regulation, section_number, title, text, part, subpart, chapter, parent_section, cross_references)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+
+      for (const section of regulation.sections) {
+        insertSection.run(
+          regulation.id,
+          section.number,
+          section.title || null,
+          section.text,
+          section.part || null,
+          section.subpart || null,
+          section.chapter || null,
+          section.parent_section || null,
+          section.cross_references ? JSON.stringify(section.cross_references) : null
+        );
+      }
+
+      // Insert definitions
+      if (regulation.definitions) {
+        const insertDefinition = db.prepare(`
+          INSERT OR IGNORE INTO definitions (regulation, term, definition, section)
+          VALUES (?, ?, ?, ?)
+        `);
+
+        for (const def of regulation.definitions) {
+          insertDefinition.run(regulation.id, def.term, def.definition, def.section);
+        }
+      }
+
+      // Update source registry with timestamps
+      const now = new Date().toISOString();
+      const sourceType = regulation.source_url?.includes('api') ? 'api' : regulation.source_url?.includes('.pdf') ? 'pdf' : 'html';
+      db.prepare(`
+        INSERT INTO source_registry (regulation, source_type, source_url, api_endpoint, last_fetched, sections_expected, sections_parsed, quality_status)
+        VALUES (?, ?, ?, ?, ?, ?, ?, 'complete')
+      `).run(
+        regulation.id,
+        sourceType,
+        regulation.source_url || '',
+        regulation.source_url?.includes('api') ? regulation.source_url : null,
+        now,
+        regulation.sections.length,
+        regulation.sections.length
+      );
+
+      console.log(`  Loaded ${regulation.sections.length} sections, ${regulation.definitions?.length || 0} definitions`);
+    }
+
+    // Note: Control mappings and applicability rules are loaded separately
+    // using scripts/load-seed-data.ts after data ingestion
+    console.log('\nNote: Run "npm run load-seed" to load control mappings and applicability rules');
+  } else {
+    console.log('No seed directory found. Database created with empty tables.');
+    console.log(`Create seed files in: ${SEED_DIR}`);
+  }
+
+  db.close();
+  console.log(`\nDatabase created at: ${DB_PATH}`);
+}
+
+buildDatabase();

package/scripts/ingest.ts

@@ -0,0 +1,214 @@
+#!/usr/bin/env npx tsx
+
+/**
+ * Ingestion Orchestrator
+ *
+ * Coordinates all three regulation adapters (HIPAA, CCPA, SOX) to populate the database.
+ * Run with: npx tsx scripts/ingest.ts
+ */
+
+import Database from 'better-sqlite3';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { createHipaaAdapter } from '../src/ingest/adapters/ecfr.js';
+import { createCcpaAdapter } from '../src/ingest/adapters/california-leginfo.js';
+import { createSoxAdapter } from '../src/ingest/adapters/regulations-gov.js';
+import type { SourceAdapter } from '../src/ingest/framework.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const DB_PATH = join(__dirname, '..', 'data', 'regulations.db');
+
+interface IngestResult {
+  regulation: string;
+  success: boolean;
+  sections_added: number;
+  definitions_added: number;
+  error?: string;
+  duration_ms: number;
+}
+
+/**
+ * Ingest a single regulation
+ */
+async function ingestRegulation(
+  db: Database.Database,
+  regulationId: string,
+  adapter: SourceAdapter
+): Promise<IngestResult> {
+  const startTime = Date.now();
+
+  try {
+    console.log(`\n📥 Ingesting ${regulationId}...`);
+
+    let sectionsAdded = 0;
+    let definitionsAdded = 0;
+
+    // Fetch all data first (async operations)
+    console.log(`  Fetching metadata...`);
+    const metadata = await adapter.fetchMetadata();
+
+    console.log(`  Fetching sections...`);
+    const allSections: typeof sectionBatch = [];
+    for await (const sectionBatch of adapter.fetchSections()) {
+      allSections.push(...sectionBatch);
+      console.log(`    ${allSections.length} sections...`);
+    }
+
+    console.log(`  Fetching definitions...`);
+    const definitions = await adapter.extractDefinitions();
+
+    // Then insert everything in one synchronous transaction
+    console.log(`  Writing to database...`);
+    const insertTransaction = db.transaction(() => {
+      // 1. Insert regulation metadata
+      db.prepare(`
+        INSERT OR REPLACE INTO regulations
+        (id, full_name, short_name, citation, effective_date, last_amended, source_url, jurisdiction, regulation_type)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `).run(
+        metadata.id,
+        metadata.full_name,
+        null, // short_name
+        metadata.citation,
+        metadata.effective_date,
+        metadata.last_amended,
+        metadata.source_url,
+        metadata.jurisdiction,
+        metadata.regulation_type
+      );
+
+      // 2. Insert sections
+      const insertSection = db.prepare(`
+        INSERT OR REPLACE INTO sections
+        (regulation, section_number, title, text, part, subpart, chapter, parent_section, cross_references)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+
+      for (const section of allSections) {
+        insertSection.run(
+          regulationId,
+          section.sectionNumber,
+          section.title || null,
+          section.text,
+          null, // part (deprecated, using chapter)
+          null, // subpart (deprecated, using chapter)
+          section.chapter || null,
+          section.parentSection || null,
+          section.crossReferences ? JSON.stringify(section.crossReferences) : null
+        );
+        sectionsAdded++;
+      }
+
+      // 3. Insert definitions
+      const insertDef = db.prepare(`
+        INSERT OR REPLACE INTO definitions (regulation, term, definition, section)
+        VALUES (?, ?, ?, ?)
+      `);
+
+      for (const def of definitions) {
+        insertDef.run(regulationId, def.term, def.definition, def.section);
+        definitionsAdded++;
+      }
+
+      // 4. Update source registry
+      db.prepare(`
+        INSERT OR REPLACE INTO source_registry
+        (regulation, source_type, source_url, last_fetched, sections_expected, sections_parsed, quality_status)
+        VALUES (?, ?, ?, ?, ?, ?, ?)
+      `).run(
+        regulationId,
+        'api',
+        metadata.source_url,
+        new Date().toISOString(),
+        sectionsAdded,
+        sectionsAdded,
+        sectionsAdded > 0 ? 'complete' : 'incomplete'
+      );
+    });
+
+    // Execute transaction
+    insertTransaction();
+
+    const duration = Date.now() - startTime;
+
+    console.log(`✅ ${regulationId}: ${sectionsAdded} sections, ${definitionsAdded} definitions (${duration}ms)`);
+
+    return {
+      regulation: regulationId,
+      success: true,
+      sections_added: sectionsAdded,
+      definitions_added: definitionsAdded,
+      duration_ms: duration,
+    };
+  } catch (error) {
+    const duration = Date.now() - startTime;
+    const errorMessage = error instanceof Error ? error.message : String(error);
+
+    console.error(`❌ ${regulationId} failed: ${errorMessage}`);
+
+    return {
+      regulation: regulationId,
+      success: false,
+      sections_added: 0,
+      definitions_added: 0,
+      error: errorMessage,
+      duration_ms: duration,
+    };
+  }
+}
+
+/**
+ * Main ingestion function
+ */
+async function ingestAll(): Promise<IngestResult[]> {
+  console.log('🚀 Starting US Compliance MCP Ingestion...\n');
+  console.log(`Database: ${DB_PATH}`);
+
+  const db = Database(DB_PATH);
+  const results: IngestResult[] = [];
+
+  const adapters: Array<{ id: string; adapter: SourceAdapter }> = [
+    { id: 'HIPAA', adapter: createHipaaAdapter() },
+    { id: 'CCPA', adapter: createCcpaAdapter() },
+    { id: 'SOX', adapter: createSoxAdapter() },
+  ];
+
+  for (const { id, adapter } of adapters) {
+    const result = await ingestRegulation(db, id, adapter);
+    results.push(result);
+  }
+
+  db.close();
+
+  console.log('\n📊 Ingestion Summary:');
+  console.table(
+    results.map(r => ({
+      Regulation: r.regulation,
+      Status: r.success ? '✅ Success' : '❌ Failed',
+      Sections: r.sections_added,
+      Definitions: r.definitions_added,
+      'Duration (s)': (r.duration_ms / 1000).toFixed(2),
+      Error: r.error || '-',
+    }))
+  );
+
+  const totalSections = results.reduce((sum, r) => sum + r.sections_added, 0);
+  const successCount = results.filter(r => r.success).length;
+
+  console.log(`\n✨ Total: ${totalSections} sections from ${successCount}/${adapters.length} regulations`);
+
+  return results;
+}
+
+// Run if executed directly
+ingestAll()
+  .then(results => {
+    const allSuccess = results.every(r => r.success);
+    process.exit(allSuccess ? 0 : 1);
+  })
+  .catch(error => {
+    console.error('Fatal error:', error);
+    process.exit(1);
+  });

package/scripts/load-seed-data.ts

@@ -0,0 +1,133 @@
+#!/usr/bin/env npx tsx
+
+/**
+ * Load Seed Data
+ *
+ * Loads pre-generated control mappings and applicability rules into the database
+ * Run with: npx tsx scripts/load-seed-data.ts
+ */
+
+import Database from 'better-sqlite3';
+import { readFileSync, readdirSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const DB_PATH = join(__dirname, '..', 'data', 'regulations.db');
+const MAPPINGS_DIR = join(__dirname, '..', 'data', 'seed', 'mappings');
+const APPLICABILITY_DIR = join(__dirname, '..', 'data', 'seed', 'applicability');
+
+interface Mapping {
+  section_number: string;
+  control_id: string;
+  control_name: string;
+  coverage: 'full' | 'partial' | 'related';
+  confidence: number;
+  rationale: string;
+  function?: string;
+  category?: string;
+  subcategory?: string;
+}
+
+interface ApplicabilityRule {
+  regulation: string;
+  sector: string;
+  subsector: string | null;
+  confidence: 'definite' | 'likely' | 'possible';
+  rationale: string;
+}
+
+console.log('🌱 Loading seed data into database...\n');
+
+const db = Database(DB_PATH);
+
+// Load control mappings
+console.log('📊 Loading control mappings...');
+let totalMappings = 0;
+
+const mappingFiles = readdirSync(MAPPINGS_DIR).filter(f => f.endsWith('.json'));
+
+for (const file of mappingFiles) {
+  const filePath = join(MAPPINGS_DIR, file);
+  const data = JSON.parse(readFileSync(filePath, 'utf-8'));
+
+  console.log(`  ${file}: ${data.mappings.length} mappings`);
+
+  const insertMapping = db.prepare(`
+    INSERT OR REPLACE INTO control_mappings
+    (framework, control_id, control_name, regulation, sections, coverage, notes, confidence, generated_by)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+
+  db.transaction(() => {
+    for (const mapping of data.mappings as Mapping[]) {
+      // For NIST CSF, construct control_id from function/category/subcategory
+      const controlId = mapping.control_id ||
+        (mapping.function && mapping.category ?
+          `${mapping.function}.${mapping.category}${mapping.subcategory ? `.${mapping.subcategory}` : ''}` :
+          'UNKNOWN');
+
+      const controlName = mapping.control_name ||
+        (mapping.category ? mapping.category : 'Unknown');
+
+      insertMapping.run(
+        data.framework,
+        controlId,
+        controlName,
+        data.regulation,
+        JSON.stringify([mapping.section_number]),
+        mapping.coverage,
+        mapping.rationale,
+        mapping.confidence,
+        data.generated_by
+      );
+      totalMappings++;
+    }
+  })();
+}
+
+console.log(`✅ Loaded ${totalMappings} control mappings\n`);
+
+// Load applicability rules
+console.log('🎯 Loading applicability rules...');
+let totalRules = 0;
+
+const applicabilityFiles = readdirSync(APPLICABILITY_DIR).filter(f => f.endsWith('.json'));
+
+for (const file of applicabilityFiles) {
+  const filePath = join(APPLICABILITY_DIR, file);
+  const data = JSON.parse(readFileSync(filePath, 'utf-8'));
+
+  console.log(`  ${file}: ${data.rules.length} rules`);
+
+  const insertRule = db.prepare(`
+    INSERT OR REPLACE INTO applicability_rules
+    (regulation, sector, subsector, applies, confidence, rationale)
+    VALUES (?, ?, ?, ?, ?, ?)
+  `);
+
+  db.transaction(() => {
+    for (const rule of data.rules as ApplicabilityRule[]) {
+      insertRule.run(
+        rule.regulation,
+        rule.sector,
+        rule.subsector,
+        1, // applies = true
+        rule.confidence,
+        rule.rationale
+      );
+      totalRules++;
+    }
+  })();
+}
+
+console.log(`✅ Loaded ${totalRules} applicability rules\n`);
+
+db.close();
+
+console.log('🎉 Seed data loading complete!');
+console.log(`\nSummary:`);
+console.log(`  Control mappings: ${totalMappings}`);
+console.log(`  Applicability rules: ${totalRules}`);