@ansvar/us-regulations-mcp 1.0.0

package/src/ingest/adapters/ecfr.ts

@@ -0,0 +1,403 @@
+/**
+ * eCFR Adapter (Electronic Code of Federal Regulations)
+ *
+ * Fetches HIPAA regulations from ecfr.gov API.
+ * Source: 45 CFR Parts 160, 162, 164 (Privacy Rule, Security Rule, Breach Notification)
+ *
+ * PRODUCTION IMPLEMENTATION
+ * Uses the eCFR API: https://www.ecfr.gov/developers/documentation/api/v1
+ */
+
+import {
+  SourceAdapter,
+  RegulationMetadata,
+  Section,
+  Definition,
+  UpdateStatus,
+} from '../framework.js';
+import { XMLParser } from 'fast-xml-parser';
+
+/**
+ * Adapter for fetching HIPAA from eCFR API
+ */
+export class EcfrAdapter implements SourceAdapter {
+  private readonly regulationId: string;
+  private readonly cfr_title: number;
+  private readonly cfr_parts: number[];
+
+  constructor(regulationId: string, cfr_title: number, cfr_parts: number[]) {
+    this.regulationId = regulationId;
+    this.cfr_title = cfr_title;
+    this.cfr_parts = cfr_parts;
+  }
+
+  /**
+   * Fetch HIPAA metadata
+   *
+   * PLACEHOLDER: Returns hardcoded HIPAA metadata
+   * TODO: Integrate with eCFR API to fetch live metadata
+   */
+  async fetchMetadata(): Promise<RegulationMetadata> {
+    // Placeholder metadata for HIPAA
+    return {
+      id: this.regulationId,
+      full_name: 'Health Insurance Portability and Accountability Act',
+      citation: '45 CFR Parts 160, 162, 164',
+      effective_date: '2003-04-14',
+      last_amended: '2013-01-25',
+      source_url: 'https://www.ecfr.gov/current/title-45',
+      jurisdiction: 'federal',
+      regulation_type: 'rule',
+    };
+  }
+
+  /**
+   * Fetch all HIPAA sections
+   *
+   * Fetches XML from eCFR API and parses sections with hierarchical structure
+   * API endpoint: https://www.ecfr.gov/api/versioner/v1/full/{date}/title-{title}.xml
+   */
+  async *fetchSections(): AsyncGenerator<Section[]> {
+    // Get the latest available date for this title
+    const date = await this.getLatestDate();
+    const url = `https://www.ecfr.gov/api/versioner/v1/full/${date}/title-${this.cfr_title}.xml`;
+
+    console.log(`Fetching eCFR Title ${this.cfr_title} from ${url}...`);
+
+    // Fetch XML with retry logic
+    const response = await this.fetchWithRetry(url);
+    const xmlText = await response.text();
+
+    // Parse XML
+    const parser = new XMLParser({
+      ignoreAttributes: false,
+      attributeNamePrefix: '@_',
+      textNodeName: '#text',
+      preserveOrder: false,
+    });
+
+    const xmlDoc = parser.parse(xmlText);
+
+    // Extract sections from relevant parts
+    const sections: Section[] = [];
+
+    // Navigate XML structure: ECFR → DIV1 (Title) → DIV5 (Part) → DIV8 (Section)
+    // The structure is: DIV1 (title) → DIV2 (subtitle) → DIV4 (subchapter) → DIV5 (part) → DIV8 (section)
+    const title = xmlDoc.ECFR?.DIV1;
+    if (!title) {
+      console.warn(`No title found in eCFR XML for Title ${this.cfr_title}`);
+      return;
+    }
+
+    // Find all DIV5 (parts) recursively
+    const findParts = (node: any): any[] => {
+      if (!node || typeof node !== 'object') return [];
+
+      const parts: any[] = [];
+
+      if (node.DIV5) {
+        const div5s = Array.isArray(node.DIV5) ? node.DIV5 : [node.DIV5];
+        parts.push(...div5s.filter(Boolean));
+      }
+
+      // Recurse into child DIVs
+      for (const key of Object.keys(node)) {
+        if (key.startsWith('DIV') && key !== 'DIV5' && typeof node[key] === 'object') {
+          const childNodes = Array.isArray(node[key]) ? node[key] : [node[key]];
+          for (const child of childNodes) {
+            parts.push(...findParts(child));
+          }
+        }
+      }
+
+      return parts;
+    };
+
+    const parts = findParts(title);
+    console.log(`  Found ${parts.length} parts in Title ${this.cfr_title}`);
+
+    for (const part of parts) {
+      const partNum = this.extractNumber(part['@_N']);
+
+      // Only process our target parts
+      if (!this.cfr_parts.includes(partNum)) {
+        continue;
+      }
+
+      console.log(`  Processing Part ${partNum}...`);
+
+      // Sections are in DIV6 (subparts) → DIV8 (sections)
+      const findSections = (node: any): any[] => {
+        if (!node || typeof node !== 'object') return [];
+
+        const secs: any[] = [];
+
+        // If this node has DIV8, collect them
+        if (node.DIV8) {
+          const div8s = Array.isArray(node.DIV8) ? node.DIV8 : [node.DIV8];
+          secs.push(...div8s.filter(Boolean));
+        }
+
+        // Recurse into DIV6 (subparts) and other child DIVs
+        for (const key of Object.keys(node)) {
+          if (key.startsWith('DIV') && key !== 'DIV8' && typeof node[key] === 'object') {
+            const childNodes = Array.isArray(node[key]) ? node[key] : [node[key]];
+            for (const child of childNodes) {
+              secs.push(...findSections(child));
+            }
+          }
+        }
+
+        return secs;
+      };
+
+      const sectionDivs = findSections(part);
+      console.log(`    Found ${sectionDivs.length} sections in Part ${partNum}`);
+
+      for (const sectionDiv of sectionDivs) {
+        const section = this.parseSection(sectionDiv, partNum, '');
+        if (section) {
+          sections.push(section);
+        }
+      }
+
+      // Yield in batches of 50
+      if (sections.length >= 50) {
+        yield sections.splice(0, 50);
+      }
+    }
+
+    // Yield remaining sections
+    if (sections.length > 0) {
+      yield sections;
+    }
+  }
+
+  /**
+   * Parse a section from XML DIV8 element
+   */
+  private parseSection(div: any, partNum: number, subpartId: string): Section | null {
+    const sectionNum = div['@_N'];
+    if (!sectionNum) return null;
+
+    // Extract section text
+    const text = this.extractText(div);
+    if (!text || text.length < 10) return null;
+
+    // Extract title (HEAD)
+    const title = div.HEAD ? this.extractText(div.HEAD) : undefined;
+
+    // Build section number - eCFR @_N already includes part number (e.g., "164.308")
+    // Don't prepend if section already starts with part number
+    const fullSectionNum = sectionNum.startsWith(`${partNum}.`)
+      ? sectionNum
+      : `${partNum}.${sectionNum}`;
+
+    // Extract cross-references (CITA tags)
+    const crossReferences = this.extractCrossReferences(div);
+
+    return {
+      sectionNumber: fullSectionNum,
+      title,
+      text,
+      chapter: `Part ${partNum}, Subpart ${subpartId}`,
+      parentSection: undefined, // TODO: implement parent detection for nested sections
+      crossReferences: crossReferences.length > 0 ? crossReferences : undefined,
+    };
+  }
+
+  /**
+   * Extract text content from XML element recursively
+   */
+  private extractText(element: any): string {
+    if (typeof element === 'string') {
+      return element.trim();
+    }
+
+    if (typeof element === 'object') {
+      if (element['#text']) {
+        const text = element['#text'];
+        return typeof text === 'string' ? text.trim() : String(text).trim();
+      }
+
+      // Recursively extract from child elements
+      const texts: string[] = [];
+      for (const key of Object.keys(element)) {
+        if (key.startsWith('@_')) continue; // Skip attributes
+        if (key === 'HEAD') continue; // Skip title
+
+        const child = element[key];
+        if (Array.isArray(child)) {
+          for (const item of child) {
+            const text = this.extractText(item);
+            if (text) texts.push(text);
+          }
+        } else {
+          const text = this.extractText(child);
+          if (text) texts.push(text);
+        }
+      }
+      return texts.join(' ').trim();
+    }
+
+    return '';
+  }
+
+  /**
+   * Extract cross-references from CITA tags
+   */
+  private extractCrossReferences(element: any): string[] {
+    const refs: string[] = [];
+
+    const findCITA = (obj: any) => {
+      if (typeof obj !== 'object') return;
+
+      for (const key of Object.keys(obj)) {
+        if (key === 'CITA') {
+          const cita = obj[key];
+          if (Array.isArray(cita)) {
+            refs.push(...cita.map(c => this.extractText(c)).filter(Boolean));
+          } else {
+            const ref = this.extractText(cita);
+            if (ref) refs.push(ref);
+          }
+        } else if (typeof obj[key] === 'object') {
+          findCITA(obj[key]);
+        }
+      }
+    };
+
+    findCITA(element);
+    return refs;
+  }
+
+  /**
+   * Extract numeric part from section number
+   */
+  private extractNumber(str: string | undefined): number {
+    if (!str) return 0;
+    const match = str.match(/\d+/);
+    return match ? parseInt(match[0]) : 0;
+  }
+
+  /**
+   * Get the latest available date for this title from eCFR titles API
+   */
+  private async getLatestDate(): Promise<string> {
+    try {
+      const response = await fetch('https://www.ecfr.gov/api/versioner/v1/titles');
+      const data = await response.json();
+
+      const titleInfo = data.titles.find((t: any) => t.number === this.cfr_title);
+      if (titleInfo && titleInfo.latest_issue_date) {
+        return titleInfo.latest_issue_date;
+      }
+
+      // Fallback to current date
+      return new Date().toISOString().split('T')[0];
+    } catch (error) {
+      console.warn('Failed to fetch latest date, using current date:', error);
+      return new Date().toISOString().split('T')[0];
+    }
+  }
+
+  /**
+   * Fetch with retry logic and exponential backoff
+   */
+  private async fetchWithRetry(url: string, maxRetries = 3): Promise<Response> {
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+      try {
+        const response = await fetch(url);
+
+        if (response.status === 429) {
+          // Rate limited - backoff
+          const delay = Math.min(1000 * 2 ** attempt + Math.random() * 1000, 30000);
+          console.warn(`Rate limited, retrying in ${delay}ms...`);
+          await new Promise(resolve => setTimeout(resolve, delay));
+          continue;
+        }
+
+        if (!response.ok) {
+          throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+        }
+
+        return response;
+      } catch (error) {
+        if (attempt === maxRetries - 1) throw error;
+        const delay = 1000 * 2 ** attempt;
+        console.warn(`Fetch failed, retrying in ${delay}ms...`, error);
+        await new Promise(resolve => setTimeout(resolve, delay));
+      }
+    }
+
+    throw new Error('Max retries exceeded');
+  }
+
+  /**
+   * Check for updates since last fetch
+   *
+   * Queries eCFR API for revision dates and compares with lastFetched
+   * eCFR updates daily from Federal Register
+   */
+  async checkForUpdates(lastFetched: Date): Promise<UpdateStatus> {
+    try {
+      // eCFR provides last-modified in HTTP headers
+      const date = new Date().toISOString().split('T')[0];
+      const url = `https://www.ecfr.gov/api/versioner/v1/full/${date}/title-${this.cfr_title}.xml`;
+
+      const response = await fetch(url, { method: 'HEAD' });
+      const lastModifiedHeader = response.headers.get('last-modified');
+
+      if (!lastModifiedHeader) {
+        console.warn('No last-modified header from eCFR');
+        return {
+          hasChanges: false,
+          lastModified: new Date(),
+          changes: [],
+        };
+      }
+
+      const lastModified = new Date(lastModifiedHeader);
+
+      return {
+        hasChanges: lastModified > lastFetched,
+        lastModified,
+        changes: lastModified > lastFetched
+          ? [`Title ${this.cfr_title} updated on ${lastModified.toISOString()}`]
+          : [],
+      };
+    } catch (error) {
+      console.error('Error checking for updates:', error);
+      return {
+        hasChanges: false,
+        lastModified: new Date(),
+        changes: [],
+      };
+    }
+  }
+
+  /**
+   * Extract definitions from HIPAA sections
+   *
+   * Future enhancement: Parse definition sections (e.g., 45 CFR 160.103, 164.103)
+   * For now, returns empty array - definitions can be added manually if needed
+   */
+  async extractDefinitions(): Promise<Definition[]> {
+    // TODO: Implement definition extraction
+    // HIPAA definitions are in:
+    // - 45 CFR 160.103 (General definitions)
+    // - 45 CFR 164.103 (Security Rule definitions)
+    // - 45 CFR 164.501 (Privacy Rule definitions)
+    //
+    // These sections have structured format: "Term means definition."
+    // Would need regex parsing or NLP to extract accurately
+    return [];
+  }
+}
+
+/**
+ * Factory function to create HIPAA adapter
+ */
+export function createHipaaAdapter(): EcfrAdapter {
+  return new EcfrAdapter('HIPAA', 45, [160, 162, 164]);
+}

package/src/ingest/adapters/regulations-gov.ts

@@ -0,0 +1,112 @@
+/**
+ * SOX Adapter
+ *
+ * Fetches SOX regulations from eCFR (for SEC implementing rules).
+ * Source: 17 CFR Part 229 (Regulation S-K, Item 308) and Part 240 (Exchange Act Rules)
+ *
+ * PRODUCTION IMPLEMENTATION
+ * Uses eCFR API for SEC regulations implementing Sarbanes-Oxley Section 404
+ */
+
+import {
+  SourceAdapter,
+  RegulationMetadata,
+  Section,
+  Definition,
+  UpdateStatus,
+} from '../framework.js';
+import { XMLParser } from 'fast-xml-parser';
+import { EcfrAdapter } from './ecfr.js';
+
+/**
+ * Adapter for fetching SOX regulations from eCFR
+ *
+ * Uses eCFR API for SEC regulations implementing Sarbanes-Oxley
+ */
+export class SoxAdapter implements SourceAdapter {
+  private readonly regulationId: string;
+  private readonly ecfrAdapter: EcfrAdapter;
+
+  constructor(regulationId: string) {
+    this.regulationId = regulationId;
+    // Use eCFR adapter for Title 17 (SEC regulations)
+    this.ecfrAdapter = new EcfrAdapter('SOX-SEC', 17, [229, 240]);
+  }
+
+  /**
+   * Fetch SOX metadata
+   */
+  async fetchMetadata(): Promise<RegulationMetadata> {
+    return {
+      id: this.regulationId,
+      full_name: 'Sarbanes-Oxley Act - SEC Implementing Regulations',
+      citation: '17 CFR Parts 229, 240 (Regulation S-K Item 308, Exchange Act Rules)',
+      effective_date: '2003-06-05',
+      last_amended: new Date().toISOString().split('T')[0],
+      source_url: 'https://www.ecfr.gov/current/title-17',
+      jurisdiction: 'federal',
+      regulation_type: 'rule',
+    };
+  }
+
+  /**
+   * Fetch all SOX-related sections from eCFR
+   *
+   * Fetches 17 CFR Parts 229 and 240, filtering to SOX-relevant sections
+   */
+  async *fetchSections(): AsyncGenerator<Section[]> {
+    console.log('Fetching SOX sections from eCFR (Title 17)...');
+
+    // Key SOX-related sections:
+    // - 17 CFR 229.308 (Item 308: Internal control over financial reporting)
+    // - 17 CFR 240.13a-15 (Controls and procedures)
+    // - 17 CFR 240.15d-15 (Controls and procedures)
+    // - 17 CFR 240.13a-14 (Certifications)
+    // - 17 CFR 240.15d-14 (Certifications)
+
+    const relevantSections = [
+      '229.308',
+      '240.13a-15',
+      '240.15d-15',
+      '240.13a-14',
+      '240.15d-14',
+    ];
+
+    // Fetch from eCFR adapter
+    for await (const sectionBatch of this.ecfrAdapter.fetchSections()) {
+      // Filter to SOX-relevant sections
+      const filtered = sectionBatch.filter(section =>
+        relevantSections.some(relevant => section.sectionNumber.includes(relevant))
+      );
+
+      if (filtered.length > 0) {
+        yield filtered;
+      }
+    }
+  }
+
+  /**
+   * Check for updates since last fetch
+   *
+   * Delegates to eCFR adapter for update checking
+   */
+  async checkForUpdates(lastFetched: Date): Promise<UpdateStatus> {
+    return this.ecfrAdapter.checkForUpdates(lastFetched);
+  }
+
+  /**
+   * Extract definitions from SOX sections
+   *
+   * Future enhancement: Parse definitions from SEC regulations
+   */
+  async extractDefinitions(): Promise<Definition[]> {
+    return this.ecfrAdapter.extractDefinitions();
+  }
+}
+
+/**
+ * Factory function to create SOX adapter
+ */
+export function createSoxAdapter(): SoxAdapter {
+  return new SoxAdapter('SOX');
+}

package/src/ingest/framework.ts

@@ -0,0 +1,92 @@
+/**
+ * Ingestion Framework
+ *
+ * Defines interfaces for regulation data ingestion from multiple sources.
+ * Supports automated fetching from APIs (regulations.gov, ecfr.gov, California LegInfo).
+ */
+
+/**
+ * Metadata about a regulation
+ */
+export interface RegulationMetadata {
+  id: string;                       // 'HIPAA', 'CCPA', 'SOX'
+  full_name: string;                // 'Health Insurance Portability and Accountability Act'
+  citation: string;                 // 'Pub. L. 104-191' or 'Cal. Civ. Code § 1798.100'
+  effective_date?: string;          // ISO 8601 date string
+  last_amended?: string;            // ISO 8601 date string
+  source_url: string;               // API endpoint or official source URL
+  jurisdiction: string;             // 'federal', 'california', 'virginia', etc.
+  regulation_type: string;          // 'statute', 'rule', 'guidance'
+}
+
+/**
+ * A single section within a regulation
+ */
+export interface Section {
+  sectionNumber: string;            // '164.308(a)(1)(ii)(A)' for HIPAA
+  title?: string;                   // Section title/heading
+  text: string;                     // Full text content
+  chapter?: string;                 // Chapter or part designation
+  parentSection?: string;           // Parent section for nested structures
+  crossReferences?: string[];       // Array of referenced section numbers
+}
+
+/**
+ * A term definition from a regulation
+ */
+export interface Definition {
+  regulation: string;               // Regulation ID
+  term: string;                     // Defined term
+  definition: string;               // Full definition text
+  section: string;                  // Section where defined
+}
+
+/**
+ * Status of potential updates from source
+ */
+export interface UpdateStatus {
+  hasChanges: boolean;              // Whether updates detected
+  lastModified?: Date;              // Last modification date from source
+  changes?: string[];               // Description of changes (if available)
+  sectionsAdded?: number;           // Count of new sections
+  sectionsModified?: number;        // Count of modified sections
+}
+
+/**
+ * Source adapter interface
+ *
+ * Each regulation source (regulations.gov, ecfr.gov, etc.) implements this interface
+ * to provide normalized access to regulation data.
+ */
+export interface SourceAdapter {
+  /**
+   * Fetch regulation metadata
+   */
+  fetchMetadata(): Promise<RegulationMetadata>;
+
+  /**
+   * Fetch all sections with pagination support
+   * Returns an async generator for memory-efficient streaming
+   */
+  fetchSections(): AsyncGenerator<Section[]>;
+
+  /**
+   * Check if source has updates since last fetch
+   */
+  checkForUpdates(lastFetched: Date): Promise<UpdateStatus>;
+
+  /**
+   * Extract definitions from regulation text
+   */
+  extractDefinitions(): Promise<Definition[]>;
+}
+
+/**
+ * Regulation source configuration
+ */
+export interface RegulationSource {
+  id: string;                       // 'HIPAA', 'CCPA', 'SOX'
+  name: string;                     // Full regulation name
+  sourceType: 'api' | 'html' | 'pdf';
+  adapter: SourceAdapter;
+}