@ansvar/us-regulations-mcp 1.0.0 → 1.2.2

package/src/ingest/adapters/connecticut-cga.ts

@@ -0,0 +1,150 @@
+/**
+ * Connecticut General Assembly Adapter
+ *
+ * Fetches Connecticut CTDPA from Connecticut General Assembly website.
+ * Source: Conn. Gen. Stat. § 42-515 to 42-524
+ *
+ * NOTE: This adapter parses a SINGLE page containing ALL sections,
+ * unlike Virginia/Colorado which fetch individual section pages.
+ */
+
+import {
+  SourceAdapter,
+  RegulationMetadata,
+  Section,
+  Definition,
+  UpdateStatus,
+} from '../framework.js';
+import * as cheerio from 'cheerio';
+import https from 'https';
+
+class ScrapingError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'ScrapingError';
+  }
+}
+
+export class ConnecticutCGAAdapter implements SourceAdapter {
+  private readonly regulationId = 'CONNECTICUT_CTDPA';
+  private readonly chapterUrl =
+    'https://www.cga.ct.gov/current/pub/chap_743jj.htm';
+
+  async fetchMetadata(): Promise<RegulationMetadata> {
+    return {
+      id: 'CONNECTICUT_CTDPA',
+      full_name: 'Connecticut Data Privacy Act',
+      citation: 'Conn. Gen. Stat. §42-515 to 42-524',
+      effective_date: '2023-07-01',
+      last_amended: '2023-07-01',
+      source_url: this.chapterUrl,
+      jurisdiction: 'connecticut',
+      regulation_type: 'statute',
+    };
+  }
+
+  async *fetchSections(): AsyncGenerator<Section[]> {
+    console.log('Fetching Connecticut CTDPA from single chapter page...');
+
+    // Connecticut's certificate has SSL issues, use https module directly
+    const html = await this.fetchWithHttps(this.chapterUrl);
+    const $ = cheerio.load(html);
+
+    const sections = this.parseSections($);
+
+    if (sections.length < 8) {
+      throw new ScrapingError(
+        `Expected at least 8 sections, got ${sections.length}`
+      );
+    }
+
+    console.log(`  ✅ Parsed ${sections.length} sections from chapter page`);
+    yield sections;
+  }
+
+  private async fetchWithHttps(url: string): Promise<string> {
+    return new Promise((resolve, reject) => {
+      https.get(url, { rejectUnauthorized: false }, (res) => {
+        if (res.statusCode !== 200) {
+          reject(new ScrapingError(`HTTP ${res.statusCode}`));
+          return;
+        }
+
+        let data = '';
+        res.on('data', (chunk) => data += chunk);
+        res.on('end', () => resolve(data));
+        res.on('error', reject);
+      }).on('error', reject);
+    });
+  }
+
+  private parseSections($: cheerio.Root): Section[] {
+    const sections: Section[] = [];
+
+    // Find all paragraphs containing section headers (span.catchln)
+    $('span.catchln[id^="sec_42-"]').each((_idx, elem) => {
+      const $span = $(elem);
+      const id = $span.attr('id');
+
+      if (!id) return;
+
+      // Extract section number from id: "sec_42-515" -> "515"
+      const idMatch = id.match(/sec_42-(\d+)/);
+      if (!idMatch) return;
+
+      const sectionNum = idMatch[1];
+
+      // Extract title from span text: "Sec. 42-515. Definitions." -> "Definitions"
+      const spanText = $span.text().trim();
+      const titleMatch = spanText.match(/^Sec\.\s+42-\d+\.\s+(.+?)\.?\s*$/);
+      const title = titleMatch ? titleMatch[1].trim() : `Section 42-${sectionNum}`;
+
+      // Get the parent paragraph and all following paragraphs until next section
+      const $startPara = $span.parent();
+      let fullText = $startPara.text().trim();
+
+      // Collect following paragraphs until we hit another section marker
+      let $next = $startPara.next('p');
+      while ($next.length > 0) {
+        // Stop if this paragraph contains another section marker
+        if ($next.find('span.catchln[id^="sec_42-"]').length > 0) {
+          break;
+        }
+
+        const nextText = $next.text().trim();
+        if (nextText.length > 0) {
+          fullText += '\n\n' + nextText;
+        }
+
+        $next = $next.next('p');
+      }
+
+      // Only include sections in the 42-515 to 42-526 range (CTDPA)
+      const num = parseInt(sectionNum);
+      if (num >= 515 && num <= 526) {
+        sections.push({
+          sectionNumber: `42-${sectionNum}`,
+          title: title,
+          text: fullText,
+        });
+
+        console.log(`  Found § 42-${sectionNum}: ${title}`);
+      }
+    });
+
+    return sections;
+  }
+
+  async extractDefinitions(): Promise<Definition[]> {
+    // Definitions are in section 42-515, could be extracted if needed
+    return [];
+  }
+
+  async checkForUpdates(lastFetched: Date): Promise<UpdateStatus> {
+    return { hasChanges: false };
+  }
+}
+
+export function createConnecticutAdapter(): SourceAdapter {
+  return new ConnecticutCGAAdapter();
+}

package/src/ingest/adapters/ecfr.ts

@@ -17,6 +17,17 @@ import {
 } from '../framework.js';
 import { XMLParser } from 'fast-xml-parser';
 
+/**
+ * Configuration for regulation metadata fallback
+ */
+export interface RegulationConfig {
+  full_name: string;
+  citation: string;
+  effective_date: string;
+  jurisdiction: 'federal' | 'state';
+  regulation_type: 'rule' | 'statute';
+}
+
 /**
  * Adapter for fetching HIPAA from eCFR API
  */
@@ -24,31 +35,85 @@ export class EcfrAdapter implements SourceAdapter {
   private readonly regulationId: string;
   private readonly cfr_title: number;
   private readonly cfr_parts: number[];
-
-  constructor(regulationId: string, cfr_title: number, cfr_parts: number[]) {
+  private readonly fallbackMetadata: RegulationConfig;
+
+  constructor(
+    regulationId: string,
+    cfr_title: number,
+    cfr_parts: number[],
+    fallbackMetadata: RegulationConfig
+  ) {
     this.regulationId = regulationId;
     this.cfr_title = cfr_title;
     this.cfr_parts = cfr_parts;
+    this.fallbackMetadata = fallbackMetadata;
   }
 
   /**
-   * Fetch HIPAA metadata
+   * Fetch metadata from eCFR API
+   * Returns effective_date and last_amended from API with fallback metadata for other fields
+   */
+  private async fetchMetadataFromAPI(): Promise<RegulationMetadata> {
+    try {
+      // Get latest available date for this CFR title
+      const latestDate = await this.getLatestDate();
+
+      // Fetch structure JSON (lighter than full XML)
+      const url = `https://www.ecfr.gov/api/versioner/v1/structure/${latestDate}/title-${this.cfr_title}.json`;
+      const response = await fetch(url);
+
+      if (!response.ok) {
+        throw new Error(`eCFR API returned ${response.status} for ${url}`);
+      }
+
+      const data = await response.json();
+
+      // Extract dates from API response
+      // Note: eCFR structure API may not have these exact fields, using latest date as proxy
+      const effectiveDate = data.effective_date || latestDate;
+      const lastAmended = data.last_amended || latestDate;
+
+      return {
+        id: this.regulationId,
+        full_name: this.fallbackMetadata.full_name,
+        citation: this.fallbackMetadata.citation,
+        effective_date: effectiveDate,
+        last_amended: lastAmended,
+        source_url: `https://www.ecfr.gov/current/title-${this.cfr_title}`,
+        jurisdiction: this.fallbackMetadata.jurisdiction,
+        regulation_type: this.fallbackMetadata.regulation_type
+      };
+    } catch (error) {
+      // Re-throw to be caught by fetchMetadata
+      throw error;
+    }
+  }
+
+  /**
+   * Fetch regulation metadata
    *
-   * PLACEHOLDER: Returns hardcoded HIPAA metadata
-   * TODO: Integrate with eCFR API to fetch live metadata
+   * Hybrid approach:
+   * 1. Try to fetch effective_date and last_amended from eCFR API
+   * 2. Fall back to hardcoded metadata if API unavailable
    */
   async fetchMetadata(): Promise<RegulationMetadata> {
-    // Placeholder metadata for HIPAA
-    return {
-      id: this.regulationId,
-      full_name: 'Health Insurance Portability and Accountability Act',
-      citation: '45 CFR Parts 160, 162, 164',
-      effective_date: '2003-04-14',
-      last_amended: '2013-01-25',
-      source_url: 'https://www.ecfr.gov/current/title-45',
-      jurisdiction: 'federal',
-      regulation_type: 'rule',
-    };
+    try {
+      return await this.fetchMetadataFromAPI();
+    } catch (error) {
+      console.warn(`⚠️  eCFR API unavailable for ${this.regulationId}, using fallback metadata`);
+      console.warn(`   Error: ${error instanceof Error ? error.message : String(error)}`);
+
+      return {
+        id: this.regulationId,
+        full_name: this.fallbackMetadata.full_name,
+        citation: this.fallbackMetadata.citation,
+        effective_date: this.fallbackMetadata.effective_date,
+        last_amended: this.fallbackMetadata.effective_date, // Use effective_date as fallback
+        source_url: `https://www.ecfr.gov/current/title-${this.cfr_title}`,
+        jurisdiction: this.fallbackMetadata.jurisdiction,
+        regulation_type: this.fallbackMetadata.regulation_type
+      };
+    }
   }
 
   /**
@@ -399,5 +464,81 @@ export class EcfrAdapter implements SourceAdapter {
  * Factory function to create HIPAA adapter
  */
 export function createHipaaAdapter(): EcfrAdapter {
-  return new EcfrAdapter('HIPAA', 45, [160, 162, 164]);
+  return new EcfrAdapter('HIPAA', 45, [160, 162, 164], {
+    full_name: 'Health Insurance Portability and Accountability Act',
+    citation: '45 CFR Parts 160, 162, 164',
+    effective_date: '2003-04-14',
+    jurisdiction: 'federal',
+    regulation_type: 'rule'
+  });
+}
+
+/**
+ * Factory function to create GLBA adapter
+ * Gramm-Leach-Bliley Act - Safeguards Rule
+ */
+export function createGlbaAdapter(): EcfrAdapter {
+  return new EcfrAdapter('GLBA', 16, [314], {
+    full_name: 'Gramm-Leach-Bliley Act - Safeguards Rule',
+    citation: '16 CFR Part 314',
+    effective_date: '2003-05-23',
+    jurisdiction: 'federal',
+    regulation_type: 'rule'
+  });
+}
+
+/**
+ * Factory function to create FERPA adapter
+ * Family Educational Rights and Privacy Act
+ */
+export function createFerpaAdapter(): EcfrAdapter {
+  return new EcfrAdapter('FERPA', 34, [99], {
+    full_name: 'Family Educational Rights and Privacy Act',
+    citation: '34 CFR Part 99',
+    effective_date: '2009-01-03',
+    jurisdiction: 'federal',
+    regulation_type: 'rule'
+  });
+}
+
+/**
+ * Factory function to create COPPA adapter
+ * Children's Online Privacy Protection Act Rule
+ */
+export function createCoppaAdapter(): EcfrAdapter {
+  return new EcfrAdapter('COPPA', 16, [312], {
+    full_name: "Children's Online Privacy Protection Act Rule",
+    citation: '16 CFR Part 312',
+    effective_date: '2013-07-01',
+    jurisdiction: 'federal',
+    regulation_type: 'rule'
+  });
+}
+
+/**
+ * Factory function to create FDA 21 CFR Part 11 adapter
+ * Electronic Records and Electronic Signatures
+ */
+export function createFdaAdapter(): EcfrAdapter {
+  return new EcfrAdapter('FDA_CFR_11', 21, [11], {
+    full_name: 'FDA Electronic Records and Electronic Signatures',
+    citation: '21 CFR Part 11',
+    effective_date: '1997-08-20',
+    jurisdiction: 'federal',
+    regulation_type: 'rule'
+  });
+}
+
+/**
+ * Factory function to create EPA RMP adapter
+ * Risk Management Plan Rule
+ */
+export function createEpaRmpAdapter(): EcfrAdapter {
+  return new EcfrAdapter('EPA_RMP', 40, [68], {
+    full_name: 'EPA Risk Management Plan Rule',
+    citation: '40 CFR Part 68',
+    effective_date: '2017-01-13',
+    jurisdiction: 'federal',
+    regulation_type: 'rule'
+  });
 }

package/src/ingest/adapters/ffiec.ts

@@ -0,0 +1,77 @@
+/**
+ * FFIEC Adapter - Seed-based for v1.1
+ * TODO: Implement PDF scraping for automated updates
+ *
+ * Source: FFIEC IT Examination Handbook
+ * URL: https://www.ffiec.gov/examination.htm
+ */
+
+import { SourceAdapter, RegulationMetadata, Section, Definition, UpdateStatus } from '../framework.js';
+import { readFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+/**
+ * Seed-based adapter for FFIEC Guidelines
+ *
+ * Future enhancement: Implement PDF scraping from ffiec.gov
+ * The IT Examination Handbook is published as a collection of PDF booklets
+ */
+export class FfiecAdapter implements SourceAdapter {
+  private seedPath: string;
+
+  constructor() {
+    this.seedPath = join(__dirname, '../../../data/seed/ffiec.json');
+  }
+
+  /**
+   * Fetch regulation metadata from seed file
+   */
+  async fetchMetadata(): Promise<RegulationMetadata> {
+    const seed = JSON.parse(readFileSync(this.seedPath, 'utf-8'));
+    return {
+      ...seed.regulation,
+      source_url: 'https://www.ffiec.gov/examination.htm',
+      last_amended: seed.regulation.effective_date
+    };
+  }
+
+  /**
+   * Fetch sections from seed file
+   * Returns all sections in a single batch
+   */
+  async *fetchSections(): AsyncGenerator<Section[]> {
+    const seed = JSON.parse(readFileSync(this.seedPath, 'utf-8'));
+    yield seed.sections;
+  }
+
+  /**
+   * Extract definitions (not implemented for seed-based adapter)
+   * Future: Extract from PDF glossary sections
+   */
+  async extractDefinitions(): Promise<Definition[]> {
+    return [];
+  }
+
+  /**
+   * Check for updates (not implemented for seed-based adapter)
+   * Future: Check PDF publication dates on ffiec.gov
+   */
+  async checkForUpdates(lastFetched: Date): Promise<UpdateStatus> {
+    return {
+      hasChanges: false,
+      lastModified: new Date(),
+      changes: []
+    };
+  }
+}
+
+/**
+ * Factory function to create FFIEC adapter
+ */
+export function createFfiecAdapter(): FfiecAdapter {
+  return new FfiecAdapter();
+}

package/src/ingest/adapters/nydfs.ts

@@ -0,0 +1,77 @@
+/**
+ * NYDFS Adapter - Seed-based for v1.1
+ * TODO: Implement HTML scraping for automated updates
+ *
+ * Source: NY DFS Cybersecurity Regulation (23 NYCRR 500)
+ * URL: https://www.dfs.ny.gov/industry_guidance/cybersecurity
+ */
+
+import { SourceAdapter, RegulationMetadata, Section, Definition, UpdateStatus } from '../framework.js';
+import { readFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+/**
+ * Seed-based adapter for NYDFS 23 NYCRR 500
+ *
+ * Future enhancement: Implement HTML scraping from NY DFS website
+ * The regulation is published online at dfs.ny.gov
+ */
+export class NydfsAdapter implements SourceAdapter {
+  private seedPath: string;
+
+  constructor() {
+    this.seedPath = join(__dirname, '../../../data/seed/nydfs.json');
+  }
+
+  /**
+   * Fetch regulation metadata from seed file
+   */
+  async fetchMetadata(): Promise<RegulationMetadata> {
+    const seed = JSON.parse(readFileSync(this.seedPath, 'utf-8'));
+    return {
+      ...seed.regulation,
+      source_url: 'https://www.dfs.ny.gov/industry_guidance/cybersecurity',
+      last_amended: seed.regulation.effective_date
+    };
+  }
+
+  /**
+   * Fetch sections from seed file
+   * Returns all sections in a single batch
+   */
+  async *fetchSections(): AsyncGenerator<Section[]> {
+    const seed = JSON.parse(readFileSync(this.seedPath, 'utf-8'));
+    yield seed.sections;
+  }
+
+  /**
+   * Extract definitions (not implemented for seed-based adapter)
+   * Future: Extract from 500.01 Definitions section
+   */
+  async extractDefinitions(): Promise<Definition[]> {
+    return [];
+  }
+
+  /**
+   * Check for updates (not implemented for seed-based adapter)
+   * Future: Check last-modified headers from NY DFS website
+   */
+  async checkForUpdates(lastFetched: Date): Promise<UpdateStatus> {
+    return {
+      hasChanges: false,
+      lastModified: new Date(),
+      changes: []
+    };
+  }
+}
+
+/**
+ * Factory function to create NYDFS adapter
+ */
+export function createNydfsAdapter(): NydfsAdapter {
+  return new NydfsAdapter();
+}

package/src/ingest/adapters/regulations-gov.ts

@@ -1,11 +1,14 @@
 /**
  * SOX Adapter
  *
- * Fetches SOX regulations from eCFR (for SEC implementing rules).
- * Source: 17 CFR Part 229 (Regulation S-K, Item 308) and Part 240 (Exchange Act Rules)
+ * Provides Sarbanes-Oxley Act compliance content including:
+ * - SOX statute sections (15 U.S.C., 18 U.S.C.)
+ * - SEC implementing regulations (17 CFR)
+ * - PCAOB auditing standards
+ * - IT General Controls guidance
  *
  * PRODUCTION IMPLEMENTATION
- * Uses eCFR API for SEC regulations implementing Sarbanes-Oxley Section 404
+ * Uses comprehensive seed data verified against official sources
  */
 
 import {
@@ -15,22 +18,25 @@ import {
   Definition,
   UpdateStatus,
 } from '../framework.js';
-import { XMLParser } from 'fast-xml-parser';
-import { EcfrAdapter } from './ecfr.js';
+import * as fs from 'fs';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
 
 /**
- * Adapter for fetching SOX regulations from eCFR
+ * Adapter for SOX compliance content
  *
- * Uses eCFR API for SEC regulations implementing Sarbanes-Oxley
+ * Provides comprehensive SOX coverage including statute, SEC rules, and PCAOB standards
  */
 export class SoxAdapter implements SourceAdapter {
   private readonly regulationId: string;
-  private readonly ecfrAdapter: EcfrAdapter;
+  private readonly seedPath: string;
 
   constructor(regulationId: string) {
     this.regulationId = regulationId;
-    // Use eCFR adapter for Title 17 (SEC regulations)
-    this.ecfrAdapter = new EcfrAdapter('SOX-SEC', 17, [229, 240]);
+    this.seedPath = path.join(__dirname, '../../../data/seed/sox.json');
   }
 
   /**
@@ -39,68 +45,63 @@ export class SoxAdapter implements SourceAdapter {
   async fetchMetadata(): Promise<RegulationMetadata> {
     return {
       id: this.regulationId,
-      full_name: 'Sarbanes-Oxley Act - SEC Implementing Regulations',
-      citation: '17 CFR Parts 229, 240 (Regulation S-K Item 308, Exchange Act Rules)',
-      effective_date: '2003-06-05',
-      last_amended: new Date().toISOString().split('T')[0],
-      source_url: 'https://www.ecfr.gov/current/title-17',
+      full_name: 'Sarbanes-Oxley Act of 2002',
+      citation: 'Pub.L. 107-204, 15 U.S.C. §§ 7201-7266, 17 CFR Parts 229, 240',
+      effective_date: '2002-07-30',
+      last_amended: '2023-01-01',
+      source_url: 'https://www.sec.gov/spotlight/sarbanes-oxley.htm',
       jurisdiction: 'federal',
-      regulation_type: 'rule',
+      regulation_type: 'statute',
     };
   }
 
   /**
-   * Fetch all SOX-related sections from eCFR
+   * Fetch all SOX sections from seed data
    *
-   * Fetches 17 CFR Parts 229 and 240, filtering to SOX-relevant sections
+   * Includes statute sections, SEC implementing rules, and PCAOB standards
    */
   async *fetchSections(): AsyncGenerator<Section[]> {
-    console.log('Fetching SOX sections from eCFR (Title 17)...');
+    console.log('Loading SOX sections from seed data...');
+
+    try {
+      const seedData = JSON.parse(fs.readFileSync(this.seedPath, 'utf-8'));
 
-    // Key SOX-related sections:
-    // - 17 CFR 229.308 (Item 308: Internal control over financial reporting)
-    // - 17 CFR 240.13a-15 (Controls and procedures)
-    // - 17 CFR 240.15d-15 (Controls and procedures)
-    // - 17 CFR 240.13a-14 (Certifications)
-    // - 17 CFR 240.15d-14 (Certifications)
+      if (!seedData.sections || !Array.isArray(seedData.sections)) {
+        throw new Error('Invalid seed data format');
+      }
 
-    const relevantSections = [
-      '229.308',
-      '240.13a-15',
-      '240.15d-15',
-      '240.13a-14',
-      '240.15d-14',
-    ];
+      const sections: Section[] = seedData.sections.map((s: any) => ({
+        sectionNumber: s.sectionNumber,
+        title: s.title,
+        text: s.text,
+        chapter: s.chapter || 'Sarbanes-Oxley Act',
+      }));
 
-    // Fetch from eCFR adapter
-    for await (const sectionBatch of this.ecfrAdapter.fetchSections()) {
-      // Filter to SOX-relevant sections
-      const filtered = sectionBatch.filter(section =>
-        relevantSections.some(relevant => section.sectionNumber.includes(relevant))
-      );
+      console.log(`  Loaded ${sections.length} SOX sections from seed data`);
 
-      if (filtered.length > 0) {
-        yield filtered;
-      }
+      yield sections;
+    } catch (error) {
+      console.error('Failed to load SOX seed data:', error);
+      throw error;
     }
   }
 
   /**
    * Check for updates since last fetch
-   *
-   * Delegates to eCFR adapter for update checking
    */
   async checkForUpdates(lastFetched: Date): Promise<UpdateStatus> {
-    return this.ecfrAdapter.checkForUpdates(lastFetched);
+    // SOX statute is stable; SEC rules update periodically
+    return {
+      hasChanges: false,
+      changes: ['SOX uses verified seed data. Check SEC.gov for regulatory updates.']
+    };
   }
 
   /**
    * Extract definitions from SOX sections
-   *
-   * Future enhancement: Parse definitions from SEC regulations
    */
   async extractDefinitions(): Promise<Definition[]> {
-    return this.ecfrAdapter.extractDefinitions();
+    return [];
   }
 }