@ansvar/eu-regulations-mcp 0.8.0 → 1.1.0

package/scripts/bulk-reingest-all.ts

@@ -1,331 +0,0 @@
-#!/usr/bin/env npx tsx
-
-/**
- * Bulk re-ingestion script for all 37 EU regulations.
- *
- * Uses Puppeteer-based browser ingestion to bypass EUR-Lex WAF.
- * Processes regulations in batches of 3 parallel browser instances
- * with 2s delays between batches for rate limiting.
- *
- * Usage:
- *   npx tsx scripts/bulk-reingest-all.ts
- *   npx tsx scripts/bulk-reingest-all.ts --dry-run  # Preview without executing
- *
- * SECURITY NOTE: Uses execFile (not exec) to prevent command injection.
- */
-
-import { execFile } from 'child_process';
-import { promisify } from 'util';
-import { readdir } from 'fs/promises';
-import { join, basename } from 'path';
-
-const execFileAsync = promisify(execFile);
-
-interface Regulation {
-  celexId: string;
-  filename: string;
-  filepath: string;
-}
-
-interface IngestionResult {
-  regulation: string;
-  success: boolean;
-  error?: string;
-  duration?: number;
-}
-
-// Configuration
-const BATCH_SIZE = 3; // Parallel browser instances
-const BATCH_DELAY_MS = 2000; // Rate limiting between batches
-const TIMEOUT_MS = 120000; // 2 minutes per regulation
-
-/**
- * Discover all regulation JSON files in data/seed/
- */
-async function discoverRegulations(): Promise<Regulation[]> {
-  const seedDir = join(process.cwd(), 'data', 'seed');
-  const files = await readdir(seedDir);
-
-  const regulations: Regulation[] = [];
-
-  for (const file of files) {
-    if (!file.endsWith('.json')) continue;
-
-    const filepath = join(seedDir, file);
-    const filename = basename(file, '.json');
-
-    // Read JSON to extract CELEX ID
-    try {
-      const { readFileSync } = await import('fs');
-      const content = readFileSync(filepath, 'utf-8');
-      const data = JSON.parse(content);
-
-      if (data.celex_id) {
-        regulations.push({
-          celexId: data.celex_id,
-          filename,
-          filepath,
-        });
-      } else {
-        console.warn(`⚠️  Warning: ${file} has no celex_id field`);
-      }
-    } catch (err) {
-      console.warn(`⚠️  Warning: Failed to parse ${file}:`, (err as Error).message);
-    }
-  }
-
-  return regulations.sort((a, b) => a.filename.localeCompare(b.filename));
-}
-
-/**
- * Ingest a single regulation using the appropriate script
- */
-async function ingestRegulation(regulation: Regulation): Promise<IngestionResult> {
-  const startTime = Date.now();
-
-  try {
-    // Determine which ingestion script to use
-    const isUNECE = regulation.celexId.startsWith('42021X');
-    const scriptName = isUNECE ? 'ingest-unece.ts' : 'ingest-eurlex.ts';
-    const scriptPath = join(process.cwd(), 'scripts', scriptName);
-
-    // SECURITY: Using execFile (not exec) to prevent command injection
-    const args = ['tsx', scriptPath, regulation.celexId, regulation.filepath];
-
-    // Add --browser flag for EUR-Lex regulations to bypass WAF
-    if (!isUNECE) {
-      args.push('--browser');
-    }
-
-    const { stdout, stderr } = await execFileAsync(
-      'npx',
-      args,
-      {
-        timeout: TIMEOUT_MS,
-        cwd: process.cwd(),
-        maxBuffer: 10 * 1024 * 1024, // 10MB buffer for large outputs
-      }
-    );
-
-    const duration = Date.now() - startTime;
-
-    // Log any warnings from stderr
-    if (stderr) {
-      console.log(`  [stderr] ${stderr.trim()}`);
-    }
-
-    return {
-      regulation: regulation.filename,
-      success: true,
-      duration,
-    };
-  } catch (err: any) {
-    const duration = Date.now() - startTime;
-
-    return {
-      regulation: regulation.filename,
-      success: false,
-      error: err.message || String(err),
-      duration,
-    };
-  }
-}
-
-/**
- * Process regulations in batches with rate limiting
- */
-async function processBatches(regulations: Regulation[]): Promise<IngestionResult[]> {
-  const results: IngestionResult[] = [];
-  const totalBatches = Math.ceil(regulations.length / BATCH_SIZE);
-
-  for (let i = 0; i < regulations.length; i += BATCH_SIZE) {
-    const batch = regulations.slice(i, i + BATCH_SIZE);
-    const batchNum = Math.floor(i / BATCH_SIZE) + 1;
-
-    console.log(`\n📦 Batch ${batchNum}/${totalBatches} (${batch.length} regulations)`);
-    console.log('─'.repeat(60));
-
-    // Process batch in parallel
-    const batchPromises = batch.map(async (reg, idx) => {
-      const regNum = i + idx + 1;
-      console.log(`[${regNum}/${regulations.length}] Starting: ${reg.filename} (${reg.celexId})`);
-
-      const result = await ingestRegulation(reg);
-
-      if (result.success) {
-        console.log(`[${regNum}/${regulations.length}] ✅ ${reg.filename} (${result.duration}ms)`);
-      } else {
-        console.log(`[${regNum}/${regulations.length}] ❌ ${reg.filename}: ${result.error}`);
-      }
-
-      return result;
-    });
-
-    const batchResults = await Promise.all(batchPromises);
-    results.push(...batchResults);
-
-    // Rate limiting: delay between batches (except after last batch)
-    if (i + BATCH_SIZE < regulations.length) {
-      console.log(`\n⏸️  Waiting ${BATCH_DELAY_MS}ms before next batch...`);
-      await new Promise(resolve => setTimeout(resolve, BATCH_DELAY_MS));
-    }
-  }
-
-  return results;
-}
-
-/**
- * Print summary report
- */
-function printSummary(results: IngestionResult[]) {
-  console.log('\n' + '═'.repeat(60));
-  console.log('📊 INGESTION SUMMARY');
-  console.log('═'.repeat(60));
-
-  const successful = results.filter(r => r.success);
-  const failed = results.filter(r => !r.success);
-
-  console.log(`\n✅ Successful: ${successful.length}/${results.length}`);
-  console.log(`❌ Failed: ${failed.length}/${results.length}`);
-
-  if (successful.length > 0) {
-    const totalDuration = successful.reduce((sum, r) => sum + (r.duration || 0), 0);
-    const avgDuration = totalDuration / successful.length;
-    console.log(`⏱️  Average duration: ${Math.round(avgDuration)}ms`);
-  }
-
-  if (failed.length > 0) {
-    console.log('\n❌ Failed regulations:');
-    for (const result of failed) {
-      console.log(`  - ${result.regulation}: ${result.error}`);
-    }
-  }
-
-  console.log('\n' + '═'.repeat(60));
-}
-
-/**
- * Rebuild database and verify recitals
- */
-async function rebuildDatabase() {
-  console.log('\n🔨 Rebuilding database...');
-
-  try {
-    const { stdout } = await execFileAsync('npm', ['run', 'build:db'], {
-      cwd: process.cwd(),
-      timeout: 60000, // 1 minute timeout
-    });
-
-    console.log(stdout);
-    console.log('✅ Database rebuilt successfully');
-  } catch (err: any) {
-    console.error('❌ Database rebuild failed:', err.message);
-    throw err;
-  }
-}
-
-/**
- * Query and display recital counts
- */
-async function verifyRecitals() {
-  console.log('\n📊 Verifying recital counts...');
-
-  try {
-    const { stdout } = await execFileAsync(
-      'sqlite3',
-      [
-        'data/regulations.db',
-        'SELECT regulation, COUNT(*) as recital_count FROM recitals GROUP BY regulation ORDER BY recital_count DESC LIMIT 10;',
-      ],
-      { cwd: process.cwd() }
-    );
-
-    console.log('\nTop 10 regulations by recital count:');
-    console.log(stdout);
-
-    // Get total count
-    const { stdout: totalStdout } = await execFileAsync(
-      'sqlite3',
-      ['data/regulations.db', 'SELECT COUNT(*) FROM recitals;'],
-      { cwd: process.cwd() }
-    );
-
-    const totalRecitals = parseInt(totalStdout.trim());
-    console.log(`\n📈 Total recitals in database: ${totalRecitals}`);
-
-    if (totalRecitals < 2000) {
-      console.warn(`⚠️  Warning: Expected ~2,500+ recitals, got ${totalRecitals}`);
-    } else {
-      console.log('✅ Recital count looks good!');
-    }
-  } catch (err: any) {
-    console.error('❌ Verification failed:', err.message);
-  }
-}
-
-/**
- * Main execution
- */
-async function main() {
-  const isDryRun = process.argv.includes('--dry-run');
-
-  console.log('🚀 EU Regulations Bulk Re-Ingestion');
-  console.log('═'.repeat(60));
-  console.log(`Mode: ${isDryRun ? 'DRY RUN' : 'LIVE'}`);
-  console.log(`Batch size: ${BATCH_SIZE} parallel instances`);
-  console.log(`Batch delay: ${BATCH_DELAY_MS}ms`);
-  console.log(`Timeout per regulation: ${TIMEOUT_MS}ms`);
-  console.log('═'.repeat(60));
-
-  // Step 1: Discover regulations
-  console.log('\n🔍 Discovering regulations...');
-  const regulations = await discoverRegulations();
-  console.log(`Found ${regulations.length} regulations in data/seed/`);
-
-  if (regulations.length === 0) {
-    console.error('❌ No regulations found. Exiting.');
-    process.exit(1);
-  }
-
-  // List regulations
-  console.log('\nRegulations to process:');
-  for (const reg of regulations) {
-    const type = reg.celexId.startsWith('42021X') ? '[UNECE]' : '[EUR-Lex]';
-    console.log(`  ${type} ${reg.filename} (${reg.celexId})`);
-  }
-
-  if (isDryRun) {
-    console.log('\n✅ Dry run complete. No ingestion performed.');
-    return;
-  }
-
-  // Confirm before starting
-  console.log('\n⚠️  This will re-ingest all regulations using browser automation.');
-  console.log(`⏱️  Estimated time: ${Math.ceil((regulations.length / BATCH_SIZE) * (BATCH_DELAY_MS / 1000))} seconds + ingestion time`);
-
-  // Step 2: Process batches
-  const results = await processBatches(regulations);
-
-  // Step 3: Print summary
-  printSummary(results);
-
-  // Step 4: Rebuild database
-  if (results.some(r => r.success)) {
-    await rebuildDatabase();
-
-    // Step 5: Verify recitals
-    await verifyRecitals();
-  } else {
-    console.error('\n❌ No successful ingestions. Skipping database rebuild.');
-  }
-
-  // Exit with appropriate code
-  const hasFailures = results.some(r => !r.success);
-  process.exit(hasFailures ? 1 : 0);
-}
-
-// Run main
-main().catch(err => {
-  console.error('💥 Fatal error:', err);
-  process.exit(1);
-});

package/scripts/check-updates.ts

@@ -1,294 +0,0 @@
-#!/usr/bin/env npx tsx
-
-/**
- * Check for updates to EU regulations from EUR-Lex.
- * Compares current database versions against EUR-Lex metadata.
- *
- * Usage: npx tsx scripts/check-updates.ts
- */
-
-import Database from 'better-sqlite3';
-import { existsSync } from 'fs';
-import { join, dirname } from 'path';
-import { fileURLToPath } from 'url';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-
-const DB_PATH = join(__dirname, '..', 'data', 'regulations.db');
-
-interface SourceRecord {
-  regulation: string;
-  celex_id: string;
-  eur_lex_version: string | null;
-  last_fetched: string | null;
-  articles_expected: number | null;
-  articles_parsed: number | null;
-  quality_status: string;
-}
-
-interface EurLexMetadata {
-  celexId: string;
-  lastModified: string;
-  title: string;
-  dateDocument: string;
-  consolidatedVersions?: string[];
-}
-
-// No hardcoded list - source_registry table IS the source of truth
-// To add a new regulation: ingest it, and it's automatically monitored
-
-async function fetchEurLexMetadata(celexId: string): Promise<EurLexMetadata | null> {
-  const infoUrl = `https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:${celexId}`;
-
-  try {
-    const response = await fetch(infoUrl, {
-      headers: {
-        'User-Agent': 'Mozilla/5.0 (compatible; EU-Compliance-MCP/1.0)',
-        'Accept': 'text/html',
-      },
-    });
-
-    if (!response.ok) {
-      console.error(`Failed to fetch metadata for ${celexId}: ${response.status}`);
-      return null;
-    }
-
-    const html = await response.text();
-
-    // Try multiple extraction methods in order of reliability:
-
-    // 1. ELI metadata (works for all document types including UNECE)
-    const eliDateDoc = html.match(/property="eli:date_document"[^>]*content="(\d{4}-\d{2}-\d{2})"/);
-    const eliDatePub = html.match(/property="eli:date_publication"[^>]*content="(\d{4}-\d{2}-\d{2})"/);
-    // Also try the reverse attribute order
-    const eliDateDoc2 = html.match(/content="(\d{4}-\d{2}-\d{2})"[^>]*property="eli:date_document"/);
-    const eliDatePub2 = html.match(/content="(\d{4}-\d{2}-\d{2})"[^>]*property="eli:date_publication"/);
-
-    // 2. Visible text patterns
-    const dateMatch = html.match(/Date of document:\s*(\d{2}\/\d{2}\/\d{4})/i);
-
-    // 3. Generic ELI date pattern (fallback)
-    const genericEli = html.match(/eli[^>]*(\d{4}-\d{2}-\d{2})/i);
-
-    const titleMatch = html.match(/<title>([^<]+)<\/title>/i);
-
-    // Use the best available date (prefer publication date for tracking updates)
-    const lastModified =
-      eliDatePub?.[1] || eliDatePub2?.[1] ||
-      eliDateDoc?.[1] || eliDateDoc2?.[1] ||
-      genericEli?.[1] ||
-      (dateMatch?.[1] ? convertDateFormat(dateMatch[1]) : null) ||
-      'unknown';
-
-    return {
-      celexId,
-      lastModified,
-      title: titleMatch?.[1]?.trim() || 'Unknown',
-      dateDocument: eliDateDoc?.[1] || eliDateDoc2?.[1] || 'unknown',
-    };
-  } catch (error) {
-    console.error(`Error fetching metadata for ${celexId}:`, error);
-    return null;
-  }
-}
-
-// Convert DD/MM/YYYY to YYYY-MM-DD
-function convertDateFormat(date: string): string {
-  const parts = date.split('/');
-  if (parts.length === 3) {
-    return `${parts[2]}-${parts[1]}-${parts[0]}`;
-  }
-  return date;
-}
-
-// Sync mode: update database with current EUR-Lex versions
-async function syncVersions(): Promise<void> {
-  console.log('Syncing EUR-Lex versions to database...\n');
-
-  if (!existsSync(DB_PATH)) {
-    console.log('Database not found. Run `npm run build:db` first.');
-    process.exit(1);
-  }
-
-  const db = new Database(DB_PATH);
-
-  const sources = db.prepare(`
-    SELECT regulation, celex_id FROM source_registry
-    WHERE celex_id IS NOT NULL AND celex_id != ''
-  `).all() as SourceRecord[];
-
-  const updateStmt = db.prepare(`
-    UPDATE source_registry SET eur_lex_version = ?, last_fetched = ?
-    WHERE regulation = ?
-  `);
-
-  const now = new Date().toISOString();
-  let updated = 0;
-
-  for (const source of sources) {
-    process.stdout.write(`${source.regulation}: `);
-    const metadata = await fetchEurLexMetadata(source.celex_id);
-
-    if (metadata && metadata.lastModified !== 'unknown') {
-      updateStmt.run(metadata.lastModified, now, source.regulation);
-      console.log(`synced to ${metadata.lastModified}`);
-      updated++;
-    } else {
-      console.log('skipped (unknown version)');
-    }
-  }
-
-  db.close();
-  console.log(`\n✓ Synced ${updated} regulation(s)`);
-}
-
-async function checkForUpdates(): Promise<void> {
-  console.log('Checking EUR-Lex for regulation updates...\n');
-
-  // Check if database exists
-  if (!existsSync(DB_PATH)) {
-    console.log('Database not found. Run `npm run build:db` first.');
-    process.exit(1);
-  }
-
-  const db = new Database(DB_PATH, { readonly: true });
-
-  // Get all regulations from source_registry - this IS the source of truth
-  const sources = db.prepare(`
-    SELECT regulation, celex_id, eur_lex_version, last_fetched, quality_status
-    FROM source_registry
-    WHERE celex_id IS NOT NULL AND celex_id != ''
-    ORDER BY regulation
-  `).all() as SourceRecord[];
-
-  if (sources.length === 0) {
-    console.log('No regulations found in source_registry.');
-    console.log('Ingest regulations first with: npx tsx scripts/ingest-eurlex.ts <CELEX_ID> <output.json>');
-    db.close();
-    process.exit(0);
-  }
-
-  console.log(`Found ${sources.length} regulation(s) to check\n`);
-  console.log('Status Report');
-  console.log('='.repeat(80));
-
-  const updates: Array<{ id: string; celex_id: string; reason: string }> = [];
-
-  for (const source of sources) {
-    process.stdout.write(`\n${source.regulation.padEnd(20)} (${source.celex_id}): `);
-
-    // Fetch current EUR-Lex metadata
-    const metadata = await fetchEurLexMetadata(source.celex_id);
-
-    if (!metadata) {
-      console.log('FETCH FAILED');
-      continue;
-    }
-
-    const lastFetched = source.last_fetched || 'never';
-    const eurLexVersion = metadata.lastModified;
-
-    // Helper to compare dates (returns true if eurLex is newer)
-    const isNewer = (eurLex: string, local: string): boolean => {
-      if (eurLex === 'unknown' || !eurLex) return false;
-      try {
-        const eurLexDate = new Date(eurLex);
-        const localDate = new Date(local);
-        return eurLexDate > localDate;
-      } catch {
-        return false;
-      }
-    };
-
-    if (eurLexVersion === 'unknown') {
-      // UNECE or non-standard documents - can't auto-check
-      console.log('MANUAL CHECK REQUIRED');
-      console.log(`  Source type: Non-standard (UNECE/consolidated)`);
-      console.log(`  Last fetched: ${lastFetched}`);
-    } else if (!source.eur_lex_version) {
-      // First time checking - record the version but don't flag as update
-      console.log('VERSION NOT TRACKED');
-      console.log(`  EUR-Lex version: ${eurLexVersion}`);
-      console.log(`  Run ingest again to record version`);
-    } else if (isNewer(eurLexVersion, source.eur_lex_version)) {
-      // EUR-Lex has a newer version
-      console.log('UPDATE AVAILABLE');
-      console.log(`  Local version:  ${source.eur_lex_version}`);
-      console.log(`  EUR-Lex version: ${eurLexVersion}`);
-      updates.push({
-        id: source.regulation,
-        celex_id: source.celex_id,
-        reason: `Newer version: ${source.eur_lex_version} -> ${eurLexVersion}`
-      });
-    } else if (source.quality_status !== 'complete') {
-      console.log(`INCOMPLETE (${source.quality_status})`);
-      updates.push({
-        id: source.regulation,
-        celex_id: source.celex_id,
-        reason: `Quality status: ${source.quality_status}`
-      });
-    } else {
-      console.log('UP TO DATE');
-      console.log(`  EUR-Lex version: ${eurLexVersion}`);
-      console.log(`  Last fetched: ${lastFetched}`);
-    }
-  }
-
-  db.close();
-
-  // Summary
-  console.log('\n' + '='.repeat(80));
-  console.log('Summary');
-  console.log('='.repeat(80));
-
-  if (updates.length === 0) {
-    console.log('\n✓ All monitored regulations are up to date.');
-  } else {
-    console.log(`\n⚠ ${updates.length} regulation(s) need attention:\n`);
-    for (const u of updates) {
-      console.log(`  - ${u.id}: ${u.reason}`);
-    }
-
-    console.log('\nTo update, run:');
-    for (const u of updates) {
-      console.log(`  npx tsx scripts/ingest-eurlex.ts ${u.celex_id} data/seed/${u.id.toLowerCase()}.json`);
-    }
-    console.log('\nThen: npm run build:db');
-  }
-
-  // Output for CI: write CELEX IDs to file for workflow to use
-  const celexList = sources.map(s => s.celex_id).join('|');
-  console.log(`\n::set-output name=celex_pattern::${celexList}`);
-}
-
-// Also provide a function to update the source registry after ingestion
-export async function updateSourceRegistry(
-  db: Database.Database,
-  regulation: string,
-  celexId: string,
-  articleCount: number
-): Promise<void> {
-  const now = new Date().toISOString();
-
-  db.prepare(`
-    INSERT OR REPLACE INTO source_registry
-    (regulation, celex_id, eur_lex_version, last_fetched, articles_expected, articles_parsed, quality_status)
-    VALUES (?, ?, ?, ?, ?, ?, 'complete')
-  `).run(regulation, celexId, now.split('T')[0], now, articleCount, articleCount);
-}
-
-// Main execution
-const args = process.argv.slice(2);
-
-if (args.includes('--sync')) {
-  syncVersions().catch(err => {
-    console.error('Error:', err);
-    process.exit(1);
-  });
-} else {
-  checkForUpdates().catch(err => {
-    console.error('Error:', err);
-    process.exit(1);
-  });
-}