@ansvar/eu-regulations-mcp 0.8.0 → 1.1.0

package/scripts/extract-eprivacy-recitals.ts

@@ -1,98 +0,0 @@
-#!/usr/bin/env npx tsx
-
-import { writeFileSync, readFileSync } from 'fs';
-import { JSDOM } from 'jsdom';
-
-async function fetchEPrivacyRecitals() {
-  const url = 'https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=CELEX:32002L0058';
-  console.log(`Fetching: ${url}`);
-
-  const response = await fetch(url, {
-    headers: {
-      'User-Agent': 'Mozilla/5.0 (compatible)',
-      'Accept': 'text/html',
-    },
-  });
-
-  if (!response.ok) {
-    throw new Error(`Failed: ${response.status}`);
-  }
-
-  const html = await response.text();
-  const dom = new JSDOM(html);
-  const doc = dom.window.document;
-
-  // Extract all paragraphs
-  const paragraphs = Array.from(doc.querySelectorAll('p'));
-
-  const recitals: Array<{ recital_number: number; text: string }> = [];
-  let inRecitals = false;
-  let currentNumber: number | null = null;
-  let currentText: string[] = [];
-
-  for (const p of paragraphs) {
-    const text = p.textContent?.trim() || '';
-
-    // Check if we've entered the recitals section
-    if (text === 'Whereas:') {
-      inRecitals = true;
-      continue;
-    }
-
-    // Check if we've left the recitals section
-    if (text.match(/^HAVE ADOPTED/i) || text.match(/^Article\s+1/i)) {
-      if (currentNumber && currentText.length > 0) {
-        recitals.push({
-          recital_number: currentNumber,
-          text: currentText.join('\n\n'),
-        });
-      }
-      break;
-    }
-
-    if (!inRecitals) continue;
-
-    // Check for recital number at start: "(1)", "(2)", etc.
-    const recitalMatch = text.match(/^\((\d+)\)/);
-
-    if (recitalMatch) {
-      // Save previous recital
-      if (currentNumber && currentText.length > 0) {
-        recitals.push({
-          recital_number: currentNumber,
-          text: currentText.join('\n\n'),
-        });
-      }
-
-      // Start new recital
-      currentNumber = parseInt(recitalMatch[1]);
-      const remainingText = text.substring(recitalMatch[0].length).trim();
-      currentText = remainingText ? [remainingText] : [];
-    } else if (currentNumber && text.length > 0) {
-      // Add to current recital
-      currentText.push(text);
-    }
-  }
-
-  // Don't forget the last one
-  if (currentNumber && currentText.length > 0) {
-    recitals.push({
-      recital_number: currentNumber,
-      text: currentText.join('\n\n'),
-    });
-  }
-
-  console.log(`Extracted ${recitals.length} recitals`);
-
-  // Load existing ePrivacy JSON
-  const existingData = JSON.parse(readFileSync('data/seed/eprivacy.json', 'utf-8'));
-  existingData.recitals = recitals;
-
-  // Save updated file
-  writeFileSync('data/seed/eprivacy.json', JSON.stringify(existingData, null, 2));
-  console.log(`Saved to: data/seed/eprivacy.json`);
-  console.log(`Articles: ${existingData.articles.length}`);
-  console.log(`Recitals: ${recitals.length}`);
-}
-
-fetchEPrivacyRecitals().catch(console.error);

package/scripts/ingest-eurlex-browser.ts

@@ -1,113 +0,0 @@
-#!/usr/bin/env npx tsx
-
-/**
- * Browser-based EUR-Lex fetcher to bypass AWS WAF challenges.
- *
- * EUR-Lex deployed AWS WAF that returns 2036-byte JavaScript challenge pages
- * instead of actual HTML when using fetch(). This script launches a headless
- * browser to wait for the challenge to complete and retrieve the real content.
- *
- * Usage:
- *   npx tsx scripts/ingest-eurlex-browser.ts <celex_id>
- *   npx tsx scripts/ingest-eurlex-browser.ts 32016R0679
- *
- * Or import as a function:
- *   import { fetchEurLexWithBrowser } from './scripts/ingest-eurlex-browser';
- *   const html = await fetchEurLexWithBrowser('32016R0679');
- */
-
-import puppeteer from 'puppeteer';
-
-/**
- * Fetches EUR-Lex regulation HTML using Puppeteer to bypass WAF challenges.
- *
- * @param celexId - CELEX identifier (e.g., '32016R0679' for GDPR)
- * @returns Full HTML content of the regulation
- * @throws Error if fetching fails or content is invalid
- */
-export async function fetchEurLexWithBrowser(celexId: string): Promise<string> {
-  const url = `https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=CELEX:${celexId}`;
-  console.log(`[Browser] Launching headless browser...`);
-  console.log(`[Browser] Fetching: ${url}`);
-
-  const browser = await puppeteer.launch({
-    headless: true,
-    args: [
-      '--no-sandbox',
-      '--disable-setuid-sandbox',
-      '--disable-dev-shm-usage',
-      '--disable-accelerated-2d-canvas',
-      '--disable-gpu',
-    ],
-  });
-
-  try {
-    const page = await browser.newPage();
-
-    // Set realistic User-Agent to appear as a normal browser
-    await page.setUserAgent(
-      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
-    );
-
-    // Set viewport
-    await page.setViewport({ width: 1920, height: 1080 });
-
-    console.log(`[Browser] Navigating to URL...`);
-    await page.goto(url, {
-      waitUntil: 'networkidle0', // Wait until network is idle
-      timeout: 30000,
-    });
-
-    console.log(`[Browser] Page loaded. Waiting for WAF challenge to complete...`);
-
-    // Wait a bit longer to ensure WAF challenge JavaScript has executed
-    await new Promise(resolve => setTimeout(resolve, 5000));
-
-    // Get the full HTML content
-    const html = await page.content();
-
-    console.log(`[Browser] Fetched ${html.length} bytes`);
-
-    // Validate that we got real content, not a WAF challenge page
-    if (html.length < 10000) {
-      throw new Error(`Fetched content is suspiciously small (${html.length} bytes). Possible WAF block.`);
-    }
-
-    if (html.includes('window.gokuProps')) {
-      throw new Error('Received AWS WAF challenge page instead of regulation content.');
-    }
-
-    if (!html.includes('Article')) {
-      console.warn('[Browser] Warning: HTML does not contain "Article" - may not be valid regulation content');
-    }
-
-    return html;
-  } finally {
-    await browser.close();
-    console.log(`[Browser] Browser closed`);
-  }
-}
-
-// CLI interface
-if (import.meta.url === `file://${process.argv[1]}`) {
-  const [,, celexId] = process.argv;
-
-  if (!celexId) {
-    console.log('Usage: npx tsx scripts/ingest-eurlex-browser.ts <celex_id>');
-    console.log('Example: npx tsx scripts/ingest-eurlex-browser.ts 32016R0679');
-    console.log('\nThis will fetch the HTML and print it to stdout.');
-    console.log('Pipe to a file: npx tsx scripts/ingest-eurlex-browser.ts 32016R0679 > output.html');
-    process.exit(1);
-  }
-
-  fetchEurLexWithBrowser(celexId)
-    .then(html => {
-      // Output HTML to stdout for piping
-      console.log('\n--- HTML Content ---\n');
-      console.log(html);
-    })
-    .catch(err => {
-      console.error('Error:', err.message);
-      process.exit(1);
-    });
-}

package/scripts/ingest-eurlex.ts

@@ -1,346 +0,0 @@
-#!/usr/bin/env npx tsx
-
-/**
- * Ingest EU regulations from EUR-Lex.
- *
- * Usage: npx tsx scripts/ingest-eurlex.ts <celex_id> <output_file> [--browser]
- * Example: npx tsx scripts/ingest-eurlex.ts 32016R0679 data/seed/gdpr.json
- * Example (with browser): npx tsx scripts/ingest-eurlex.ts 32016R0679 data/seed/gdpr.json --browser
- */
-
-import { writeFileSync } from 'fs';
-import { JSDOM } from 'jsdom';
-import { fetchEurLexWithBrowser } from './ingest-eurlex-browser.js';
-
-interface Article {
-  number: string;
-  title?: string;
-  text: string;
-  chapter?: string;
-}
-
-interface Definition {
-  term: string;
-  definition: string;
-  article: string;
-}
-
-interface Recital {
-  recital_number: number;
-  text: string;
-  related_articles?: string;
-}
-
-interface RegulationData {
-  id: string;
-  full_name: string;
-  celex_id: string;
-  effective_date?: string;
-  eur_lex_url: string;
-  articles: Article[];
-  definitions: Definition[];
-  recitals?: Recital[];
-}
-
-const REGULATION_METADATA: Record<string, { id: string; full_name: string; effective_date?: string }> = {
-  '32016R0679': { id: 'GDPR', full_name: 'General Data Protection Regulation', effective_date: '2018-05-25' },
-  '32022L2555': { id: 'NIS2', full_name: 'Directive on measures for a high common level of cybersecurity across the Union', effective_date: '2024-10-17' },
-  '32022R2554': { id: 'DORA', full_name: 'Digital Operational Resilience Act', effective_date: '2025-01-17' },
-  '32024R1689': { id: 'AI_ACT', full_name: 'Artificial Intelligence Act', effective_date: '2024-08-01' },
-  '32024R2847': { id: 'CRA', full_name: 'Cyber Resilience Act', effective_date: '2024-12-10' },
-  '32019R0881': { id: 'CYBERSECURITY_ACT', full_name: 'EU Cybersecurity Act', effective_date: '2019-06-27' },
-  '32024R1183': { id: 'EIDAS2', full_name: 'European Digital Identity Framework (eIDAS 2.0)', effective_date: '2024-05-20' },
-  '02014R0910-20241018': { id: 'EIDAS2', full_name: 'European Digital Identity Framework (eIDAS 2.0)', effective_date: '2024-05-20' },
-  // Digital Single Market regulations
-  '32023R2854': { id: 'DATA_ACT', full_name: 'Data Act', effective_date: '2025-09-12' },
-  '32022R2065': { id: 'DSA', full_name: 'Digital Services Act', effective_date: '2024-02-17' },
-  '32022R1925': { id: 'DMA', full_name: 'Digital Markets Act', effective_date: '2023-05-02' },
-  // UN Regulations (adopted by EU)
-  '42021X0387': { id: 'UN_R155', full_name: 'UN Regulation No. 155 - Cyber security and cyber security management system', effective_date: '2021-01-22' },
-  '42025X0005': { id: 'UN_R155', full_name: 'UN Regulation No. 155 - Cyber security and cyber security management system (Supplement 3)', effective_date: '2025-01-10' },
-};
-
-async function fetchEurLexHtml(celexId: string, useBrowser = false): Promise<string> {
-  if (useBrowser) {
-    console.log('Using Puppeteer to bypass WAF...');
-    return fetchEurLexWithBrowser(celexId);
-  }
-
-  // Fallback to direct fetch (will fail with WAF)
-  const url = `https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=CELEX:${celexId}`;
-  console.log(`Fetching: ${url}`);
-
-  const response = await fetch(url, {
-    headers: {
-      'User-Agent': 'Mozilla/5.0 (compatible; EU-Compliance-MCP/1.0; +https://github.com/Ansvar-Systems/EU_compliance_MCP)',
-      'Accept': 'text/html',
-    },
-  });
-
-  if (!response.ok) {
-    throw new Error(`Failed to fetch: ${response.status} ${response.statusText}`);
-  }
-
-  return response.text();
-}
-
-function parseRecitals(html: string): Recital[] {
-  const dom = new JSDOM(html);
-  const doc = dom.window.document;
-
-  const recitals: Recital[] = [];
-  const allText = doc.body?.textContent || '';
-  const lines = allText.split('\n').map(l => l.trim()).filter(l => l);
-
-  let inRecitalsSection = false;
-  let currentRecital: { number: number; lines: string[] } | null = null;
-
-  for (const line of lines) {
-    // Detect start of recitals section
-    if (line.match(/^Having regard to/i) || line.match(/^Whereas:/i)) {
-      inRecitalsSection = true;
-      continue;
-    }
-
-    // Detect end of recitals (usually "HAVE ADOPTED" or "Article 1")
-    if (line.match(/^HAVE ADOPTED/i) || line.match(/^Article\s+1$/i)) {
-      inRecitalsSection = false;
-      if (currentRecital && currentRecital.lines.length > 0) {
-        recitals.push({
-          recital_number: currentRecital.number,
-          text: currentRecital.lines.join('\n\n'),
-        });
-      }
-      break;
-    }
-
-    if (!inRecitalsSection) continue;
-
-    // Match recital number: "(1)", "(123)", etc.
-    const recitalMatch = line.match(/^\((\d+)\)/);
-    if (recitalMatch) {
-      // Save previous recital
-      if (currentRecital && currentRecital.lines.length > 0) {
-        recitals.push({
-          recital_number: currentRecital.number,
-          text: currentRecital.lines.join('\n\n'),
-        });
-      }
-
-      // Start new recital
-      currentRecital = {
-        number: parseInt(recitalMatch[1]),
-        lines: [],
-      };
-
-      // Add remaining text after number
-      const textAfterNumber = line.substring(recitalMatch[0].length).trim();
-      if (textAfterNumber) {
-        currentRecital.lines.push(textAfterNumber);
-      }
-      continue;
-    }
-
-    // Add line to current recital
-    if (currentRecital && line.length > 0) {
-      currentRecital.lines.push(line);
-    }
-  }
-
-  // Don't forget the last recital
-  if (currentRecital && currentRecital.lines.length > 0) {
-    recitals.push({
-      recital_number: currentRecital.number,
-      text: currentRecital.lines.join('\n\n'),
-    });
-  }
-
-  return recitals;
-}
-
-function parseArticles(html: string, celexId: string): { articles: Article[]; definitions: Definition[] } {
-  const dom = new JSDOM(html);
-  const doc = dom.window.document;
-
-  const articles: Article[] = [];
-  const definitions: Definition[] = [];
-  let currentChapter = '';
-
-  // Get all text content and split by article markers
-  const allText = doc.body?.textContent || '';
-  const lines = allText.split('\n').map(l => l.trim()).filter(l => l);
-
-  let currentArticle: { number: string; title?: string; lines: string[] } | null = null;
-
-  for (const line of lines) {
-    const articleStart = line.match(/^Article\s+(\d+[a-z]?)$/i);
-    if (articleStart) {
-      if (currentArticle && currentArticle.lines.length > 0) {
-        articles.push({
-          number: currentArticle.number,
-          title: currentArticle.title,
-          text: currentArticle.lines.join('\n\n'),
-          chapter: currentChapter || undefined,
-        });
-      }
-      currentArticle = { number: articleStart[1], lines: [] };
-      continue;
-    }
-
-    const chapterStart = line.match(/^CHAPTER\s+([IVXLC]+)/i);
-    if (chapterStart) {
-      currentChapter = chapterStart[1];
-      continue;
-    }
-
-    if (currentArticle) {
-      // Check if this is a title line (short, no period at end)
-      if (!currentArticle.title && currentArticle.lines.length === 0 && line.length < 100 && !line.endsWith('.')) {
-        currentArticle.title = line;
-      } else if (line.length > 0) {
-        currentArticle.lines.push(line);
-      }
-    }
-  }
-
-  // Don't forget the last article
-  if (currentArticle && currentArticle.lines.length > 0) {
-    articles.push({
-      number: currentArticle.number,
-      title: currentArticle.title,
-      text: currentArticle.lines.join('\n\n'),
-      chapter: currentChapter || undefined,
-    });
-  }
-
-  // Deduplicate articles - keep the one with the most content for each number
-  const articleMap = new Map<string, Article>();
-  for (const article of articles) {
-    const existing = articleMap.get(article.number);
-    if (!existing || article.text.length > existing.text.length) {
-      articleMap.set(article.number, article);
-    }
-  }
-  const deduplicatedArticles = Array.from(articleMap.values())
-    .sort((a, b) => {
-      // Extract numeric and letter parts (e.g., "5a" -> [5, "a"])
-      const matchA = a.number.match(/^(\d+)([a-z]?)$/);
-      const matchB = b.number.match(/^(\d+)([a-z]?)$/);
-      if (!matchA || !matchB) return 0;
-      
-      const numA = parseInt(matchA[1]);
-      const numB = parseInt(matchB[1]);
-      
-      // Sort by number first
-      if (numA !== numB) return numA - numB;
-      
-      // Then by letter (empty string sorts before letters)
-      return (matchA[2] || '').localeCompare(matchB[2] || '');
-    });
-
-  // Extract definitions from Article 4 (or similar definitions article)
-  // Find definitions article from deduplicated list
-  const defsArticle = deduplicatedArticles.find(a =>
-    a.title?.toLowerCase().includes('definition')
-  );
-
-  if (defsArticle && defsArticle.text.includes('means')) {
-    // Normalize text: collapse whitespace and normalize quotes
-    const normalizedText = defsArticle.text
-      .replace(/\s+/g, ' ')
-      .replace(/[\u2018\u2019]/g, "'"); // Curly quotes to straight
-
-    // Parse definitions by extracting content between consecutive numbered entries
-    // This handles:
-    // - Complex definitions with internal periods/semicolons
-    // - 'term' or 'alternate' means... patterns (NIS2 Art 6)
-    // - 'term1', 'term2' and 'term3' mean... patterns (CRA Art 3)
-    // - 'term' of the something means... patterns (GDPR Art 4)
-    // - mean, respectively... patterns (CRA Art 3)
-    // - means: (a) ... patterns (complex definitions with sub-parts)
-    const defRegex = /\((\d+)\)\s*'([^']+)'(?:[^(]*?)means?[,:;]?\s+(.+?)(?=\(\d+\)\s*'|$)/g;
-    let defMatch;
-    while ((defMatch = defRegex.exec(normalizedText)) !== null) {
-      const term = defMatch[2].trim().toLowerCase();
-      const definition = defMatch[3].trim();
-      // Only add if we got meaningful content
-      if (term.length > 0 && definition.length > 10) {
-        definitions.push({
-          term,
-          definition,
-          article: defsArticle.number,
-        });
-      }
-    }
-  }
-
-  return { articles: deduplicatedArticles, definitions };
-}
-
-async function ingestRegulation(celexId: string, outputPath: string, useBrowser = false): Promise<void> {
-  const metadata = REGULATION_METADATA[celexId];
-  if (!metadata) {
-    console.warn(`Unknown CELEX ID: ${celexId}. Using generic metadata.`);
-  }
-
-  const html = await fetchEurLexHtml(celexId, useBrowser);
-  console.log(`Fetched ${html.length} bytes`);
-
-  // Parse recitals BEFORE articles
-  const recitals = parseRecitals(html);
-  console.log(`Parsed ${recitals.length} recitals`);
-
-  const { articles, definitions } = parseArticles(html, celexId);
-  console.log(`Parsed ${articles.length} articles, ${definitions.length} definitions`);
-
-  if (articles.length === 0) {
-    console.error('No articles found! The HTML structure may have changed.');
-    console.log('Saving raw HTML for debugging...');
-    writeFileSync(outputPath.replace('.json', '.html'), html);
-    return;
-  }
-
-  const regulation: RegulationData = {
-    id: metadata?.id || celexId,
-    full_name: metadata?.full_name || `Regulation ${celexId}`,
-    celex_id: celexId,
-    effective_date: metadata?.effective_date,
-    eur_lex_url: `https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:${celexId}`,
-    articles,
-    definitions,
-    recitals,
-  };
-
-  writeFileSync(outputPath, JSON.stringify(regulation, null, 2));
-  console.log(`\nSaved to: ${outputPath}`);
-  console.log(`Articles: ${articles.length}`);
-  console.log(`Definitions: ${definitions.length}`);
-  console.log(`Recitals: ${recitals.length}`);
-}
-
-// Main
-const args = process.argv.slice(2);
-const useBrowser = args.includes('--browser');
-const [celexId, outputPath] = args.filter(arg => arg !== '--browser');
-
-if (!celexId || !outputPath) {
-  console.log('Usage: npx tsx scripts/ingest-eurlex.ts <celex_id> <output_file> [--browser]');
-  console.log('Example: npx tsx scripts/ingest-eurlex.ts 32016R0679 data/seed/gdpr.json');
-  console.log('Example (with browser): npx tsx scripts/ingest-eurlex.ts 32016R0679 data/seed/gdpr.json --browser');
-  console.log('\nOptions:');
-  console.log('  --browser    Use Puppeteer to bypass EUR-Lex WAF challenges');
-  console.log('\nKnown CELEX IDs:');
-  Object.entries(REGULATION_METADATA).forEach(([id, meta]) => {
-    console.log(`  ${id} - ${meta.id} (${meta.full_name})`);
-  });
-  process.exit(1);
-}
-
-if (useBrowser) {
-  console.log('Browser mode enabled - using Puppeteer to fetch content\n');
-}
-
-ingestRegulation(celexId, outputPath, useBrowser).catch(err => {
-  console.error('Error:', err);
-  process.exit(1);
-});