@ansvar/eu-regulations-mcp 0.8.0 → 1.1.0

package/scripts/ingest-unece.ts

@@ -1,382 +0,0 @@
-#!/usr/bin/env npx tsx
-
-/**
- * Ingest UN/ECE regulations from EUR-Lex.
- * UN regulations use numbered sections (1., 2., etc.) instead of "Article X".
- *
- * Usage: npx tsx scripts/ingest-unece.ts <celex_id> <output_file>
- * Example: npx tsx scripts/ingest-unece.ts 42021X0387 data/seed/un-r155.json
- */
-
-import { writeFileSync } from 'fs';
-import { JSDOM } from 'jsdom';
-
-interface Article {
-  number: string;
-  title?: string;
-  text: string;
-  chapter?: string;
-}
-
-interface Definition {
-  term: string;
-  definition: string;
-  article: string;
-}
-
-interface RegulationData {
-  id: string;
-  full_name: string;
-  celex_id: string;
-  effective_date?: string;
-  eur_lex_url: string;
-  articles: Article[];
-  definitions: Definition[];
-}
-
-const UN_REGULATION_METADATA: Record<string, { id: string; full_name: string; effective_date?: string }> = {
-  '42021X0387': {
-    id: 'UN_R155',
-    full_name: 'UN Regulation No. 155 - Cyber security and cyber security management system',
-    effective_date: '2021-01-22',
-  },
-  '42025X0005': {
-    id: 'UN_R155',
-    full_name: 'UN Regulation No. 155 - Cyber security and cyber security management system (Supplement 3)',
-    effective_date: '2025-01-10',
-  },
-  '42021X0388': {
-    id: 'UN_R156',
-    full_name: 'UN Regulation No. 156 - Software update and software update management system',
-    effective_date: '2021-01-22',
-  },
-};
-
-// Section titles for UN regulations (most are shared, some differ)
-const COMMON_SECTION_TITLES: Record<string, string> = {
-  '1': 'Scope',
-  '2': 'Definitions',
-  '3': 'Application for approval',
-  '4': 'Markings',
-  '5': 'Approval',
-  '7': 'Specifications',
-  '8': 'Modification of vehicle type and extension of type approval',
-  '9': 'Conformity of production',
-  '10': 'Penalties for non-conformity of production',
-  '11': 'Production definitively discontinued',
-  '12': 'Names and addresses of Technical Services responsible for conducting approval tests, and of Type Approval Authorities',
-};
-
-// Regulation-specific section titles (for section 6 which differs)
-const REGULATION_SECTION_TITLES: Record<string, Record<string, string>> = {
-  UN_R155: {
-    '6': 'Certificate of Compliance for Cybersecurity Management System',
-  },
-  UN_R156: {
-    '6': 'Certificate of Compliance for Software Update Management System',
-  },
-};
-
-function getSectionTitle(sectionNum: string, regulationId: string): string {
-  const regSpecific = REGULATION_SECTION_TITLES[regulationId]?.[sectionNum];
-  if (regSpecific) return regSpecific;
-  return COMMON_SECTION_TITLES[sectionNum] || `Section ${sectionNum}`;
-}
-
-async function fetchEurLexHtml(celexId: string): Promise<string> {
-  const url = `https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=CELEX:${celexId}`;
-  console.log(`Fetching: ${url}`);
-
-  const response = await fetch(url, {
-    headers: {
-      'User-Agent': 'Mozilla/5.0 (compatible; EU-Compliance-MCP/1.0; +https://github.com/Ansvar-Systems/EU_compliance_MCP)',
-      Accept: 'text/html',
-    },
-  });
-
-  if (!response.ok) {
-    throw new Error(`Failed to fetch: ${response.status} ${response.statusText}`);
-  }
-
-  return response.text();
-}
-
-function parseUnRegulation(html: string, celexId: string): { articles: Article[]; definitions: Definition[] } {
-  const dom = new JSDOM(html);
-  const doc = dom.window.document;
-
-  const articles: Article[] = [];
-  const definitions: Definition[] = [];
-
-  // Strategy: Get all text elements (p, span, td) and process sequentially
-  // UN regulations use numbered sections with content in tables/spans
-  const allElements = Array.from(doc.querySelectorAll('p, span, td'));
-
-  let currentSection: { number: string; title: string; lines: string[] } | null = null;
-  let currentAnnex: { number: string; title: string; lines: string[] } | null = null;
-  let inAnnex = false;
-  let seenSections = new Set<string>();
-  let inTableOfContents = true; // Skip TOC at start
-
-  for (const el of allElements) {
-    const text = el.textContent?.trim() || '';
-    if (!text || text.length < 2) continue;
-
-    // Detect end of table of contents - when we see the actual section header format
-    const mainSectionHeader = text.match(/^(\d{1,2})\.\s+[A-Z][A-Z\s]+$/);
-    if (mainSectionHeader && el.classList?.contains('oj-ti-grseq-1')) {
-      inTableOfContents = false;
-    }
-
-    // Skip if still in table of contents
-    if (inTableOfContents && !el.classList?.contains('oj-ti-grseq-1')) {
-      continue;
-    }
-
-    // Check for main section headers (format: "8.   MODIFICATION OF VEHICLE TYPE...")
-    // Note: some titles have hyphens (e.g., "NON-CONFORMITY")
-    const sectionHeaderMatch = text.match(/^(\d{1,2})\.\s+([A-Z][A-Z\s,\-]+)$/);
-    if (sectionHeaderMatch && el.classList?.contains('oj-ti-grseq-1')) {
-      const sectionNum = sectionHeaderMatch[1];
-
-      // Save current section if exists
-      if (currentSection && currentSection.lines.length > 0 && !seenSections.has(currentSection.number)) {
-        articles.push({
-          number: currentSection.number,
-          title: currentSection.title,
-          text: currentSection.lines.join('\n\n'),
-        });
-        seenSections.add(currentSection.number);
-      }
-
-      currentSection = {
-        number: sectionNum,
-        title: getSectionTitle(sectionNum, metadata?.id || 'UN_R155') || sectionHeaderMatch[2].trim(),
-        lines: [],
-      };
-      currentAnnex = null;
-      inAnnex = false;
-      continue;
-    }
-
-    // Check for Annex headers
-    const annexMatch = text.match(/^Annex\s+(\d+)/i) || text.match(/^ANNEX\s+(\d+)/i);
-    if (annexMatch || (el.classList?.contains('oj-doc-ti') && text.includes('Annex'))) {
-      // Save current section/annex if exists
-      if (currentSection && currentSection.lines.length > 0 && !seenSections.has(currentSection.number)) {
-        articles.push({
-          number: currentSection.number,
-          title: currentSection.title,
-          text: currentSection.lines.join('\n\n'),
-        });
-        seenSections.add(currentSection.number);
-      }
-      if (currentAnnex && currentAnnex.lines.length > 0 && !seenSections.has(`Annex ${currentAnnex.number}`)) {
-        articles.push({
-          number: `Annex ${currentAnnex.number}`,
-          title: currentAnnex.title,
-          text: currentAnnex.lines.join('\n\n'),
-          chapter: 'Annexes',
-        });
-        seenSections.add(`Annex ${currentAnnex.number}`);
-      }
-
-      const annexNum = annexMatch?.[1] || text.match(/Annex\s+(\d+)/i)?.[1];
-      if (annexNum) {
-        inAnnex = true;
-        currentAnnex = { number: annexNum, title: extractAnnexTitle(text), lines: [] };
-        currentSection = null;
-      }
-      continue;
-    }
-
-    // Skip metadata and navigation elements
-    if (
-      text.includes('Official Journal') ||
-      text.includes('EUR-Lex') ||
-      text.includes('CONTENTS') ||
-      text.match(/^[A-Z]+$/) ||
-      text.match(/^L\s+\d+\/\d+$/) ||
-      text.match(/^\d+\.\d+\.\d+\s+EN$/)
-    ) {
-      continue;
-    }
-
-    // Add content to current section or annex
-    if (inAnnex && currentAnnex) {
-      currentAnnex.lines.push(text);
-    } else if (currentSection) {
-      currentSection.lines.push(text);
-    }
-  }
-
-  // Don't forget last section/annex
-  if (currentSection && currentSection.lines.length > 0 && !seenSections.has(currentSection.number)) {
-    articles.push({
-      number: currentSection.number,
-      title: currentSection.title,
-      text: currentSection.lines.join('\n\n'),
-    });
-  }
-  if (currentAnnex && currentAnnex.lines.length > 0 && !seenSections.has(`Annex ${currentAnnex.number}`)) {
-    articles.push({
-      number: `Annex ${currentAnnex.number}`,
-      title: currentAnnex.title,
-      text: currentAnnex.lines.join('\n\n'),
-      chapter: 'Annexes',
-    });
-  }
-
-  // Extract definitions from Section 2
-  // UN regulations use format: 2.1. 'term' means/refers to definition
-  // Note: Uses curly quotes (Unicode 8216/8217) not straight quotes
-  const defsSection = articles.find((a) => a.number === '2');
-  if (defsSection) {
-    // Normalize text: collapse newlines, handle both straight and curly quotes
-    const normalizedText = defsSection.text
-      .replace(/\n+/g, ' ')
-      .replace(/\s+/g, ' ')
-      .replace(/[\u2018\u2019]/g, "'"); // Convert curly quotes to straight
-
-    // Match patterns like: 2.1. 'Vehicle type' means/refers to ...
-    // Some definitions use "means", others use "refers to"
-    const defRegex = /(\d+\.\d+\.)\s*'([^']+)'\s+(?:means|refers to)\s+(.+?)(?=\d+\.\d+\.\s*'|$)/g;
-    const defMatches = normalizedText.matchAll(defRegex);
-    for (const match of defMatches) {
-      const term = match[2].trim().toLowerCase();
-      let definition = match[3].trim();
-      // Clean up the definition - remove trailing section numbers and punctuation
-      definition = definition.replace(/\s*\d+\.\d+\.\s*$/, '').replace(/[;.]$/, '').trim();
-      if (term && definition.length > 10) {
-        definitions.push({
-          term,
-          definition,
-          article: '2',
-        });
-      }
-    }
-  }
-
-  // Deduplicate and sort articles
-  const articleMap = new Map<string, Article>();
-  for (const article of articles) {
-    const existing = articleMap.get(article.number);
-    if (!existing || article.text.length > existing.text.length) {
-      articleMap.set(article.number, article);
-    }
-  }
-
-  const sortedArticles = Array.from(articleMap.values()).sort((a, b) => {
-    // Sort numbered sections first, then annexes
-    const aIsAnnex = a.number.startsWith('Annex');
-    const bIsAnnex = b.number.startsWith('Annex');
-    if (aIsAnnex && !bIsAnnex) return 1;
-    if (!aIsAnnex && bIsAnnex) return -1;
-    if (aIsAnnex && bIsAnnex) {
-      // Extract numeric part from "Annex 5" or "Annex 5a"
-      const matchA = a.number.match(/Annex (\d+)([a-z]?)/);
-      const matchB = b.number.match(/Annex (\d+)([a-z]?)/);
-      if (!matchA || !matchB) return 0;
-      const numA = parseInt(matchA[1]);
-      const numB = parseInt(matchB[1]);
-      if (numA !== numB) return numA - numB;
-      return (matchA[2] || '').localeCompare(matchB[2] || '');
-    }
-    // Regular numbered sections - handle sub-sections like "5a"
-    const matchA = a.number.match(/^(\d+)([a-z]?)$/);
-    const matchB = b.number.match(/^(\d+)([a-z]?)$/);
-    if (!matchA || !matchB) return 0;
-    const numA = parseInt(matchA[1]);
-    const numB = parseInt(matchB[1]);
-    if (numA !== numB) return numA - numB;
-    return (matchA[2] || '').localeCompare(matchB[2] || '');
-  });
-
-  return { articles: sortedArticles, definitions };
-}
-
-function extractAnnexTitle(text: string): string {
-  // Extract title after "Annex X"
-  const match = text.match(/Annex\s+\d+\s*[–—-]?\s*(.*)/i);
-  if (match && match[1]) {
-    return match[1].trim();
-  }
-
-  // Common annex titles for R155
-  const annexTitles: Record<string, string> = {
-    '1': 'Information document',
-    '2': 'Communication',
-    '3': 'Arrangements of the approval mark',
-    '4': 'Certificate of Compliance for CSMS',
-    '5': 'List of threats and corresponding mitigations',
-  };
-
-  const annexNum = text.match(/Annex\s+(\d+)/i)?.[1];
-  if (annexNum && annexTitles[annexNum]) {
-    return annexTitles[annexNum];
-  }
-
-  return '';
-}
-
-async function ingestUnRegulation(celexId: string, outputPath: string): Promise<void> {
-  const metadata = UN_REGULATION_METADATA[celexId];
-  if (!metadata) {
-    console.warn(`Unknown CELEX ID: ${celexId}. Using generic metadata.`);
-  }
-
-  const html = await fetchEurLexHtml(celexId);
-  console.log(`Fetched ${html.length} bytes`);
-
-  // Save HTML for debugging
-  writeFileSync(outputPath.replace('.json', '.html'), html);
-
-  const { articles, definitions } = parseUnRegulation(html, celexId);
-  console.log(`Parsed ${articles.length} articles/sections, ${definitions.length} definitions`);
-
-  if (articles.length === 0) {
-    console.error('No sections found! The HTML structure may have changed.');
-    return;
-  }
-
-  const regulation: RegulationData = {
-    id: metadata?.id || celexId,
-    full_name: metadata?.full_name || `UN Regulation ${celexId}`,
-    celex_id: celexId,
-    effective_date: metadata?.effective_date,
-    eur_lex_url: `https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:${celexId}`,
-    articles,
-    definitions,
-  };
-
-  writeFileSync(outputPath, JSON.stringify(regulation, null, 2));
-  console.log(`\nSaved to: ${outputPath}`);
-  console.log(`Sections: ${articles.filter((a) => !a.number.startsWith('Annex')).length}`);
-  console.log(`Annexes: ${articles.filter((a) => a.number.startsWith('Annex')).length}`);
-  console.log(`Definitions: ${definitions.length}`);
-
-  // Print summary
-  console.log('\nSections found:');
-  for (const article of articles) {
-    const preview = article.text.substring(0, 60).replace(/\n/g, ' ');
-    console.log(`  ${article.number}: ${article.title || '(no title)'} - ${preview}...`);
-  }
-}
-
-// Main
-const [, , celexId, outputPath] = process.argv;
-
-if (!celexId || !outputPath) {
-  console.log('Usage: npx tsx scripts/ingest-unece.ts <celex_id> <output_file>');
-  console.log('Example: npx tsx scripts/ingest-unece.ts 42021X0387 data/seed/un-r155.json');
-  console.log('\nKnown UN/ECE CELEX IDs:');
-  Object.entries(UN_REGULATION_METADATA).forEach(([id, meta]) => {
-    console.log(`  ${id} - ${meta.id} (${meta.full_name})`);
-  });
-  process.exit(1);
-}
-
-ingestUnRegulation(celexId, outputPath).catch((err) => {
-  console.error('Error:', err);
-  process.exit(1);
-});