iev 0.4.7 → 0.4.8

data/lib/iev/scraper/browser.rb

@@ -6,6 +6,10 @@ module Iev
   class Scraper
     # Shared headless browser utilities for fetching pages behind AWS WAF.
     module Browser
+      # Each profile is tagged with the host platform it can run on so
+      # that navigator.platform (set by Chrome from the real OS) agrees
+      # with the User-Agent and Sec-Ch-Ua-Platform headers. AWS WAF
+      # fingerprints this mismatch and refuses to clear its challenge.
       USER_AGENT_PROFILES = [
         {
           user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
@@ -13,6 +17,15 @@ module Iev
                       "Chrome/131.0.0.0 Safari/537.36",
           platform: '"macOS"',
           chrome_version: "131",
+          host_platform: :mac,
+        },
+        {
+          user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/129.0.0.0 Safari/537.36",
+          platform: '"macOS"',
+          chrome_version: "129",
+          host_platform: :mac,
         },
         {
           user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
@@ -20,85 +33,177 @@ module Iev
                       "Chrome/130.0.0.0 Safari/537.36",
           platform: '"Windows"',
           chrome_version: "130",
+          host_platform: :windows,
         },
         {
-          user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
+          user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
                       "AppleWebKit/537.36 (KHTML, like Gecko) " \
                       "Chrome/131.0.0.0 Safari/537.36",
-          platform: '"Linux"',
+          platform: '"Windows"',
           chrome_version: "131",
+          host_platform: :windows,
         },
         {
-          user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
-                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                      "Chrome/129.0.0.0 Safari/537.36",
-          platform: '"macOS"',
-          chrome_version: "129",
-        },
-        {
-          user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
+          user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
                       "AppleWebKit/537.36 (KHTML, like Gecko) " \
                       "Chrome/131.0.0.0 Safari/537.36",
-          platform: '"Windows"',
+          platform: '"Linux"',
           chrome_version: "131",
+          host_platform: :linux,
         },
       ].freeze
 
-      # Fetch a URL using headless Chrome, returning the page HTML.
-      # Handles AWS WAF challenge pages by waiting for JS execution.
-      def self.fetch(url, browser_opts: {})
-        browser = Ferrum::Browser.new(
-          headless: "new",
-          timeout: 30,
-          window_size: [1366, 768],
-          browser_options: {
-            "disable-blink-features" => "AutomationControlled",
-          },
-          **browser_opts,
-        )
-
-        browser.headers.set(random_headers)
-        browser.go_to(url)
-        browser.network.wait_for_idle(timeout: 15)
-        html = browser.body
-
-        if html.include?("403 ERROR") || html.include?("Request blocked")
-          warn "IEV: AWS WAF blocked request for #{url}"
-          return nil
-        end
+      DEFAULT_LANG = "en-US,en"
+      DEFAULT_BROWSER_OPTIONS = {
+        "disable-blink-features" => "AutomationControlled",
+        "lang" => DEFAULT_LANG,
+      }.freeze
 
-        html
-      rescue Ferrum::Error, Ferrum::BrowserError => e
-        warn "IEV: Browser error fetching #{url}: #{e.message}"
-        nil
-      ensure
-        browser&.quit
+      # One-shot fetch. Each call spins up a fresh headless Chrome, fetches,
+      # and tears it down. Suitable for ad-hoc use; the WAF cookie does not
+      # survive between calls. Batch callers (Fetcher::Mirror) should use
+      # Session instead so the cookie set on the first successful challenge
+      # is reused across requests.
+      def self.fetch(url, **_browser_opts)
+        Session.new.fetch(url)
       end
 
+      # Returns request headers that match what real Chrome sends on a
+      # fresh address-bar navigation. AWS WAF fingerprints inconsistencies
+      # between these headers and the browser's runtime state, so we:
+      #   - omit Sec-Fetch-* (Chrome computes those itself from the
+      #     navigation context; setting them via Ferrum's Network domain
+      #     overrides the real values and is detectable), and
+      #   - keep Sec-Ch-Ua-Platform aligned with the host OS, which is
+      #     what Chrome reports via navigator.platform.
       def self.random_headers
-        profile = USER_AGENT_PROFILES.sample
-        sec_ch_ua = "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
-                    "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
-                    "\"Not_A Brand\";v=\"24\""
+        profile = profile_for_host
+        static_headers.merge(headers_from_profile(profile))
+      end
 
+      def self.static_headers
         {
           "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
                       "image/avif,image/webp,image/apng,*/*;q=0.8," \
                       "application/signed-exchange;v=b3;q=0.7",
-          "Accept-Language" => "en-GB,en-US;q=0.9,en;q=0.8",
+          "Accept-Language" => "en-US,en;q=0.9",
           "Cache-Control" => "no-cache",
           "Pragma" => "no-cache",
-          "Sec-Ch-Ua" => sec_ch_ua,
           "Sec-Ch-Ua-Mobile" => "?0",
-          "Sec-Ch-Ua-Platform" => profile[:platform],
-          "Sec-Fetch-Dest" => "document",
-          "Sec-Fetch-Mode" => "navigate",
-          "Sec-Fetch-Site" => "cross-site",
-          "Sec-Fetch-User" => "?1",
           "Upgrade-Insecure-Requests" => "1",
+        }
+      end
+
+      def self.headers_from_profile(profile)
+        {
+          "Sec-Ch-Ua" => sec_ch_ua_for(profile),
+          "Sec-Ch-Ua-Platform" => profile[:platform],
           "User-Agent" => profile[:user_agent],
         }
       end
+
+      def self.profile_for_host
+        USER_AGENT_PROFILES.select do |profile|
+          profile[:host_platform] == host_platform
+        end.sample
+      end
+
+      def self.host_platform
+        case RUBY_PLATFORM
+        when /darwin/ then :mac
+        when /mswin|mingw|cygwin|bccwin|wince|emx/ then :windows
+        else :linux
+        end
+      end
+
+      def self.sec_ch_ua_for(profile)
+        "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
+          "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
+          "\"Not_A Brand\";v=\"24\""
+      end
+
+      # A long-lived headless Chrome session. Cookies persist across
+      # fetches, so once the AWS WAF challenge is cleared on the first
+      # request, subsequent requests reuse the token and succeed at
+      # near-100% rate. The Mirror creates one Session per run and
+      # shares it across all SequentialProbe iterations.
+      #
+      # Chrome leaks ~1MB per page load (mostly V8 heap that doesn't get
+      # GC'd between navigations). After ~1000 fetches the process is at
+      # ~1GB and the OOM risk climbs sharply. #restart quits the browser
+      # and starts a fresh one with a new cookie jar; the WAF challenge
+      # will need to be cleared again on the next fetch.
+      class Session
+        def initialize
+          @browser = Ferrum::Browser.new(
+            headless: "new",
+            timeout: 30,
+            window_size: [1366, 768],
+            browser_options: Browser::DEFAULT_BROWSER_OPTIONS,
+          )
+          @browser.headers.set(Browser.random_headers)
+        end
+
+        def fetch(url)
+          @browser.go_to(url)
+          @browser.network.wait_for_idle(timeout: 15)
+          reject_blocked(url, @browser.body)
+        rescue Ferrum::DeadBrowserError, Ferrum::NoSuchPageError,
+               Ferrum::NoSuchTargetError => e
+          # Chrome process or page/tab has crashed. Restart once, retry
+          # once. If the restart itself fails, surface as nil so the
+          # probe silently skips and the run continues.
+          if restart
+            retry
+          else
+            warn "IEV: Browser crashed, restart failed: #{e.message}"
+            nil
+          end
+        rescue Ferrum::BrowserError => e
+          if fetch_dead?(e.message) && restart
+            retry
+          else
+            warn "IEV: Browser error fetching #{url}: #{e.message}"
+            nil
+          end
+        rescue Ferrum::Error => e
+          warn "IEV: Browser error fetching #{url}: #{e.message}"
+          nil
+        end
+
+        def fetch_dead?(message)
+          message.include?("Browser is dead") ||
+            message.include?("given window is closed") ||
+            message.include?("Target closed")
+        end
+
+        def reject_blocked(url, html)
+          if html.include?("403 ERROR") || html.include?("Request blocked")
+            warn "IEV: AWS WAF blocked request for #{url}"
+            nil
+          else
+            html
+          end
+        end
+
+        # Quit the current browser and start a fresh one. The WAF cookie
+        # is lost, so the next fetch will go through the challenge cycle
+        # again. Call this after N fetches to bound Ferrum's memory growth,
+        # or rely on #fetch to call it automatically when Chrome dies.
+        # Returns true on success, false if the new browser fails to start.
+        def restart
+          quit
+          initialize
+          true
+        rescue StandardError => e
+          warn "IEV: Session restart failed: #{e.message}"
+          false
+        end
+
+        def quit
+          @browser&.quit
+        end
+      end
     end
   end
 end

data/lib/iev/scraper/page_parser.rb

@@ -4,13 +4,14 @@ module Iev
   class Scraper
     # Parses an Electropedia HTML page into a concept data hash.
     #
-    # The Electropedia HTML structure is a table with rows for each language:
-    # - Language row: <div align="center"><font color="#800080">en</font></div>
-    # - Term cell: <b>term text</b> in the third <td>
-    # - Definition row: next row's third <td> (if present)
-    # - Empty/separator rows with <hr> or spacer images
+    # All extracted text (terms, definitions, notes, examples) is run
+    # through Iev::Utilities' semantic enrichment pipeline, converting
+    # HTML to AsciiDoc notation: <i>x</i> → stem:[x], <b>x</b> → *x*,
+    # <a href=IEV...> → {{urn:...}}, MathML → AsciiMath. This ensures
+    # the parsed output matches the format used in termbase.yaml.
     class PageParser
-      # Map Electropedia HTML language codes to ISO 639-2/3 three-char codes.
+      include Utilities
+
       LANG_CODE_MAP = {
         "en" => "eng",
         "fr" => "fra",
@@ -54,28 +55,48 @@ module Iev
       private
 
       def find_iev_ref
-        # Find the IEV reference cell to confirm the page is valid
         @doc.at_css("b:contains('#{@code}')") ||
           @doc.at_xpath("//td/b[contains(text(), '#{@code}')]")
       end
 
+      def term_domain
+        @code.rpartition("-").first
+      end
+
       def localized_concepts
         result = {}
-        lang_sections.each do |lang, term_row, def_row|
+        lang_sections.each do |lang, area, term_row, def_row|
           term = extract_term(term_row)
           next unless term
 
-          entry = { "term" => term }
+          entry = result[lang] || { "terms" => [] }
+          term_entry = { "designation" => term }
+          term_entry["geographical_area"] = area if area
+          entry["terms"] << term_entry
+
           definition = extract_definition(def_row)
-          entry["definition"] = definition if definition
+          entry["definition"] = definition if definition && !entry.key?("definition")
 
           result[lang] = entry
         end
+
+        # Convert terms array to single "term" field for backward compat
+        # when there's only one term and no area
+        result.each_value do |entry|
+          terms = entry.delete("terms")
+          if terms.size == 1 && !terms.first["geographical_area"]
+            entry["term"] = terms.first["designation"]
+          else
+            entry["term"] = terms.map { |t| t["designation"] }.join("\n")
+            entry["term_areas"] = terms.each_with_object({}) do |t, h|
+              area = t["geographical_area"]
+              h[t["designation"]] = area if area
+            end
+          end
+        end
         result
       end
 
-      # Finds all language sections in the table.
-      # Returns array of [lang_code, term_row, definition_row] tuples.
       def lang_sections
         sections = []
         rows = content_rows
@@ -84,17 +105,30 @@ module Iev
           lang = extract_lang(row)
           next unless lang
 
-          # The definition is in the next non-empty, non-separator row
+          area = extract_area(row)
           def_row = find_definition_row(rows, idx + 1)
-          sections << [lang, row, def_row]
+          sections << [lang, area, row, def_row]
         end
 
         sections
       end
 
+      def extract_area(row)
+        tds = row.css("td")
+        return nil if tds.length < 2
+
+        area_cell = tds[1]
+        font = area_cell.at_css("font[color='#800080']")
+        return nil unless font
+
+        text = font.text.strip.downcase
+        return nil if text.empty?
+        return nil if text.match?(/\A(nl|be|nb|nn|br|pt|fr|de|us|gb|ch|at|ca|au|nz)\z/i) == false
+
+        text
+      end
+
       def content_rows
-        # Find the main content table (the one with language data)
-        # It's the largest table with IEV data
         tables = @doc.css("table")
         content_table = tables.max_by { |t| t.css("tr").length }
         content_table ? content_table.css("tr").to_a : []
@@ -109,15 +143,30 @@ module Iev
       end
 
       def extract_term(row)
-        # Term is in the third <td> — may be in a <b> tag (en, fr) or plain text
         tds = row.css("td")
         return nil if tds.length < 3
 
         content_td = tds[2]
+        return nil if area_marker_cell?(content_td)
+
         bold = content_td.at_css("b")
 
-        term = bold ? bold.text.strip : content_td.text.strip
-        term.empty? ? nil : term
+        html = if bold
+                 bold.inner_html.strip
+               else
+                 content_td.inner_html.strip
+               end
+        return nil if html.empty?
+
+        enrich(html)
+      end
+
+      # True when the cell contains only a geographical-area marker font
+      # tag (e.g. <font color="#800080">nb</font>), not a real term.
+      def area_marker_cell?(cell)
+        font = cell.at_css("font[color='#800080']")
+        return false unless font
+        cell.at_css("b").nil? && cell.text.strip == font.text.strip && font.text.strip.length <= 3
       end
 
       def extract_definition(row)
@@ -127,15 +176,59 @@ module Iev
         return nil if tds.length < 3
 
         content_td = tds[2]
-        # The definition is the text content, which may include MathML
         html = content_td.inner_html.strip
         return nil if html.empty? || html.match?(/\A<img.*ecblank/)
 
-        html
+        enrich(html)
+      end
+
+      def enrich(html_text)
+        text = html_text.to_s
+          .gsub(/&lt;i&gt;/, "<i>")
+          .gsub(/&lt;\/i&gt;/, "</i>")
+        mathml_blocks, text = extract_mathml_blocks(text)
+        result = parse_anchor_tag(text, term_domain)
+        result = restore_and_convert_mathml(result, mathml_blocks)
+        result = replace_newlines(result)
+        result = convert_literal_italic(result)
+        result.strip
+      end
+
+      MATHML_PLACEHOLDER = "@@MATHML_%d@@"
+
+      def extract_mathml_blocks(text)
+        blocks = []
+        result = text.gsub(/<math[^>]*>.*?<\/math>/m) do |match|
+          blocks << match
+          format(MATHML_PLACEHOLDER, blocks.size - 1)
+        end
+        [blocks, result]
+      end
+
+      # Restore MathML blocks and convert each to AsciiMath individually.
+      # If the converter returns empty (unsupported expression like
+      # <mtable>), keep the original MathML so the formula is not lost.
+      def restore_and_convert_mathml(text, blocks)
+        return text if blocks.empty?
+
+        blocks.each_with_index do |mathml, i|
+          converted = Iev::Converter.mathml_to_asciimath(mathml).strip
+          replacement = converted.empty? ? mathml : converted
+          text = text.sub(format(MATHML_PLACEHOLDER, i), replacement)
+        end
+        text
+      end
+
+      # After Nokogiri processing, &lt;i&gt;entity-encoded tags appear as
+      # literal <i>...</i> text. Convert them to stem:[...] just like
+      # real <i> elements are converted by convert_italic.
+      def convert_literal_italic(text)
+        text.gsub(/<i>([^<]*)<\/i>/) do
+          inner = Regexp.last_match(1)
+          convert_italic(inner)
+        end
       end
 
-      # Find the definition row following a language row.
-      # Skip separator rows (empty, <hr>, or spacer images).
       def find_definition_row(rows, start_idx)
         return nil if start_idx >= rows.length
 
@@ -149,7 +242,6 @@ module Iev
         content = tds[2].inner_html.strip
         return nil if content.empty?
 
-        # Skip rows that are only spacer images (unless they have <b> content)
         if content.match?(/\A<img.*ecblank/) && !content.include?("<b>")
           return nil
         end

data/lib/iev/source_parser.rb

@@ -406,7 +406,18 @@ module Iev
       return nil unless item
 
       item.source("src")
-    rescue Relaton::RequestError, Socket::ResolutionError, SocketError => e
+      # Free-text refs like "ITU-R Rec. 431 MOD" are not parseable pubids.
+      # relaton 3 (pubid 2) raises Pubid::Errors::ParseError (a
+      # Parslet::ParseFailed) eagerly when building the search, whereas
+      # relaton 2 swallowed it. Link resolution is best-effort: warn and
+      # skip instead of failing the whole source parse.
+      # relaton 3.0.0.pre.alpha.6 (#205 strict routing) also raises
+      # UnknownReferenceError for spellings no flavor claims (the French
+      # "CEI", agency refs like "IAEA 4") where alpha.5's permissive
+      # parse answered nil. Same best-effort contract: warn and skip.
+    rescue Relaton::RequestError, Relaton::UnknownReferenceError,
+           Socket::ResolutionError, SocketError,
+           Parslet::ParseFailed => e
       warn e.message
       nil
     end

data/lib/iev/subject_area_concepts.rb

@@ -128,10 +128,12 @@ module Iev
 
       # --- RelatedConcept factory methods ---
 
+      # RelatedConcept#content is a lang-keyed hash since glossarist 2.13
+      # (L10N invariant: all relationship free-text is { "eng" => ... }).
       def broader_relation(target_uri)
         Glossarist::RelatedConcept.new(
           type: "broader",
-          content: target_uri,
+          content: { "eng" => target_uri },
           ref: Glossarist::ConceptRef.new(source: "IEV", id: target_uri),
         )
       end
@@ -139,7 +141,7 @@ module Iev
       def narrower_relation(target_uri)
         Glossarist::RelatedConcept.new(
           type: "narrower",
-          content: target_uri,
+          content: { "eng" => target_uri },
           ref: Glossarist::ConceptRef.new(source: "IEV", id: target_uri),
         )
       end

data/lib/iev/utilities.rb

@@ -92,7 +92,9 @@ module Iev
       when "img"
         src = node["src"] || node.attributes.keys.first.to_s
         "#{IMAGE_PATH_PREFIX}/#{term_domain}/#{src}[]"
-      when "p", "div", "span"
+      when "p"
+        "#{inner}\n"
+      when "div", "span"
         inner
       when "i"
         convert_italic(inner)
@@ -101,9 +103,9 @@ module Iev
       when "sup"
         inner.empty? ? "" : "^#{inner}^"
       when "ol"
-        convert_list(node, ". ")
+        convert_list(node, ". ", term_domain)
       when "ul"
-        convert_list(node, "* ")
+        convert_list(node, "* ", term_domain)
       when "li"
         inner
       when "font"
@@ -124,8 +126,8 @@ module Iev
       end
     end
 
-    def convert_list(node, prefix)
-      node.css("li").map { |li| "#{prefix}#{li.text}" }.join
+    def convert_list(node, prefix, term_domain)
+      node.css("li").map { |li| "#{prefix}#{nodes_to_adoc(li.children, term_domain)}" }.join("\n")
     end
 
     def convert_font(node, inner)

data/lib/iev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Iev
-  VERSION = "0.4.7"
+  VERSION = "0.4.8"
 end

data/lib/iev.rb

@@ -32,7 +32,9 @@ module Iev
   autoload :FigureBuilder, "iev/figure_builder"
   autoload :IevCode, "iev/iev_code"
   autoload :Iso639Code, "iev/iso_639_code"
+  autoload :MultiDocYaml, "iev/multi_doc_yaml"
   autoload :Profiler, "iev/profiler"
+  autoload :Reconciler, "iev/reconciler"
   autoload :RelatonDb, "iev/relaton_db"
   autoload :Scraper, "iev/scraper"
   autoload :Section, "iev/section"

data/scripts/find_unparseable.rb

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+# Identify cached HTML pages that PageParser cannot turn into a concept.
+# Run: bundle exec ruby -Ilib scripts/find_unparseable.rb
+require "iev"
+
+store = Iev::Fetcher::PageStore.new
+ok = 0
+failed = []
+store.each_concept(scope: Iev::Fetcher::Scope.all).each do |code, html|
+  doc = Nokogiri::HTML(html)
+  if Iev::Scraper::PageParser.new(doc, code).parse
+    ok += 1
+  else
+    failed << code
+  end
+end
+
+puts "Parsed OK:  #{ok}"
+puts "Failed:     #{failed.size}"
+puts
+puts "First 20 failed codes:"
+failed.take(20).each { |c| puts "  #{c}" }