iev 0.4.6 → 0.4.8

data/lib/iev/reconciler/term_marker_parser.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module Iev
+  module Reconciler
+    # Extracts grammatical markers, context qualifiers, and part-of-speech
+    # tags that are inline in Electropedia's live HTML term text, matching
+    # how the termbase stores them in separate fields.
+    #
+    # Two-pass approach:
+    #   Pass 1: extract context qualifiers and IEV cross-references from
+    #           angle brackets (<text>, <相关条目：IEV xxx>)
+    #   Pass 2: extract gender/number/POS/prefix markers from remaining text
+    #
+    # Marker formats handled:
+    #   Gender:      , f  , m  , n  (also multiple: , f, n)
+    #                , ж  , м  , с  (Serbian Cyrillic)
+    #                , m/f  (slash = both apply)
+    #   Number:      , pl , sg  (also Serbian: јд, мн)
+    #                , f pl  (space-separated gender+number)
+    #   Usage info:  <text> → usage_info field
+    #   Part of speech: , 名詞 , 명사 , noun , verb , adj. , agg , etc.
+    #   Prefix:      , Präfix , 접두사 , 接頭語 , etc. → prefix flag
+    class TermMarkerParser
+      GENDER_MAP = {
+        "f" => "feminine", "m" => "masculine", "n" => "neuter",
+        "ж" => "feminine", "м" => "masculine", "с" => "neuter",
+      }.freeze
+
+      NUMBER_MAP = {
+        "јд" => "singular", "мн" => "plural",
+        "sg" => "singular", "pl" => "plural",
+      }.freeze
+
+      PART_OF_SPEECH_MAP = {
+        "noun" => "noun",
+        "명사" => "noun", "名詞" => "noun", "اسم" => "noun",
+        "именица" => "noun",
+        "verb" => "verb", "verbo" => "verb",
+        "동사" => "verb", "動詞" => "verb", "فعل" => "verb",
+        "глагол" => "verb",
+        "adj" => "adj", "adj." => "adj", "adjective" => "adj",
+        "agg" => "adj", "agg." => "adj",
+        "형용사" => "adj", "形容詞" => "adj", "صفة" => "adj",
+        "adjektiv" => "adj", "придев" => "adj",
+        "adv" => "adv", "adv." => "adv", "adverb" => "adv",
+        "부사" => "adv", "副詞" => "adv",
+        "прилог" => "adv",
+      }.freeze
+
+      PREFIX_KEYWORDS = %w[
+        Präfix prefix préfixe 접두사 接頭語 接尾語
+      ].freeze
+
+      DOMAIN_RE = /<([^>]+)>/
+      USAGE_INFO_RE = /\(([^)]+)\)\s*(?=[,]?\s*(?:[fmnжмс]\b|[fmn]\/[fmn]|sg|pl|јд|мн|\z))/i
+      IEV_XREF_RE = /<[^>]*IEV\s*(\d{3}-\d{2}-\d{2,3})[^>]*>/i
+
+      SERBIAN_RE = /,\s*([жмс])\s+(јд|мн)\s*\z/
+      WESTERN_GENDER_NUMBER_RE = /,\s*([fmn])\s+(sg|pl)\.?\s*\z/i
+      WESTERN_NUMBER_RE = /,\s*(sg|pl)\.?\s*\z/i
+      SLASH_GENDER_RE = /,\s*([fmn])\/([fmn])\s*\z/i
+      WESTERN_RE = /[,]?\s+([fmn])\s*\z/
+      SPACE_GENDER_RE = /\s+([fmn])\s*\z/
+
+      POS_RE = /,\s*(#{PART_OF_SPEECH_MAP.keys.map { |k| Regexp.escape(k) }.join("|")})\s*\z/i
+      PREFIX_RE = /,\s*(#{PREFIX_KEYWORDS.map { |k| Regexp.escape(k) }.join("|")})\s*\z/
+
+      TRAILING_PUNCT_RE = /[,;\s]+$/
+
+      Result = Struct.new(:designation, :genders, :numbers, :domain,
+                          :usage_info, :part_of_speech, :related_refs,
+                          :is_prefix, keyword_init: true)
+
+      class << self
+        def parse(term)
+          return Result.new(designation: nil) unless term
+
+          text = decode_entities(term).strip.gsub(/\s+/, " ").gsub(/　/, " ")
+          related_refs, text = extract_iev_xrefs(text)
+          domain, text = extract_domain(text)
+          usage_info, text = extract_usage_info(text)
+          designation, genders, numbers, pos, is_prefix = extract_markers(text)
+
+          Result.new(
+            designation: designation,
+            genders: genders,
+            numbers: numbers,
+            domain: domain,
+            usage_info: usage_info,
+            part_of_speech: pos,
+            related_refs: related_refs,
+            is_prefix: is_prefix,
+          )
+        end
+
+        def parse_multiple(term_text)
+          return [] unless term_text
+
+          term_text
+            .split(/\n+/)
+            .map(&:strip)
+            .reject(&:empty?)
+            .map { |part| parse(part) }
+        end
+
+        private
+
+        def decode_entities(str)
+          str
+            .gsub(/&lt;/, "<")
+            .gsub(/&gt;/, ">")
+            .gsub(/&amp;/, "&")
+            .gsub(/&quot;/, '"')
+            .gsub(/&#39;/, "'")
+        end
+
+        # Extract domain qualifier from angle brackets: <text> -> domain
+        def extract_domain(text)
+          match = text.match(DOMAIN_RE)
+          return [nil, text] unless match
+
+          domain = match[1].strip
+          remaining = text.sub(match[0], "").gsub(TRAILING_PUNCT_RE, "").strip
+          remaining = remaining.sub(/^[,;\s]+/, "").strip
+          [domain, remaining]
+        end
+
+        def extract_iev_xrefs(text)
+          refs = []
+          remaining = text
+          while (match = remaining.match(IEV_XREF_RE))
+            refs << match[1]
+            remaining = remaining.sub(match[0], "").gsub(TRAILING_PUNCT_RE, "").strip
+            remaining = remaining.sub(/^[,;\s]+/, "").strip
+          end
+          [refs.empty? ? nil : refs, remaining]
+        end
+
+        # Extract usage_info from parentheses that precede trailing markers.
+        # E.g. "Orientierung (einer Kurve) f" -> usage_info="einer Kurve"
+        def extract_usage_info(text)
+          match = text.match(USAGE_INFO_RE)
+          return [nil, text] unless match
+
+          usage = match[1].strip
+          remaining = text.sub(match[0], "").gsub(/\s+/, " ").strip
+          [usage, remaining]
+        end
+
+        def extract_markers(text)
+          designation = text
+          genders = []
+          numbers = []
+          pos = nil
+          is_prefix = false
+
+          loop do
+            break if designation.nil? || designation.empty?
+
+            if (m = designation.match(SERBIAN_RE))
+              designation = designation[0, m.begin(0)].strip
+              genders << GENDER_MAP[m[1]] if GENDER_MAP[m[1]]
+              numbers << NUMBER_MAP[m[2]] if NUMBER_MAP[m[2]]
+            elsif (m = designation.match(SLASH_GENDER_RE))
+              designation = designation[0, m.begin(0)].strip
+              genders << GENDER_MAP[m[1]] if GENDER_MAP[m[1]]
+              genders << GENDER_MAP[m[2]] if GENDER_MAP[m[2]]
+            elsif (m = designation.match(WESTERN_GENDER_NUMBER_RE))
+              designation = designation[0, m.begin(0)].strip
+              genders << GENDER_MAP[m[1]] if GENDER_MAP[m[1]]
+              num_key = m[2].downcase.sub(/\.$/, "")
+              numbers << NUMBER_MAP[num_key] if NUMBER_MAP[num_key]
+            elsif (m = designation.match(WESTERN_NUMBER_RE))
+              designation = designation[0, m.begin(0)].strip
+              num_key = m[1].downcase.sub(/\.$/, "")
+              numbers << NUMBER_MAP[num_key] if NUMBER_MAP[num_key]
+            elsif (m = designation.match(POS_RE))
+              designation = designation[0, m.begin(0)].strip
+              pos = PART_OF_SPEECH_MAP[m[1].downcase]
+            elsif (m = designation.match(PREFIX_RE))
+              designation = designation[0, m.begin(0)].strip
+              is_prefix = true
+            elsif (m = designation.match(WESTERN_RE))
+              designation = designation[0, m.begin(0)].strip
+              genders << GENDER_MAP[m[1]] if GENDER_MAP[m[1]]
+            else
+              break
+            end
+          end
+
+          [designation, genders.uniq, numbers.uniq, pos, is_prefix]
+        end
+      end
+    end
+  end
+end

data/lib/iev/reconciler/termbase_loader.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Iev
+  module Reconciler
+    # Loads termbase.yaml into a raw hash, then builds
+    # Glossarist::ManagedConcept objects on demand per code.
+    # This avoids the cost of constructing 22k model objects upfront.
+    class TermbaseLoader
+      # @param path [String, Pathname] path to termbase.yaml
+      def initialize(path)
+        @path = path.to_s
+        @data = nil
+      end
+
+      # @return [Hash<String, Hash>] raw termbase data keyed by code
+      def raw
+        @data ||= YAML.load_file(@path)
+      end
+
+      # @return [Array<String>] all codes in the termbase
+      def codes
+        raw.keys.map(&:to_s)
+      end
+
+      # Build a single ManagedConcept for the given code.
+      # @param code [String]
+      # @return [Glossarist::ManagedConcept, nil]
+      def get(code)
+        entry = raw[code] || raw[code.to_sym]
+        return nil unless entry.is_a?(Hash)
+
+        build_concept(code.to_s, entry)
+      end
+
+      private
+
+      def build_concept(code, entry)
+        concept = Glossarist::ManagedConcept.of_yaml(
+          "id" => code, "data" => { "id" => code },
+        )
+        concept.status = concept_status(entry)
+        concept.dates = concept_dates(entry)
+        concept.schema_version = "3"
+
+        build_localized(code, entry).each do |lc|
+          concept.add_l10n(lc)
+        end
+
+        concept
+      end
+
+      def concept_status(entry)
+        lang_data = first_language(entry)
+        StatusMapper.call(lang_data&.dig("entry_status"))
+      end
+
+      def concept_dates(entry)
+        lang_data = first_language(entry)
+        return [] unless lang_data
+
+        dates = []
+        if lang_data["date_accepted"]
+          dates << Glossarist::ConceptDate.new(
+            type: "accepted",
+            date: lang_data["date_accepted"],
+          )
+        end
+        if lang_data["date_amended"] && lang_data["date_amended"] != lang_data["date_accepted"]
+          dates << Glossarist::ConceptDate.new(
+            type: "amended",
+            date: lang_data["date_amended"],
+          )
+        end
+        dates
+      end
+
+      def first_language(entry)
+        entry.values.find { |v| v.is_a?(Hash) && v["terms"] }
+      end
+
+      def build_localized(code, entry)
+        result = []
+        entry.each do |lang, data|
+          next unless data.is_a?(Hash) && data["terms"]
+          lc = build_localized_concept(code, lang, data)
+          result << lc if lc
+        end
+        result
+      end
+
+      def build_localized_concept(code, lang, data)
+        cdata = Glossarist::ConceptData.new
+        cdata.id = code
+        cdata.language_code = lang
+        cdata.entry_status = StatusMapper.call(data["entry_status"])
+
+        cdata.terms = (data["terms"] || []).map do |t|
+          build_term_expression(t)
+        end
+
+        if data["definition"] && !data["definition"].empty?
+          cdata.definition = [Glossarist::DetailedDefinition.new(content: data["definition"])]
+        end
+
+        cdata.notes = Array(data["notes"]).map { |n| Glossarist::DetailedDefinition.new(content: n) }
+        cdata.examples = Array(data["examples"]).map { |e| Glossarist::DetailedDefinition.new(content: e) }
+
+        lc = Glossarist::LocalizedConcept.new
+        lc.id = code
+        lc.data = cdata
+        lc
+      end
+
+      def build_term_expression(term_data)
+        designation = term_data["designation"].to_s
+        parsed = TermMarkerParser.parse(designation)
+
+        expr = Glossarist::Designation::Expression.new(
+          designation: parsed.designation,
+          normative_status: term_data["normative_status"] || "preferred",
+          type: term_data["type"] || "expression",
+        )
+        expr.geographical_area = term_data["geographical_area"] if term_data["geographical_area"]
+        expr.field_of_application = term_data["usage_info"] || parsed.domain
+        expr.usage_info = parsed.usage_info if parsed.usage_info
+        expr.prefix = parsed.is_prefix if parsed.is_prefix
+
+        grammar = Glossarist::Designation::GrammarInfo.new
+        merged_genders = (Array(term_data["gender"]) + parsed.genders).flatten.compact.uniq
+        merged_numbers = (Array(term_data["plurality"]) + parsed.numbers).flatten.compact.uniq
+        grammar.gender = merged_genders if merged_genders.any?
+        grammar.number = merged_numbers if merged_numbers.any?
+        grammar.part_of_speech = parsed.part_of_speech if parsed.part_of_speech
+        expr.grammar_info = [grammar] if grammar.gender&.any? || grammar.number&.any? || grammar.part_of_speech
+        expr
+      end
+    end
+  end
+end

data/lib/iev/reconciler.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Iev
+  module Reconciler
+    autoload :Change,            "iev/reconciler/change"
+    autoload :ChangeSet,         "iev/reconciler/change_set"
+    autoload :ConceptMerger,     "iev/reconciler/concept_merger"
+    autoload :ContentDiffer,     "iev/reconciler/content_differ"
+    autoload :LiveLoader,        "iev/reconciler/live_loader"
+    autoload :Pipeline,          "iev/reconciler/pipeline"
+    autoload :ReconciledConcept, "iev/reconciler/reconciled_concept"
+    autoload :Report,            "iev/reconciler/report"
+    autoload :StatusMapper,      "iev/reconciler/status_mapper"
+    autoload :TermMarkerParser,  "iev/reconciler/term_marker_parser"
+    autoload :TermbaseLoader,    "iev/reconciler/termbase_loader"
+  end
+end

data/lib/iev/scraper/browser.rb

@@ -6,6 +6,10 @@ module Iev
   class Scraper
     # Shared headless browser utilities for fetching pages behind AWS WAF.
     module Browser
+      # Each profile is tagged with the host platform it can run on so
+      # that navigator.platform (set by Chrome from the real OS) agrees
+      # with the User-Agent and Sec-Ch-Ua-Platform headers. AWS WAF
+      # fingerprints this mismatch and refuses to clear its challenge.
       USER_AGENT_PROFILES = [
         {
           user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
@@ -13,6 +17,15 @@ module Iev
                       "Chrome/131.0.0.0 Safari/537.36",
           platform: '"macOS"',
           chrome_version: "131",
+          host_platform: :mac,
+        },
+        {
+          user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/129.0.0.0 Safari/537.36",
+          platform: '"macOS"',
+          chrome_version: "129",
+          host_platform: :mac,
         },
         {
           user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
@@ -20,85 +33,177 @@ module Iev
                       "Chrome/130.0.0.0 Safari/537.36",
           platform: '"Windows"',
           chrome_version: "130",
+          host_platform: :windows,
         },
         {
-          user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
+          user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
                       "AppleWebKit/537.36 (KHTML, like Gecko) " \
                       "Chrome/131.0.0.0 Safari/537.36",
-          platform: '"Linux"',
+          platform: '"Windows"',
           chrome_version: "131",
+          host_platform: :windows,
         },
         {
-          user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
-                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                      "Chrome/129.0.0.0 Safari/537.36",
-          platform: '"macOS"',
-          chrome_version: "129",
-        },
-        {
-          user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
+          user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
                       "AppleWebKit/537.36 (KHTML, like Gecko) " \
                       "Chrome/131.0.0.0 Safari/537.36",
-          platform: '"Windows"',
+          platform: '"Linux"',
           chrome_version: "131",
+          host_platform: :linux,
         },
       ].freeze
 
-      # Fetch a URL using headless Chrome, returning the page HTML.
-      # Handles AWS WAF challenge pages by waiting for JS execution.
-      def self.fetch(url, browser_opts: {})
-        browser = Ferrum::Browser.new(
-          headless: "new",
-          timeout: 30,
-          window_size: [1366, 768],
-          browser_options: {
-            "disable-blink-features" => "AutomationControlled",
-          },
-          **browser_opts,
-        )
-
-        browser.headers.set(random_headers)
-        browser.go_to(url)
-        browser.network.wait_for_idle(timeout: 15)
-        html = browser.body
-
-        if html.include?("403 ERROR") || html.include?("Request blocked")
-          warn "IEV: AWS WAF blocked request for #{url}"
-          return nil
-        end
+      DEFAULT_LANG = "en-US,en"
+      DEFAULT_BROWSER_OPTIONS = {
+        "disable-blink-features" => "AutomationControlled",
+        "lang" => DEFAULT_LANG,
+      }.freeze
 
-        html
-      rescue Ferrum::Error, Ferrum::BrowserError => e
-        warn "IEV: Browser error fetching #{url}: #{e.message}"
-        nil
-      ensure
-        browser&.quit
+      # One-shot fetch. Each call spins up a fresh headless Chrome, fetches,
+      # and tears it down. Suitable for ad-hoc use; the WAF cookie does not
+      # survive between calls. Batch callers (Fetcher::Mirror) should use
+      # Session instead so the cookie set on the first successful challenge
+      # is reused across requests.
+      def self.fetch(url, **_browser_opts)
+        Session.new.fetch(url)
       end
 
+      # Returns request headers that match what real Chrome sends on a
+      # fresh address-bar navigation. AWS WAF fingerprints inconsistencies
+      # between these headers and the browser's runtime state, so we:
+      #   - omit Sec-Fetch-* (Chrome computes those itself from the
+      #     navigation context; setting them via Ferrum's Network domain
+      #     overrides the real values and is detectable), and
+      #   - keep Sec-Ch-Ua-Platform aligned with the host OS, which is
+      #     what Chrome reports via navigator.platform.
       def self.random_headers
-        profile = USER_AGENT_PROFILES.sample
-        sec_ch_ua = "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
-                    "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
-                    "\"Not_A Brand\";v=\"24\""
+        profile = profile_for_host
+        static_headers.merge(headers_from_profile(profile))
+      end
 
+      def self.static_headers
         {
           "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
                       "image/avif,image/webp,image/apng,*/*;q=0.8," \
                       "application/signed-exchange;v=b3;q=0.7",
-          "Accept-Language" => "en-GB,en-US;q=0.9,en;q=0.8",
+          "Accept-Language" => "en-US,en;q=0.9",
           "Cache-Control" => "no-cache",
           "Pragma" => "no-cache",
-          "Sec-Ch-Ua" => sec_ch_ua,
           "Sec-Ch-Ua-Mobile" => "?0",
-          "Sec-Ch-Ua-Platform" => profile[:platform],
-          "Sec-Fetch-Dest" => "document",
-          "Sec-Fetch-Mode" => "navigate",
-          "Sec-Fetch-Site" => "cross-site",
-          "Sec-Fetch-User" => "?1",
           "Upgrade-Insecure-Requests" => "1",
+        }
+      end
+
+      def self.headers_from_profile(profile)
+        {
+          "Sec-Ch-Ua" => sec_ch_ua_for(profile),
+          "Sec-Ch-Ua-Platform" => profile[:platform],
           "User-Agent" => profile[:user_agent],
         }
       end
+
+      def self.profile_for_host
+        USER_AGENT_PROFILES.select do |profile|
+          profile[:host_platform] == host_platform
+        end.sample
+      end
+
+      def self.host_platform
+        case RUBY_PLATFORM
+        when /darwin/ then :mac
+        when /mswin|mingw|cygwin|bccwin|wince|emx/ then :windows
+        else :linux
+        end
+      end
+
+      def self.sec_ch_ua_for(profile)
+        "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
+          "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
+          "\"Not_A Brand\";v=\"24\""
+      end
+
+      # A long-lived headless Chrome session. Cookies persist across
+      # fetches, so once the AWS WAF challenge is cleared on the first
+      # request, subsequent requests reuse the token and succeed at
+      # near-100% rate. The Mirror creates one Session per run and
+      # shares it across all SequentialProbe iterations.
+      #
+      # Chrome leaks ~1MB per page load (mostly V8 heap that doesn't get
+      # GC'd between navigations). After ~1000 fetches the process is at
+      # ~1GB and the OOM risk climbs sharply. #restart quits the browser
+      # and starts a fresh one with a new cookie jar; the WAF challenge
+      # will need to be cleared again on the next fetch.
+      class Session
+        def initialize
+          @browser = Ferrum::Browser.new(
+            headless: "new",
+            timeout: 30,
+            window_size: [1366, 768],
+            browser_options: Browser::DEFAULT_BROWSER_OPTIONS,
+          )
+          @browser.headers.set(Browser.random_headers)
+        end
+
+        def fetch(url)
+          @browser.go_to(url)
+          @browser.network.wait_for_idle(timeout: 15)
+          reject_blocked(url, @browser.body)
+        rescue Ferrum::DeadBrowserError, Ferrum::NoSuchPageError,
+               Ferrum::NoSuchTargetError => e
+          # Chrome process or page/tab has crashed. Restart once, retry
+          # once. If the restart itself fails, surface as nil so the
+          # probe silently skips and the run continues.
+          if restart
+            retry
+          else
+            warn "IEV: Browser crashed, restart failed: #{e.message}"
+            nil
+          end
+        rescue Ferrum::BrowserError => e
+          if fetch_dead?(e.message) && restart
+            retry
+          else
+            warn "IEV: Browser error fetching #{url}: #{e.message}"
+            nil
+          end
+        rescue Ferrum::Error => e
+          warn "IEV: Browser error fetching #{url}: #{e.message}"
+          nil
+        end
+
+        def fetch_dead?(message)
+          message.include?("Browser is dead") ||
+            message.include?("given window is closed") ||
+            message.include?("Target closed")
+        end
+
+        def reject_blocked(url, html)
+          if html.include?("403 ERROR") || html.include?("Request blocked")
+            warn "IEV: AWS WAF blocked request for #{url}"
+            nil
+          else
+            html
+          end
+        end
+
+        # Quit the current browser and start a fresh one. The WAF cookie
+        # is lost, so the next fetch will go through the challenge cycle
+        # again. Call this after N fetches to bound Ferrum's memory growth,
+        # or rely on #fetch to call it automatically when Chrome dies.
+        # Returns true on success, false if the new browser fails to start.
+        def restart
+          quit
+          initialize
+          true
+        rescue StandardError => e
+          warn "IEV: Session restart failed: #{e.message}"
+          false
+        end
+
+        def quit
+          @browser&.quit
+        end
+      end
     end
   end
 end