iev 0.4.5 → 0.4.6

data/lib/iev/scraper/browser.rb

@@ -3,100 +3,102 @@
 require "ferrum"
 
 module Iev
-  # Shared headless browser utilities for fetching pages behind AWS WAF.
-  module ScraperBrowser
-    USER_AGENT_PROFILES = [
-      {
-        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"macOS"',
-        chrome_version: "131",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/130.0.0.0 Safari/537.36",
-        platform: '"Windows"',
-        chrome_version: "130",
-      },
-      {
-        user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"Linux"',
-        chrome_version: "131",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/129.0.0.0 Safari/537.36",
-        platform: '"macOS"',
-        chrome_version: "129",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"Windows"',
-        chrome_version: "131",
-      },
-    ].freeze
-
-    # Fetch a URL using headless Chrome, returning the page HTML.
-    # Handles AWS WAF challenge pages by waiting for JS execution.
-    def self.fetch(url, browser_opts: {})
-      browser = Ferrum::Browser.new(
-        headless: "new",
-        timeout: 30,
-        window_size: [1366, 768],
-        browser_options: {
-          "disable-blink-features" => "AutomationControlled",
+  class Scraper
+    # Shared headless browser utilities for fetching pages behind AWS WAF.
+    module Browser
+      USER_AGENT_PROFILES = [
+        {
+          user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/131.0.0.0 Safari/537.36",
+          platform: '"macOS"',
+          chrome_version: "131",
+        },
+        {
+          user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/130.0.0.0 Safari/537.36",
+          platform: '"Windows"',
+          chrome_version: "130",
+        },
+        {
+          user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/131.0.0.0 Safari/537.36",
+          platform: '"Linux"',
+          chrome_version: "131",
+        },
+        {
+          user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/129.0.0.0 Safari/537.36",
+          platform: '"macOS"',
+          chrome_version: "129",
+        },
+        {
+          user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/131.0.0.0 Safari/537.36",
+          platform: '"Windows"',
+          chrome_version: "131",
         },
-        **browser_opts,
-      )
+      ].freeze
 
-      browser.headers.set(random_headers)
-      browser.go_to(url)
-      browser.network.wait_for_idle(timeout: 15)
-      html = browser.body
+      # Fetch a URL using headless Chrome, returning the page HTML.
+      # Handles AWS WAF challenge pages by waiting for JS execution.
+      def self.fetch(url, browser_opts: {})
+        browser = Ferrum::Browser.new(
+          headless: "new",
+          timeout: 30,
+          window_size: [1366, 768],
+          browser_options: {
+            "disable-blink-features" => "AutomationControlled",
+          },
+          **browser_opts,
+        )
 
-      if html.include?("403 ERROR") || html.include?("Request blocked")
-        warn "IEV: AWS WAF blocked request for #{url}"
-        return nil
-      end
+        browser.headers.set(random_headers)
+        browser.go_to(url)
+        browser.network.wait_for_idle(timeout: 15)
+        html = browser.body
 
-      html
-    rescue Ferrum::Error, Ferrum::BrowserError => e
-      warn "IEV: Browser error fetching #{url}: #{e.message}"
-      nil
-    ensure
-      browser&.quit
-    end
+        if html.include?("403 ERROR") || html.include?("Request blocked")
+          warn "IEV: AWS WAF blocked request for #{url}"
+          return nil
+        end
 
-    def self.random_headers
-      profile = USER_AGENT_PROFILES.sample
-      sec_ch_ua = "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
-                  "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
-                  "\"Not_A Brand\";v=\"24\""
+        html
+      rescue Ferrum::Error, Ferrum::BrowserError => e
+        warn "IEV: Browser error fetching #{url}: #{e.message}"
+        nil
+      ensure
+        browser&.quit
+      end
 
-      {
-        "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
-                    "image/avif,image/webp,image/apng,*/*;q=0.8," \
-                    "application/signed-exchange;v=b3;q=0.7",
-        "Accept-Language" => "en-GB,en-US;q=0.9,en;q=0.8",
-        "Cache-Control" => "no-cache",
-        "Pragma" => "no-cache",
-        "Sec-Ch-Ua" => sec_ch_ua,
-        "Sec-Ch-Ua-Mobile" => "?0",
-        "Sec-Ch-Ua-Platform" => profile[:platform],
-        "Sec-Fetch-Dest" => "document",
-        "Sec-Fetch-Mode" => "navigate",
-        "Sec-Fetch-Site" => "cross-site",
-        "Sec-Fetch-User" => "?1",
-        "Upgrade-Insecure-Requests" => "1",
-        "User-Agent" => profile[:user_agent],
-      }
+      def self.random_headers
+        profile = USER_AGENT_PROFILES.sample
+        sec_ch_ua = "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
+                    "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
+                    "\"Not_A Brand\";v=\"24\""
+
+        {
+          "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
+                      "image/avif,image/webp,image/apng,*/*;q=0.8," \
+                      "application/signed-exchange;v=b3;q=0.7",
+          "Accept-Language" => "en-GB,en-US;q=0.9,en;q=0.8",
+          "Cache-Control" => "no-cache",
+          "Pragma" => "no-cache",
+          "Sec-Ch-Ua" => sec_ch_ua,
+          "Sec-Ch-Ua-Mobile" => "?0",
+          "Sec-Ch-Ua-Platform" => profile[:platform],
+          "Sec-Fetch-Dest" => "document",
+          "Sec-Fetch-Mode" => "navigate",
+          "Sec-Fetch-Site" => "cross-site",
+          "Sec-Fetch-User" => "?1",
+          "Upgrade-Insecure-Requests" => "1",
+          "User-Agent" => profile[:user_agent],
+        }
+      end
     end
   end
 end

data/lib/iev/scraper.rb

@@ -4,6 +4,9 @@ require "nokogiri"
 
 module Iev
   class Scraper
+    autoload :Browser, "iev/scraper/browser"
+    autoload :PageParser, "iev/scraper/page_parser"
+
     BASE_URL = "https://www.electropedia.org/iev/iev.nsf/" \
                "display?openform&ievref="
 
@@ -14,7 +17,8 @@ module Iev
     # Fetch the Electropedia page HTML for a given IEV code.
     # Returns a Nokogiri document.
     def fetch_page(code)
-      html = ScraperBrowser.fetch("#{BASE_URL}#{code}", browser_opts: @browser_opts)
+      html = Browser.fetch("#{BASE_URL}#{code}",
+                           browser_opts: @browser_opts)
       return nil unless html
 
       Nokogiri::HTML(html)
@@ -30,6 +34,3 @@ module Iev
     end
   end
 end
-
-require_relative "scraper/browser"
-require_relative "scraper/page_parser"

data/lib/iev/source_parser.rb

@@ -12,6 +12,7 @@ module Iev
   #   SourceParser.new(cell_data_string).parsed_sources
   class SourceParser
     include Utilities
+
     using DataConversions
 
     # When false, obtain_source_link skips Relaton network calls.
@@ -112,13 +113,13 @@ module Iev
       # IEC 62313:2009, 3.6, modifié
 
       str
-        .gsub(/CEI/, "IEC")
-        .gsub(/Guide IEC/, "IEC Guide")
+        .gsub("CEI", "IEC")
+        .gsub("Guide IEC", "IEC Guide")
         .gsub(%r{Guide ISO/IEC}, "ISO/IEC Guide")
-        .gsub(/VEI/, "IEV")
-        .gsub(/UIT/, "ITU")
-        .gsub(/IUT-R/, "ITU-R")
-        .gsub(/UTI-R/, "ITU-R")
+        .gsub("VEI", "IEV")
+        .gsub("UIT", "ITU")
+        .gsub("IUT-R", "ITU-R")
+        .gsub("UTI-R", "ITU-R")
         .gsub(/Recomm[ea]ndation ITU-T/, "ITU-T Recommendation")
         .gsub(/ITU-T (\w.\d{3}):(\d{4})/, 'ITU-T Recommendation \1 (\2)')
         .gsub(/ITU-R Rec. (\d+)/, 'ITU-R Recommendation \1')
@@ -290,17 +291,15 @@ module Iev
       ].map do |regex, _rule|
         # TODO: Rubocop complains about unused rule -- need to make sure
         # that no one forgot about something.
-        res = []
         # puts "str is '#{str}'"
         # puts "regex is '#{regex.to_s}'"
-        str.scan(regex).each do |result|
+        str.scan(regex).map do |result|
           # puts "result is #{result.first}"
-          res << {
+          {
             index: $LAST_MATCH_INFO.offset(0)[0],
             clause: result.first.strip,
           }
         end
-        res
         # sort by index and also the length of match
       end.flatten.sort_by { |hash| [hash[:index], -hash[:clause].length] }
 

data/lib/iev/subject_area_concepts.rb

@@ -15,13 +15,11 @@ module Iev
   #     (added by Exporter)
   #
   # Classification (separate from hierarchy):
-  #   - Each concept's ManagedConceptData#domains includes area and
-  #     section ConceptReferences
-  #   - Each concept's ConceptData#domain references its section URI
+  #   - Each concept's ManagedConceptData#domains includes domain and
+  #     section ConceptReferences (per ConceptReferenceType)
   #   - Each section concept's ConceptData#domain references parent area
+  #     title text (a LocalizedString, not a URI)
   module SubjectAreaConcepts
-    IEV_SOURCE = "urn:iec:std:iec:60050"
-
     class << self
       # Build all area and section concepts and add them to the collection.
       #
@@ -41,14 +39,6 @@ module Iev
 
       private
 
-      def domain_ref(concept_id)
-        Glossarist::ConceptReference.new(
-          concept_id: concept_id,
-          source: IEV_SOURCE,
-          ref_type: "domain",
-        )
-      end
-
       def build_area_concept(area)
         id = area.uri
 
@@ -56,12 +46,16 @@ module Iev
           data: Glossarist::ManagedConceptData.new(
             id: id,
             domains: [domain_ref(id)],
+            tags: [area.title],
           ),
         )
         mc.uuid = id
+        mc.schema_version = "3"
 
-        mc.add_localization(build_localization(id, area.title, "eng"))
-        mc.related = area.sections.map { |s| build_narrower_relation(s.uri) }
+        mc.add_localization(
+          build_localization(id, build_concept_data(id, area.title, "eng")),
+        )
+        mc.related = area.sections.map { |s| narrower_relation(s.uri) }
         mc.related = nil if mc.related.empty?
 
         mc
@@ -75,18 +69,22 @@ module Iev
             id: id,
             domains: [
               domain_ref(area.uri),
-              domain_ref(id),
+              section_ref(id),
             ],
+            tags: [area.title, section.title],
           ),
         )
         mc.uuid = id
+        mc.schema_version = "3"
 
         cd = build_concept_data(id, section.title, "eng")
-        cd.domain = area.uri
-
-        mc.add_localization(build_localization_from_data(id, cd))
+        # ConceptData#domain is a LocalizedString — use the area title text,
+        # not a URI. The structural relationship is expressed via domains[]
+        # and related[].
+        cd.domain = area.title
 
-        mc.related = [build_broader_relation(area.uri)]
+        mc.add_localization(build_localization(id, cd))
+        mc.related = [broader_relation(area.uri)]
 
         mc
       end
@@ -105,27 +103,32 @@ module Iev
         )
       end
 
-      def build_localization(id, title, lang_code)
-        cd = build_concept_data(id, title, lang_code)
-
+      def build_localization(id, concept_data)
         l10n = Glossarist::LocalizedConcept.new
-        l10n.data = cd
+        l10n.data = concept_data
         l10n.id = id
         l10n.entry_status = "valid"
         l10n.data.review_decision_event = "published"
         l10n
       end
 
-      def build_localization_from_data(id, concept_data)
-        l10n = Glossarist::LocalizedConcept.new
-        l10n.data = concept_data
-        l10n.id = id
-        l10n.entry_status = "valid"
-        l10n.data.review_decision_event = "published"
-        l10n
+      # --- ConceptReference factory methods ---
+
+      def domain_ref(concept_id)
+        ref = Glossarist::ConceptReference.domain(concept_id)
+        ref.source = IEV_SOURCE
+        ref
       end
 
-      def build_broader_relation(target_uri)
+      def section_ref(concept_id)
+        ref = Glossarist::ConceptReference.section(concept_id)
+        ref.source = IEV_SOURCE
+        ref
+      end
+
+      # --- RelatedConcept factory methods ---
+
+      def broader_relation(target_uri)
         Glossarist::RelatedConcept.new(
           type: "broader",
           content: target_uri,
@@ -133,7 +136,7 @@ module Iev
         )
       end
 
-      def build_narrower_relation(target_uri)
+      def narrower_relation(target_uri)
         Glossarist::RelatedConcept.new(
           type: "narrower",
           content: target_uri,

data/lib/iev/subject_areas.rb

@@ -3,7 +3,6 @@
 require "yaml"
 require "nokogiri"
 require "fileutils"
-require "iev/config"
 
 module Iev
   module SubjectAreas
@@ -44,7 +43,7 @@ module Iev
       # Return all subject areas with their sections.
       # @return [Array<SubjectArea>]
       def all
-        @typed_areas ||= raw_data["areas"].map { |h| build_area(h) }
+        @all ||= raw_data["areas"].map { |h| build_area(h) }
       end
 
       # Find a single subject area by its numeric code. O(1) indexed.
@@ -105,7 +104,7 @@ module Iev
         puts "Found #{fresh_areas.length} areas (#{areas.length} cached)" if $stdout.tty?
 
         # Merge: keep existing sections, add new areas
-        existing = areas.each_with_object({}) { |a, h| h[a["code"]] = a }
+        existing = areas.to_h { |a| [a["code"], a] }
         fresh_areas.each do |fa|
           existing[fa["code"]] ||= fa
         end
@@ -119,13 +118,13 @@ module Iev
             area["fetched"] = true
           rescue FetchError
             area["sections"] ||= []
-            warn "IEV: Skipping area #{area["code"]} due to WAF"
+            warn "IEV: Skipping area #{area['code']} due to WAF"
           end
 
-          puts "[#{i + 1}/#{areas.length}] #{area["code"]}: #{area["title"]} — #{area["sections"].length} sections" if $stdout.tty?
+          puts "[#{i + 1}/#{areas.length}] #{area['code']}: #{area['title']} — #{area['sections'].length} sections" if $stdout.tty?
 
           # Save progress every 10 areas so partial results survive WAF failures
-          if (i + 1) % 10 == 0
+          if ((i + 1) % 10).zero?
             write_cache("subject_areas.yaml", { "areas" => areas })
           end
 
@@ -201,7 +200,8 @@ module Iev
         @raw_data ||= begin
           path = File.exist?(DATA_FILE) ? DATA_FILE : nil
           if path
-            YAML.safe_load(File.read(path, encoding: "utf-8")) || { "areas" => [] }
+            YAML.safe_load(File.read(path,
+                                     encoding: "utf-8")) || { "areas" => [] }
           else
             { "areas" => [] }
           end
@@ -209,7 +209,7 @@ module Iev
       end
 
       def area_index
-        @area_index ||= all.each_with_object({}) { |a, h| h[a.code] = a }
+        @area_index ||= all.to_h { |a| [a.code, a] }
       end
 
       def section_index
@@ -233,10 +233,8 @@ module Iev
       end
 
       def fetch_page_with_retry(url, retries: MAX_RETRIES)
-        require "iev/scraper/browser"
-
         retries.times do |attempt|
-          html = ScraperBrowser.fetch(url)
+          html = Scraper::Browser.fetch(url)
           raise FetchError, "Failed to fetch #{url}" unless html
 
           unless captcha_page?(html)

data/lib/iev/term_attrs_parser.rb

@@ -68,7 +68,7 @@ module Iev
       extract_usage_info(curr_str)
       extract_prefix(curr_str)
 
-      return unless /\p{Word}/.match?(curr_str)
+      nil unless /\p{Word}/.match?(curr_str)
 
       # Term attributes could not be parsed completely
     end

data/lib/iev/term_builder.rb

@@ -7,6 +7,7 @@ module Iev
   class TermBuilder
     include Cli::Ui
     include Utilities
+
     using DataConversions
 
     def initialize(data)
@@ -102,18 +103,22 @@ module Iev
     end
 
     # Derives the domain (subject area section) from the IEVREF identifier.
-    # IEVREF format: "AAA-BB-CC" where AAA = area, AAA-BB = section.
-    # Returns a URI reference to the section concept (e.g. "section-103-01").
+    #
+    # Returns the section or area title text as a localized string.
+    # Per the concept model, ConceptData#domain is a LocalizedString
+    # (the domain name), not a URI. Structural membership is expressed
+    # via ManagedConceptData#domains[] with ConceptReference objects.
     def extract_domain
       return nil unless term_id
 
-      section_code = term_id.split("-")[0..1].join("-")
-      section = Iev.find_section(section_code)
-      return SubjectAreas.section_uri(section_code) if section
+      code = IevCode.new(term_id)
+      section = Iev.find_section(code.section_code) if code.section_code
+      return section.title if section
 
-      area_code = term_id.split("-")[0]
-      SubjectAreas.area_uri(area_code)
-    rescue StandardError
+      area = Iev.find_subject_area(code.area_code)
+      area&.title
+    rescue StandardError => e
+      warn "IEV: extract_domain failed for #{term_id}: #{e.message}"
       nil
     end
 
@@ -142,7 +147,7 @@ module Iev
             Note \d+\sto\sentry: |
             Note\s*\d+\sto\sthe\sentry: |
             Note\sto\sentry\s*\d+: |
-            Note\s*\d+?\sà\sl['']article: |
+            Note\s*\d+?\sà\sl'article: |
             <NOTE/?>?\s*\d?\s+[–-]\s* |
             NOTE(?:\s+-)?\s* |
             Note\s+\d+\s[–-]\s* |

data/lib/iev/utilities.rb

@@ -5,6 +5,10 @@ module Iev
     IMAGE_PATH_PREFIX = "image::/assets/images/parts"
     IEV_CODE_RE = /\A(IEV)?\s*(\d{2,3}-\d{2,3}-\d{2,3})\z/
 
+    # Pattern matching an anchor's inner text that is just an IEV code
+    # (not a meaningful term designation).
+    IEV_CODE_TEXT_RE = /\A\s*IEV\s*\d{2,3}-\d{2,3}-\d{2,3}\s*\z/
+
     # SIMG/Figure patterns — custom IEV XML, pre-processed before Nokogiri.
     # Uses [^>] and [^<] instead of . to avoid polynomial backtracking.
     SIMG_PATH_REGEX = /<simg [^>]*\/\$file\/([\d\-\w.]+)>/
@@ -134,12 +138,36 @@ module Iev
 
       if href.match?(IEV_CODE_RE)
         iev_code = href.sub(/\AIEV\s*/, "")
-        "{{#{inner}, urn:iec:std:iec:60050-#{iev_code}}}"
+        display = render_term_for(inner, iev_code)
+        "{{urn:iec:std:iec:60050-#{iev_code}, #{display}}}"
       elsif !href.empty?
         "#{href}[#{inner}]"
       else
         inner
       end
     end
+
+    # Resolve the display (render) text for an IEV cross-reference.
+    #
+    # When the anchor text is already a meaningful term (e.g. "adjective"),
+    # use it directly. When it's just a bare IEV code (e.g. "IEV 102-01-10"),
+    # try to look up the actual term designation via DataSource.
+    #
+    # @param inner_text [String] the anchor element's inner text
+    # @param iev_code [String] the extracted numeric IEV code (e.g. "102-01-10")
+    # @return [String] the term designation to use as render text
+    def render_term_for(inner_text, iev_code)
+      stripped = inner_text.strip
+      return stripped unless iev_code_only?(stripped)
+
+      Iev.get(iev_code, "en") || stripped
+    rescue StandardError
+      stripped
+    end
+
+    # True when the anchor text is just a raw IEV code, not a term designation.
+    def iev_code_only?(text)
+      text.match?(IEV_CODE_TEXT_RE)
+    end
   end
 end

data/lib/iev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Iev
-  VERSION = "0.4.5"
+  VERSION = "0.4.6"
 end

data/lib/iev.rb

@@ -1,14 +1,10 @@
 # frozen_string_literal: true
 
 require "iev/version"
-require "iev/config"
-require "iev/data_source"
-
-require "yaml"
 
 # plurimath and unitsml both depend on mml, which has a transitive
 # dependency version mismatch with lutaml-model in some environments.
-# Load them when available; the DataSource APIs work without them.
+# Load them when available; the DataSource/Db APIs work without them.
 begin
   require "plurimath"
 rescue LoadError
@@ -22,6 +18,9 @@ rescue LoadError
 end
 
 module Iev
+  # IEV dataset URN — single source of truth for all concept references.
+  IEV_SOURCE = "urn:iec:std:iec:60050"
+
   autoload :Cli, "iev/cli"
   autoload :Config, "iev/config"
   autoload :Converter, "iev/converter"
@@ -44,6 +43,26 @@ module Iev
   autoload :TermBuilder, "iev/term_builder"
   autoload :Utilities, "iev/utilities"
 
+  # --- Configuration ---
+
+  # @return [Config]
+  def self.config
+    @config ||= Config.new
+  end
+
+  # Yield the config object for initialization.
+  # @yield [Config]
+  def self.configure
+    yield(config) if block_given?
+  end
+
+  # Reset config (useful in tests).
+  def self.reset_config!
+    @config = nil
+  end
+
+  # --- Data fetching ---
+
   # Fetch term designation from IEV data.
   #
   # @param [String] code for example "103-01-02"
@@ -51,7 +70,6 @@ module Iev
   #
   # @return [String, nil] if found then term,
   #   if code or language not found then nil.
-  #
   def self.get(code, lang)
     DataSource.fetch_term_designation(code, lang)
   rescue DataSource::NotFoundError
@@ -77,6 +95,8 @@ module Iev
     DataSource.fetch_term(code, lang)
   end
 
+  # --- Scraping ---
+
   # Scrape concept data from Electropedia for a given IEV code.
   # Uses Ferrum (headless Chrome) to handle AWS WAF challenge.
   #
@@ -86,6 +106,8 @@ module Iev
     Scraper.new.fetch_concept(code)
   end
 
+  # --- Subject area / section queries ---
+
   # Return all IEV subject areas with their sections (from bundled data).
   # @return [Array<SubjectArea>]
   def self.subject_areas