iev 0.4.3 → 0.4.5

data/exe/iev

@@ -9,7 +9,7 @@ require "creek"
 require "glossarist"
 require "nokogiri"
 require "relaton"
-require "relaton_bib"
+require "relaton/bib"
 require "sequel"
 require "thor"
 

data/iev.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = Gem::Requirement.new(">= 3.2.0")
 
   spec.add_dependency "creek", "~> 2.6"
-  spec.add_dependency "glossarist", ">= 2.3.0"
+  spec.add_dependency "glossarist", "~> 2.6", ">= 2.6.7"
   spec.add_dependency "ferrum", "~> 0.15"
   spec.add_dependency "nokogiri", "~> 1.19"
   spec.add_dependency "plurimath"

data/lib/iev/cli/command.rb

@@ -142,6 +142,30 @@ module Iev
         summary
       end
 
+      desc "subject_areas", "Fetch IEV subject areas and sections from Electropedia."
+      option :output, desc: "Output YAML file (default: stdout)", aliases: :o
+      option :refresh, type: :boolean, default: false,
+                       desc: "Force re-fetch even if cached"
+      def subject_areas
+        if options[:refresh]
+          cache_path = File.join(Iev.config.cache_dir, "subject_areas.yaml")
+          FileUtils.rm_f(cache_path) if File.exist?(cache_path)
+        end
+
+        result = Iev::SubjectAreas.fetch
+
+        yaml = YAML.dump(result)
+        if options[:output]
+          File.write(options[:output], yaml, encoding: "utf-8")
+          puts "Written to #{options[:output]}"
+        else
+          puts yaml
+        end
+      rescue Iev::SubjectAreas::FetchError => e
+        error e.message
+        exit 1
+      end
+
       desc "fetch CODE", "Fetch an IEV concept and output YAML to stdout."
       option :scrape, type: :boolean, default: false,
                       desc: "Scrape from Electropedia instead of using cached data"

data/lib/iev/cli/command_helper.rb

@@ -111,8 +111,7 @@ module Iev
 
         definition = entry["definition"]
         if definition
-          content = definition.is_a?(String) ? definition : definition
-          cd.definition = [Glossarist::DetailedDefinition.new(content: content)]
+          cd.definition = [Glossarist::DetailedDefinition.new(content: definition)]
         end
 
         l10n = Glossarist::LocalizedConcept.new

data/lib/iev/exporter.rb

@@ -28,16 +28,19 @@ module Iev
     # @param only_concepts [String, nil] SQL LIKE pattern for IEVREF filtering
     # @param only_languages [String, nil] comma-separated language codes
     # @param fetch_relaton_links [Boolean] fetch source URLs via Relaton
+    # @param include_areas [Boolean] create area/section hierarchy concepts
     # @param on_progress [Proc, nil] callback (current, total) during build
     def initialize(input_path, output_dir: Dir.pwd,
                    only_concepts: nil, only_languages: nil,
                    fetch_relaton_links: false,
+                   include_areas: true,
                    on_progress: nil)
       @input_path = Pathname.new(input_path)
       validate_input!
 
       @output_dir = Pathname.new(output_dir)
       @fetch_relaton_links = fetch_relaton_links
+      @include_areas = include_areas
       @on_progress = on_progress
       @filters = {
         only_concepts: only_concepts,
@@ -51,6 +54,8 @@ module Iev
       start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       dataset = load_dataset
       collection = build_collection(dataset)
+      add_subject_area_concepts(collection) if @include_areas
+      build_section_narrower_relations(collection) if @include_areas
       save_collection(collection)
       elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
 
@@ -81,7 +86,7 @@ module Iev
 
       exts = (XLSX_EXTENSIONS + SQLITE_EXTENSIONS).join(", ")
       raise ArgumentError,
-        "Unsupported format: #{input_path.extname}. Supported: #{exts}"
+            "Unsupported format: #{input_path.extname}. Supported: #{exts}"
     end
 
     def input_format
@@ -137,10 +142,16 @@ module Iev
 
         concept = concept_index[term.id] ||= begin
           c = Glossarist::ManagedConcept.new(data: { "id" => term.id })
+          c.uuid = term.id
+          c.data.domains = domain_references_for(term.id)
+          add_section_broader(c, term.id)
           collection.store(c)
           c
         end
         concept.add_l10n(term)
+
+        promote_supersession(concept, term)
+        set_managed_status(concept, term)
       end
 
       collection
@@ -148,14 +159,109 @@ module Iev
       SourceParser.relaton_enabled = true
     end
 
+    def add_subject_area_concepts(collection)
+      SubjectAreaConcepts.add_to(collection)
+    end
+
     def save_collection(collection)
       concepts_dir = output_dir.expand_path.join("concepts")
       FileUtils.mkdir_p(concepts_dir)
-      collection.save_to_files(concepts_dir.to_s)
+      collection.save_grouped_concepts_to_files(concepts_dir.to_s)
     end
 
     def localized_count(collection)
       collection.sum { |c| c.localized_concepts.count }
     end
+
+    IEV_SOURCE = "urn:iec:std:iec:60050"
+
+    def domain_references_for(ievref)
+      code = IevCode.new(ievref.to_s)
+      refs = []
+      if code.area_code
+        refs << Glossarist::ConceptReference.new(
+          concept_id: code.area_uri,
+          source: IEV_SOURCE,
+          ref_type: "domain",
+        )
+      end
+      if code.section_code
+        refs << Glossarist::ConceptReference.new(
+          concept_id: code.section_uri,
+          source: IEV_SOURCE,
+          ref_type: "domain",
+        )
+      end
+      refs
+    end
+
+    def add_section_broader(concept, ievref)
+      code = IevCode.new(ievref.to_s)
+      return unless code.section_uri
+
+      concept.related ||= []
+      return if concept.related.any? do |r|
+        r.type == "broader" && r.ref&.id == code.section_uri
+      end
+
+      concept.related << Glossarist::RelatedConcept.new(
+        type: "broader",
+        content: code.section_uri,
+        ref: Glossarist::ConceptRef.new(source: "IEV", id: code.section_uri),
+      )
+    end
+
+    def build_section_narrower_relations(collection)
+      mc_index = collection.each_with_object({}) do |c, h|
+        h[c.data&.id] = c if c.data&.id
+      end
+
+      section_children = {}
+      mc_index.each_key do |concept_id|
+        code = IevCode.new(concept_id)
+        next unless code.section_uri
+
+        (section_children[code.section_uri] ||= []) << concept_id
+      end
+
+      section_children.each do |section_uri, child_ids|
+        section_mc = mc_index[section_uri]
+        next unless section_mc
+
+        narrower = child_ids.sort.map do |child_id|
+          Glossarist::RelatedConcept.new(
+            type: "narrower",
+            content: child_id,
+            ref: Glossarist::ConceptRef.new(source: "IEV", id: child_id),
+          )
+        end
+
+        section_mc.related ||= []
+        section_mc.related.concat(narrower)
+      end
+    end
+
+    # Promote supersedes relations from localized ConceptData to managed level.
+    # Supersession is language-independent (REPLACES column is per-concept).
+    def promote_supersession(concept, term)
+      related = term.data&.related
+      return unless related&.any?
+
+      concept.related ||= []
+      related.each do |r|
+        next if concept.related.any? { |er| er.type == r.type && er.ref&.id == r.ref&.id }
+
+        concept.related << r
+      end
+      term.data.related = nil
+    end
+
+    # Derive managed concept status from the localization's entry_status.
+    def set_managed_status(concept, term)
+      return if concept.status
+
+      status = term.entry_status
+      concept.status = status if status && !status.empty?
+    end
   end
 end

data/lib/iev/iev_code.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Iev
+  # Immutable value object that decomposes an IEV concept code
+  # into its structural parts: area code, section code, and number.
+  #
+  # The IEV code format is AAA-BB-CC where:
+  #   AAA = area code (e.g. "103")
+  #   BB  = section sub-code (e.g. "01")
+  #   CC  = concept number (e.g. "02")
+  #
+  # @example Full concept code
+  #   code = Iev::IevCode.new("103-01-02")
+  #   code.area_code    #=> "103"
+  #   code.section_code #=> "103-01"
+  #   code.number       #=> "02"
+  #   code.area_uri     #=> "area-103"
+  #   code.section_uri  #=> "section-103-01"
+  #
+  # @example Section code (no concept number)
+  #   code = Iev::IevCode.new("103-01")
+  #   code.area_code    #=> "103"
+  #   code.section_code #=> "103-01"
+  #   code.number       #=> nil
+  #   code.section_uri  #=> "section-103-01"
+  #
+  class IevCode
+    include Comparable
+
+    attr_reader :raw, :area_code, :section_code, :number
+
+    # @param code [#to_s] IEV reference, e.g. "103-01-02"
+    def initialize(code)
+      @raw = code.to_s
+      parts = @raw.split("-")
+      @area_code = parts[0]
+      @section_code = parts.length >= 2 ? "#{parts[0]}-#{parts[1]}" : nil
+      @number = parts.length >= 3 ? parts[2] : nil
+      freeze
+    end
+
+    def area_uri
+      "area-#{area_code}"
+    end
+
+    def section_uri
+      "section-#{section_code}" if section_code
+    end
+
+    def to_s
+      @raw
+    end
+
+    def to_str
+      @raw
+    end
+
+    def ==(other)
+      other.is_a?(self.class) && raw == other.raw
+    end
+    alias_method :eql?, :==
+
+    def hash
+      raw.hash
+    end
+
+    def <=>(other)
+      to_s <=> other.to_s
+    end
+
+    # Safe constructor that returns nil for codes that don't parse.
+    # @param code [#to_s]
+    # @return [IevCode, nil]
+    def self.parse(code)
+      new(code)
+    rescue ArgumentError
+      nil
+    end
+  end
+end

data/lib/iev/iso_639_code.rb

@@ -6,7 +6,7 @@
 module Iev
   # @todo This needs to be rewritten.
   class Iso639Code
-    COUNTRY_CODES = YAML.load(IO.read(File.join(__dir__, "iso_639_2.yaml")))
+    COUNTRY_CODES = YAML.safe_load(IO.read(File.join(__dir__, "iso_639_2.yaml")), permitted_classes: [Symbol]).freeze
     # rubocop:disable Style/MutableConstant
     THREE_CHAR_MEMO = {} # Memoization cache, must be mutable
     # rubocop:enable Style/MutableConstant

data/lib/iev/scraper/browser.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "ferrum"
+
+module Iev
+  # Shared headless browser utilities for fetching pages behind AWS WAF.
+  module ScraperBrowser
+    USER_AGENT_PROFILES = [
+      {
+        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                    "Chrome/131.0.0.0 Safari/537.36",
+        platform: '"macOS"',
+        chrome_version: "131",
+      },
+      {
+        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
+                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                    "Chrome/130.0.0.0 Safari/537.36",
+        platform: '"Windows"',
+        chrome_version: "130",
+      },
+      {
+        user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
+                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                    "Chrome/131.0.0.0 Safari/537.36",
+        platform: '"Linux"',
+        chrome_version: "131",
+      },
+      {
+        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                    "Chrome/129.0.0.0 Safari/537.36",
+        platform: '"macOS"',
+        chrome_version: "129",
+      },
+      {
+        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
+                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                    "Chrome/131.0.0.0 Safari/537.36",
+        platform: '"Windows"',
+        chrome_version: "131",
+      },
+    ].freeze
+
+    # Fetch a URL using headless Chrome, returning the page HTML.
+    # Handles AWS WAF challenge pages by waiting for JS execution.
+    def self.fetch(url, browser_opts: {})
+      browser = Ferrum::Browser.new(
+        headless: "new",
+        timeout: 30,
+        window_size: [1366, 768],
+        browser_options: {
+          "disable-blink-features" => "AutomationControlled",
+        },
+        **browser_opts,
+      )
+
+      browser.headers.set(random_headers)
+      browser.go_to(url)
+      browser.network.wait_for_idle(timeout: 15)
+      html = browser.body
+
+      if html.include?("403 ERROR") || html.include?("Request blocked")
+        warn "IEV: AWS WAF blocked request for #{url}"
+        return nil
+      end
+
+      html
+    rescue Ferrum::Error, Ferrum::BrowserError => e
+      warn "IEV: Browser error fetching #{url}: #{e.message}"
+      nil
+    ensure
+      browser&.quit
+    end
+
+    def self.random_headers
+      profile = USER_AGENT_PROFILES.sample
+      sec_ch_ua = "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
+                  "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
+                  "\"Not_A Brand\";v=\"24\""
+
+      {
+        "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
+                    "image/avif,image/webp,image/apng,*/*;q=0.8," \
+                    "application/signed-exchange;v=b3;q=0.7",
+        "Accept-Language" => "en-GB,en-US;q=0.9,en;q=0.8",
+        "Cache-Control" => "no-cache",
+        "Pragma" => "no-cache",
+        "Sec-Ch-Ua" => sec_ch_ua,
+        "Sec-Ch-Ua-Mobile" => "?0",
+        "Sec-Ch-Ua-Platform" => profile[:platform],
+        "Sec-Fetch-Dest" => "document",
+        "Sec-Fetch-Mode" => "navigate",
+        "Sec-Fetch-Site" => "cross-site",
+        "Sec-Fetch-User" => "?1",
+        "Upgrade-Insecure-Requests" => "1",
+        "User-Agent" => profile[:user_agent],
+      }
+    end
+  end
+end

data/lib/iev/scraper.rb

@@ -1,59 +1,12 @@
 # frozen_string_literal: true
 
+require "nokogiri"
+
 module Iev
-  # Scrapes IEV term data from Electropedia (electropedia.org).
-  #
-  # Electropedia is behind AWS WAF which requires JavaScript execution,
-  # so a headless browser (via Ferrum/Chrome) is used to handle the challenge.
-  #
-  # @example
-  #   scraper = Iev::Scraper.new
-  #   concept = scraper.fetch_concept("103-01-02")
-  #   doc = scraper.fetch_page("103-01-02")
   class Scraper
     BASE_URL = "https://www.electropedia.org/iev/iev.nsf/" \
                "display?openform&ievref="
 
-    # Pool of realistic Chrome User-Agent strings with matching platform hints.
-    # Rotated per request to reduce fingerprinting by AWS WAF.
-    USER_AGENT_PROFILES = [
-      {
-        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"macOS"',
-        chrome_version: "131",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/130.0.0.0 Safari/537.36",
-        platform: '"Windows"',
-        chrome_version: "130",
-      },
-      {
-        user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"Linux"',
-        chrome_version: "131",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/129.0.0.0 Safari/537.36",
-        platform: '"macOS"',
-        chrome_version: "129",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"Windows"',
-        chrome_version: "131",
-      },
-    ].freeze
-
     def initialize(browser_opts: {})
       @browser_opts = browser_opts
     end
@@ -61,37 +14,10 @@ module Iev
     # Fetch the Electropedia page HTML for a given IEV code.
     # Returns a Nokogiri document.
     def fetch_page(code)
-      require "ferrum"
-      require "nokogiri"
-
-      url = "#{BASE_URL}#{code}"
-      browser = Ferrum::Browser.new(
-        headless: "new",
-        timeout: 30,
-        window_size: [1366, 768],
-        browser_options: {
-          "disable-blink-features" => "AutomationControlled",
-        },
-        **@browser_opts,
-      )
-
-      browser.headers.set(random_headers)
-      browser.go_to(url)
-      browser.network.wait_for_idle(timeout: 15)
-      html = browser.body
-
-      # Check if we got a real page or a WAF block
-      if html.include?("403 ERROR") || html.include?("Request blocked")
-        warn "IEV Scraper: AWS WAF blocked request for #{code}"
-        return nil
-      end
+      html = ScraperBrowser.fetch("#{BASE_URL}#{code}", browser_opts: @browser_opts)
+      return nil unless html
 
       Nokogiri::HTML(html)
-    rescue Ferrum::Error, Ferrum::BrowserError => e
-      warn "IEV Scraper error for #{code}: #{e.message}"
-      nil
-    ensure
-      browser&.quit
     end
 
     # Fetch and parse concept data for an IEV code.
@@ -102,34 +28,8 @@ module Iev
 
       PageParser.new(doc, code).parse
     end
-
-    private
-
-    def random_headers
-      profile = USER_AGENT_PROFILES.sample
-      sec_ch_ua = "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
-                  "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
-                  "\"Not_A Brand\";v=\"24\""
-
-      {
-        "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
-                    "image/avif,image/webp,image/apng,*/*;q=0.8," \
-                    "application/signed-exchange;v=b3;q=0.7",
-        "Accept-Language" => "en-GB,en-US;q=0.9,en;q=0.8",
-        "Cache-Control" => "no-cache",
-        "Pragma" => "no-cache",
-        "Sec-Ch-Ua" => sec_ch_ua,
-        "Sec-Ch-Ua-Mobile" => "?0",
-        "Sec-Ch-Ua-Platform" => profile[:platform],
-        "Sec-Fetch-Dest" => "document",
-        "Sec-Fetch-Mode" => "navigate",
-        "Sec-Fetch-Site" => "cross-site",
-        "Sec-Fetch-User" => "?1",
-        "Upgrade-Insecure-Requests" => "1",
-        "User-Agent" => profile[:user_agent],
-      }
-    end
   end
 end
 
+require_relative "scraper/browser"
 require_relative "scraper/page_parser"

data/lib/iev/section.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Iev
+  # Immutable value object representing an IEV section (e.g. "103-01").
+  #
+  # A section belongs to exactly one area, identified by +area_code+.
+  class Section
+    attr_reader :code, :title, :area_code
+
+    # @param code [#to_s] section code, e.g. "103-01"
+    # @param title [#to_s] section title, e.g. "General concepts on functions"
+    # @param area_code [#to_s] parent area code, e.g. "103"
+    def initialize(code:, title:, area_code:)
+      @code = code.to_s
+      @title = title.to_s
+      @area_code = area_code.to_s
+      freeze
+    end
+
+    def uri
+      "section-#{code}"
+    end
+
+    def to_h
+      { "code" => code, "title" => title }
+    end
+
+    def ==(other)
+      other.is_a?(self.class) && code == other.code
+    end
+    alias_method :eql?, :==
+
+    def hash
+      code.hash
+    end
+  end
+end

data/lib/iev/source_parser.rb

@@ -79,10 +79,11 @@ module Iev
       relationship = extract_source_relationship(raw_ref)
       clean_ref = normalize_ref_string(raw_ref)
       source_ref = extract_source_ref(clean_ref)
+      ref_source, ref_id = split_ref(source_ref)
       clause = extract_source_clause(clean_ref)
 
       origin = Glossarist::Citation.new(
-        ref: source_ref,
+        ref: Glossarist::Citation::Ref.new(source: ref_source, id: ref_id),
         locality: build_locality(clause),
         link: obtain_source_link(source_ref),
         original: Iev::Converter.mathml_to_asciimath(
@@ -351,6 +352,52 @@ module Iev
       )
     end
 
+    # Splits a normalized bibliographic reference into [source, id] for
+    # structured Citation::Ref construction.  The full string is still
+    # passed to Relaton for link resolution — only the Citation::Ref
+    # model receives the split form.
+    #
+    #   "IEC 62302:2007"           → ["IEC",          "62302:2007"]
+    #   "ISO/IEC 2382:2015"        → ["ISO/IEC",      "2382:2015"]
+    #   "ISO/TS 14812:2022"        → ["ISO/TS",       "14812:2022"]
+    #   "IEC CISPR 16-1:2003"      → ["IEC CISPR",    "16-1:2003"]
+    #   "ITU-T Recommendation F.791 (11/2015)" → ["ITU-T Recommendation", "F.791 (11/2015)"]
+    #   "IEV"                      → ["IEV",          nil]
+    def split_ref(full_ref)
+      case full_ref
+      when /\A(ISO\/IEC\/IEEE)\s+(.+)/
+        [$1, $2]
+      when /\A(ISO\/IEC\s+Guide)\s+(.+)/
+        [$1, $2]
+      when /\A(ISO\/IEC)\s+(.+)/
+        [$1, $2]
+      when /\A(IEC\/IEEE)\s+(.+)/
+        [$1, $2]
+      when %r{\A((?:ISO|IEC)/(?:PAS|TR|TS))\s+(.+)}
+        [$1, $2]
+      when /\A(IEC\s+CISPR)\s+(.+)/
+        [$1, $2]
+      when /\A(ITU-T\s+Recommendation)\s+(.+)/
+        [$1, $2]
+      when /\A(ITU-R\s+Recommendation)\s+(.+)/
+        [$1, $2]
+      when /\A(ITU-R)\s+(.+)/
+        [$1, $2]
+      when /\A((?:ISO|IEC)\s+Guide)\s+(.+)/
+        [$1, $2]
+      when /\A(ISO|IEC|IAEA)\s+(.+)/
+        [$1, $2]
+      when /\AIEV\z/
+        ["IEV", nil]
+      when /\A(JCGM)\s+(VIM)\z/
+        [$1, $2]
+      when /\ABBIPM/
+        ["BIPM", "SI Brochure"]
+      else
+        [full_ref, nil]
+      end
+    end
+
     # Uses Relaton to obtain link for given source ref.
     def obtain_source_link(ref)
       return nil unless self.class.relaton_enabled