iev 0.4.4 → 0.4.6

data/lib/iev/exporter.rb

@@ -55,7 +55,9 @@ module Iev
       dataset = load_dataset
       collection = build_collection(dataset)
       add_subject_area_concepts(collection) if @include_areas
+      build_section_narrower_relations(collection) if @include_areas
       save_collection(collection)
+      save_register
       elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
 
       @stats = {
@@ -85,7 +87,7 @@ module Iev
 
       exts = (XLSX_EXTENSIONS + SQLITE_EXTENSIONS).join(", ")
       raise ArgumentError,
-        "Unsupported format: #{input_path.extname}. Supported: #{exts}"
+            "Unsupported format: #{input_path.extname}. Supported: #{exts}"
     end
 
     def input_format
@@ -139,13 +141,23 @@ module Iev
         term = TermBuilder.build_from(row)
         next unless term
 
+        # Parse IevCode once per concept — used by all helpers below.
+        code = IevCode.new(term.id)
+
         concept = concept_index[term.id] ||= begin
           c = Glossarist::ManagedConcept.new(data: { "id" => term.id })
-          c.data.domains = domain_references_for(term.id)
+          c.uuid = term.id
+          c.schema_version = "3"
+          c.data.domains = domain_references_for(code)
+          c.data.tags = tags_for(code)
+          add_section_broader(c, code)
           collection.store(c)
           c
         end
         concept.add_l10n(term)
+
+        promote_supersession(concept, term)
+        set_managed_status(concept, term)
       end
 
       collection
@@ -160,21 +172,181 @@ module Iev
     def save_collection(collection)
       concepts_dir = output_dir.expand_path.join("concepts")
       FileUtils.mkdir_p(concepts_dir)
-      collection.save_to_files(concepts_dir.to_s)
+      collection.save_grouped_concepts_to_files(concepts_dir.to_s)
+    end
+
+    def save_register
+      areas = SubjectAreas.all
+      sections = build_section_tree(areas)
+
+      register = Glossarist::DatasetRegister.new(
+        schema_type: "glossarist",
+        schema_version: "3",
+        id: "iev",
+        ref: "IEC 60050:2011",
+        year: 2011,
+        urn: IEV_SOURCE,
+        urn_aliases: ["#{IEV_SOURCE}*"],
+        status: "current",
+        owner: "IEC",
+        source_repo: "https://github.com/glossarist/iev-data",
+        tags: %w[electrotechnical vocabulary iec],
+        languages: %w[eng fra],
+        language_order: %w[eng fra],
+        ordering: "systematic",
+        sections: sections,
+      )
+
+      register_path = output_dir.expand_path.join("register.yaml")
+      FileUtils.mkdir_p(register_path.dirname)
+      File.write(register_path, register.to_yaml, encoding: "utf-8")
+      puts "Written register.yaml with #{sections.length} areas" if $stdout.tty?
+    end
+
+    def build_section_tree(areas)
+      areas.sort_by { |a| a.code.to_i }.map do |area|
+        children = area.sections.sort_by do |s|
+          s.code.split("-").map(&:to_i)
+        end.map do |sec|
+          Glossarist::Section.new(
+            id: sec.code,
+            names: { "eng" => sec.title },
+          )
+        end
+
+        Glossarist::Section.new(
+          id: area.code,
+          names: { "eng" => area.title },
+          children: children.empty? ? nil : children,
+        )
+      end
     end
 
     def localized_count(collection)
       collection.sum { |c| c.localized_concepts.count }
     end
 
-    def domain_references_for(ievref)
-      parts = ievref.to_s.split("-")
-      return [] unless parts.length >= 2
+    # Build domain ConceptReferences for a concept.
+    #
+    # Per the concept model, ConceptReferenceType distinguishes:
+    #   - "domain"  → thematic/subject-area classification (area level)
+    #   - "section" → structural section membership (section level)
+    #
+    # Every concept gets both: a "domain" ref to its area and a "section"
+    # ref to its section. Concepts with only an area code (no section)
+    # get only a "domain" ref.
+    #
+    # @param code [IevCode] pre-parsed IEV code
+    # @return [Array<Glossarist::ConceptReference>]
+    def domain_references_for(code)
+      refs = []
+
+      # Domain reference: thematic classification at the area level
+      refs << domain_ref(code.area_uri)
+
+      # Section reference: structural membership in the section
+      if code.section_code
+        refs << section_ref(code.section_uri)
+      end
+
+      refs
+    end
+
+    # @param code [IevCode] pre-parsed IEV code
+    # @return [Array<String>]
+    def tags_for(code)
+      tags = []
+      area = SubjectAreas.find_area(code.area_code)
+      tags << area.title if area
+      section = code.section_code && SubjectAreas.find_section(code.section_code)
+      tags << section.title if section
+      tags
+    end
+
+    # @param concept [Glossarist::ManagedConcept]
+    # @param code [IevCode] pre-parsed IEV code
+    def add_section_broader(concept, code)
+      return unless code.section_uri
+
+      concept.related ||= []
+      return if concept.related.any? do |r|
+        r.type == "broader" && r.ref&.id == code.section_uri
+      end
+
+      concept.related << Glossarist::RelatedConcept.new(
+        type: "broader",
+        content: code.section_uri,
+        ref: Glossarist::ConceptRef.new(source: "IEV", id: code.section_uri),
+      )
+    end
+
+    def build_section_narrower_relations(collection)
+      mc_index = collection.each_with_object({}) do |c, h|
+        h[c.data&.id] = c if c.data&.id
+      end
+
+      section_children = {}
+      mc_index.each_key do |concept_id|
+        code = IevCode.new(concept_id)
+        next unless code.section_uri
+
+        (section_children[code.section_uri] ||= []) << concept_id
+      end
+
+      section_children.each do |section_uri, child_ids|
+        section_mc = mc_index[section_uri]
+        next unless section_mc
+
+        narrower = child_ids.sort.map do |child_id|
+          Glossarist::RelatedConcept.new(
+            type: "narrower",
+            content: child_id,
+            ref: Glossarist::ConceptRef.new(source: "IEV", id: child_id),
+          )
+        end
+
+        section_mc.related ||= []
+        section_mc.related.concat(narrower)
+      end
+    end
+
+    # Promote supersedes relations from localized ConceptData to managed level.
+    # Supersession is language-independent (REPLACES column is per-concept).
+    def promote_supersession(concept, term)
+      related = term.data&.related
+      return unless related&.any?
+
+      concept.related ||= []
+      related.each do |r|
+        next if concept.related.any? do |er|
+          er.type == r.type && er.ref&.id == r.ref&.id
+        end
+
+        concept.related << r
+      end
+      term.data.related = nil
+    end
+
+    # Derive managed concept status from the localization's entry_status.
+    def set_managed_status(concept, term)
+      return if concept.status
+
+      status = term.entry_status
+      concept.status = status if status && !status.empty?
+    end
+
+    # --- ConceptReference factory helpers ---
+
+    def domain_ref(concept_id)
+      ref = Glossarist::ConceptReference.domain(concept_id)
+      ref.source = IEV_SOURCE
+      ref
+    end
 
-      [
-        SubjectAreas.area_uri(parts[0]),
-        SubjectAreas.section_uri(parts[0..1].join("-")),
-      ].map { |id| Glossarist::ConceptReference.domain(id) }
+    def section_ref(concept_id)
+      ref = Glossarist::ConceptReference.section(concept_id)
+      ref.source = IEV_SOURCE
+      ref
     end
   end
 end

data/lib/iev/iev_code.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Iev
+  # Immutable value object that decomposes an IEV concept code
+  # into its structural parts: area code, section code, and number.
+  #
+  # The IEV code format is AAA-BB-CC where:
+  #   AAA = area code (e.g. "103")
+  #   BB  = section sub-code (e.g. "01")
+  #   CC  = concept number (e.g. "02")
+  #
+  # @example Full concept code
+  #   code = Iev::IevCode.new("103-01-02")
+  #   code.area_code    #=> "103"
+  #   code.section_code #=> "103-01"
+  #   code.number       #=> "02"
+  #   code.area_uri     #=> "area-103"
+  #   code.section_uri  #=> "section-103-01"
+  #
+  # @example Section code (no concept number)
+  #   code = Iev::IevCode.new("103-01")
+  #   code.area_code    #=> "103"
+  #   code.section_code #=> "103-01"
+  #   code.number       #=> nil
+  #   code.section_uri  #=> "section-103-01"
+  #
+  class IevCode
+    include Comparable
+
+    attr_reader :raw, :area_code, :section_code, :number
+
+    # @param code [#to_s] IEV reference, e.g. "103-01-02"
+    def initialize(code)
+      @raw = code.to_s
+      parts = @raw.split("-")
+      @area_code = parts[0]
+      @section_code = parts.length >= 2 ? "#{parts[0]}-#{parts[1]}" : nil
+      @number = parts.length >= 3 ? parts[2] : nil
+      freeze
+    end
+
+    def area_uri
+      "area-#{area_code}"
+    end
+
+    def section_uri
+      "section-#{section_code}" if section_code
+    end
+
+    def to_s
+      @raw
+    end
+
+    def to_str
+      @raw
+    end
+
+    def ==(other)
+      other.is_a?(self.class) && raw == other.raw
+    end
+    alias_method :eql?, :==
+
+    def hash
+      raw.hash
+    end
+
+    def <=>(other)
+      to_s <=> other.to_s
+    end
+
+    # Safe constructor that returns nil for codes that don't parse.
+    # @param code [#to_s]
+    # @return [IevCode, nil]
+    def self.parse(code)
+      new(code)
+    rescue ArgumentError
+      nil
+    end
+  end
+end

data/lib/iev/iso_639_code.rb

@@ -6,7 +6,8 @@
 module Iev
   # @todo This needs to be rewritten.
   class Iso639Code
-    COUNTRY_CODES = YAML.load(IO.read(File.join(__dir__, "iso_639_2.yaml")))
+    COUNTRY_CODES = YAML.safe_load_file(File.join(__dir__,
+                                                  "iso_639_2.yaml"), permitted_classes: [Symbol]).freeze
     # rubocop:disable Style/MutableConstant
     THREE_CHAR_MEMO = {} # Memoization cache, must be mutable
     # rubocop:enable Style/MutableConstant

data/lib/iev/relaton_db.rb

@@ -36,7 +36,7 @@ module Iev
       rescue StandardError
         raise unless curr_attempt <= attempts
 
-        sleep(2**curr_attempt * 0.1)
+        sleep((2**curr_attempt) * 0.1)
         curr_attempt += 1
         retry
       end

data/lib/iev/scraper/browser.rb

@@ -3,100 +3,102 @@
 require "ferrum"
 
 module Iev
-  # Shared headless browser utilities for fetching pages behind AWS WAF.
-  module ScraperBrowser
-    USER_AGENT_PROFILES = [
-      {
-        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"macOS"',
-        chrome_version: "131",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/130.0.0.0 Safari/537.36",
-        platform: '"Windows"',
-        chrome_version: "130",
-      },
-      {
-        user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"Linux"',
-        chrome_version: "131",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/129.0.0.0 Safari/537.36",
-        platform: '"macOS"',
-        chrome_version: "129",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"Windows"',
-        chrome_version: "131",
-      },
-    ].freeze
-
-    # Fetch a URL using headless Chrome, returning the page HTML.
-    # Handles AWS WAF challenge pages by waiting for JS execution.
-    def self.fetch(url, browser_opts: {})
-      browser = Ferrum::Browser.new(
-        headless: "new",
-        timeout: 30,
-        window_size: [1366, 768],
-        browser_options: {
-          "disable-blink-features" => "AutomationControlled",
+  class Scraper
+    # Shared headless browser utilities for fetching pages behind AWS WAF.
+    module Browser
+      USER_AGENT_PROFILES = [
+        {
+          user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/131.0.0.0 Safari/537.36",
+          platform: '"macOS"',
+          chrome_version: "131",
+        },
+        {
+          user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/130.0.0.0 Safari/537.36",
+          platform: '"Windows"',
+          chrome_version: "130",
+        },
+        {
+          user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/131.0.0.0 Safari/537.36",
+          platform: '"Linux"',
+          chrome_version: "131",
+        },
+        {
+          user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/129.0.0.0 Safari/537.36",
+          platform: '"macOS"',
+          chrome_version: "129",
+        },
+        {
+          user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/131.0.0.0 Safari/537.36",
+          platform: '"Windows"',
+          chrome_version: "131",
         },
-        **browser_opts,
-      )
+      ].freeze
 
-      browser.headers.set(random_headers)
-      browser.go_to(url)
-      browser.network.wait_for_idle(timeout: 15)
-      html = browser.body
+      # Fetch a URL using headless Chrome, returning the page HTML.
+      # Handles AWS WAF challenge pages by waiting for JS execution.
+      def self.fetch(url, browser_opts: {})
+        browser = Ferrum::Browser.new(
+          headless: "new",
+          timeout: 30,
+          window_size: [1366, 768],
+          browser_options: {
+            "disable-blink-features" => "AutomationControlled",
+          },
+          **browser_opts,
+        )
 
-      if html.include?("403 ERROR") || html.include?("Request blocked")
-        warn "IEV: AWS WAF blocked request for #{url}"
-        return nil
-      end
+        browser.headers.set(random_headers)
+        browser.go_to(url)
+        browser.network.wait_for_idle(timeout: 15)
+        html = browser.body
 
-      html
-    rescue Ferrum::Error, Ferrum::BrowserError => e
-      warn "IEV: Browser error fetching #{url}: #{e.message}"
-      nil
-    ensure
-      browser&.quit
-    end
+        if html.include?("403 ERROR") || html.include?("Request blocked")
+          warn "IEV: AWS WAF blocked request for #{url}"
+          return nil
+        end
 
-    def self.random_headers
-      profile = USER_AGENT_PROFILES.sample
-      sec_ch_ua = "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
-                  "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
-                  "\"Not_A Brand\";v=\"24\""
+        html
+      rescue Ferrum::Error, Ferrum::BrowserError => e
+        warn "IEV: Browser error fetching #{url}: #{e.message}"
+        nil
+      ensure
+        browser&.quit
+      end
 
-      {
-        "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
-                    "image/avif,image/webp,image/apng,*/*;q=0.8," \
-                    "application/signed-exchange;v=b3;q=0.7",
-        "Accept-Language" => "en-GB,en-US;q=0.9,en;q=0.8",
-        "Cache-Control" => "no-cache",
-        "Pragma" => "no-cache",
-        "Sec-Ch-Ua" => sec_ch_ua,
-        "Sec-Ch-Ua-Mobile" => "?0",
-        "Sec-Ch-Ua-Platform" => profile[:platform],
-        "Sec-Fetch-Dest" => "document",
-        "Sec-Fetch-Mode" => "navigate",
-        "Sec-Fetch-Site" => "cross-site",
-        "Sec-Fetch-User" => "?1",
-        "Upgrade-Insecure-Requests" => "1",
-        "User-Agent" => profile[:user_agent],
-      }
+      def self.random_headers
+        profile = USER_AGENT_PROFILES.sample
+        sec_ch_ua = "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
+                    "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
+                    "\"Not_A Brand\";v=\"24\""
+
+        {
+          "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
+                      "image/avif,image/webp,image/apng,*/*;q=0.8," \
+                      "application/signed-exchange;v=b3;q=0.7",
+          "Accept-Language" => "en-GB,en-US;q=0.9,en;q=0.8",
+          "Cache-Control" => "no-cache",
+          "Pragma" => "no-cache",
+          "Sec-Ch-Ua" => sec_ch_ua,
+          "Sec-Ch-Ua-Mobile" => "?0",
+          "Sec-Ch-Ua-Platform" => profile[:platform],
+          "Sec-Fetch-Dest" => "document",
+          "Sec-Fetch-Mode" => "navigate",
+          "Sec-Fetch-Site" => "cross-site",
+          "Sec-Fetch-User" => "?1",
+          "Upgrade-Insecure-Requests" => "1",
+          "User-Agent" => profile[:user_agent],
+        }
+      end
     end
   end
 end

data/lib/iev/scraper.rb

@@ -4,6 +4,9 @@ require "nokogiri"
 
 module Iev
   class Scraper
+    autoload :Browser, "iev/scraper/browser"
+    autoload :PageParser, "iev/scraper/page_parser"
+
     BASE_URL = "https://www.electropedia.org/iev/iev.nsf/" \
                "display?openform&ievref="
 
@@ -14,7 +17,8 @@ module Iev
     # Fetch the Electropedia page HTML for a given IEV code.
     # Returns a Nokogiri document.
     def fetch_page(code)
-      html = ScraperBrowser.fetch("#{BASE_URL}#{code}", browser_opts: @browser_opts)
+      html = Browser.fetch("#{BASE_URL}#{code}",
+                           browser_opts: @browser_opts)
       return nil unless html
 
       Nokogiri::HTML(html)
@@ -30,6 +34,3 @@ module Iev
     end
   end
 end
-
-require_relative "scraper/browser"
-require_relative "scraper/page_parser"

data/lib/iev/section.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Iev
+  # Immutable value object representing an IEV section (e.g. "103-01").
+  #
+  # A section belongs to exactly one area, identified by +area_code+.
+  class Section
+    attr_reader :code, :title, :area_code
+
+    # @param code [#to_s] section code, e.g. "103-01"
+    # @param title [#to_s] section title, e.g. "General concepts on functions"
+    # @param area_code [#to_s] parent area code, e.g. "103"
+    def initialize(code:, title:, area_code:)
+      @code = code.to_s
+      @title = title.to_s
+      @area_code = area_code.to_s
+      freeze
+    end
+
+    def uri
+      "section-#{code}"
+    end
+
+    def to_h
+      { "code" => code, "title" => title }
+    end
+
+    def ==(other)
+      other.is_a?(self.class) && code == other.code
+    end
+    alias_method :eql?, :==
+
+    def hash
+      code.hash
+    end
+  end
+end