iev 0.4.5 → 0.4.7

data/lib/iev/exporter.rb

@@ -56,12 +56,18 @@ module Iev
       collection = build_collection(dataset)
       add_subject_area_concepts(collection) if @include_areas
       build_section_narrower_relations(collection) if @include_areas
+      figures = FigureBuilder.extract!(collection)
+      enrich_references(collection)
       save_collection(collection)
+      save_figures(figures)
+      save_bibliography(BibliographyBuilder.build(collection))
+      save_register
       elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
 
       @stats = {
         concept_count: collection.count,
         localized_count: localized_count(collection),
+        figure_count: figures.length,
         elapsed_seconds: elapsed,
       }
       collection
@@ -140,11 +146,16 @@ module Iev
         term = TermBuilder.build_from(row)
         next unless term
 
+        # Parse IevCode once per concept — used by all helpers below.
+        code = IevCode.new(term.id)
+
         concept = concept_index[term.id] ||= begin
           c = Glossarist::ManagedConcept.new(data: { "id" => term.id })
           c.uuid = term.id
-          c.data.domains = domain_references_for(term.id)
-          add_section_broader(c, term.id)
+          c.schema_version = "3"
+          c.data.domains = domain_references_for(code)
+          c.data.tags = tags_for(code)
+          add_section_broader(c, code)
           collection.store(c)
           c
         end
@@ -169,34 +180,125 @@ module Iev
       collection.save_grouped_concepts_to_files(concepts_dir.to_s)
     end
 
-    def localized_count(collection)
-      collection.sum { |c| c.localized_concepts.count }
+    def save_figures(figures)
+      return if figures.empty?
+
+      figures_dir = output_dir.expand_path.join("figures")
+      FileUtils.mkdir_p(figures_dir)
+      figures.each do |figure|
+        path = figures_dir.join("#{figure.id}.yaml")
+        File.write(path, figure.to_yaml, encoding: "utf-8")
+      end
+      puts "Written #{figures.length} figures to figures/" if $stdout.tty?
     end
 
-    IEV_SOURCE = "urn:iec:std:iec:60050"
+    def save_bibliography(bibliography)
+      return if bibliography.entries.empty?
 
-    def domain_references_for(ievref)
-      code = IevCode.new(ievref.to_s)
-      refs = []
-      if code.area_code
-        refs << Glossarist::ConceptReference.new(
-          concept_id: code.area_uri,
-          source: IEV_SOURCE,
-          ref_type: "domain",
+      path = output_dir.expand_path.join("bibliography.yaml")
+      FileUtils.mkdir_p(path.dirname)
+      File.write(path, bibliography.to_yaml, encoding: "utf-8")
+      count = bibliography.entries.length
+      puts "Written bibliography.yaml with #{count} entries" if $stdout.tty?
+    end
+
+    def enrich_references(collection)
+      return if collection.none?
+
+      Glossarist::ConceptEnricher.new.inject_references(collection.to_a)
+    end
+
+    def save_register
+      areas = SubjectAreas.all
+      sections = build_section_tree(areas)
+
+      register = Glossarist::DatasetRegister.new(
+        schema_type: "glossarist",
+        schema_version: "3",
+        id: "iev",
+        ref: "IEC 60050:2011",
+        year: 2011,
+        urn: IEV_SOURCE,
+        urn_aliases: ["#{IEV_SOURCE}*"],
+        status: "current",
+        owner: "IEC",
+        source_repo: "https://github.com/glossarist/iev-data",
+        tags: %w[electrotechnical vocabulary iec],
+        languages: %w[eng fra],
+        language_order: %w[eng fra],
+        ordering: "systematic",
+        sections: sections,
+      )
+
+      register_path = output_dir.expand_path.join("register.yaml")
+      FileUtils.mkdir_p(register_path.dirname)
+      File.write(register_path, register.to_yaml, encoding: "utf-8")
+      puts "Written register.yaml with #{sections.length} areas" if $stdout.tty?
+    end
+
+    def build_section_tree(areas)
+      areas.sort_by { |a| a.code.to_i }.map do |area|
+        children = area.sections.sort_by do |s|
+          s.code.split("-").map(&:to_i)
+        end.map do |sec|
+          Glossarist::Section.new(
+            id: sec.code,
+            names: { "eng" => sec.title },
+          )
+        end
+
+        Glossarist::Section.new(
+          id: area.code,
+          names: { "eng" => area.title },
+          children: children.empty? ? nil : children,
         )
       end
+    end
+
+    def localized_count(collection)
+      collection.sum { |c| c.localized_concepts.count }
+    end
+
+    # Build domain ConceptReferences for a concept.
+    #
+    # Per the concept model, ConceptReferenceType distinguishes:
+    #   - "domain"  → thematic/subject-area classification (area level)
+    #   - "section" → structural section membership (section level)
+    #
+    # Every concept gets both: a "domain" ref to its area and a "section"
+    # ref to its section. Concepts with only an area code (no section)
+    # get only a "domain" ref.
+    #
+    # @param code [IevCode] pre-parsed IEV code
+    # @return [Array<Glossarist::ConceptReference>]
+    def domain_references_for(code)
+      refs = []
+
+      # Domain reference: thematic classification at the area level
+      refs << domain_ref(code.area_uri)
+
+      # Section reference: structural membership in the section
       if code.section_code
-        refs << Glossarist::ConceptReference.new(
-          concept_id: code.section_uri,
-          source: IEV_SOURCE,
-          ref_type: "domain",
-        )
+        refs << section_ref(code.section_uri)
       end
+
       refs
     end
 
-    def add_section_broader(concept, ievref)
-      code = IevCode.new(ievref.to_s)
+    # @param code [IevCode] pre-parsed IEV code
+    # @return [Array<String>]
+    def tags_for(code)
+      tags = []
+      area = SubjectAreas.find_area(code.area_code)
+      tags << area.title if area
+      section = code.section_code && SubjectAreas.find_section(code.section_code)
+      tags << section.title if section
+      tags
+    end
+
+    # @param concept [Glossarist::ManagedConcept]
+    # @param code [IevCode] pre-parsed IEV code
+    def add_section_broader(concept, code)
       return unless code.section_uri
 
       concept.related ||= []
@@ -249,7 +351,9 @@ module Iev
 
       concept.related ||= []
       related.each do |r|
-        next if concept.related.any? { |er| er.type == r.type && er.ref&.id == r.ref&.id }
+        next if concept.related.any? do |er|
+          er.type == r.type && er.ref&.id == r.ref&.id
+        end
 
         concept.related << r
       end
@@ -263,5 +367,19 @@ module Iev
       status = term.entry_status
       concept.status = status if status && !status.empty?
     end
+
+    # --- ConceptReference factory helpers ---
+
+    def domain_ref(concept_id)
+      ref = Glossarist::ConceptReference.domain(concept_id)
+      ref.source = IEV_SOURCE
+      ref
+    end
+
+    def section_ref(concept_id)
+      ref = Glossarist::ConceptReference.section(concept_id)
+      ref.source = IEV_SOURCE
+      ref
+    end
   end
 end

data/lib/iev/figure_builder.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+module Iev
+  # Hoists IEV figure references into dataset-shared Figure entities.
+  #
+  # IEV source data carries figures as inline SIMG tags, which Utilities
+  # rewrites to AsciiDoc image macros (+image::/assets/images/parts/{area}/
+  # FILE[Figure N - caption]+). This builder walks every concept's
+  # localizations, finds those image macros, promotes each to a
+  # dataset-shared Glossarist::Figure entity, and rewrites the inline text
+  # to a V3 figure mention (+{{fig:id, display}}+).
+  #
+  # The Figure entity is shared across concepts and languages — captions
+  # from different localizations merge into the same {lang => text} hash.
+  # The structural link from concept to figure is a FigureReference entry
+  # on ManagedConceptData#figures.
+  #
+  # Extraction is destructive: it mutates DetailedDefinition#content and
+  # appends FigureReference entries. Returns the unique Figure entities so
+  # the exporter can persist them to figures/{id}.yaml.
+  module FigureBuilder
+    # URL path prefix emitted by Utilities when converting SIMG tags.
+    # Kept in sync with Utilities::IMAGE_PATH_PREFIX (without the macro).
+    PATH_PREFIX = "/assets/images/parts"
+    private_constant :PATH_PREFIX
+
+    # Matches AsciiDoc image macros emitted by Utilities#process_simg_figures.
+    IMAGE_MACRO_REGEX = /
+      image::#{Regexp.escape(PATH_PREFIX)}
+      \/(?<area>\d+)\/(?<file>[\w.-]+)\[(?<caption>[^\]]*)\]
+    /x
+
+    # Captures "Figure N" label and the trailing caption text.
+    CAPTION_REGEX = /\A(?<label>Figure\s+\d+)\s*[–-]\s*(?<text>.+)\z/m
+
+    module_function
+
+    # @param collection [Glossarist::ManagedConceptCollection]
+    # @return [Array<Glossarist::Figure>] unique figures, sorted by id
+    def extract!(collection)
+      figures_by_id = {}
+
+      collection.each do |concept|
+        concept.localizations.each do |l10n|
+          process_localization(l10n, concept, figures_by_id)
+        end
+      end
+
+      figures_by_id.values.sort_by(&:id)
+    end
+
+    def process_localization(l10n, concept, figures_by_id)
+      lang = l10n.data&.language_code
+      return unless lang && lang.length == 3
+
+      Glossarist::ConceptData.detailed_definition_fields.each do |field|
+        process_field(l10n, field, lang, concept, figures_by_id)
+      end
+    end
+    private_class_method :process_localization
+
+    def process_field(l10n, field, lang, concept, figures_by_id)
+      l10n.data.public_send(field).each do |dd|
+        next unless dd.content&.include?("image::")
+
+        rewritten, hits = extract_from_text(dd.content, lang)
+        next if hits.empty?
+
+        dd.content = rewritten
+        hits.each { |hit| register_figure(hit, concept, figures_by_id) }
+      end
+    end
+    private_class_method :process_field
+
+    # @return [Array<(String, Array<Hash>)>] rewritten text and per-match
+    #   figure descriptors ({ id:, identifier:, caption:, lang:, image: })
+    def extract_from_text(text, lang)
+      hits = []
+      rewritten = text.gsub(IMAGE_MACRO_REGEX) do
+        hit = build_hit(Regexp.last_match, lang)
+        hits << hit
+        mention_for(hit)
+      end
+      [rewritten, hits]
+    end
+    private_class_method :extract_from_text
+
+    def build_hit(match, lang)
+      identifier, caption = parse_caption(match[:caption])
+      {
+        id: figure_id_for(match[:file]),
+        identifier: identifier,
+        caption: caption,
+        lang: lang,
+        image: build_image(match[:area], match[:file]),
+      }
+    end
+    private_class_method :build_hit
+
+    def build_image(area, file)
+      Glossarist::FigureImage.new(
+        src: "#{PATH_PREFIX}/#{area}/#{file}",
+        format: format_for(file),
+      )
+    end
+    private_class_method :build_image
+
+    def parse_caption(bracket)
+      stripped = bracket.to_s.strip
+      return [nil, nil] if stripped.empty?
+
+      if (m = stripped.match(CAPTION_REGEX))
+        label = m[:label].gsub(/\s+/, " ")
+        [label, m[:text].strip]
+      elsif stripped.match?(/\AFigure\s+\d+\z/)
+        [stripped, nil]
+      else
+        [nil, stripped]
+      end
+    end
+    private_class_method :parse_caption
+
+    def figure_id_for(file)
+      "fig-#{file.sub(/\.[^.]+\z/, '')}"
+    end
+    private_class_method :figure_id_for
+
+    def format_for(file)
+      File.extname(file).delete_prefix(".").downcase
+    end
+    private_class_method :format_for
+
+    def mention_for(hit)
+      parts = [hit[:identifier], hit[:caption]].compact
+      return "{{fig:#{hit[:id]}}}" if parts.empty?
+
+      "{{fig:#{hit[:id]}, #{parts.join(' - ')}}}"
+    end
+    private_class_method :mention_for
+
+    # Add or merge a figure descriptor into the shared index, and ensure the
+    # concept carries a FigureReference to it.
+    def register_figure(hit, concept, figures_by_id)
+      figure = figures_by_id[hit[:id]] ||= build_figure(hit)
+      merge_caption!(figure, hit)
+      add_image_if_missing(figure, hit[:image])
+      add_figure_reference(concept, hit[:id], hit[:identifier])
+    end
+    private_class_method :register_figure
+
+    def build_figure(hit)
+      Glossarist::Figure.new(
+        id: hit[:id],
+        identifier: hit[:identifier],
+        images: [],
+        caption: {},
+      )
+    end
+    private_class_method :build_figure
+
+    def merge_caption!(figure, hit)
+      return unless hit[:caption]
+
+      figure.caption ||= {}
+      figure.caption[hit[:lang]] ||= hit[:caption]
+    end
+    private_class_method :merge_caption!
+
+    def add_image_if_missing(figure, image)
+      return if figure.images.any? { |i| i.src == image.src }
+
+      figure.images << image
+    end
+    private_class_method :add_image_if_missing
+
+    def add_figure_reference(concept, figure_id, display)
+      refs = Array(concept.data.figures)
+      return if refs.any? { |r| r.entity_id == figure_id }
+
+      concept.data.figures = refs + [
+        Glossarist::FigureReference.new(entity_id: figure_id, display: display),
+      ]
+    end
+    private_class_method :add_figure_reference
+  end
+end

data/lib/iev/iso_639_code.rb

@@ -6,7 +6,8 @@
 module Iev
   # @todo This needs to be rewritten.
   class Iso639Code
-    COUNTRY_CODES = YAML.safe_load(IO.read(File.join(__dir__, "iso_639_2.yaml")), permitted_classes: [Symbol]).freeze
+    COUNTRY_CODES = YAML.safe_load_file(File.join(__dir__,
+                                                  "iso_639_2.yaml"), permitted_classes: [Symbol]).freeze
     # rubocop:disable Style/MutableConstant
     THREE_CHAR_MEMO = {} # Memoization cache, must be mutable
     # rubocop:enable Style/MutableConstant

data/lib/iev/relaton_db.rb

@@ -36,7 +36,7 @@ module Iev
       rescue StandardError
         raise unless curr_attempt <= attempts
 
-        sleep(2**curr_attempt * 0.1)
+        sleep((2**curr_attempt) * 0.1)
         curr_attempt += 1
         retry
       end

data/lib/iev/scraper/browser.rb

@@ -3,100 +3,102 @@
 require "ferrum"
 
 module Iev
-  # Shared headless browser utilities for fetching pages behind AWS WAF.
-  module ScraperBrowser
-    USER_AGENT_PROFILES = [
-      {
-        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"macOS"',
-        chrome_version: "131",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/130.0.0.0 Safari/537.36",
-        platform: '"Windows"',
-        chrome_version: "130",
-      },
-      {
-        user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"Linux"',
-        chrome_version: "131",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/129.0.0.0 Safari/537.36",
-        platform: '"macOS"',
-        chrome_version: "129",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"Windows"',
-        chrome_version: "131",
-      },
-    ].freeze
-
-    # Fetch a URL using headless Chrome, returning the page HTML.
-    # Handles AWS WAF challenge pages by waiting for JS execution.
-    def self.fetch(url, browser_opts: {})
-      browser = Ferrum::Browser.new(
-        headless: "new",
-        timeout: 30,
-        window_size: [1366, 768],
-        browser_options: {
-          "disable-blink-features" => "AutomationControlled",
+  class Scraper
+    # Shared headless browser utilities for fetching pages behind AWS WAF.
+    module Browser
+      USER_AGENT_PROFILES = [
+        {
+          user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/131.0.0.0 Safari/537.36",
+          platform: '"macOS"',
+          chrome_version: "131",
+        },
+        {
+          user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/130.0.0.0 Safari/537.36",
+          platform: '"Windows"',
+          chrome_version: "130",
+        },
+        {
+          user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/131.0.0.0 Safari/537.36",
+          platform: '"Linux"',
+          chrome_version: "131",
+        },
+        {
+          user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/129.0.0.0 Safari/537.36",
+          platform: '"macOS"',
+          chrome_version: "129",
+        },
+        {
+          user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/131.0.0.0 Safari/537.36",
+          platform: '"Windows"',
+          chrome_version: "131",
         },
-        **browser_opts,
-      )
+      ].freeze
 
-      browser.headers.set(random_headers)
-      browser.go_to(url)
-      browser.network.wait_for_idle(timeout: 15)
-      html = browser.body
+      # Fetch a URL using headless Chrome, returning the page HTML.
+      # Handles AWS WAF challenge pages by waiting for JS execution.
+      def self.fetch(url, browser_opts: {})
+        browser = Ferrum::Browser.new(
+          headless: "new",
+          timeout: 30,
+          window_size: [1366, 768],
+          browser_options: {
+            "disable-blink-features" => "AutomationControlled",
+          },
+          **browser_opts,
+        )
 
-      if html.include?("403 ERROR") || html.include?("Request blocked")
-        warn "IEV: AWS WAF blocked request for #{url}"
-        return nil
-      end
+        browser.headers.set(random_headers)
+        browser.go_to(url)
+        browser.network.wait_for_idle(timeout: 15)
+        html = browser.body
 
-      html
-    rescue Ferrum::Error, Ferrum::BrowserError => e
-      warn "IEV: Browser error fetching #{url}: #{e.message}"
-      nil
-    ensure
-      browser&.quit
-    end
+        if html.include?("403 ERROR") || html.include?("Request blocked")
+          warn "IEV: AWS WAF blocked request for #{url}"
+          return nil
+        end
 
-    def self.random_headers
-      profile = USER_AGENT_PROFILES.sample
-      sec_ch_ua = "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
-                  "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
-                  "\"Not_A Brand\";v=\"24\""
+        html
+      rescue Ferrum::Error, Ferrum::BrowserError => e
+        warn "IEV: Browser error fetching #{url}: #{e.message}"
+        nil
+      ensure
+        browser&.quit
+      end
 
-      {
-        "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
-                    "image/avif,image/webp,image/apng,*/*;q=0.8," \
-                    "application/signed-exchange;v=b3;q=0.7",
-        "Accept-Language" => "en-GB,en-US;q=0.9,en;q=0.8",
-        "Cache-Control" => "no-cache",
-        "Pragma" => "no-cache",
-        "Sec-Ch-Ua" => sec_ch_ua,
-        "Sec-Ch-Ua-Mobile" => "?0",
-        "Sec-Ch-Ua-Platform" => profile[:platform],
-        "Sec-Fetch-Dest" => "document",
-        "Sec-Fetch-Mode" => "navigate",
-        "Sec-Fetch-Site" => "cross-site",
-        "Sec-Fetch-User" => "?1",
-        "Upgrade-Insecure-Requests" => "1",
-        "User-Agent" => profile[:user_agent],
-      }
+      def self.random_headers
+        profile = USER_AGENT_PROFILES.sample
+        sec_ch_ua = "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
+                    "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
+                    "\"Not_A Brand\";v=\"24\""
+
+        {
+          "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
+                      "image/avif,image/webp,image/apng,*/*;q=0.8," \
+                      "application/signed-exchange;v=b3;q=0.7",
+          "Accept-Language" => "en-GB,en-US;q=0.9,en;q=0.8",
+          "Cache-Control" => "no-cache",
+          "Pragma" => "no-cache",
+          "Sec-Ch-Ua" => sec_ch_ua,
+          "Sec-Ch-Ua-Mobile" => "?0",
+          "Sec-Ch-Ua-Platform" => profile[:platform],
+          "Sec-Fetch-Dest" => "document",
+          "Sec-Fetch-Mode" => "navigate",
+          "Sec-Fetch-Site" => "cross-site",
+          "Sec-Fetch-User" => "?1",
+          "Upgrade-Insecure-Requests" => "1",
+          "User-Agent" => profile[:user_agent],
+        }
+      end
     end
   end
 end

data/lib/iev/scraper.rb

@@ -4,6 +4,9 @@ require "nokogiri"
 
 module Iev
   class Scraper
+    autoload :Browser, "iev/scraper/browser"
+    autoload :PageParser, "iev/scraper/page_parser"
+
     BASE_URL = "https://www.electropedia.org/iev/iev.nsf/" \
                "display?openform&ievref="
 
@@ -14,7 +17,8 @@ module Iev
     # Fetch the Electropedia page HTML for a given IEV code.
     # Returns a Nokogiri document.
     def fetch_page(code)
-      html = ScraperBrowser.fetch("#{BASE_URL}#{code}", browser_opts: @browser_opts)
+      html = Browser.fetch("#{BASE_URL}#{code}",
+                           browser_opts: @browser_opts)
       return nil unless html
 
       Nokogiri::HTML(html)
@@ -30,6 +34,3 @@ module Iev
     end
   end
 end
-
-require_relative "scraper/browser"
-require_relative "scraper/page_parser"