iev 0.4.2 → 0.4.4

data/iev.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "nokogiri", "~> 1.19"
   spec.add_dependency "plurimath"
   spec.add_dependency "lutaml-model", "~> 0.8.0"
-  spec.add_dependency "relaton", "~> 1.18"
+  spec.add_dependency "relaton", ">= 2.0.0", "< 3"
   spec.add_dependency "sequel", "~> 5.40"
   spec.add_dependency "sqlite3", "~> 1.7"
   spec.add_dependency "thor", "~> 1.0"

data/lib/iev/cli/command.rb

@@ -142,6 +142,30 @@ module Iev
         summary
       end
 
+      desc "subject_areas", "Fetch IEV subject areas and sections from Electropedia."
+      option :output, desc: "Output YAML file (default: stdout)", aliases: :o
+      option :refresh, type: :boolean, default: false,
+                       desc: "Force re-fetch even if cached"
+      def subject_areas
+        if options[:refresh]
+          cache_path = File.join(Iev.config.cache_dir, "subject_areas.yaml")
+          FileUtils.rm_f(cache_path) if File.exist?(cache_path)
+        end
+
+        result = Iev::SubjectAreas.fetch
+
+        yaml = YAML.dump(result)
+        if options[:output]
+          File.write(options[:output], yaml, encoding: "utf-8")
+          puts "Written to #{options[:output]}"
+        else
+          puts yaml
+        end
+      rescue Iev::SubjectAreas::FetchError => e
+        error e.message
+        exit 1
+      end
+
       desc "fetch CODE", "Fetch an IEV concept and output YAML to stdout."
       option :scrape, type: :boolean, default: false,
                       desc: "Scrape from Electropedia instead of using cached data"

data/lib/iev/exporter.rb

@@ -28,16 +28,19 @@ module Iev
     # @param only_concepts [String, nil] SQL LIKE pattern for IEVREF filtering
     # @param only_languages [String, nil] comma-separated language codes
     # @param fetch_relaton_links [Boolean] fetch source URLs via Relaton
+    # @param include_areas [Boolean] create area/section hierarchy concepts
     # @param on_progress [Proc, nil] callback (current, total) during build
     def initialize(input_path, output_dir: Dir.pwd,
                    only_concepts: nil, only_languages: nil,
                    fetch_relaton_links: false,
+                   include_areas: true,
                    on_progress: nil)
       @input_path = Pathname.new(input_path)
       validate_input!
 
       @output_dir = Pathname.new(output_dir)
       @fetch_relaton_links = fetch_relaton_links
+      @include_areas = include_areas
       @on_progress = on_progress
       @filters = {
         only_concepts: only_concepts,
@@ -51,6 +54,7 @@ module Iev
       start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       dataset = load_dataset
       collection = build_collection(dataset)
+      add_subject_area_concepts(collection) if @include_areas
       save_collection(collection)
       elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
 
@@ -137,6 +141,7 @@ module Iev
 
         concept = concept_index[term.id] ||= begin
           c = Glossarist::ManagedConcept.new(data: { "id" => term.id })
+          c.data.domains = domain_references_for(term.id)
           collection.store(c)
           c
         end
@@ -148,6 +153,10 @@ module Iev
       SourceParser.relaton_enabled = true
     end
 
+    def add_subject_area_concepts(collection)
+      SubjectAreaConcepts.add_to(collection)
+    end
+
     def save_collection(collection)
       concepts_dir = output_dir.expand_path.join("concepts")
       FileUtils.mkdir_p(concepts_dir)
@@ -157,5 +166,15 @@ module Iev
     def localized_count(collection)
       collection.sum { |c| c.localized_concepts.count }
     end
+
+    def domain_references_for(ievref)
+      parts = ievref.to_s.split("-")
+      return [] unless parts.length >= 2
+
+      [
+        SubjectAreas.area_uri(parts[0]),
+        SubjectAreas.section_uri(parts[0..1].join("-")),
+      ].map { |id| Glossarist::ConceptReference.domain(id) }
+    end
   end
 end

data/lib/iev/scraper/browser.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "ferrum"
+
+module Iev
+  # Shared headless browser utilities for fetching pages behind AWS WAF.
+  module ScraperBrowser
+    USER_AGENT_PROFILES = [
+      {
+        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                    "Chrome/131.0.0.0 Safari/537.36",
+        platform: '"macOS"',
+        chrome_version: "131",
+      },
+      {
+        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
+                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                    "Chrome/130.0.0.0 Safari/537.36",
+        platform: '"Windows"',
+        chrome_version: "130",
+      },
+      {
+        user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
+                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                    "Chrome/131.0.0.0 Safari/537.36",
+        platform: '"Linux"',
+        chrome_version: "131",
+      },
+      {
+        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                    "Chrome/129.0.0.0 Safari/537.36",
+        platform: '"macOS"',
+        chrome_version: "129",
+      },
+      {
+        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
+                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                    "Chrome/131.0.0.0 Safari/537.36",
+        platform: '"Windows"',
+        chrome_version: "131",
+      },
+    ].freeze
+
+    # Fetch a URL using headless Chrome, returning the page HTML.
+    # Handles AWS WAF challenge pages by waiting for JS execution.
+    def self.fetch(url, browser_opts: {})
+      browser = Ferrum::Browser.new(
+        headless: "new",
+        timeout: 30,
+        window_size: [1366, 768],
+        browser_options: {
+          "disable-blink-features" => "AutomationControlled",
+        },
+        **browser_opts,
+      )
+
+      browser.headers.set(random_headers)
+      browser.go_to(url)
+      browser.network.wait_for_idle(timeout: 15)
+      html = browser.body
+
+      if html.include?("403 ERROR") || html.include?("Request blocked")
+        warn "IEV: AWS WAF blocked request for #{url}"
+        return nil
+      end
+
+      html
+    rescue Ferrum::Error, Ferrum::BrowserError => e
+      warn "IEV: Browser error fetching #{url}: #{e.message}"
+      nil
+    ensure
+      browser&.quit
+    end
+
+    def self.random_headers
+      profile = USER_AGENT_PROFILES.sample
+      sec_ch_ua = "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
+                  "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
+                  "\"Not_A Brand\";v=\"24\""
+
+      {
+        "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
+                    "image/avif,image/webp,image/apng,*/*;q=0.8," \
+                    "application/signed-exchange;v=b3;q=0.7",
+        "Accept-Language" => "en-GB,en-US;q=0.9,en;q=0.8",
+        "Cache-Control" => "no-cache",
+        "Pragma" => "no-cache",
+        "Sec-Ch-Ua" => sec_ch_ua,
+        "Sec-Ch-Ua-Mobile" => "?0",
+        "Sec-Ch-Ua-Platform" => profile[:platform],
+        "Sec-Fetch-Dest" => "document",
+        "Sec-Fetch-Mode" => "navigate",
+        "Sec-Fetch-Site" => "cross-site",
+        "Sec-Fetch-User" => "?1",
+        "Upgrade-Insecure-Requests" => "1",
+        "User-Agent" => profile[:user_agent],
+      }
+    end
+  end
+end

data/lib/iev/scraper.rb

@@ -1,59 +1,12 @@
 # frozen_string_literal: true
 
+require "nokogiri"
+
 module Iev
-  # Scrapes IEV term data from Electropedia (electropedia.org).
-  #
-  # Electropedia is behind AWS WAF which requires JavaScript execution,
-  # so a headless browser (via Ferrum/Chrome) is used to handle the challenge.
-  #
-  # @example
-  #   scraper = Iev::Scraper.new
-  #   concept = scraper.fetch_concept("103-01-02")
-  #   doc = scraper.fetch_page("103-01-02")
   class Scraper
     BASE_URL = "https://www.electropedia.org/iev/iev.nsf/" \
                "display?openform&ievref="
 
-    # Pool of realistic Chrome User-Agent strings with matching platform hints.
-    # Rotated per request to reduce fingerprinting by AWS WAF.
-    USER_AGENT_PROFILES = [
-      {
-        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"macOS"',
-        chrome_version: "131",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/130.0.0.0 Safari/537.36",
-        platform: '"Windows"',
-        chrome_version: "130",
-      },
-      {
-        user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"Linux"',
-        chrome_version: "131",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/129.0.0.0 Safari/537.36",
-        platform: '"macOS"',
-        chrome_version: "129",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"Windows"',
-        chrome_version: "131",
-      },
-    ].freeze
-
     def initialize(browser_opts: {})
       @browser_opts = browser_opts
     end
@@ -61,37 +14,10 @@ module Iev
     # Fetch the Electropedia page HTML for a given IEV code.
     # Returns a Nokogiri document.
     def fetch_page(code)
-      require "ferrum"
-      require "nokogiri"
-
-      url = "#{BASE_URL}#{code}"
-      browser = Ferrum::Browser.new(
-        headless: "new",
-        timeout: 30,
-        window_size: [1366, 768],
-        browser_options: {
-          "disable-blink-features" => "AutomationControlled",
-        },
-        **@browser_opts,
-      )
-
-      browser.headers.set(random_headers)
-      browser.go_to(url)
-      browser.network.wait_for_idle(timeout: 15)
-      html = browser.body
-
-      # Check if we got a real page or a WAF block
-      if html.include?("403 ERROR") || html.include?("Request blocked")
-        warn "IEV Scraper: AWS WAF blocked request for #{code}"
-        return nil
-      end
+      html = ScraperBrowser.fetch("#{BASE_URL}#{code}", browser_opts: @browser_opts)
+      return nil unless html
 
       Nokogiri::HTML(html)
-    rescue Ferrum::Error, Ferrum::BrowserError => e
-      warn "IEV Scraper error for #{code}: #{e.message}"
-      nil
-    ensure
-      browser&.quit
     end
 
     # Fetch and parse concept data for an IEV code.
@@ -102,34 +28,8 @@ module Iev
 
       PageParser.new(doc, code).parse
     end
-
-    private
-
-    def random_headers
-      profile = USER_AGENT_PROFILES.sample
-      sec_ch_ua = "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
-                  "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
-                  "\"Not_A Brand\";v=\"24\""
-
-      {
-        "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
-                    "image/avif,image/webp,image/apng,*/*;q=0.8," \
-                    "application/signed-exchange;v=b3;q=0.7",
-        "Accept-Language" => "en-GB,en-US;q=0.9,en;q=0.8",
-        "Cache-Control" => "no-cache",
-        "Pragma" => "no-cache",
-        "Sec-Ch-Ua" => sec_ch_ua,
-        "Sec-Ch-Ua-Mobile" => "?0",
-        "Sec-Ch-Ua-Platform" => profile[:platform],
-        "Sec-Fetch-Dest" => "document",
-        "Sec-Fetch-Mode" => "navigate",
-        "Sec-Fetch-Site" => "cross-site",
-        "Sec-Fetch-User" => "?1",
-        "Upgrade-Insecure-Requests" => "1",
-        "User-Agent" => profile[:user_agent],
-      }
-    end
   end
 end
 
+require_relative "scraper/browser"
 require_relative "scraper/page_parser"

data/lib/iev/source_parser.rb

@@ -95,7 +95,7 @@ module Iev
         origin: origin,
         modification: relationship[:modification],
       )
-    rescue ::RelatonBib::RequestError, Socket::ResolutionError, SocketError => e
+    rescue Relaton::RequestError, Socket::ResolutionError, SocketError => e
       warn e.message
     end
 
@@ -356,8 +356,11 @@ module Iev
       return nil unless self.class.relaton_enabled
       return nil unless defined?(RelatonDb)
 
-      RelatonDb.instance.fetch(ref)&.url
-    rescue ::RelatonBib::RequestError, Socket::ResolutionError, SocketError => e
+      item = RelatonDb.instance.fetch(ref)
+      return nil unless item
+
+      item.source("src")
+    rescue Relaton::RequestError, Socket::ResolutionError, SocketError => e
       warn e.message
       nil
     end

data/lib/iev/subject_area_concepts.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Iev
+  # Creates ManagedConcept entries for the IEV subject area hierarchy.
+  #
+  # The hierarchy has two levels:
+  #   - Area (e.g., "102" = "Mathematics - General concepts and linear algebra")
+  #   - Section (e.g., "102-01" = "Sets and operations")
+  #
+  # Linking:
+  #   - Each IEV concept's ConceptData#domain references its section URI
+  #   - Each IEV concept's ManagedConceptData#domains includes area and section codes
+  #   - Each section concept has a "broader" relation to its parent area
+  #   - Each area concept has "narrower" relations to its sections
+  module SubjectAreaConcepts
+    class << self
+      # Build all area and section concepts and add them to the collection.
+      #
+      # @param collection [Glossarist::ManagedConceptCollection]
+      # @return [void]
+      def add_to(collection)
+        Iev.subject_areas.each do |area|
+          area_mc = build_area_concept(area)
+          collection.store(area_mc)
+
+          (area["sections"] || []).each do |section|
+            section_mc = build_section_concept(section, area)
+            collection.store(section_mc)
+          end
+        end
+      end
+
+      private
+
+      def build_area_concept(area)
+        id = SubjectAreas.area_uri(area["code"])
+
+        mc = Glossarist::ManagedConcept.new(
+          data: Glossarist::ManagedConceptData.new(
+            id: id,
+            domains: [Glossarist::ConceptReference.domain(id)],
+          ),
+        )
+
+        mc.add_localization(build_localization(id, area["title"], "eng"))
+
+        narrower = (area["sections"] || []).map { |s| build_narrower_ref(s["code"]) }
+        mc.related = narrower unless narrower.empty?
+
+        mc
+      end
+
+      def build_section_concept(section, area)
+        id = SubjectAreas.section_uri(section["code"])
+
+        mc = Glossarist::ManagedConcept.new(
+          data: Glossarist::ManagedConceptData.new(
+            id: id,
+            domains: [
+              Glossarist::ConceptReference.domain(SubjectAreas.area_uri(area["code"])),
+              Glossarist::ConceptReference.domain(id),
+            ],
+          ),
+        )
+
+        cd = build_concept_data(id, section["title"], "eng")
+        cd.domain = SubjectAreas.area_uri(area["code"])
+        cd.related = [build_broader_ref(area["code"])]
+
+        mc.add_localization(build_localization_from_data(id, cd))
+        mc
+      end
+
+      def build_concept_data(id, title, lang_code)
+        Glossarist::ConceptData.new(
+          id: id,
+          language_code: lang_code,
+          terms: [
+            Glossarist::Designation::Expression.new(
+              type: "expression",
+              designation: title,
+              normative_status: "preferred",
+            ),
+          ],
+        )
+      end
+
+      def build_localization(id, title, lang_code)
+        cd = build_concept_data(id, title, lang_code)
+
+        l10n = Glossarist::LocalizedConcept.new
+        l10n.data = cd
+        l10n.id = id
+        l10n.entry_status = "valid"
+        l10n.data.review_decision_event = "published"
+        l10n
+      end
+
+      def build_localization_from_data(id, concept_data)
+        l10n = Glossarist::LocalizedConcept.new
+        l10n.data = concept_data
+        l10n.id = id
+        l10n.entry_status = "valid"
+        l10n.data.review_decision_event = "published"
+        l10n
+      end
+
+      def build_broader_ref(area_code)
+        Glossarist::RelatedConcept.new(
+          type: "broader",
+          content: SubjectAreas.area_uri(area_code),
+        )
+      end
+
+      def build_narrower_ref(section_code)
+        Glossarist::RelatedConcept.new(
+          type: "narrower",
+          content: SubjectAreas.section_uri(section_code),
+        )
+      end
+    end
+  end
+end