iev 0.4.3 → 0.4.4

data/lib/iev/cli/command.rb

@@ -142,6 +142,30 @@ module Iev
         summary
       end
 
+      desc "subject_areas", "Fetch IEV subject areas and sections from Electropedia."
+      option :output, desc: "Output YAML file (default: stdout)", aliases: :o
+      option :refresh, type: :boolean, default: false,
+                       desc: "Force re-fetch even if cached"
+      def subject_areas
+        if options[:refresh]
+          cache_path = File.join(Iev.config.cache_dir, "subject_areas.yaml")
+          FileUtils.rm_f(cache_path) if File.exist?(cache_path)
+        end
+
+        result = Iev::SubjectAreas.fetch
+
+        yaml = YAML.dump(result)
+        if options[:output]
+          File.write(options[:output], yaml, encoding: "utf-8")
+          puts "Written to #{options[:output]}"
+        else
+          puts yaml
+        end
+      rescue Iev::SubjectAreas::FetchError => e
+        error e.message
+        exit 1
+      end
+
       desc "fetch CODE", "Fetch an IEV concept and output YAML to stdout."
       option :scrape, type: :boolean, default: false,
                       desc: "Scrape from Electropedia instead of using cached data"

data/lib/iev/exporter.rb

@@ -28,16 +28,19 @@ module Iev
     # @param only_concepts [String, nil] SQL LIKE pattern for IEVREF filtering
     # @param only_languages [String, nil] comma-separated language codes
     # @param fetch_relaton_links [Boolean] fetch source URLs via Relaton
+    # @param include_areas [Boolean] create area/section hierarchy concepts
     # @param on_progress [Proc, nil] callback (current, total) during build
     def initialize(input_path, output_dir: Dir.pwd,
                    only_concepts: nil, only_languages: nil,
                    fetch_relaton_links: false,
+                   include_areas: true,
                    on_progress: nil)
       @input_path = Pathname.new(input_path)
       validate_input!
 
       @output_dir = Pathname.new(output_dir)
       @fetch_relaton_links = fetch_relaton_links
+      @include_areas = include_areas
       @on_progress = on_progress
       @filters = {
         only_concepts: only_concepts,
@@ -51,6 +54,7 @@ module Iev
       start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       dataset = load_dataset
       collection = build_collection(dataset)
+      add_subject_area_concepts(collection) if @include_areas
       save_collection(collection)
       elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
 
@@ -137,6 +141,7 @@ module Iev
 
         concept = concept_index[term.id] ||= begin
           c = Glossarist::ManagedConcept.new(data: { "id" => term.id })
+          c.data.domains = domain_references_for(term.id)
           collection.store(c)
           c
         end
@@ -148,6 +153,10 @@ module Iev
       SourceParser.relaton_enabled = true
     end
 
+    def add_subject_area_concepts(collection)
+      SubjectAreaConcepts.add_to(collection)
+    end
+
     def save_collection(collection)
       concepts_dir = output_dir.expand_path.join("concepts")
       FileUtils.mkdir_p(concepts_dir)
@@ -157,5 +166,15 @@ module Iev
     def localized_count(collection)
       collection.sum { |c| c.localized_concepts.count }
     end
+
+    def domain_references_for(ievref)
+      parts = ievref.to_s.split("-")
+      return [] unless parts.length >= 2
+
+      [
+        SubjectAreas.area_uri(parts[0]),
+        SubjectAreas.section_uri(parts[0..1].join("-")),
+      ].map { |id| Glossarist::ConceptReference.domain(id) }
+    end
   end
 end

data/lib/iev/scraper/browser.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "ferrum"
+
+module Iev
+  # Shared headless browser utilities for fetching pages behind AWS WAF.
+  module ScraperBrowser
+    USER_AGENT_PROFILES = [
+      {
+        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                    "Chrome/131.0.0.0 Safari/537.36",
+        platform: '"macOS"',
+        chrome_version: "131",
+      },
+      {
+        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
+                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                    "Chrome/130.0.0.0 Safari/537.36",
+        platform: '"Windows"',
+        chrome_version: "130",
+      },
+      {
+        user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
+                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                    "Chrome/131.0.0.0 Safari/537.36",
+        platform: '"Linux"',
+        chrome_version: "131",
+      },
+      {
+        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                    "Chrome/129.0.0.0 Safari/537.36",
+        platform: '"macOS"',
+        chrome_version: "129",
+      },
+      {
+        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
+                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                    "Chrome/131.0.0.0 Safari/537.36",
+        platform: '"Windows"',
+        chrome_version: "131",
+      },
+    ].freeze
+
+    # Fetch a URL using headless Chrome, returning the page HTML.
+    # Handles AWS WAF challenge pages by waiting for JS execution.
+    def self.fetch(url, browser_opts: {})
+      browser = Ferrum::Browser.new(
+        headless: "new",
+        timeout: 30,
+        window_size: [1366, 768],
+        browser_options: {
+          "disable-blink-features" => "AutomationControlled",
+        },
+        **browser_opts,
+      )
+
+      browser.headers.set(random_headers)
+      browser.go_to(url)
+      browser.network.wait_for_idle(timeout: 15)
+      html = browser.body
+
+      if html.include?("403 ERROR") || html.include?("Request blocked")
+        warn "IEV: AWS WAF blocked request for #{url}"
+        return nil
+      end
+
+      html
+    rescue Ferrum::Error, Ferrum::BrowserError => e
+      warn "IEV: Browser error fetching #{url}: #{e.message}"
+      nil
+    ensure
+      browser&.quit
+    end
+
+    def self.random_headers
+      profile = USER_AGENT_PROFILES.sample
+      sec_ch_ua = "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
+                  "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
+                  "\"Not_A Brand\";v=\"24\""
+
+      {
+        "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
+                    "image/avif,image/webp,image/apng,*/*;q=0.8," \
+                    "application/signed-exchange;v=b3;q=0.7",
+        "Accept-Language" => "en-GB,en-US;q=0.9,en;q=0.8",
+        "Cache-Control" => "no-cache",
+        "Pragma" => "no-cache",
+        "Sec-Ch-Ua" => sec_ch_ua,
+        "Sec-Ch-Ua-Mobile" => "?0",
+        "Sec-Ch-Ua-Platform" => profile[:platform],
+        "Sec-Fetch-Dest" => "document",
+        "Sec-Fetch-Mode" => "navigate",
+        "Sec-Fetch-Site" => "cross-site",
+        "Sec-Fetch-User" => "?1",
+        "Upgrade-Insecure-Requests" => "1",
+        "User-Agent" => profile[:user_agent],
+      }
+    end
+  end
+end

data/lib/iev/scraper.rb

@@ -1,59 +1,12 @@
 # frozen_string_literal: true
 
+require "nokogiri"
+
 module Iev
-  # Scrapes IEV term data from Electropedia (electropedia.org).
-  #
-  # Electropedia is behind AWS WAF which requires JavaScript execution,
-  # so a headless browser (via Ferrum/Chrome) is used to handle the challenge.
-  #
-  # @example
-  #   scraper = Iev::Scraper.new
-  #   concept = scraper.fetch_concept("103-01-02")
-  #   doc = scraper.fetch_page("103-01-02")
   class Scraper
     BASE_URL = "https://www.electropedia.org/iev/iev.nsf/" \
                "display?openform&ievref="
 
-    # Pool of realistic Chrome User-Agent strings with matching platform hints.
-    # Rotated per request to reduce fingerprinting by AWS WAF.
-    USER_AGENT_PROFILES = [
-      {
-        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"macOS"',
-        chrome_version: "131",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/130.0.0.0 Safari/537.36",
-        platform: '"Windows"',
-        chrome_version: "130",
-      },
-      {
-        user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"Linux"',
-        chrome_version: "131",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/129.0.0.0 Safari/537.36",
-        platform: '"macOS"',
-        chrome_version: "129",
-      },
-      {
-        user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
-                    "AppleWebKit/537.36 (KHTML, like Gecko) " \
-                    "Chrome/131.0.0.0 Safari/537.36",
-        platform: '"Windows"',
-        chrome_version: "131",
-      },
-    ].freeze
-
     def initialize(browser_opts: {})
       @browser_opts = browser_opts
     end
@@ -61,37 +14,10 @@ module Iev
     # Fetch the Electropedia page HTML for a given IEV code.
     # Returns a Nokogiri document.
     def fetch_page(code)
-      require "ferrum"
-      require "nokogiri"
-
-      url = "#{BASE_URL}#{code}"
-      browser = Ferrum::Browser.new(
-        headless: "new",
-        timeout: 30,
-        window_size: [1366, 768],
-        browser_options: {
-          "disable-blink-features" => "AutomationControlled",
-        },
-        **@browser_opts,
-      )
-
-      browser.headers.set(random_headers)
-      browser.go_to(url)
-      browser.network.wait_for_idle(timeout: 15)
-      html = browser.body
-
-      # Check if we got a real page or a WAF block
-      if html.include?("403 ERROR") || html.include?("Request blocked")
-        warn "IEV Scraper: AWS WAF blocked request for #{code}"
-        return nil
-      end
+      html = ScraperBrowser.fetch("#{BASE_URL}#{code}", browser_opts: @browser_opts)
+      return nil unless html
 
       Nokogiri::HTML(html)
-    rescue Ferrum::Error, Ferrum::BrowserError => e
-      warn "IEV Scraper error for #{code}: #{e.message}"
-      nil
-    ensure
-      browser&.quit
     end
 
     # Fetch and parse concept data for an IEV code.
@@ -102,34 +28,8 @@ module Iev
 
       PageParser.new(doc, code).parse
     end
-
-    private
-
-    def random_headers
-      profile = USER_AGENT_PROFILES.sample
-      sec_ch_ua = "\"Google Chrome\";v=\"#{profile[:chrome_version]}\", " \
-                  "\"Chromium\";v=\"#{profile[:chrome_version]}\", " \
-                  "\"Not_A Brand\";v=\"24\""
-
-      {
-        "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
-                    "image/avif,image/webp,image/apng,*/*;q=0.8," \
-                    "application/signed-exchange;v=b3;q=0.7",
-        "Accept-Language" => "en-GB,en-US;q=0.9,en;q=0.8",
-        "Cache-Control" => "no-cache",
-        "Pragma" => "no-cache",
-        "Sec-Ch-Ua" => sec_ch_ua,
-        "Sec-Ch-Ua-Mobile" => "?0",
-        "Sec-Ch-Ua-Platform" => profile[:platform],
-        "Sec-Fetch-Dest" => "document",
-        "Sec-Fetch-Mode" => "navigate",
-        "Sec-Fetch-Site" => "cross-site",
-        "Sec-Fetch-User" => "?1",
-        "Upgrade-Insecure-Requests" => "1",
-        "User-Agent" => profile[:user_agent],
-      }
-    end
   end
 end
 
+require_relative "scraper/browser"
 require_relative "scraper/page_parser"

data/lib/iev/subject_area_concepts.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Iev
+  # Creates ManagedConcept entries for the IEV subject area hierarchy.
+  #
+  # The hierarchy has two levels:
+  #   - Area (e.g., "102" = "Mathematics - General concepts and linear algebra")
+  #   - Section (e.g., "102-01" = "Sets and operations")
+  #
+  # Linking:
+  #   - Each IEV concept's ConceptData#domain references its section URI
+  #   - Each IEV concept's ManagedConceptData#domains includes area and section codes
+  #   - Each section concept has a "broader" relation to its parent area
+  #   - Each area concept has "narrower" relations to its sections
+  module SubjectAreaConcepts
+    class << self
+      # Build all area and section concepts and add them to the collection.
+      #
+      # @param collection [Glossarist::ManagedConceptCollection]
+      # @return [void]
+      def add_to(collection)
+        Iev.subject_areas.each do |area|
+          area_mc = build_area_concept(area)
+          collection.store(area_mc)
+
+          (area["sections"] || []).each do |section|
+            section_mc = build_section_concept(section, area)
+            collection.store(section_mc)
+          end
+        end
+      end
+
+      private
+
+      def build_area_concept(area)
+        id = SubjectAreas.area_uri(area["code"])
+
+        mc = Glossarist::ManagedConcept.new(
+          data: Glossarist::ManagedConceptData.new(
+            id: id,
+            domains: [Glossarist::ConceptReference.domain(id)],
+          ),
+        )
+
+        mc.add_localization(build_localization(id, area["title"], "eng"))
+
+        narrower = (area["sections"] || []).map { |s| build_narrower_ref(s["code"]) }
+        mc.related = narrower unless narrower.empty?
+
+        mc
+      end
+
+      def build_section_concept(section, area)
+        id = SubjectAreas.section_uri(section["code"])
+
+        mc = Glossarist::ManagedConcept.new(
+          data: Glossarist::ManagedConceptData.new(
+            id: id,
+            domains: [
+              Glossarist::ConceptReference.domain(SubjectAreas.area_uri(area["code"])),
+              Glossarist::ConceptReference.domain(id),
+            ],
+          ),
+        )
+
+        cd = build_concept_data(id, section["title"], "eng")
+        cd.domain = SubjectAreas.area_uri(area["code"])
+        cd.related = [build_broader_ref(area["code"])]
+
+        mc.add_localization(build_localization_from_data(id, cd))
+        mc
+      end
+
+      def build_concept_data(id, title, lang_code)
+        Glossarist::ConceptData.new(
+          id: id,
+          language_code: lang_code,
+          terms: [
+            Glossarist::Designation::Expression.new(
+              type: "expression",
+              designation: title,
+              normative_status: "preferred",
+            ),
+          ],
+        )
+      end
+
+      def build_localization(id, title, lang_code)
+        cd = build_concept_data(id, title, lang_code)
+
+        l10n = Glossarist::LocalizedConcept.new
+        l10n.data = cd
+        l10n.id = id
+        l10n.entry_status = "valid"
+        l10n.data.review_decision_event = "published"
+        l10n
+      end
+
+      def build_localization_from_data(id, concept_data)
+        l10n = Glossarist::LocalizedConcept.new
+        l10n.data = concept_data
+        l10n.id = id
+        l10n.entry_status = "valid"
+        l10n.data.review_decision_event = "published"
+        l10n
+      end
+
+      def build_broader_ref(area_code)
+        Glossarist::RelatedConcept.new(
+          type: "broader",
+          content: SubjectAreas.area_uri(area_code),
+        )
+      end
+
+      def build_narrower_ref(section_code)
+        Glossarist::RelatedConcept.new(
+          type: "narrower",
+          content: SubjectAreas.section_uri(section_code),
+        )
+      end
+    end
+  end
+end

data/lib/iev/subject_areas.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "nokogiri"
+require "fileutils"
+require "iev/config"
+
+module Iev
+  module SubjectAreas
+    DATA_FILE = File.expand_path("../../data/subject_areas.yaml", __dir__)
+
+    AREAS_URL = "https://electropedia.org/iev/iev.nsf/" \
+                "6d6bdd8667c378f7c12581fa003d80e7?OpenForm"
+    SECTIONS_URL_TEMPLATE = "https://electropedia.org/iev/iev.nsf/" \
+                            "index?openform&part=%<part>s"
+
+    MIN_PAGE_SIZE = 15_000
+
+    FETCH_DELAY = 5
+    RETRY_DELAY = 30
+    MAX_RETRIES = 2
+
+    class FetchError < StandardError; end
+
+    class << self
+      # --- URI scheme ---
+
+      # URI for a subject area concept.
+      # @param code [String, Integer] e.g. "102"
+      # @return [String] e.g. "area-102"
+      def area_uri(code)
+        "area-#{code}"
+      end
+
+      # URI for a section concept.
+      # @param code [String] e.g. "103-01"
+      # @return [String] e.g. "section-103-01"
+      def section_uri(code)
+        "section-#{code}"
+      end
+
+      # --- Query API (reads from bundled data) ---
+
+      # Return all subject areas with their sections.
+      # @return [Array<Hash>] each hash has "code", "title", "sections"
+      def all
+        data["areas"]
+      end
+
+      # Find a single subject area by its numeric code.
+      # @param code [String, Integer] e.g. "102" or 102
+      # @return [Hash, nil]
+      def find_area(code)
+        all.find { |a| a["code"] == code.to_s }
+      end
+
+      # Return all sections for a given area code.
+      # @param code [String, Integer] area code, e.g. "102"
+      # @return [Array<Hash>] each hash has "code", "title"
+      def sections_for(code)
+        area = find_area(code)
+        area ? area["sections"] : []
+      end
+
+      # Find a single section by its section code.
+      # @param section_code [String] e.g. "102-01"
+      # @return [Hash, nil]
+      def find_section(section_code)
+        sc = section_code.to_s
+        all.each do |area|
+          found = area["sections"]&.find { |s| s["code"] == sc }
+          return found if found
+        end
+        nil
+      end
+
+      # Return the parent area for a given section code.
+      # @param section_code [String] e.g. "102-01"
+      # @return [Hash, nil]
+      def area_for_section(section_code)
+        sc = section_code.to_s
+        all.find do |area|
+          area["sections"]&.any? { |s| s["code"] == sc }
+        end
+      end
+
+      # --- Fetching (network, writes to bundled data file) ---
+
+      def fetch
+        cached = read_cache("subject_areas.yaml")
+        return cached if cached && complete?(cached)
+
+        areas = cached ? cached["areas"] : []
+        fresh_areas = fetch_areas
+        puts "Found #{fresh_areas.length} areas (#{areas.length} cached)" if $stdout.tty?
+
+        # Merge: keep existing sections, add new areas
+        existing = areas.each_with_object({}) { |a, h| h[a["code"]] = a }
+        fresh_areas.each do |fa|
+          existing[fa["code"]] ||= fa
+        end
+        areas = fresh_areas.map { |fa| existing[fa["code"]] || fa }
+
+        areas.each_with_index do |area, i|
+          next if area["fetched"]
+
+          begin
+            area["sections"] = fetch_sections(area["code"])
+            area["fetched"] = true
+          rescue FetchError
+            area["sections"] ||= []
+            warn "IEV: Skipping area #{area["code"]} due to WAF"
+          end
+
+          puts "[#{i + 1}/#{areas.length}] #{area["code"]}: #{area["title"]} — #{area["sections"].length} sections" if $stdout.tty?
+
+          # Save progress every 10 areas so partial results survive WAF failures
+          if (i + 1) % 10 == 0
+            write_cache("subject_areas.yaml", { "areas" => areas })
+          end
+
+          sleep FETCH_DELAY unless i == areas.length - 1
+        end
+
+        result = { "areas" => areas }
+        write_cache("subject_areas.yaml", result)
+        result
+      end
+
+      def fetch_areas
+        html = fetch_page_with_retry(AREAS_URL)
+        doc = Nokogiri::HTML(html)
+
+        areas = []
+        doc.css("a").each do |link|
+          href = link["href"].to_s
+          next unless href.include?("part=")
+
+          code = href.match(/part=(\d+)/)&.[](1)
+          next unless code
+
+          title = link.text.strip
+          next if title.empty?
+
+          areas << { "code" => code, "title" => title, "sections" => [] }
+        end
+
+        areas.uniq { |a| a["code"] }
+      end
+
+      def fetch_sections(part)
+        url = format(SECTIONS_URL_TEMPLATE, part: part)
+        html = fetch_page_with_retry(url)
+        doc = Nokogiri::HTML(html)
+
+        sections = []
+        doc.css("td").each do |td|
+          text = td.text.strip
+          if (m = text.match(/\ASection\s+([\d-]+):\s*(.+)\z/))
+            sections << { "code" => m[1], "title" => m[2].strip }
+          end
+        end
+
+        sections.uniq { |s| s["code"] }
+      end
+
+      private
+
+      def data
+        @data ||= begin
+          path = File.exist?(DATA_FILE) ? DATA_FILE : nil
+          if path
+            YAML.safe_load(File.read(path, encoding: "utf-8")) || { "areas" => [] }
+          else
+            { "areas" => [] }
+          end
+        end
+      end
+
+      def complete?(data)
+        areas = data["areas"]
+        return false unless areas&.length&.>= 99
+
+        areas.all? { |a| a["fetched"] == true }
+      end
+
+      def captcha_page?(html)
+        html.length < MIN_PAGE_SIZE ||
+          html.include?("Confirm you are human") ||
+          html.include?("solve a puzzle") ||
+          html.include?("security check before continuing")
+      end
+
+      def fetch_page_with_retry(url, retries: MAX_RETRIES)
+        require "iev/scraper/browser"
+
+        retries.times do |attempt|
+          html = ScraperBrowser.fetch(url)
+          raise FetchError, "Failed to fetch #{url}" unless html
+
+          unless captcha_page?(html)
+            return html
+          end
+
+          if attempt < retries - 1
+            wait = RETRY_DELAY * (attempt + 1)
+            warn "IEV: WAF challenge for #{url}, retrying in #{wait}s (attempt #{attempt + 1}/#{retries})"
+            sleep wait
+          else
+            raise FetchError, "WAF challenge for #{url}"
+          end
+        end
+      end
+
+      def read_cache(filename)
+        cache_path = File.join(Iev.config.cache_dir, filename)
+        return nil unless File.exist?(cache_path)
+
+        d = YAML.safe_load(File.read(cache_path, encoding: "utf-8"))
+        return nil unless d&.dig("areas")&.any?
+
+        d
+      end
+
+      def write_cache(filename, d)
+        cache_path = File.join(Iev.config.cache_dir, filename)
+        FileUtils.mkdir_p(File.dirname(cache_path))
+        File.write(cache_path, YAML.dump(d), encoding: "utf-8")
+      end
+    end
+  end
+end