iev 0.4.3 → 0.4.5

data/lib/iev/subject_area.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Iev
+  # Immutable value object representing an IEV subject area (e.g. "102").
+  #
+  # A subject area is the aggregate root for its sections.
+  # Navigation: area → sections (direct), section → area (via registry).
+  class SubjectArea
+    attr_reader :code, :title, :sections
+
+    # @param code [#to_s] area code, e.g. "103"
+    # @param title [#to_s] area title, e.g. "Mathematics - Functions"
+    # @param sections [Array<Iev::Section>] child sections
+    def initialize(code:, title:, sections: [])
+      @code = code.to_s
+      @title = title.to_s
+      @sections = sections
+      freeze
+    end
+
+    def uri
+      "area-#{code}"
+    end
+
+    def section(section_code)
+      sections.find { |s| s.code == section_code.to_s }
+    end
+
+    def to_h
+      {
+        "code" => code,
+        "title" => title,
+        "sections" => sections.map(&:to_h),
+      }
+    end
+
+    def ==(other)
+      other.is_a?(self.class) && code == other.code
+    end
+    alias_method :eql?, :==
+
+    def hash
+      code.hash
+    end
+  end
+end

data/lib/iev/subject_area_concepts.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module Iev
+  # Creates ManagedConcept entries for the IEV subject area hierarchy.
+  #
+  # The hierarchy has two levels:
+  #   - Area (e.g., "102" = "Mathematics - General concepts")
+  #   - Section (e.g., "102-01" = "Sets and operations")
+  #
+  # Linking (all at ManagedConcept#related level):
+  #   - Each area has "narrower" relations to its sections
+  #   - Each section has "broader" relation to parent area
+  #   - Each section gets "narrower" to child concepts (added by Exporter)
+  #   - Each regular IEV concept gets "broader" to its section
+  #     (added by Exporter)
+  #
+  # Classification (separate from hierarchy):
+  #   - Each concept's ManagedConceptData#domains includes area and
+  #     section ConceptReferences
+  #   - Each concept's ConceptData#domain references its section URI
+  #   - Each section concept's ConceptData#domain references parent area
+  module SubjectAreaConcepts
+    IEV_SOURCE = "urn:iec:std:iec:60050"
+
+    class << self
+      # Build all area and section concepts and add them to the collection.
+      #
+      # @param collection [Glossarist::ManagedConceptCollection]
+      # @return [void]
+      def add_to(collection)
+        Iev.subject_areas.each do |area|
+          area_mc = build_area_concept(area)
+          collection.store(area_mc)
+
+          area.sections.each do |section|
+            section_mc = build_section_concept(section, area)
+            collection.store(section_mc)
+          end
+        end
+      end
+
+      private
+
+      def domain_ref(concept_id)
+        Glossarist::ConceptReference.new(
+          concept_id: concept_id,
+          source: IEV_SOURCE,
+          ref_type: "domain",
+        )
+      end
+
+      def build_area_concept(area)
+        id = area.uri
+
+        mc = Glossarist::ManagedConcept.new(
+          data: Glossarist::ManagedConceptData.new(
+            id: id,
+            domains: [domain_ref(id)],
+          ),
+        )
+        mc.uuid = id
+
+        mc.add_localization(build_localization(id, area.title, "eng"))
+        mc.related = area.sections.map { |s| build_narrower_relation(s.uri) }
+        mc.related = nil if mc.related.empty?
+
+        mc
+      end
+
+      def build_section_concept(section, area)
+        id = section.uri
+
+        mc = Glossarist::ManagedConcept.new(
+          data: Glossarist::ManagedConceptData.new(
+            id: id,
+            domains: [
+              domain_ref(area.uri),
+              domain_ref(id),
+            ],
+          ),
+        )
+        mc.uuid = id
+
+        cd = build_concept_data(id, section.title, "eng")
+        cd.domain = area.uri
+
+        mc.add_localization(build_localization_from_data(id, cd))
+
+        mc.related = [build_broader_relation(area.uri)]
+
+        mc
+      end
+
+      def build_concept_data(id, title, lang_code)
+        Glossarist::ConceptData.new(
+          id: id,
+          language_code: lang_code,
+          terms: [
+            Glossarist::Designation::Expression.new(
+              type: "expression",
+              designation: title,
+              normative_status: "preferred",
+            ),
+          ],
+        )
+      end
+
+      def build_localization(id, title, lang_code)
+        cd = build_concept_data(id, title, lang_code)
+
+        l10n = Glossarist::LocalizedConcept.new
+        l10n.data = cd
+        l10n.id = id
+        l10n.entry_status = "valid"
+        l10n.data.review_decision_event = "published"
+        l10n
+      end
+
+      def build_localization_from_data(id, concept_data)
+        l10n = Glossarist::LocalizedConcept.new
+        l10n.data = concept_data
+        l10n.id = id
+        l10n.entry_status = "valid"
+        l10n.data.review_decision_event = "published"
+        l10n
+      end
+
+      def build_broader_relation(target_uri)
+        Glossarist::RelatedConcept.new(
+          type: "broader",
+          content: target_uri,
+          ref: Glossarist::ConceptRef.new(source: "IEV", id: target_uri),
+        )
+      end
+
+      def build_narrower_relation(target_uri)
+        Glossarist::RelatedConcept.new(
+          type: "narrower",
+          content: target_uri,
+          ref: Glossarist::ConceptRef.new(source: "IEV", id: target_uri),
+        )
+      end
+    end
+  end
+end

data/lib/iev/subject_areas.rb

@@ -0,0 +1,273 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "nokogiri"
+require "fileutils"
+require "iev/config"
+
+module Iev
+  module SubjectAreas
+    DATA_FILE = File.expand_path("../../data/subject_areas.yaml", __dir__)
+
+    AREAS_URL = "https://electropedia.org/iev/iev.nsf/" \
+                "6d6bdd8667c378f7c12581fa003d80e7?OpenForm"
+    SECTIONS_URL_TEMPLATE = "https://electropedia.org/iev/iev.nsf/" \
+                            "index?openform&part=%<part>s"
+
+    MIN_PAGE_SIZE = 15_000
+
+    FETCH_DELAY = 5
+    RETRY_DELAY = 30
+    MAX_RETRIES = 2
+
+    class FetchError < StandardError; end
+
+    class << self
+      # --- URI scheme ---
+
+      # URI for a subject area concept.
+      # @param code [String, Integer] e.g. "102"
+      # @return [String] e.g. "area-102"
+      def area_uri(code)
+        "area-#{code}"
+      end
+
+      # URI for a section concept.
+      # @param code [String] e.g. "103-01"
+      # @return [String] e.g. "section-103-01"
+      def section_uri(code)
+        "section-#{code}"
+      end
+
+      # --- Query API (returns typed objects) ---
+
+      # Return all subject areas with their sections.
+      # @return [Array<SubjectArea>]
+      def all
+        @typed_areas ||= raw_data["areas"].map { |h| build_area(h) }
+      end
+
+      # Find a single subject area by its numeric code. O(1) indexed.
+      # @param code [String, Integer] e.g. "102" or 102
+      # @return [SubjectArea, nil]
+      def find_area(code)
+        area_index[code.to_s]
+      end
+
+      # Return all sections for a given area code.
+      # @param code [String, Integer] area code, e.g. "102"
+      # @return [Array<Section>]
+      def sections_for(code)
+        find_area(code)&.sections || []
+      end
+
+      # Find a single section by its section code. O(1) indexed.
+      # @param section_code [String] e.g. "102-01"
+      # @return [Section, nil]
+      def find_section(section_code)
+        section_index[section_code.to_s]
+      end
+
+      # Return the parent area for a given section code.
+      # @param section_code [String] e.g. "102-01"
+      # @return [SubjectArea, nil]
+      def area_for_section(section_code)
+        sec = find_section(section_code)
+        sec ? find_area(sec.area_code) : nil
+      end
+
+      # --- Navigation from IEV reference ---
+
+      # Find the subject area for any IEV reference.
+      # @param ievref [String] e.g. "103-01-02"
+      # @return [SubjectArea, nil]
+      def area_for(ievref)
+        code = IevCode.new(ievref)
+        find_area(code.area_code)
+      end
+
+      # Find the section for any IEV reference.
+      # @param ievref [String] e.g. "103-01-02"
+      # @return [Section, nil]
+      def section_for(ievref)
+        code = IevCode.new(ievref)
+        code.section_code ? find_section(code.section_code) : nil
+      end
+
+      # --- Fetching (network, writes to bundled data file) ---
+
+      def fetch
+        cached = read_cache("subject_areas.yaml")
+        return cached if cached && complete?(cached)
+
+        areas = cached ? cached["areas"] : []
+        fresh_areas = fetch_areas
+        puts "Found #{fresh_areas.length} areas (#{areas.length} cached)" if $stdout.tty?
+
+        # Merge: keep existing sections, add new areas
+        existing = areas.each_with_object({}) { |a, h| h[a["code"]] = a }
+        fresh_areas.each do |fa|
+          existing[fa["code"]] ||= fa
+        end
+        areas = fresh_areas.map { |fa| existing[fa["code"]] || fa }
+
+        areas.each_with_index do |area, i|
+          next if area["fetched"]
+
+          begin
+            area["sections"] = fetch_sections(area["code"])
+            area["fetched"] = true
+          rescue FetchError
+            area["sections"] ||= []
+            warn "IEV: Skipping area #{area["code"]} due to WAF"
+          end
+
+          puts "[#{i + 1}/#{areas.length}] #{area["code"]}: #{area["title"]} — #{area["sections"].length} sections" if $stdout.tty?
+
+          # Save progress every 10 areas so partial results survive WAF failures
+          if (i + 1) % 10 == 0
+            write_cache("subject_areas.yaml", { "areas" => areas })
+          end
+
+          sleep FETCH_DELAY unless i == areas.length - 1
+        end
+
+        result = { "areas" => areas }
+        write_cache("subject_areas.yaml", result)
+        result
+      end
+
+      def fetch_areas
+        html = fetch_page_with_retry(AREAS_URL)
+        doc = Nokogiri::HTML(html)
+
+        areas = []
+        doc.css("a").each do |link|
+          href = link["href"].to_s
+          next unless href.include?("part=")
+
+          code = href.match(/part=(\d+)/)&.[](1)
+          next unless code
+
+          title = link.text.strip
+          next if title.empty?
+
+          areas << { "code" => code, "title" => title, "sections" => [] }
+        end
+
+        areas.uniq { |a| a["code"] }
+      end
+
+      def fetch_sections(part)
+        url = format(SECTIONS_URL_TEMPLATE, part: part)
+        html = fetch_page_with_retry(url)
+        doc = Nokogiri::HTML(html)
+
+        sections = []
+        doc.css("td").each do |td|
+          text = td.text.strip
+          if (m = text.match(/\ASection\s+([\d-]+):\s*(.+)\z/))
+            sections << { "code" => m[1], "title" => m[2].strip }
+          end
+        end
+
+        sections.uniq { |s| s["code"] }
+      end
+
+      # Clear cached typed objects (useful after fetch updates raw data).
+      def reload!
+        @typed_areas = nil
+        @area_index = nil
+        @section_index = nil
+        @raw_data = nil
+      end
+
+      private
+
+      def build_area(hash)
+        area_code = hash["code"]
+        sections = (hash["sections"] || []).map do |s|
+          Section.new(code: s["code"], title: s["title"], area_code: area_code)
+        end
+
+        SubjectArea.new(
+          code: area_code,
+          title: hash["title"],
+          sections: sections,
+        )
+      end
+
+      def raw_data
+        @raw_data ||= begin
+          path = File.exist?(DATA_FILE) ? DATA_FILE : nil
+          if path
+            YAML.safe_load(File.read(path, encoding: "utf-8")) || { "areas" => [] }
+          else
+            { "areas" => [] }
+          end
+        end
+      end
+
+      def area_index
+        @area_index ||= all.each_with_object({}) { |a, h| h[a.code] = a }
+      end
+
+      def section_index
+        @section_index ||= all.each_with_object({}) do |area, h|
+          area.sections.each { |s| h[s.code] = s }
+        end
+      end
+
+      def complete?(data)
+        areas = data["areas"]
+        return false unless areas&.length&.>= 99
+
+        areas.all? { |a| a["fetched"] == true }
+      end
+
+      def captcha_page?(html)
+        html.length < MIN_PAGE_SIZE ||
+          html.include?("Confirm you are human") ||
+          html.include?("solve a puzzle") ||
+          html.include?("security check before continuing")
+      end
+
+      def fetch_page_with_retry(url, retries: MAX_RETRIES)
+        require "iev/scraper/browser"
+
+        retries.times do |attempt|
+          html = ScraperBrowser.fetch(url)
+          raise FetchError, "Failed to fetch #{url}" unless html
+
+          unless captcha_page?(html)
+            return html
+          end
+
+          if attempt < retries - 1
+            wait = RETRY_DELAY * (attempt + 1)
+            warn "IEV: WAF challenge for #{url}, retrying in #{wait}s (attempt #{attempt + 1}/#{retries})"
+            sleep wait
+          else
+            raise FetchError, "WAF challenge for #{url}"
+          end
+        end
+      end
+
+      def read_cache(filename)
+        cache_path = File.join(Iev.config.cache_dir, filename)
+        return nil unless File.exist?(cache_path)
+
+        d = YAML.safe_load(File.read(cache_path, encoding: "utf-8"))
+        return nil unless d&.dig("areas")&.any?
+
+        d
+      end
+
+      def write_cache(filename, d)
+        cache_path = File.join(Iev.config.cache_dir, filename)
+        FileUtils.mkdir_p(File.dirname(cache_path))
+        File.write(cache_path, YAML.dump(d), encoding: "utf-8")
+      end
+    end
+  end
+end

data/lib/iev/supersession_parser.rb

@@ -54,10 +54,9 @@ module Iev
     def relation_from_match(match_data)
       Glossarist::RelatedConcept.new(
         type: "supersedes",
-        ref: Glossarist::Citation.new(
+        ref: Glossarist::ConceptRef.new(
           source: "IEV",
           id: match_data[:ref],
-          version: match_data[:version],
         ),
       )
     end

data/lib/iev/term_builder.rb

@@ -77,6 +77,9 @@ module Iev
       cd.notes = extract_notes
       cd.terms = extract_terms
 
+      domain = extract_domain
+      cd.domain = domain if domain
+
       sources = extract_authoritative_source
       cd.sources = sources if sources&.any?
 
@@ -98,6 +101,22 @@ module Iev
       @term_language ||= find_value_for("LANGUAGE").to_three_char_code
     end
 
+    # Derives the domain (subject area section) from the IEVREF identifier.
+    # IEVREF format: "AAA-BB-CC" where AAA = area, AAA-BB = section.
+    # Returns a URI reference to the section concept (e.g. "section-103-01").
+    def extract_domain
+      return nil unless term_id
+
+      section_code = term_id.split("-")[0..1].join("-")
+      section = Iev.find_section(section_code)
+      return SubjectAreas.section_uri(section_code) if section
+
+      area_code = term_id.split("-")[0]
+      SubjectAreas.area_uri(area_code)
+    rescue StandardError
+      nil
+    end
+
     # Splits unified definition (from the spreadsheet) into separate
     # definition, examples, and notes strings (for YAMLs).
     #

data/lib/iev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Iev
-  VERSION = "0.4.3"
+  VERSION = "0.4.5"
 end

data/lib/iev.rb

@@ -29,11 +29,16 @@ module Iev
   autoload :DataSource, "iev/data_source"
   autoload :DbWriter, "iev/db_writer"
   autoload :Exporter, "iev/exporter"
+  autoload :IevCode, "iev/iev_code"
   autoload :Iso639Code, "iev/iso_639_code"
   autoload :Profiler, "iev/profiler"
   autoload :RelatonDb, "iev/relaton_db"
   autoload :Scraper, "iev/scraper"
+  autoload :Section, "iev/section"
   autoload :SourceParser, "iev/source_parser"
+  autoload :SubjectArea, "iev/subject_area"
+  autoload :SubjectAreas, "iev/subject_areas"
+  autoload :SubjectAreaConcepts, "iev/subject_area_concepts"
   autoload :SupersessionParser, "iev/supersession_parser"
   autoload :TermAttrsParser, "iev/term_attrs_parser"
   autoload :TermBuilder, "iev/term_builder"
@@ -80,4 +85,45 @@ module Iev
   def self.scrape_concept(code)
     Scraper.new.fetch_concept(code)
   end
+
+  # Return all IEV subject areas with their sections (from bundled data).
+  # @return [Array<SubjectArea>]
+  def self.subject_areas
+    SubjectAreas.all
+  end
+
+  # Find a subject area by code.
+  # @param code [String, Integer] e.g. "102"
+  # @return [SubjectArea, nil]
+  def self.find_subject_area(code)
+    SubjectAreas.find_area(code)
+  end
+
+  # Find a section by its section code.
+  # @param section_code [String] e.g. "102-01"
+  # @return [Section, nil]
+  def self.find_section(section_code)
+    SubjectAreas.find_section(section_code)
+  end
+
+  # Return sections for a given area code.
+  # @param code [String, Integer] e.g. "102"
+  # @return [Array<Section>]
+  def self.sections_for(code)
+    SubjectAreas.sections_for(code)
+  end
+
+  # Return the parent subject area for a given section code.
+  # @param section_code [String] e.g. "102-01"
+  # @return [SubjectArea, nil]
+  def self.area_for_section(section_code)
+    SubjectAreas.area_for_section(section_code)
+  end
+
+  # Parse an IEV code into its structural components.
+  # @param code [String] e.g. "103-01-02"
+  # @return [IevCode, nil] nil if the code is blank
+  def self.parse_code(code)
+    IevCode.parse(code)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iev
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.4.5
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-03 00:00:00.000000000 Z
+date: 2026-05-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: creek
@@ -28,16 +28,22 @@ dependencies:
   name: glossarist
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.6.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.6.7
 - !ruby/object:Gem::Dependency
   name: ferrum
   requirement: !ruby/object:Gem::Requirement
@@ -191,6 +197,7 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- data/subject_areas.yaml
 - exe/iev
 - iev.gemspec
 - lib/iev.rb
@@ -205,13 +212,19 @@ files:
 - lib/iev/data_source.rb
 - lib/iev/db_writer.rb
 - lib/iev/exporter.rb
+- lib/iev/iev_code.rb
 - lib/iev/iso_639_2.yaml
 - lib/iev/iso_639_code.rb
 - lib/iev/profiler.rb
 - lib/iev/relaton_db.rb
 - lib/iev/scraper.rb
+- lib/iev/scraper/browser.rb
 - lib/iev/scraper/page_parser.rb
+- lib/iev/section.rb
 - lib/iev/source_parser.rb
+- lib/iev/subject_area.rb
+- lib/iev/subject_area_concepts.rb
+- lib/iev/subject_areas.rb
 - lib/iev/supersession_parser.rb
 - lib/iev/term_attrs_parser.rb
 - lib/iev/term_builder.rb