iev 0.4.6 → 0.4.8

data/lib/iev/scraper/page_parser.rb

@@ -4,13 +4,14 @@ module Iev
   class Scraper
     # Parses an Electropedia HTML page into a concept data hash.
     #
-    # The Electropedia HTML structure is a table with rows for each language:
-    # - Language row: <div align="center"><font color="#800080">en</font></div>
-    # - Term cell: <b>term text</b> in the third <td>
-    # - Definition row: next row's third <td> (if present)
-    # - Empty/separator rows with <hr> or spacer images
+    # All extracted text (terms, definitions, notes, examples) is run
+    # through Iev::Utilities' semantic enrichment pipeline, converting
+    # HTML to AsciiDoc notation: <i>x</i> → stem:[x], <b>x</b> → *x*,
+    # <a href=IEV...> → {{urn:...}}, MathML → AsciiMath. This ensures
+    # the parsed output matches the format used in termbase.yaml.
     class PageParser
-      # Map Electropedia HTML language codes to ISO 639-2/3 three-char codes.
+      include Utilities
+
       LANG_CODE_MAP = {
         "en" => "eng",
         "fr" => "fra",
@@ -54,28 +55,48 @@ module Iev
       private
 
       def find_iev_ref
-        # Find the IEV reference cell to confirm the page is valid
         @doc.at_css("b:contains('#{@code}')") ||
           @doc.at_xpath("//td/b[contains(text(), '#{@code}')]")
       end
 
+      def term_domain
+        @code.rpartition("-").first
+      end
+
       def localized_concepts
         result = {}
-        lang_sections.each do |lang, term_row, def_row|
+        lang_sections.each do |lang, area, term_row, def_row|
           term = extract_term(term_row)
           next unless term
 
-          entry = { "term" => term }
+          entry = result[lang] || { "terms" => [] }
+          term_entry = { "designation" => term }
+          term_entry["geographical_area"] = area if area
+          entry["terms"] << term_entry
+
           definition = extract_definition(def_row)
-          entry["definition"] = definition if definition
+          entry["definition"] = definition if definition && !entry.key?("definition")
 
           result[lang] = entry
         end
+
+        # Convert terms array to single "term" field for backward compat
+        # when there's only one term and no area
+        result.each_value do |entry|
+          terms = entry.delete("terms")
+          if terms.size == 1 && !terms.first["geographical_area"]
+            entry["term"] = terms.first["designation"]
+          else
+            entry["term"] = terms.map { |t| t["designation"] }.join("\n")
+            entry["term_areas"] = terms.each_with_object({}) do |t, h|
+              area = t["geographical_area"]
+              h[t["designation"]] = area if area
+            end
+          end
+        end
         result
       end
 
-      # Finds all language sections in the table.
-      # Returns array of [lang_code, term_row, definition_row] tuples.
       def lang_sections
         sections = []
         rows = content_rows
@@ -84,17 +105,30 @@ module Iev
           lang = extract_lang(row)
           next unless lang
 
-          # The definition is in the next non-empty, non-separator row
+          area = extract_area(row)
           def_row = find_definition_row(rows, idx + 1)
-          sections << [lang, row, def_row]
+          sections << [lang, area, row, def_row]
         end
 
         sections
       end
 
+      def extract_area(row)
+        tds = row.css("td")
+        return nil if tds.length < 2
+
+        area_cell = tds[1]
+        font = area_cell.at_css("font[color='#800080']")
+        return nil unless font
+
+        text = font.text.strip.downcase
+        return nil if text.empty?
+        return nil if text.match?(/\A(nl|be|nb|nn|br|pt|fr|de|us|gb|ch|at|ca|au|nz)\z/i) == false
+
+        text
+      end
+
       def content_rows
-        # Find the main content table (the one with language data)
-        # It's the largest table with IEV data
         tables = @doc.css("table")
         content_table = tables.max_by { |t| t.css("tr").length }
         content_table ? content_table.css("tr").to_a : []
@@ -109,15 +143,30 @@ module Iev
       end
 
       def extract_term(row)
-        # Term is in the third <td> — may be in a <b> tag (en, fr) or plain text
         tds = row.css("td")
         return nil if tds.length < 3
 
         content_td = tds[2]
+        return nil if area_marker_cell?(content_td)
+
         bold = content_td.at_css("b")
 
-        term = bold ? bold.text.strip : content_td.text.strip
-        term.empty? ? nil : term
+        html = if bold
+                 bold.inner_html.strip
+               else
+                 content_td.inner_html.strip
+               end
+        return nil if html.empty?
+
+        enrich(html)
+      end
+
+      # True when the cell contains only a geographical-area marker font
+      # tag (e.g. <font color="#800080">nb</font>), not a real term.
+      def area_marker_cell?(cell)
+        font = cell.at_css("font[color='#800080']")
+        return false unless font
+        cell.at_css("b").nil? && cell.text.strip == font.text.strip && font.text.strip.length <= 3
       end
 
       def extract_definition(row)
@@ -127,15 +176,59 @@ module Iev
         return nil if tds.length < 3
 
         content_td = tds[2]
-        # The definition is the text content, which may include MathML
         html = content_td.inner_html.strip
         return nil if html.empty? || html.match?(/\A<img.*ecblank/)
 
-        html
+        enrich(html)
+      end
+
+      def enrich(html_text)
+        text = html_text.to_s
+          .gsub(/&lt;i&gt;/, "<i>")
+          .gsub(/&lt;\/i&gt;/, "</i>")
+        mathml_blocks, text = extract_mathml_blocks(text)
+        result = parse_anchor_tag(text, term_domain)
+        result = restore_and_convert_mathml(result, mathml_blocks)
+        result = replace_newlines(result)
+        result = convert_literal_italic(result)
+        result.strip
+      end
+
+      MATHML_PLACEHOLDER = "@@MATHML_%d@@"
+
+      def extract_mathml_blocks(text)
+        blocks = []
+        result = text.gsub(/<math[^>]*>.*?<\/math>/m) do |match|
+          blocks << match
+          format(MATHML_PLACEHOLDER, blocks.size - 1)
+        end
+        [blocks, result]
+      end
+
+      # Restore MathML blocks and convert each to AsciiMath individually.
+      # If the converter returns empty (unsupported expression like
+      # <mtable>), keep the original MathML so the formula is not lost.
+      def restore_and_convert_mathml(text, blocks)
+        return text if blocks.empty?
+
+        blocks.each_with_index do |mathml, i|
+          converted = Iev::Converter.mathml_to_asciimath(mathml).strip
+          replacement = converted.empty? ? mathml : converted
+          text = text.sub(format(MATHML_PLACEHOLDER, i), replacement)
+        end
+        text
+      end
+
+      # After Nokogiri processing, &lt;i&gt;entity-encoded tags appear as
+      # literal <i>...</i> text. Convert them to stem:[...] just like
+      # real <i> elements are converted by convert_italic.
+      def convert_literal_italic(text)
+        text.gsub(/<i>([^<]*)<\/i>/) do
+          inner = Regexp.last_match(1)
+          convert_italic(inner)
+        end
       end
 
-      # Find the definition row following a language row.
-      # Skip separator rows (empty, <hr>, or spacer images).
       def find_definition_row(rows, start_idx)
         return nil if start_idx >= rows.length
 
@@ -149,7 +242,6 @@ module Iev
         content = tds[2].inner_html.strip
         return nil if content.empty?
 
-        # Skip rows that are only spacer images (unless they have <b> content)
         if content.match?(/\A<img.*ecblank/) && !content.include?("<b>")
           return nil
         end

data/lib/iev/source_parser.rb

@@ -406,7 +406,18 @@ module Iev
       return nil unless item
 
       item.source("src")
-    rescue Relaton::RequestError, Socket::ResolutionError, SocketError => e
+      # Free-text refs like "ITU-R Rec. 431 MOD" are not parseable pubids.
+      # relaton 3 (pubid 2) raises Pubid::Errors::ParseError (a
+      # Parslet::ParseFailed) eagerly when building the search, whereas
+      # relaton 2 swallowed it. Link resolution is best-effort: warn and
+      # skip instead of failing the whole source parse.
+      # relaton 3.0.0.pre.alpha.6 (#205 strict routing) also raises
+      # UnknownReferenceError for spellings no flavor claims (the French
+      # "CEI", agency refs like "IAEA 4") where alpha.5's permissive
+      # parse answered nil. Same best-effort contract: warn and skip.
+    rescue Relaton::RequestError, Relaton::UnknownReferenceError,
+           Socket::ResolutionError, SocketError,
+           Parslet::ParseFailed => e
       warn e.message
       nil
     end

data/lib/iev/subject_area_concepts.rb

@@ -128,10 +128,12 @@ module Iev
 
       # --- RelatedConcept factory methods ---
 
+      # RelatedConcept#content is a lang-keyed hash since glossarist 2.13
+      # (L10N invariant: all relationship free-text is { "eng" => ... }).
       def broader_relation(target_uri)
         Glossarist::RelatedConcept.new(
           type: "broader",
-          content: target_uri,
+          content: { "eng" => target_uri },
           ref: Glossarist::ConceptRef.new(source: "IEV", id: target_uri),
         )
       end
@@ -139,7 +141,7 @@ module Iev
       def narrower_relation(target_uri)
         Glossarist::RelatedConcept.new(
           type: "narrower",
-          content: target_uri,
+          content: { "eng" => target_uri },
           ref: Glossarist::ConceptRef.new(source: "IEV", id: target_uri),
         )
       end

data/lib/iev/utilities.rb

@@ -92,7 +92,9 @@ module Iev
       when "img"
         src = node["src"] || node.attributes.keys.first.to_s
         "#{IMAGE_PATH_PREFIX}/#{term_domain}/#{src}[]"
-      when "p", "div", "span"
+      when "p"
+        "#{inner}\n"
+      when "div", "span"
         inner
       when "i"
         convert_italic(inner)
@@ -101,9 +103,9 @@ module Iev
       when "sup"
         inner.empty? ? "" : "^#{inner}^"
       when "ol"
-        convert_list(node, ". ")
+        convert_list(node, ". ", term_domain)
       when "ul"
-        convert_list(node, "* ")
+        convert_list(node, "* ", term_domain)
       when "li"
         inner
       when "font"
@@ -124,8 +126,8 @@ module Iev
       end
     end
 
-    def convert_list(node, prefix)
-      node.css("li").map { |li| "#{prefix}#{li.text}" }.join
+    def convert_list(node, prefix, term_domain)
+      node.css("li").map { |li| "#{prefix}#{nodes_to_adoc(li.children, term_domain)}" }.join("\n")
     end
 
     def convert_font(node, inner)

data/lib/iev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Iev
-  VERSION = "0.4.6"
+  VERSION = "0.4.8"
 end

data/lib/iev.rb

@@ -21,6 +21,7 @@ module Iev
   # IEV dataset URN — single source of truth for all concept references.
   IEV_SOURCE = "urn:iec:std:iec:60050"
 
+  autoload :BibliographyBuilder, "iev/bibliography_builder"
   autoload :Cli, "iev/cli"
   autoload :Config, "iev/config"
   autoload :Converter, "iev/converter"
@@ -28,9 +29,12 @@ module Iev
   autoload :DataSource, "iev/data_source"
   autoload :DbWriter, "iev/db_writer"
   autoload :Exporter, "iev/exporter"
+  autoload :FigureBuilder, "iev/figure_builder"
   autoload :IevCode, "iev/iev_code"
   autoload :Iso639Code, "iev/iso_639_code"
+  autoload :MultiDocYaml, "iev/multi_doc_yaml"
   autoload :Profiler, "iev/profiler"
+  autoload :Reconciler, "iev/reconciler"
   autoload :RelatonDb, "iev/relaton_db"
   autoload :Scraper, "iev/scraper"
   autoload :Section, "iev/section"

data/scripts/find_unparseable.rb

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+# Identify cached HTML pages that PageParser cannot turn into a concept.
+# Run: bundle exec ruby -Ilib scripts/find_unparseable.rb
+require "iev"
+
+store = Iev::Fetcher::PageStore.new
+ok = 0
+failed = []
+store.each_concept(scope: Iev::Fetcher::Scope.all).each do |code, html|
+  doc = Nokogiri::HTML(html)
+  if Iev::Scraper::PageParser.new(doc, code).parse
+    ok += 1
+  else
+    failed << code
+  end
+end
+
+puts "Parsed OK:  #{ok}"
+puts "Failed:     #{failed.size}"
+puts
+puts "First 20 failed codes:"
+failed.take(20).each { |c| puts "  #{c}" }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iev
 version: !ruby/object:Gem::Version
-  version: 0.4.6
+  version: 0.4.8
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-06-13 00:00:00.000000000 Z
+date: 2026-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: creek
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.8.2
+        version: 2.8.15
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.8.2
+        version: 2.8.15
 - !ruby/object:Gem::Dependency
   name: lutaml-model
   requirement: !ruby/object:Gem::Requirement
@@ -103,7 +103,7 @@ dependencies:
         version: 2.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -113,7 +113,21 @@ dependencies:
         version: 2.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +192,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".claude/scheduled_tasks.lock"
 - ".github/workflows/rake.yml"
 - ".github/workflows/release.yml"
 - ".gitignore"
@@ -189,12 +204,30 @@ files:
 - LICENSE.txt
 - README.adoc
 - Rakefile
+- TODO.reconcile/01-design.md
+- TODO.reconcile/02-change-models.md
+- TODO.reconcile/02-termbase-loader.md
+- TODO.reconcile/03-live-loader.md
+- TODO.reconcile/03-status-mapper.md
+- TODO.reconcile/04-concept-merger.md
+- TODO.reconcile/04-termbase-loader.md
+- TODO.reconcile/05-content-diff.md
+- TODO.reconcile/05-live-loader.md
+- TODO.reconcile/06-content-differ.md
+- TODO.reconcile/06-v3-serializer.md
+- TODO.reconcile/07-cli-and-run.md
+- TODO.reconcile/07-concept-merger.md
+- TODO.reconcile/08-report.md
+- TODO.reconcile/09-pipeline.md
+- TODO.reconcile/10-script-and-run.md
 - bin/console
 - bin/setup
+- data/locales/glossarist_enums.yml
 - data/subject_areas.yaml
 - exe/iev
 - iev.gemspec
 - lib/iev.rb
+- lib/iev/bibliography_builder.rb
 - lib/iev/cli.rb
 - lib/iev/cli/command.rb
 - lib/iev/cli/command_helper.rb
@@ -206,10 +239,24 @@ files:
 - lib/iev/data_source.rb
 - lib/iev/db_writer.rb
 - lib/iev/exporter.rb
+- lib/iev/figure_builder.rb
 - lib/iev/iev_code.rb
 - lib/iev/iso_639_2.yaml
 - lib/iev/iso_639_code.rb
+- lib/iev/multi_doc_yaml.rb
 - lib/iev/profiler.rb
+- lib/iev/reconciler.rb
+- lib/iev/reconciler/change.rb
+- lib/iev/reconciler/change_set.rb
+- lib/iev/reconciler/concept_merger.rb
+- lib/iev/reconciler/content_differ.rb
+- lib/iev/reconciler/live_loader.rb
+- lib/iev/reconciler/pipeline.rb
+- lib/iev/reconciler/reconciled_concept.rb
+- lib/iev/reconciler/report.rb
+- lib/iev/reconciler/status_mapper.rb
+- lib/iev/reconciler/term_marker_parser.rb
+- lib/iev/reconciler/termbase_loader.rb
 - lib/iev/relaton_db.rb
 - lib/iev/scraper.rb
 - lib/iev/scraper/browser.rb
@@ -224,6 +271,7 @@ files:
 - lib/iev/term_builder.rb
 - lib/iev/utilities.rb
 - lib/iev/version.rb
+- scripts/find_unparseable.rb
 homepage: https://github.com/glossarist/iev
 licenses:
 - BSD-2-Clause