iev 0.4.0 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 382780f439ab289a11d44053962bb7dbf043688fd68cb2328b8746429c19c6d3
-  data.tar.gz: 72151b973c479bfc82e61b3d0da4648f11ac54e9344b8c71e3bbf5d91a3a3c3b
+  metadata.gz: 5100fcef07e984496be43178ab8ebaf48fb6b77a2144ccd722a1d7acc1135e57
+  data.tar.gz: 724f02e6431f8a3534a1f28e6894dd5462d65ad67ce736695b8e5f8c1b7f04ad
 SHA512:
-  metadata.gz: d36fad5c1d853d39ffa5d8d67e98ecaedd3c40206237706ccb177ff14957ebea9616420bd41878b84ac34cdab1e9c5c7890143024b0a3070fb6c8a994d769518
-  data.tar.gz: 8c5e7f0a9510fd562fc92db4ba225e5f9b7580e0e703525ec062b3d404ab57c804da824f94af0e3047738033577aaae8394970d47c349b1ac6b1979f7fbfa02d
+  metadata.gz: c239187ba2d7d06645dbebca3478bda513dd5e6da8937ff4c9fc51be1647bf2866338d6b79f865e9b3462359b45dce67999e78e8d0cd486c2264c3d3060b6abc
+  data.tar.gz: 88aded4d4dcedc76afb3160f7adb7f1746088d250a34750bb681bb69bc8c4f3d025b8d2f4c2d50df4aa66dfb4543e8699c9261ef5a3fd31ee84a6f5600828d52

data/.github/workflows/rake.yml

@@ -1,7 +1,7 @@
 name: rake
 
 permissions:
-  contents: read
+  contents: write
 
 on:
   push:

data/README.adoc

@@ -85,19 +85,21 @@ Electropedia administrator.
 Iev.get("103-01-02", "en")
 => "functional"
 
-# If code not found
+# If code not found, returns nil (does not raise)
 Iev.get("111-11-11", "en")
-=> ""
+=> nil
 
-# If language not found
+# If language not found, returns nil
 Iev.get("103-01-02", "eee")
 => nil
 
 # Fetch full concept data (all languages)
+# Raises Iev::DataSource::NotFoundError if code not found
 Iev.fetch_concept("103-01-02")
 => { "id" => "103-01-02", "data" => { ... } }
 
 # Fetch localized term data
+# Raises Iev::DataSource::NotFoundError if code not found
 Iev.fetch_term("103-01-02", "en")
 => { "term" => "functional", ... }
 ----

data/lib/iev/cli/command.rb

@@ -8,6 +8,11 @@ module Iev
     class Command < Thor
       include CommandHelper
 
+      desc "version", "Show iev gem version"
+      def version
+        puts "iev #{Iev::VERSION}"
+      end
+
       desc "export FILE", "Export IEV data to Glossarist YAML format"
       long_desc <<~DESC
         Exports IEV data from an Excel (.xlsx/.xls) or SQLite (.sqlite3/.sqlite/.db)
@@ -34,20 +39,30 @@ module Iev
                              desc: "Enables debug messages about authoritative sources recognition"
       option :debug_relaton, type: :boolean, default: false,
                              desc: "Enables debug messages about Relaton integration"
+      option :relaton, type: :boolean, default: false,
+                       desc: "Fetch source URLs via Relaton (slow, makes network requests)"
       def export(file)
         handle_generic_options(options)
 
-        Iev::Exporter.new(
+        exporter = Iev::Exporter.new(
           file,
           output_dir: options[:output],
           only_concepts: options[:only_concepts],
           only_languages: options[:only_languages],
-        ).export
-
-        info "Done!"
+          fetch_relaton_links: options[:relaton],
+          on_progress: method(:export_progress),
+        )
+        exporter.export
+        print_export_summary(exporter.stats)
+      rescue ArgumentError => e
+        error e.message
+        exit 1
+      rescue Sequel::Error => e
+        error "Database error: #{e.message}"
+        exit 1
       end
 
-      desc "xlsx2yaml FILE", "Converts Excel IEV exports to YAMLs."
+      desc "xlsx2yaml FILE", "[DEPRECATED] Use 'export' instead."
       option :output, desc: "Output directory", aliases: :o, default: Dir.pwd
       option :only_concepts,
              desc: "Only process concepts with IEVREF matching this argument, " \
@@ -66,6 +81,7 @@ module Iev
       option :debug_sources, type: :boolean, default: false
       option :debug_relaton, type: :boolean, default: false
       def xlsx2yaml(file)
+        warn "[DEPRECATED] 'xlsx2yaml' is deprecated. Use 'export' instead."
         handle_generic_options(options)
 
         Iev::Exporter.new(
@@ -78,7 +94,7 @@ module Iev
         summary
       end
 
-      desc "db2yaml DB_FILE", "Exports SQLite to IEV YAMLs."
+      desc "db2yaml DB_FILE", "[DEPRECATED] Use 'export' instead."
       option :output, desc: "Output directory", aliases: :o, default: Dir.pwd
       option :only_concepts,
              desc: "Only process concepts with IEVREF matching this argument, " \
@@ -97,6 +113,7 @@ module Iev
       option :debug_sources, type: :boolean, default: false
       option :debug_relaton, type: :boolean, default: false
       def db2yaml(dbfile)
+        warn "[DEPRECATED] 'db2yaml' is deprecated. Use 'export' instead."
         handle_generic_options(options)
 
         Iev::Exporter.new(
@@ -135,13 +152,14 @@ module Iev
                 DataSource.fetch_concept(code)
               end
 
-        unless raw
-          warn "IEV: concept #{code} not found."
-          exit 1
-        end
-
         concept = build_concept_from_raw(code, raw)
         print_concept_grouped_yaml(concept)
+      rescue Iev::DataSource::NotFoundError
+        error "IEV concept not found: #{code}"
+        exit 1
+      rescue Ferrum::Error => e
+        error "Scraping failed: #{e.message}"
+        exit 1
       end
 
       def self.exit_on_failure?

data/lib/iev/cli/command_helper.rb

@@ -24,6 +24,36 @@ module Iev
         info "Done!"
       end
 
+      def export_progress(current, total)
+        return unless $IEV_PROGRESS
+        return if total <= 1 # single-row dataset, skip progress
+
+        if current == total
+          Ui.info "" # clear progress line
+        else
+          Ui.progress "Processing #{current}/#{total}..."
+        end
+      end
+
+      def print_export_summary(stats)
+        return unless stats
+
+        s = stats
+        elapsed = format_elapsed(s[:elapsed_seconds])
+        info "Exported #{s[:concept_count]} concepts " \
+             "(#{s[:localized_count]} localized) in #{elapsed}"
+      end
+
+      def format_elapsed(seconds)
+        if seconds < 60
+          "%.1fs" % seconds
+        else
+          mins = (seconds / 60).to_i
+          secs = (seconds % 60).round
+          "#{mins}m #{secs}s"
+        end
+      end
+
       def handle_generic_options(options)
         $IEV_PROFILE = options[:profile]
         $IEV_PROGRESS = options.fetch(:progress, !ENV["CI"])

data/lib/iev/cli/ui.rb

@@ -32,6 +32,11 @@ module Iev
         print "#{Helper.clear_progress}#{message}\n"
       end
 
+      # Prints error message to stderr.
+      def error(message)
+        Kernel.warn "Error: #{message}"
+      end
+
       # Sets an UI tag which will be prepended to messages printed with
       # #debug and #warn.
       def set_ui_tag(str)

data/lib/iev/converter/mathml_to_asciimath.rb

@@ -48,8 +48,17 @@ module Iev
           mathml_to_asciimath(input)
         end
 
+        # Clear the Plurimath expression cache. Call between export runs.
+        def clear_cache
+          @math_cache = nil
+        end
+
         private
 
+        def math_cache
+          @math_cache ||= {}
+        end
+
         def mathml_to_asciimath(input)
           return input unless input&.match?(/<|&/)
 
@@ -58,17 +67,18 @@ module Iev
           to_asciimath = Nokogiri::HTML.fragment(input, "UTF-8")
 
           to_asciimath.css("math").each do |math_element|
-            asciimath = Plurimath::Math.parse(
-              math_element.to_xml, :mathml
-            ).to_asciimath.strip
+            math_xml = math_element.to_xml
+            asciimath = math_cache[math_xml] ||= begin
+              Plurimath::Math.parse(math_xml, :mathml).to_asciimath.strip
+            rescue Plurimath::Math::ParseError
+              ""
+            end
 
             if asciimath.empty?
               math_element.remove
             else
               math_element.replace "stem:[#{asciimath}]"
             end
-          rescue Plurimath::Math::ParseError
-            math_element.remove
           end
 
           html_to_asciimath(
@@ -79,6 +89,13 @@ module Iev
         def html_to_asciimath(input)
           return input if input.nil? || input.empty?
 
+          # Fast path: if no HTML elements remain that need Nokogiri processing
+          # (after parse_anchor_tag handles <i>/<sub>/<sup>/<ol>/<ul>/<font>),
+          # just do the Greek entity replacement.
+          unless input.match?(/<([iI]|sub|sup|ol|ul|font)\b/)
+            return html_entities_to_stem(input)
+          end
+
           to_asciimath = Nokogiri::HTML.fragment(input, "UTF-8")
 
           to_asciimath.css("i").each do |math_element|

data/lib/iev/data_source.rb

@@ -12,19 +12,21 @@ module Iev
       # Fetch full concept data (all languages) for a given IEV code.
       #
       # @param code [String] IEV code, e.g. "103-01-02"
-      # @return [Hash, nil] concept data hash or nil if not found
+      # @return [Hash] concept data hash
+      # @raise [NotFoundError] if the concept does not exist
       def fetch_concept(code)
-        fetch_concept_data(code)
+        fetch_concept_data(code) ||
+          raise(NotFoundError, "IEV concept not found: #{code}")
       end
 
       # Fetch localized term data for a given IEV code and language.
       #
       # @param code [String] IEV code, e.g. "103-01-02"
       # @param lang [String] language code, e.g. "en" or "eng"
-      # @return [Hash, nil] localized concept data or nil
+      # @return [Hash, nil] localized concept data or nil if language not found
+      # @raise [NotFoundError] if the concept does not exist
       def fetch_term(code, lang)
         concept = fetch_concept(code)
-        return nil unless concept
 
         lang_key = normalize_lang(lang)
         concept[lang_key]
@@ -35,7 +37,8 @@ module Iev
       #
       # @param code [String] IEV code, e.g. "103-01-02"
       # @param lang [String] language code, e.g. "en"
-      # @return [String, nil] term designation or nil
+      # @return [String, nil] term designation or nil if not found
+      # @raise [NotFoundError] if the concept does not exist
       def fetch_term_designation(code, lang)
         term_data = fetch_term(code, lang)
         return nil unless term_data

data/lib/iev/exporter.rb

@@ -27,12 +27,18 @@ module Iev
     # @param output_dir [String, Pathname] destination for YAML files
     # @param only_concepts [String, nil] SQL LIKE pattern for IEVREF filtering
     # @param only_languages [String, nil] comma-separated language codes
+    # @param fetch_relaton_links [Boolean] fetch source URLs via Relaton
+    # @param on_progress [Proc, nil] callback (current, total) during build
     def initialize(input_path, output_dir: Dir.pwd,
-                   only_concepts: nil, only_languages: nil)
+                   only_concepts: nil, only_languages: nil,
+                   fetch_relaton_links: false,
+                   on_progress: nil)
       @input_path = Pathname.new(input_path)
       validate_input!
 
       @output_dir = Pathname.new(output_dir)
+      @fetch_relaton_links = fetch_relaton_links
+      @on_progress = on_progress
       @filters = {
         only_concepts: only_concepts,
         only_languages: only_languages,
@@ -42,12 +48,23 @@ module Iev
     # Run the export pipeline: load → transform → save.
     # @return [Glossarist::ManagedConceptCollection]
     def export
+      start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       dataset = load_dataset
       collection = build_collection(dataset)
       save_collection(collection)
+      elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
+
+      @stats = {
+        concept_count: collection.count,
+        localized_count: localized_count(collection),
+        elapsed_seconds: elapsed,
+      }
       collection
     end
 
+    # @return [Hash, nil] stats from last export, or nil if export hasn't run
+    attr_reader :stats
+
     private
 
     def supported_format?
@@ -102,15 +119,33 @@ module Iev
     end
 
     def build_collection(dataset)
-      Glossarist::ManagedConceptCollection.new.tap do |collection|
-        dataset.each do |row|
-          term = TermBuilder.build_from(row)
-          next unless term
-
-          concept = collection.fetch_or_initialize(term.id)
-          concept.add_l10n(term)
+      SourceParser.relaton_enabled = @fetch_relaton_links
+
+      # Use a hash index for O(1) concept lookup instead of
+      # Glossarist's O(n) fetch_or_initialize which does linear scan.
+      concept_index = {}
+      collection = Glossarist::ManagedConceptCollection.new
+      row_count = dataset.count
+      current = 0
+
+      dataset.each do |row|
+        current += 1
+        @on_progress&.call(current, row_count)
+
+        term = TermBuilder.build_from(row)
+        next unless term
+
+        concept = concept_index[term.id] ||= begin
+          c = Glossarist::ManagedConcept.new(data: { "id" => term.id })
+          collection.store(c)
+          c
         end
+        concept.add_l10n(term)
       end
+
+      collection
+    ensure
+      SourceParser.relaton_enabled = true
     end
 
     def save_collection(collection)
@@ -118,5 +153,9 @@ module Iev
       FileUtils.mkdir_p(concepts_dir)
       collection.save_to_files(concepts_dir.to_s)
     end
+
+    def localized_count(collection)
+      collection.sum { |c| c.localized_concepts.count }
+    end
   end
 end

data/lib/iev/source_parser.rb

@@ -14,6 +14,12 @@ module Iev
     include Utilities
     using DataConversions
 
+    # When false, obtain_source_link skips Relaton network calls.
+    @relaton_enabled = true
+    class << self
+      attr_accessor :relaton_enabled
+    end
+
     attr_reader :src_split, :parsed_sources, :raw_str, :src_str
 
     def initialize(source_str, term_domain)
@@ -89,7 +95,7 @@ module Iev
         origin: origin,
         modification: relationship[:modification],
       )
-    rescue ::RelatonBib::RequestError => e
+    rescue ::RelatonBib::RequestError, Socket::ResolutionError, SocketError => e
       warn e.message
     end
 
@@ -347,10 +353,11 @@ module Iev
 
     # Uses Relaton to obtain link for given source ref.
     def obtain_source_link(ref)
+      return nil unless self.class.relaton_enabled
       return nil unless defined?(RelatonDb)
 
       RelatonDb.instance.fetch(ref)&.url
-    rescue ::RelatonBib::RequestError => e
+    rescue ::RelatonBib::RequestError, Socket::ResolutionError, SocketError => e
       warn e.message
       nil
     end

data/lib/iev/term_builder.rb

@@ -298,6 +298,8 @@ module Iev
     end
 
     def strip_html_comments(str)
+      return str unless str.include?("<!--")
+
       doc = Nokogiri::HTML::DocumentFragment.parse(str)
       comments = doc.children.select(&:comment?)
       return str if comments.empty?

data/lib/iev/utilities.rb

@@ -13,10 +13,14 @@ module Iev
 
     def parse_anchor_tag(text, term_domain)
       return nil if text.nil?
+      return text unless text.include?("<")
 
       text = process_simg_figures(text, term_domain)
       text = fix_unquoted_href(text)
 
+      # Second check: regex substitutions may have consumed all tags
+      return text unless text.include?("<")
+
       doc = Nokogiri::HTML::DocumentFragment.parse(text)
       nodes_to_adoc(doc.children, term_domain)
     end
@@ -86,11 +90,45 @@ module Iev
         "#{IMAGE_PATH_PREFIX}/#{term_domain}/#{src}[]"
       when "p", "div", "span"
         inner
+      when "i"
+        convert_italic(inner)
+      when "sub"
+        inner.empty? ? "" : "~#{inner}~"
+      when "sup"
+        inner.empty? ? "" : "^#{inner}^"
+      when "ol"
+        convert_list(node, ". ")
+      when "ul"
+        convert_list(node, "* ")
+      when "li"
+        inner
+      when "font"
+        convert_font(node, inner)
       else
         node.to_s
       end
     end
 
+    def convert_italic(text)
+      case text.length
+      when 0
+        ""
+      when 1..12
+        "stem:[#{text}]"
+      else
+        "_#{text}_"
+      end
+    end
+
+    def convert_list(node, prefix)
+      node.css("li").map { |li| "#{prefix}#{li.text}" }.join
+    end
+
+    def convert_font(node, inner)
+      style = node["style"].to_s
+      style.include?("sans-serif") ? "`#{inner}`" : inner
+    end
+
     def convert_link(node, inner)
       href = (node["href"] || "").to_s.strip
 

data/lib/iev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Iev
-  VERSION = "0.4.0"
+  VERSION = "0.4.2"
 end

data/lib/iev.rb

@@ -45,17 +45,19 @@ module Iev
   # @param [String] lang language code, for example "en"
   #
   # @return [String, nil] if found then term,
-  # if code not found then nil,
-  #   if language not found then nil.
+  #   if code or language not found then nil.
   #
   def self.get(code, lang)
     DataSource.fetch_term_designation(code, lang)
+  rescue DataSource::NotFoundError
+    nil
   end
 
   # Fetch full concept data (all languages) for a given IEV code.
   #
   # @param [String] code IEV code, e.g. "103-01-02"
-  # @return [Hash, nil] concept data hash with all languages
+  # @return [Hash] concept data hash with all languages
+  # @raise [DataSource::NotFoundError] if concept not found
   def self.fetch_concept(code)
     DataSource.fetch_concept(code)
   end
@@ -64,7 +66,8 @@ module Iev
   #
   # @param [String] code IEV code, e.g. "103-01-02"
   # @param [String] lang language code, e.g. "en" or "eng"
-  # @return [Hash, nil] localized concept data
+  # @return [Hash, nil] localized concept data or nil if not found
+  # @raise [DataSource::NotFoundError] if concept not found
   def self.fetch_term(code, lang)
     DataSource.fetch_term(code, lang)
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iev
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-04-27 00:00:00.000000000 Z
+date: 2026-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: creek